codemining.js.codeutils.JavascriptTokenizer Java Examples

The following examples show how to use codemining.js.codeutils.JavascriptTokenizer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AbstractJavascriptNameBindingsExtractor.java    From api-mining with GNU General Public License v3.0 6 votes vote down vote up 
/**
 * Get the token bindings given the ASTNode bindings and the source code
 * positions.
 *
 * @param sourceCode
 * @param nodeBindings
 * @return
 */
public static List<TokenNameBinding> getTokenBindings(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final List<TokenNameBinding> bindings = Lists.newArrayList();

	for (final Set<ASTNode> boundName : nodeBindings) {
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.add(new TokenNameBinding(Sets.newTreeSet(boundPositions),
				tokens, getFeatures(boundName)));
	}

	return bindings;
}
Example #2
Source File: AbstractJavascriptNameBindingsExtractor.java    From tassal with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
/**
 * Get the token bindings given the ASTNode bindings and the source code
 * positions.
 *
 * @param sourceCode
 * @param nodeBindings
 * @return
 */
public static List<TokenNameBinding> getTokenBindings(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final List<TokenNameBinding> bindings = Lists.newArrayList();

	for (final Set<ASTNode> boundName : nodeBindings) {
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.add(new TokenNameBinding(Sets.newTreeSet(boundPositions),
				tokens, getFeatures(boundName)));
	}

	return bindings;
}
Example #3
Source File: AbstractJavascriptNameBindingsExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
/**
 * Get the token bindings given the ASTNode bindings and the source code
 * positions.
 *
 * @param sourceCode
 * @param nodeBindings
 * @return
 */
public static List<TokenNameBinding> getTokenBindings(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final List<TokenNameBinding> bindings = Lists.newArrayList();

	for (final Set<ASTNode> boundName : nodeBindings) {
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.add(new TokenNameBinding(Sets.newTreeSet(boundPositions),
				tokens, getFeatures(boundName)));
	}

	return bindings;
}
Example #4
Source File: AbstractJavascriptNameBindingsExtractor.java    From api-mining with GNU General Public License v3.0 5 votes vote down vote up 
public static ResolvedSourceCode getResolvedSourceCode(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final ArrayListMultimap<String, TokenNameBinding> bindings = ArrayListMultimap
			.create();

	for (final Set<ASTNode> boundName : nodeBindings) {
		if (boundName.isEmpty()) {
			continue;
		}
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.put(tokens.get(boundPositions.get(0)),
				new TokenNameBinding(Sets.newTreeSet(boundPositions),
						tokens, getFeatures(boundName)));
	}

	return new ResolvedSourceCode(tokens, bindings);
}
Example #5
Source File: AbstractJavascriptNameBindingsExtractor.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public static ResolvedSourceCode getResolvedSourceCode(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final ArrayListMultimap<String, TokenNameBinding> bindings = ArrayListMultimap
			.create();

	for (final Set<ASTNode> boundName : nodeBindings) {
		if (boundName.isEmpty()) {
			continue;
		}
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.put(tokens.get(boundPositions.get(0)),
				new TokenNameBinding(Sets.newTreeSet(boundPositions),
						tokens, getFeatures(boundName)));
	}

	return new ResolvedSourceCode(tokens, bindings);
}
Example #6
Source File: AbstractJavascriptNameBindingsExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public static ResolvedSourceCode getResolvedSourceCode(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final ArrayListMultimap<String, TokenNameBinding> bindings = ArrayListMultimap
			.create();

	for (final Set<ASTNode> boundName : nodeBindings) {
		if (boundName.isEmpty()) {
			continue;
		}
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.put(tokens.get(boundPositions.get(0)),
				new TokenNameBinding(Sets.newTreeSet(boundPositions),
						tokens, getFeatures(boundName)));
	}

	return new ResolvedSourceCode(tokens, bindings);
}