codemining.languagetools.bindings.TokenNameBinding Java Examples

The following examples show how to use codemining.languagetools.bindings.TokenNameBinding. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AbstractJavaNameBindingsExtractor.java    From tassal with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
/**
 * Get the token bindings given the ASTNode bindings and the source code
 * positions.
 *
 * @param sourceCode
 * @param nodeBindings
 * @return
 */
public List<TokenNameBinding> getTokenBindings(
		final SortedMap<Integer, String> tokenPositions,
		final Set<Set<ASTNode>> nodeBindings) {
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final List<TokenNameBinding> bindings = Lists.newArrayList();

	for (final Set<ASTNode> boundName : nodeBindings) {
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.add(new TokenNameBinding(Sets.newTreeSet(boundPositions),
				tokens, getFeatures(boundName)));
	}

	return bindings;
}
Example #2
Source File: AbstractJavascriptNameBindingsExtractor.java    From api-mining with GNU General Public License v3.0 6 votes vote down vote up 
/**
 * Get the token bindings given the ASTNode bindings and the source code
 * positions.
 *
 * @param sourceCode
 * @param nodeBindings
 * @return
 */
public static List<TokenNameBinding> getTokenBindings(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final List<TokenNameBinding> bindings = Lists.newArrayList();

	for (final Set<ASTNode> boundName : nodeBindings) {
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.add(new TokenNameBinding(Sets.newTreeSet(boundPositions),
				tokens, getFeatures(boundName)));
	}

	return bindings;
}
Example #3
Source File: JavascriptApproximateVariableBindingExtractorTest.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
@Test
public void testClassBindings() throws IOException {
	final JavascriptApproximateVariableBindingExtractor jabe = new JavascriptApproximateVariableBindingExtractor();
	final JavascriptExactVariableBindingsExtractor jbe = new JavascriptExactVariableBindingsExtractor();

	final List<TokenNameBinding> classVariableBindings = jabe
			.getNameBindings(classContent);
	final List<TokenNameBinding> classVariableBindingsExact = jbe
			.getNameBindings(classContent);

	BindingTester.checkAllBindings(classVariableBindings);
	assertEquals(classVariableBindings.size(), 3);

	final List<TokenNameBinding> classVariableBindings2 = jabe
			.getNameBindings(classContent2);
	final List<TokenNameBinding> classVariableBindings2Exact = jbe
			.getNameBindings(classContent2);

	assertEquals(classVariableBindings2.size(), 13);

	allAreContained(classVariableBindingsExact, classVariableBindings);
	allAreContained(classVariableBindings2Exact, classVariableBindings2);
}
Example #4
Source File: AbstractJavaNameBindingsExtractor.java    From api-mining with GNU General Public License v3.0 6 votes vote down vote up 
/**
 * Get the token bindings given the ASTNode bindings and the source code
 * positions.
 *
 * @param sourceCode
 * @param nodeBindings
 * @return
 */
public List<TokenNameBinding> getTokenBindings(
		final SortedMap<Integer, String> tokenPositions,
		final Set<Set<ASTNode>> nodeBindings) {
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final List<TokenNameBinding> bindings = Lists.newArrayList();

	for (final Set<ASTNode> boundName : nodeBindings) {
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.add(new TokenNameBinding(Sets.newTreeSet(boundPositions),
				tokens, getFeatures(boundName)));
	}

	return bindings;
}
Example #5
Source File: JavaMethodBindingExtractorTest.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
@Test
public void testClassLevelBindings() throws IOException {
	final JavaMethodInvocationBindingExtractor jame = new JavaMethodInvocationBindingExtractor();

	final List<TokenNameBinding> classMethodBindings = jame
			.getNameBindings(classContent);

	BindingTester.checkAllBindings(classMethodBindings);
	assertEquals(classMethodBindings.size(), 7);

	final List<TokenNameBinding> classMethodBindings2 = jame
			.getNameBindings(classContent2);
	BindingTester.checkAllBindings(classMethodBindings2);

	assertEquals(classMethodBindings2.size(), 6);
}
Example #6
Source File: AbstractJavaNameBindingsExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
/**
 * Get the token bindings given the ASTNode bindings and the source code
 * positions.
 *
 * @param sourceCode
 * @param nodeBindings
 * @return
 */
public List<TokenNameBinding> getTokenBindings(
		final SortedMap<Integer, String> tokenPositions,
		final Set<Set<ASTNode>> nodeBindings) {
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final List<TokenNameBinding> bindings = Lists.newArrayList();

	for (final Set<ASTNode> boundName : nodeBindings) {
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.add(new TokenNameBinding(Sets.newTreeSet(boundPositions),
				tokens, getFeatures(boundName)));
	}

	return bindings;
}
Example #7
Source File: AbstractJavascriptNameBindingsExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
/**
 * Get the token bindings given the ASTNode bindings and the source code
 * positions.
 *
 * @param sourceCode
 * @param nodeBindings
 * @return
 */
public static List<TokenNameBinding> getTokenBindings(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final List<TokenNameBinding> bindings = Lists.newArrayList();

	for (final Set<ASTNode> boundName : nodeBindings) {
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.add(new TokenNameBinding(Sets.newTreeSet(boundPositions),
				tokens, getFeatures(boundName)));
	}

	return bindings;
}
Example #8
Source File: JavascriptApproximateVariableBindingExtractorTest.java    From tassal with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
@Test
public void testClassBindings() throws IOException {
	final JavascriptApproximateVariableBindingExtractor jabe = new JavascriptApproximateVariableBindingExtractor();
	final JavascriptExactVariableBindingsExtractor jbe = new JavascriptExactVariableBindingsExtractor();

	final List<TokenNameBinding> classVariableBindings = jabe
			.getNameBindings(classContent);
	final List<TokenNameBinding> classVariableBindingsExact = jbe
			.getNameBindings(classContent);

	BindingTester.checkAllBindings(classVariableBindings);
	assertEquals(classVariableBindings.size(), 3);

	final List<TokenNameBinding> classVariableBindings2 = jabe
			.getNameBindings(classContent2);
	final List<TokenNameBinding> classVariableBindings2Exact = jbe
			.getNameBindings(classContent2);

	assertEquals(classVariableBindings2.size(), 13);

	allAreContained(classVariableBindingsExact, classVariableBindings);
	allAreContained(classVariableBindings2Exact, classVariableBindings2);
}
Example #9
Source File: JavaApproximateVariableBindingExtractorTest.java    From api-mining with GNU General Public License v3.0 6 votes vote down vote up 
@Test
public void testClassBindings() throws IOException {
	final JavaApproximateVariableBindingExtractor jabe = new JavaApproximateVariableBindingExtractor();
	final JavaExactVariableBindingsExtractor jbe = new JavaExactVariableBindingsExtractor();

	final List<TokenNameBinding> classVariableBindings = jabe
			.getNameBindings(classContent);
	final List<TokenNameBinding> classVariableBindingsExact = jbe
			.getNameBindings(classContent);

	BindingTester
			.checkAllBindings(classVariableBindings);
	assertEquals(classVariableBindings.size(), 5);

	final List<TokenNameBinding> classVariableBindings2 = jabe
			.getNameBindings(classContent2);
	final List<TokenNameBinding> classVariableBindings2Exact = jbe
			.getNameBindings(classContent2);

	assertEquals(classVariableBindings2.size(), 9);

	allAreContained(classVariableBindingsExact, classVariableBindings);
	allAreContained(classVariableBindings2Exact, classVariableBindings2);
}
Example #10
Source File: JavaMethodBindingExtractorTest.java    From api-mining with GNU General Public License v3.0 6 votes vote down vote up 
@Test
public void testClassLevelBindings() throws IOException {
	final JavaMethodInvocationBindingExtractor jame = new JavaMethodInvocationBindingExtractor();

	final List<TokenNameBinding> classMethodBindings = jame
			.getNameBindings(classContent);

	BindingTester.checkAllBindings(classMethodBindings);
	assertEquals(classMethodBindings.size(), 7);

	final List<TokenNameBinding> classMethodBindings2 = jame
			.getNameBindings(classContent2);
	BindingTester.checkAllBindings(classMethodBindings2);

	assertEquals(classMethodBindings2.size(), 6);
}
Example #11
Source File: JavaMethodBindingExtractorTest.java    From tassal with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
@Test
public void testClassLevelBindings() throws IOException {
	final JavaMethodInvocationBindingExtractor jame = new JavaMethodInvocationBindingExtractor();

	final List<TokenNameBinding> classMethodBindings = jame
			.getNameBindings(classContent);

	BindingTester.checkAllBindings(classMethodBindings);
	assertEquals(classMethodBindings.size(), 7);

	final List<TokenNameBinding> classMethodBindings2 = jame
			.getNameBindings(classContent2);
	BindingTester.checkAllBindings(classMethodBindings2);

	assertEquals(classMethodBindings2.size(), 6);
}
Example #12
Source File: AbstractJavascriptNameBindingsExtractor.java    From tassal with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
/**
 * Get the token bindings given the ASTNode bindings and the source code
 * positions.
 *
 * @param sourceCode
 * @param nodeBindings
 * @return
 */
public static List<TokenNameBinding> getTokenBindings(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final List<TokenNameBinding> bindings = Lists.newArrayList();

	for (final Set<ASTNode> boundName : nodeBindings) {
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.add(new TokenNameBinding(Sets.newTreeSet(boundPositions),
				tokens, getFeatures(boundName)));
	}

	return bindings;
}
Example #13
Source File: JavaApproximateVariableBindingExtractorTest.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
@Test
public void testClassBindings() throws IOException {
	final JavaApproximateVariableBindingExtractor jabe = new JavaApproximateVariableBindingExtractor();
	final JavaExactVariableBindingsExtractor jbe = new JavaExactVariableBindingsExtractor();

	final List<TokenNameBinding> classVariableBindings = jabe
			.getNameBindings(classContent);
	final List<TokenNameBinding> classVariableBindingsExact = jbe
			.getNameBindings(classContent);

	BindingTester
			.checkAllBindings(classVariableBindings);
	assertEquals(classVariableBindings.size(), 5);

	final List<TokenNameBinding> classVariableBindings2 = jabe
			.getNameBindings(classContent2);
	final List<TokenNameBinding> classVariableBindings2Exact = jbe
			.getNameBindings(classContent2);

	assertEquals(classVariableBindings2.size(), 9);

	allAreContained(classVariableBindingsExact, classVariableBindings);
	allAreContained(classVariableBindings2Exact, classVariableBindings2);
}
Example #14
Source File: JavaTypeBindingExtractorTest.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
@Test
public void testClassLevelBindings() throws IOException {
	final JavaTypeDeclarationBindingExtractor jame = new JavaTypeDeclarationBindingExtractor();

	final List<TokenNameBinding> classTypeindings = jame
			.getNameBindings(classContent);

	BindingTester.checkAllBindings(classTypeindings);
	assertEquals(classTypeindings.size(), 1);

	final List<TokenNameBinding> classTypeBindings2 = jame
			.getNameBindings(classContent2);
	BindingTester.checkAllBindings(classTypeBindings2);

	assertEquals(classTypeBindings2.size(), 1);
}
Example #15
Source File: JavaApproximateVariableBindingExtractorTest.java    From tassal with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
@Test
public void testClassBindings() throws IOException {
	final JavaApproximateVariableBindingExtractor jabe = new JavaApproximateVariableBindingExtractor();
	final JavaExactVariableBindingsExtractor jbe = new JavaExactVariableBindingsExtractor();

	final List<TokenNameBinding> classVariableBindings = jabe
			.getNameBindings(classContent);
	final List<TokenNameBinding> classVariableBindingsExact = jbe
			.getNameBindings(classContent);

	BindingTester
			.checkAllBindings(classVariableBindings);
	assertEquals(classVariableBindings.size(), 5);

	final List<TokenNameBinding> classVariableBindings2 = jabe
			.getNameBindings(classContent2);
	final List<TokenNameBinding> classVariableBindings2Exact = jbe
			.getNameBindings(classContent2);

	assertEquals(classVariableBindings2.size(), 9);

	allAreContained(classVariableBindingsExact, classVariableBindings);
	allAreContained(classVariableBindings2Exact, classVariableBindings2);
}
Example #16
Source File: JavascriptApproximateVariableBindingExtractorTest.java    From api-mining with GNU General Public License v3.0 6 votes vote down vote up 
@Test
public void testClassBindings() throws IOException {
	final JavascriptApproximateVariableBindingExtractor jabe = new JavascriptApproximateVariableBindingExtractor();
	final JavascriptExactVariableBindingsExtractor jbe = new JavascriptExactVariableBindingsExtractor();

	final List<TokenNameBinding> classVariableBindings = jabe
			.getNameBindings(classContent);
	final List<TokenNameBinding> classVariableBindingsExact = jbe
			.getNameBindings(classContent);

	BindingTester.checkAllBindings(classVariableBindings);
	assertEquals(classVariableBindings.size(), 3);

	final List<TokenNameBinding> classVariableBindings2 = jabe
			.getNameBindings(classContent2);
	final List<TokenNameBinding> classVariableBindings2Exact = jbe
			.getNameBindings(classContent2);

	assertEquals(classVariableBindings2.size(), 13);

	allAreContained(classVariableBindingsExact, classVariableBindings);
	allAreContained(classVariableBindings2Exact, classVariableBindings2);
}
Example #17
Source File: JavaTypeBindingExtractorTest.java    From tassal with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
@Test
public void testClassLevelBindings() throws IOException {
	final JavaTypeDeclarationBindingExtractor jame = new JavaTypeDeclarationBindingExtractor();

	final List<TokenNameBinding> classTypeindings = jame
			.getNameBindings(classContent);

	BindingTester.checkAllBindings(classTypeindings);
	assertEquals(classTypeindings.size(), 1);

	final List<TokenNameBinding> classTypeBindings2 = jame
			.getNameBindings(classContent2);
	BindingTester.checkAllBindings(classTypeBindings2);

	assertEquals(classTypeBindings2.size(), 1);
}
Example #18
Source File: JavaTypeBindingExtractorTest.java    From api-mining with GNU General Public License v3.0 6 votes vote down vote up 
@Test
public void testClassLevelBindings() throws IOException {
	final JavaTypeDeclarationBindingExtractor jame = new JavaTypeDeclarationBindingExtractor();

	final List<TokenNameBinding> classTypeindings = jame
			.getNameBindings(classContent);

	BindingTester.checkAllBindings(classTypeindings);
	assertEquals(classTypeindings.size(), 1);

	final List<TokenNameBinding> classTypeBindings2 = jame
			.getNameBindings(classContent2);
	BindingTester.checkAllBindings(classTypeBindings2);

	assertEquals(classTypeBindings2.size(), 1);
}
Example #19
Source File: BindingTester.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public static void checkAllBindings(final List<TokenNameBinding> bindings) {
	final Set<Integer> indexes = Sets.newHashSet();
	for (final TokenNameBinding binding : bindings) {
		BindingTester.checkBinding(binding);
		assertFalse("Indexes appear only once",
				indexes.removeAll(binding.nameIndexes));
		indexes.addAll(binding.nameIndexes);
	}
}
Example #20
Source File: JavascriptApproximateVariableBindingExtractorTest.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Test
public void testMethodBinding() {
	final JavascriptApproximateVariableBindingExtractor jabe = new JavascriptApproximateVariableBindingExtractor();
	final List<TokenNameBinding> methodVariableBindings = jabe
			.getNameBindings(methodContent);
	BindingTester.checkAllBindings(methodVariableBindings);
	assertEquals(methodVariableBindings.size(), 1);

}
Example #21
Source File: AbstractJavascriptNameBindingsExtractor.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public static ResolvedSourceCode getResolvedSourceCode(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final ArrayListMultimap<String, TokenNameBinding> bindings = ArrayListMultimap
			.create();

	for (final Set<ASTNode> boundName : nodeBindings) {
		if (boundName.isEmpty()) {
			continue;
		}
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.put(tokens.get(boundPositions.get(0)),
				new TokenNameBinding(Sets.newTreeSet(boundPositions),
						tokens, getFeatures(boundName)));
	}

	return new ResolvedSourceCode(tokens, bindings);
}
Example #22
Source File: AbstractJavaNameBindingsExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public ResolvedSourceCode getResolvedSourceCode(final String sourceCode,
		final Set<Set<ASTNode>> nodeBindings, final String filename,
		final Predicate<ASTNode> includeNode) {
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final ArrayListMultimap<String, TokenNameBinding> bindings = ArrayListMultimap
			.create();

	for (final Set<ASTNode> boundName : nodeBindings) {
		if (boundName.isEmpty()
				|| boundName.stream().noneMatch(includeNode)) {
			continue;
		}
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.put(tokens.get(boundPositions.get(0)),
				new TokenNameBinding(Sets.newTreeSet(boundPositions),
						tokens, getFeatures(boundName)));
	}

	return new ResolvedSourceCode(filename, tokens, bindings);
}
Example #23
Source File: AbstractJavaNameBindingsExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Override
public List<TokenNameBinding> getNameBindings(final String code) {
	final JavaASTExtractor ex = createExtractor();
	try {
		return getNameBindings(ex.getBestEffortAstNode(code), code);
	} catch (final Exception e) {
		throw new IllegalArgumentException(e);
	}
}
Example #24
Source File: BindingTester.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public static void checkAllBindings(final List<TokenNameBinding> bindings) {
	final Set<Integer> indexes = Sets.newHashSet();
	for (final TokenNameBinding binding : bindings) {
		BindingTester.checkBinding(binding);
		assertFalse("Indexes appear only once",
				indexes.removeAll(binding.nameIndexes));
		indexes.addAll(binding.nameIndexes);
	}
}
Example #25
Source File: AbstractJavascriptNameBindingsExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Override
public List<TokenNameBinding> getNameBindings(final String code) {
	final JavascriptASTExtractor ex = createExtractor();
	try {
		return getNameBindings(ex.getCompilationUnitAstNode(code), code);
	} catch (final Exception e) {
		throw new IllegalArgumentException(e);
	}
}
Example #26
Source File: AbstractJavascriptNameBindingsExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public static ResolvedSourceCode getResolvedSourceCode(
		final String sourceCode, final Set<Set<ASTNode>> nodeBindings) {
	final JavascriptTokenizer tokenizer = new JavascriptTokenizer();
	final SortedMap<Integer, String> tokenPositions = tokenizer
			.tokenListWithPos(sourceCode.toCharArray());
	final SortedMap<Integer, Integer> positionToIndex = getTokenIndexForPostion(tokenPositions);
	final List<String> tokens = Lists.newArrayList(tokenPositions.values());

	final ArrayListMultimap<String, TokenNameBinding> bindings = ArrayListMultimap
			.create();

	for (final Set<ASTNode> boundName : nodeBindings) {
		if (boundName.isEmpty()) {
			continue;
		}
		final List<Integer> boundPositions = Lists.newArrayList();
		for (final ASTNode name : boundName) {
			// Convert position to token index and add
			final int tokenIdx = positionToIndex.get(name
					.getStartPosition());
			boundPositions.add(tokenIdx);
		}
		bindings.put(tokens.get(boundPositions.get(0)),
				new TokenNameBinding(Sets.newTreeSet(boundPositions),
						tokens, getFeatures(boundName)));
	}

	return new ResolvedSourceCode(tokens, bindings);
}
Example #27
Source File: JavaExactVariableBindingsExtractorTest.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Test
public void testClassBindings() throws IOException {
	final JavaExactVariableBindingsExtractor jbe = new JavaExactVariableBindingsExtractor();
	final List<TokenNameBinding> classVariableBindings = jbe
			.getNameBindings(classContent);
	BindingTester.checkAllBindings(classVariableBindings);
	assertEquals(classVariableBindings.size(), 5);

	final List<TokenNameBinding> classVariableBindings2 = jbe
			.getNameBindings(classContent2);

	assertEquals(classVariableBindings2.size(), 9);
}
Example #28
Source File: AbstractJavaNameBindingsExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public final List<TokenNameBinding> getNameBindings(final ASTNode node,
		final File file) throws IOException {
	final Set<Set<ASTNode>> nodeBindings = getNameBindings(node);
	final SortedMap<Integer, String> tokenPositions = Maps.transformValues(
			tokenizer.tokenListWithPos(file),
			FullToken.TOKEN_NAME_CONVERTER);
	return getTokenBindings(tokenPositions, nodeBindings);
}
Example #29
Source File: JavaApproximateVariableBindingExtractorTest.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Test
public void testMethodBinding() {
	final JavaApproximateVariableBindingExtractor jabe = new JavaApproximateVariableBindingExtractor();
	final List<TokenNameBinding> methodVariableBindings = jabe
			.getNameBindings(methodContent);
	BindingTester
			.checkAllBindings(methodVariableBindings);
	assertEquals(methodVariableBindings.size(), 3);

}
Example #30
Source File: JavaBindingsToJson.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
protected SerializableResolvedSourceCode(final ResolvedSourceCode rsc) {
	codeTokens = rsc.codeTokens;
	boundVariables = Lists.newArrayList();
	boundVariableFeatures = Lists.newArrayList();
	for (final TokenNameBinding binding : rsc.getAllBindings()) {
		boundVariables.add(new ArrayList<Integer>(binding.nameIndexes));
		boundVariableFeatures.add(new ArrayList<String>(
				binding.features));
	}
	provenance = rsc.name;
}