codemining.languagetools.Scope Java Examples

The following examples show how to use codemining.languagetools.Scope. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MethodScopeExtractor.java    From tassal with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
public static Multimap<Scope, String> getScopeSnippets(final ASTNode node,
		final boolean methodAsRoots) {
	final ScopeFinder scopeFinder = new ScopeFinder(methodAsRoots);
	node.accept(scopeFinder);

	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final Entry<ASTNode, Method> method : scopeFinder.methods
			.entries()) {
		scopes.put(new Scope(method.getKey().toString(),
				method.getValue().type, METHOD_CALL, 0, 0), method
				.getValue().name);
	}

	return scopes;

}
Example #2
Source File: MethodScopeExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
public static Multimap<Scope, String> getScopeSnippets(final ASTNode node,
		final boolean methodAsRoots) {
	final ScopeFinder scopeFinder = new ScopeFinder(methodAsRoots);
	node.accept(scopeFinder);

	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final Entry<ASTNode, Method> method : scopeFinder.methods
			.entries()) {
		scopes.put(new Scope(method.getKey().toString(),
				method.getValue().type, METHOD_CALL, 0, 0), method
				.getValue().name);
	}

	return scopes;

}
Example #3
Source File: VariableScopeExtractor.java    From tassal with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
/**
 * Return a multimap containing all the (local) variables of the given
 * scope.
 * 
 * @param cu
 * @return Multimap<Snippet, VariableName>
 */
public static Multimap<Scope, String> getScopeSnippets(final ASTNode cu) {
	final VariableScopeFinder scopeFinder = new VariableScopeFinder();
	cu.accept(scopeFinder);

	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final Entry<ASTNode, Variable> variable : scopeFinder.variableScopes
			.entries()) {
		final int astNodeType = variable.getKey().getNodeType();
		final int astNodeParentType;
		if (variable.getKey().getParent() == null) {
			astNodeParentType = -1;
		} else {
			astNodeParentType = variable.getKey().getParent().getNodeType();
		}
		scopes.put(
				new Scope(variable.getKey().toString(),
						variable.getValue().scope,
						variable.getValue().type, astNodeType,
						astNodeParentType), variable.getValue().name);
	}

	return scopes;

}
Example #4
Source File: TypenameScopeExtractor.java    From api-mining with GNU General Public License v3.0 6 votes vote down vote up 
private Multimap<Scope, String> getClassnames(final ASTNode node) {
	final ClassnameFinder cf = new ClassnameFinder(methodsAsRoots);
	node.accept(cf);

	final Multimap<Scope, String> classnames = TreeMultimap.create();

	for (final Entry<ASTNode, String> classname : cf.types.entries()) {
		final ASTNode parentNode = classname.getKey();
		final Scope sc = new Scope(
				classname.getKey().toString(),
				parentNode.getNodeType() == ASTNode.METHOD_DECLARATION ? ScopeType.SCOPE_METHOD
						: ScopeType.SCOPE_CLASS, TYPENAME,
				parentNode.getNodeType(), -1);
		classnames.put(sc, classname.getValue());
	}
	return classnames;
}
Example #5
Source File: TypenameScopeExtractor.java    From tassal with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
private Multimap<Scope, String> getClassnames(final ASTNode node) {
	final ClassnameFinder cf = new ClassnameFinder(methodsAsRoots);
	node.accept(cf);

	final Multimap<Scope, String> classnames = TreeMultimap.create();

	for (final Entry<ASTNode, String> classname : cf.types.entries()) {
		final ASTNode parentNode = classname.getKey();
		final Scope sc = new Scope(
				classname.getKey().toString(),
				parentNode.getNodeType() == ASTNode.METHOD_DECLARATION ? ScopeType.SCOPE_METHOD
						: ScopeType.SCOPE_CLASS, TYPENAME,
				parentNode.getNodeType(), -1);
		classnames.put(sc, classname.getValue());
	}
	return classnames;
}
Example #6
Source File: JavaVariableGrammarPrior.java    From naturalize with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
public static JavaVariableGrammarPrior buildFromFiles(
		final Collection<File> files) {
	final JavaVariableGrammarPrior gp = new JavaVariableGrammarPrior();
	for (final File f : files) {
		try {
			for (final Entry<Scope, String> variable : VariableScopeExtractor
					.getScopeSnippets(f).entries()) {
				gp.parentPrior.addElement(variable.getValue(),
						variable.getKey().astNodeType);
				gp.grandParentPrior.addElement(variable.getValue(),
						variable.getKey().astParentNodeType);
			}
		} catch (final IOException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	}

	return gp;
}
Example #7
Source File: CommonNameRenamingEvaluator.java    From naturalize with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
/**
 * @param renamer
 * @param scopes
 */
private void evaluateJunkRenamings(
		final AbstractIdentifierRenamings renamer,
		final Multimap<Scope, String> scopes) {
	for (final Entry<Scope, String> variable : scopes.entries()) {
		try {
			final SortedSet<Renaming> renamings = renamer.getRenamings(
					variable.getKey(), variable.getValue());
			final boolean weServeJunk = junkVariables
					.contains(renamings.first().name);
			final boolean variableWasJunk = junkVariables
					.contains(variable.getValue());
			updateResults(variableWasJunk, weServeJunk);
		} catch (Throwable e) {
			LOGGER.warning("Failed to evaluate renaming " + variable
					+ " " + ExceptionUtils.getFullStackTrace(e));
		}
	}
}
Example #8
Source File: VariableScopeExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
/**
 * Return a multimap containing all the (local) variables of the given
 * scope.
 * 
 * @param cu
 * @return Multimap<Snippet, VariableName>
 */
public static Multimap<Scope, String> getScopeSnippets(final ASTNode cu) {
	final VariableScopeFinder scopeFinder = new VariableScopeFinder();
	cu.accept(scopeFinder);

	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final Entry<ASTNode, Variable> variable : scopeFinder.variableScopes
			.entries()) {
		final int astNodeType = variable.getKey().getNodeType();
		final int astNodeParentType;
		if (variable.getKey().getParent() == null) {
			astNodeParentType = -1;
		} else {
			astNodeParentType = variable.getKey().getParent().getNodeType();
		}
		scopes.put(
				new Scope(variable.getKey().toString(),
						variable.getValue().scope,
						variable.getValue().type, astNodeType,
						astNodeParentType), variable.getValue().name);
	}

	return scopes;

}
Example #9
Source File: PerturbationEvaluator.java    From naturalize with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
void evaluateFile(final File testFile, final Collection<File> trainFiles)
		throws IllegalArgumentException, SecurityException,
		InstantiationException, IllegalAccessException,
		InvocationTargetException, NoSuchMethodException,
		ClassNotFoundException, IOException {
	final AbstractIdentifierRenamings renamer = (AbstractIdentifierRenamings) Class
			.forName(renamerClass).getDeclaredConstructor(ITokenizer.class)
			.newInstance(tokenizer);
	renamer.buildRenamingModel(trainFiles);
	final Multimap<Scope, String> scopes = scopeExtractor
			.getFromFile(testFile);
	final String targetPertubedName = "mblamblambla";

	for (final Entry<Scope, String> entry : scopes.entries()) {
		// TODO, here instead of reading again, give the method
		final Multimap<Scope, String> perturbed = perturbedScopes(
				FileUtils.readFileToString(testFile), entry.getKey(),
				entry.getValue(), targetPertubedName);
		final SegmentRenamingSuggestion rn = new SegmentRenamingSuggestion(
				renamer, true);
		final SortedSet<Suggestion> sg = rn.rankSuggestions(perturbed);
		pushResults(sg, targetPertubedName);
	}
}
Example #10
Source File: SegmentRenamingSuggestion.java    From naturalize with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
private Suggestion addRenamingSuggestion(
		final SortedSet<Renaming> renamings, final String idName,
		final Scope scope) {
	final double topSuggestionXEnt = renamings.first().score;
	double currentNameXent = Double.NaN;

	for (final Renaming renaming : renamings) {
		if (renaming.name.equals(idName)
				|| (renaming.name.equals("UNK_SYMBOL") && useUNK)) {
			currentNameXent = renaming.score;
			break;
		}
	}

	checkArgument(!Double.isNaN(currentNameXent));

	final double confidence = topSuggestionXEnt - currentNameXent;
	return new Suggestion(idName, scope, confidence, renamings);
}
Example #11
Source File: TypenameScopeExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
private Multimap<Scope, String> getClassnames(final ASTNode node) {
	final ClassnameFinder cf = new ClassnameFinder(methodsAsRoots);
	node.accept(cf);

	final Multimap<Scope, String> classnames = TreeMultimap.create();

	for (final Entry<ASTNode, String> classname : cf.types.entries()) {
		final ASTNode parentNode = classname.getKey();
		final Scope sc = new Scope(
				classname.getKey().toString(),
				parentNode.getNodeType() == ASTNode.METHOD_DECLARATION ? ScopeType.SCOPE_METHOD
						: ScopeType.SCOPE_CLASS, TYPENAME,
				parentNode.getNodeType(), -1);
		classnames.put(sc, classname.getValue());
	}
	return classnames;
}
Example #12
Source File: SegmentRenamingSuggestion.java    From naturalize with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
/**
 * @param identifiers
 * @return
 */
public SortedSet<Suggestion> rankSuggestions(
		final Multimap<Scope, String> identifiers) {
	final SortedSet<Suggestion> suggestions = Sets.newTreeSet();
	for (final Entry<Scope, String> s : identifiers.entries()) {
		try {
			final SortedSet<Renaming> renamings = renamer.getRenamings(
					s.getKey(), s.getValue());
			suggestions.add(addRenamingSuggestion(renamings, s.getValue(),
					s.getKey()));
		} catch (final Throwable e) {
			LOGGER.warning("Failed to get suggestions for " + s
					+ ExceptionUtils.getFullStackTrace(e));
		}
	}

	return suggestions;
}
Example #13
Source File: VariableUsageStatistics.java    From naturalize with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
public void extractStats() throws IOException {
	for (final File f : allFiles) {
		final Multimap<Scope, String> scopes = scopeExtractor
				.getFromFile(f);
		for (final Entry<Scope, String> entry : scopes.entries()) {
			final String varName = entry.getValue();
			final List<String> tokens = tokenizer.tokenListFromCode(entry
					.getKey().code.toCharArray());
			int count = 0;
			for (final String token : tokens) {
				if (token.equals(varName)) {
					count++;
				}
			}
			final UsageStats stats;
			if (statistics.containsKey(varName)) {
				stats = statistics.get(varName);
			} else {
				stats = new UsageStats();
				statistics.put(varName, stats);
			}
			stats.sumContexts += count;
			stats.timesSeen++;
		}
	}
}
Example #14
Source File: AbstractIdentifierRenamings.java    From naturalize with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
@Override
public SortedSet<Renaming> calculateScores(
		final Multiset<NGram<String>> ngrams,
		final Set<String> alternatives, final Scope scope) {
	final SortedSet<Renaming> scoreMap = Sets.newTreeSet();

	for (final String identifierName : alternatives) {
		double score = 0;
		for (final Entry<NGram<String>> ngram : ngrams.entrySet()) {
			try {
				final NGram<String> identNGram = NGram.substituteTokenWith(
						ngram.getElement(), WILDCARD_TOKEN, identifierName);
				final double ngramScore = scoreNgram(identNGram);
				score += DoubleMath.log2(ngramScore) * ngram.getCount();
			} catch (final Throwable e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		}
		scoreMap.add(new Renaming(identifierName, (addScopePriors(
				identifierName, scope) - score) / ngrams.size(), ngrams
				.size() / ngramLM.getN(), scope));
	}

	return scoreMap;
}
Example #15
Source File: FormattingRenamings.java    From naturalize with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
/**
 * Predict the max-likelihood tokens given the ngrams.
 * 
 * @param ngrams
 * @param alternatives
 * @return
 */
@Override
public SortedSet<Renaming> calculateScores(
		final Multiset<NGram<String>> ngrams,
		final Set<String> alternatives, final Scope scope) {
	final SortedSet<Renaming> suggestions = Sets.newTreeSet();

	for (final String alternative : alternatives) {
		double score = 0;
		for (final NGram<String> ngram : ngrams) {
			score += DoubleMath.log2(getNgramLM().getProbabilityFor(
					NGram.substituteTokenWith(ngram, WILDCARD_TOKEN,
							alternative)));
		}
		suggestions.add(new Renaming(alternative, -score, 1, null));
	}
	return suggestions;
}
Example #16
Source File: SegmentRenamingSuggestion.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
Suggestion(final String idName, final Scope scope,
		final double confidence, final SortedSet<Renaming> alternatives) {
	identifierName = idName;
	this.scope = scope;
	renamings = alternatives;
	confidenceGap = confidence;
}
Example #17
Source File: AllScopeExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Override
public Multimap<Scope, String> getFromString(final String file,
		final ParseType parseType) {
	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final IScopeExtractor extractor : allExtractors) {
		scopes.putAll(extractor.getFromString(file, parseType));
	}
	return scopes;
}
Example #18
Source File: AllScopeExtractor.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Override
public Multimap<Scope, String> getFromFile(final File file)
		throws IOException {
	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final IScopeExtractor extractor : allExtractors) {
		scopes.putAll(extractor.getFromFile(file));
	}
	return scopes;
}
Example #19
Source File: LocalTypedIdentifierRenamings.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Override
public double addScopePriors(final String identifierName, final Scope scope) {
	final double prob = tp.getMLProbability(identifierName, scope.type);
	if (prob > 0) {
		return -DoubleMath.log2(prob);
	} else if (!this.isTrueUNK(identifierName)) {
		return 100;
	} else {
		return 0;
	}
}
Example #20
Source File: INGramIdentifierRenamer.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public Renaming(final String id, final double xEntropy,
		final int contexts, final Scope renamingScope) {
	name = id;
	score = xEntropy;
	nContexts = contexts;
	scope = renamingScope;
}
Example #21
Source File: MethodScopeExtractor.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Override
public final Multimap<Scope, String> getFromFile(final File f) {
	try {
		return getScopeSnippets(f, methodAsRoots);
	} catch (IOException e) {
		LOGGER.severe("Unable to extract method scope snippets from file "
				+ f.getName());
		throw new IllegalArgumentException(
				"Unable to extract method scope snippets from file");
	}
}
Example #22
Source File: MethodScopeExtractor.java    From api-mining with GNU General Public License v3.0 5 votes vote down vote up 
@Override
public final Multimap<Scope, String> getFromFile(final File f) {
	try {
		return getScopeSnippets(f, methodAsRoots);
	} catch (IOException e) {
		LOGGER.severe("Unable to extract method scope snippets from file "
				+ f.getName());
		throw new IllegalArgumentException(
				"Unable to extract method scope snippets from file");
	}
}
Example #23
Source File: GrammarPriorIdentifierRenaming.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Override
protected double addScopePriors(final String identifierName,
		final Scope scope) {
	final double prob = gp.getMLProbability(identifierName,
			Pair.create(scope.astNodeType, scope.astParentNodeType));
	if (prob > 0) {
		return -DoubleMath.log2(prob);
	} else if (!this.isTrueUNK(identifierName)) {
		return 100;
	} else {
		return 0;
	}
}
Example #24
Source File: AllPriorIdentifierRenaming.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
private double addTypePrior(final String identifierName, final Scope scope) {
	if (!USE_TYPES) {
		return 0;
	}
	final double prob = tp.getMLProbability(identifierName, scope.type);
	if (prob > 0) {
		return -DoubleMath.log2(prob);
	} else if (!this.isTrueUNK(identifierName)) {
		return 6;
	} else {
		return 0;
	}
}
Example #25
Source File: AllPriorIdentifierRenaming.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
private double addGrammarPrior(final String identifierName,
		final Scope scope) {
	if (!USE_GRAMMAR) {
		return 0;
	}
	final double prob = gp.getMLProbability(identifierName,
			Pair.create(scope.astNodeType, scope.astParentNodeType));
	if (prob > 0) {
		return -DoubleMath.log2(prob);
	} else if (!this.isTrueUNK(identifierName)) {
		return 6;
	} else {
		return 0;
	}
}
Example #26
Source File: ScopedIdentifierRenaming.java    From api-mining with GNU General Public License v3.0 5 votes vote down vote up 
public Multimap<Scope, String> getRenamedScopes(
		final String originalScopeCode, final String from, final String to,
		final String wholeFile) {
	final String code = getRenamedCode(originalScopeCode, from, to,
			wholeFile);
	return scopeExtractor.getFromString(code, parseKindToUseOnOriginal);
}
Example #27
Source File: AllScopeExtractor.java    From api-mining with GNU General Public License v3.0 5 votes vote down vote up 
@Override
public Multimap<Scope, String> getFromFile(final File file)
		throws IOException {
	final Multimap<Scope, String> scopes = TreeMultimap.create();
	for (final IScopeExtractor extractor : allExtractors) {
		scopes.putAll(extractor.getFromFile(file));
	}
	return scopes;
}
Example #28
Source File: PerturbationEvaluator.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public Multimap<Scope, String> perturbedScopes(final String fileContent,
		final Scope scope, final String from, final String to) {
	final Multimap<Scope, String> copy = varRenamer.getRenamedScopes(scope,
			from, to, fileContent);

	return copy;
}
Example #29
Source File: DynamicRangeEval.java    From naturalize with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
private void pushStatsFor(final BaseIdentifierRenamings renamer,
		final MethodDeclaration method, final AbstractNGramLM lm,
		final List<String> allToks) throws IOException {
	final JavaASTExtractor ex = new JavaASTExtractor(false);
	final Multimap<Scope, String> scopes = scopeExtractor
			.getFromNode(method);
	final ScopedIdentifierRenaming identifierRenamer = new ScopedIdentifierRenaming(
			scopeExtractor, ParseType.METHOD);
	final List<String> renamedVars = Lists.newArrayList(scopes.values());
	Collections.shuffle(renamedVars);

	checkArgument(!renamedVars.isEmpty());

	final SnippetSuggestions ssBefore = SnippetScorer.scoreSnippet(method,
			renamer, scopeExtractor, false, false);

	final Map<String, String> renamingPlan = Maps.newTreeMap();
	final String targetName = getRandomName(allToks, renamer.getLM()
			.getTokenizer());
	renamingPlan.put(renamedVars.get(0), targetName);

	final String renamedMethod = identifierRenamer.getRenamedCode(
			method.toString(), method.toString(), renamingPlan);
	final ASTNode renamedMethodNode = ex.getASTNode(renamedMethod,
			ParseType.METHOD);

	final SnippetSuggestions ssAfter = SnippetScorer.scoreSnippet(
			renamedMethodNode, renamer, scopeExtractor, false, false);

	scoreBest.add(new BeforeAfterScore<Double>(-ssBefore.suggestions
			.first().getConfidence(), -ssAfter.suggestions.first()
			.getConfidence()));

	if (DEBUG_OUTPUT) {
		printDebugOutput(method, lm, renamedVars, ssBefore, targetName,
				renamedMethodNode, ssAfter);
	}
}
Example #30
Source File: ScopedIdentifierRenaming.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public Multimap<Scope, String> getRenamedScopes(
		final String originalScopeCode, final String from, final String to,
		final String wholeFile) {
	final String code = getRenamedCode(originalScopeCode, from, to,
			wholeFile);
	return scopeExtractor.getFromString(code, parseKindToUseOnOriginal);
}