codemining.languagetools.ITokenizer Java Examples

/**
 * @param args
 */
public static void main(String[] args) {
	if (args.length < 5) {
		System.err.println("Usage <folder> <junkNames> ...");
		return;
	}

	final File directory = new File(args[0]);

	final ITokenizer tokenizer = new JavaTokenizer();

	final Class<? extends AbstractNGramLM> smoothedNgramClass = StupidBackoff.class;

	final CommonNameRenamingEvaluator evaluator = new CommonNameRenamingEvaluator(
			directory, tokenizer, smoothedNgramClass,
			Sets.newTreeSet(Arrays.asList(args).subList(1, args.length)));
	evaluator.evaluate();
	evaluator.printResults();

}
 

/**
 * Crudely rename the name of an identifier by searching for similarly named
 * tokens.
 * 
 * @param snippet
 * @param variableMapping
 *            from, to
 * @return
 */
private String renameVariableInSnippet(final String snippet,
		final Map<String, String> variableMapping) {
	final List<String> tokens = tokenizer.tokenListFromCode(snippet
			.toCharArray());

	final StringBuffer bf = new StringBuffer();
	for (final String token : tokens) {
		if (variableMapping.containsKey(token)) {
			bf.append(variableMapping.get(token));
		} else if (token.equals(ITokenizer.SENTENCE_START)
				|| token.equals(ITokenizer.SENTENCE_END)) {
			continue;
		} else {
			bf.append(token);
		}
		bf.append(" ");
	}
	return bf.toString();

}
 

/**
 * Hacky way to compare snippets.
 *
 * @param snippet
 * @return
 */
private String normalizeCode(final char[] snippet) {
	final List<String> tokens = (new JavaTokenizer())
			.tokenListFromCode(snippet);

	final StringBuffer bf = new StringBuffer();
	for (final String token : tokens) {
		if (token.equals(ITokenizer.SENTENCE_START)
				|| token.equals(ITokenizer.SENTENCE_END)) {
			continue;
		} else {
			bf.append(token);
		}
		bf.append(" ");
	}
	return bf.toString();

}
 

/**
 * @param args
 * @throws IllegalAccessException
 * @throws InstantiationException
 * @throws ClassNotFoundException
 */
public static void main(String[] args) throws InstantiationException,
		IllegalAccessException, ClassNotFoundException {
	if (args.length < 5) {
		System.err
				.println("Usage: <projectDir> <tokenizerClass> variable|method|class examplesToGenerate <renamerClass> [renamerParams]");
		return;
	}

	final File directory = new File(args[0]);
	final IScopeExtractor extractor = ScopesTUI
			.getScopeExtractorByName(args[2]);
	final long nExamples = Long.parseLong(args[3]);

	final Class<? extends ITokenizer> tokenizerName = (Class<? extends ITokenizer>) Class
			.forName(args[1]);
	final ITokenizer tokenizer = tokenizerName.newInstance();

	final String renamerClass = args[4];

	final HumanEvaluationOutput heo = new HumanEvaluationOutput(directory,
			extractor, renamerClass, args.length == 5 ? null : args[5],
			tokenizer, nExamples);
	heo.getOutput();

}
 

public static SortedSet<Suggestion> getVariableSuggestions(
		final File currentFile, final File directory, final boolean useUNK)
		throws IOException {
	final ITokenizer tokenizer = new JavaTokenizer();

	final AbstractIdentifierRenamings renamer = new BaseIdentifierRenamings(
			tokenizer);

	final Collection<java.io.File> trainingFiles = FileUtils.listFiles(
			directory, tokenizer.getFileFilter(),
			DirectoryFileFilter.DIRECTORY);

	trainingFiles.remove(currentFile);

	renamer.buildRenamingModel(trainingFiles);

	final IScopeExtractor scopeExtractor = new VariableScopeExtractor.VariableScopeSnippetExtractor();

	final SegmentRenamingSuggestion suggestion = new SegmentRenamingSuggestion(
			renamer, scopeExtractor, useUNK);

	return suggestion.rankSuggestions(currentFile);
}
 

public static void main(final String[] args)
		throws IllegalArgumentException, SecurityException,
		InstantiationException, IllegalAccessException,
		InvocationTargetException, NoSuchMethodException,
		ClassNotFoundException, IOException {
	if (args.length < 4) {
		System.err
				.println("Usage <TestFile> <TrainDirectory> <renamerClass> variable|method");
		return;
	}

	final ITokenizer tokenizer = new JavaTokenizer();

	final AbstractIdentifierRenamings renamer = (AbstractIdentifierRenamings) Class
			.forName(args[2]).getDeclaredConstructor(ITokenizer.class)
			.newInstance(tokenizer);

	renamer.buildRenamingModel(FileUtils.listFiles(new File(args[1]),
			tokenizer.getFileFilter(), DirectoryFileFilter.DIRECTORY));

	final IScopeExtractor scopeExtractor = ScopesTUI
			.getScopeExtractorByName(args[3]);
	final SegmentRenamingSuggestion suggestion = new SegmentRenamingSuggestion(
			renamer, scopeExtractor, true);

	System.out.println(suggestion.rankSuggestions(new File(args[0])));

}
 

@Override
public FullToken getTokenFromString(final String token) {
	if (token.equals(ITokenizer.SENTENCE_START)) {
		return new FullToken(ITokenizer.SENTENCE_START,
				ITokenizer.SENTENCE_START);
	}

	if (token.equals(ITokenizer.SENTENCE_END)) {
		return new FullToken(ITokenizer.SENTENCE_END,
				ITokenizer.SENTENCE_END);
	}
	return getTokenListFromCode(token.toCharArray()).get(1);
}
 

@Override
public FullToken getTokenFromString(final String token) {
	if (token.equals(ITokenizer.SENTENCE_START)) {
		return new FullToken(ITokenizer.SENTENCE_START,
				ITokenizer.SENTENCE_START);
	}

	if (token.equals(ITokenizer.SENTENCE_END)) {
		return new FullToken(ITokenizer.SENTENCE_END,
				ITokenizer.SENTENCE_END);
	}
	return getTokenListFromCode(token.toCharArray()).get(1);
}
 

/**
 * @param tokenizer
 */
protected void testSample2(ITokenizer tokenizer) {
	final List<String> tok = tokenizer.tokenListFromCode(CODE_SAMPLE2);

	for (int i = 0; i < TOKENS_SAMPLE2.length; i++) {
		assertEquals(tok.get(i), TOKENS_SAMPLE2[i]);
	}
}
 

@Test
public void testTokenTypes() {
	ITokenizer tokenizer = new JavaTokenizer();
	assertEquals(
			tokenizer.getTokenFromString("hello"),
			new ITokenizer.FullToken("hello", tokenizer.getIdentifierType()));
	assertEquals(
			tokenizer.getTokenFromString("{"),
			new ITokenizer.FullToken("{", Integer
					.toString(ITerminalSymbols.TokenNameLBRACE)));

}
 

/**
 * @param tokenizer
 */
protected void testSample3(final ITokenizer tokenizer) {
	final List<String> tok = tokenizer.tokenListFromCode(CODE_SAMPLE3);
	for (int i = 0; i < TOKENS_SAMPLE1.length; i++) {
		assertEquals(tok.get(i), TOKENS_SAMPLE1[i]);
	}
	assertEquals(tok.size(), TOKENS_SAMPLE1.length);
}
 

/**
 * @param tokenizer
 */
protected void testSample3(ITokenizer tokenizer) {
	final List<String> tok = tokenizer.tokenListFromCode(CODE_SAMPLE3);
	for (int i = 0; i < TOKENS_SAMPLE1.length; i++) {
		assertEquals(tok.get(i), TOKENS_SAMPLE1[i]);
	}
	assertEquals(tok.size(), TOKENS_SAMPLE1.length);
}
 

/**
 * 
 */
public DynamicRangeEval(final File directory, final ITokenizer tokenizer,
		final IScopeExtractor ex) {
	allFiles = FileUtils.listFiles(directory, tokenizer.getFileFilter(),
			DirectoryFileFilter.DIRECTORY);
	scopeExtractor = ex;
}
 

public PerturbationEvaluator(final File directory,
		final ITokenizer tokenizer, final IScopeExtractor scopeExtractor,
		final String renamerClass) {
	allFiles = FileUtils.listFiles(directory, tokenizer.getFileFilter(),
			DirectoryFileFilter.DIRECTORY);
	this.tokenizer = tokenizer;
	this.scopeExtractor = scopeExtractor;
	this.renamerClass = renamerClass;
	varRenamer = new ScopedIdentifierRenaming(scopeExtractor,
			ParseType.COMPILATION_UNIT);
}
 

@Override
public FullToken getTokenFromString(final String token) {
	if (token.equals(ITokenizer.SENTENCE_START)) {
		return new FullToken(ITokenizer.SENTENCE_START,
				ITokenizer.SENTENCE_START);
	}

	if (token.equals(ITokenizer.SENTENCE_END)) {
		return new FullToken(ITokenizer.SENTENCE_END,
				ITokenizer.SENTENCE_END);
	}
	return getTokenListFromCode(token.toCharArray()).get(1);
}
 

public JavaMethodDeclarationBindingExtractor(final ITokenizer tokenizer,
		final boolean includeOverrides, final File inputFolder) {
	super(tokenizer);
	this.includeOverrides = includeOverrides;
	if (!includeOverrides) {
		pti = buildProjectTypeInformation(inputFolder);
	} else {
		pti = null;
	}
}
 

@Override
public FullToken getTokenFromString(final String token) {
	if (token.equals(ITokenizer.SENTENCE_START)) {
		return new FullToken(ITokenizer.SENTENCE_START,
				ITokenizer.SENTENCE_START);
	}

	if (token.equals(ITokenizer.SENTENCE_END)) {
		return new FullToken(ITokenizer.SENTENCE_END,
				ITokenizer.SENTENCE_END);
	}
	return getTokenListFromCode(token.toCharArray()).get(1);
}
 

/**
 * @param smoothedNgramClass
 * @param tokenizer
 * @param directory
 * 
 */
public CommonNameRenamingEvaluator(File directory, ITokenizer tokenizer,
		Class<? extends AbstractNGramLM> smoothedNgramClass,
		Set<String> junkVariables) {
	allFiles = FileUtils.listFiles(directory, tokenizer.getFileFilter(),
			DirectoryFileFilter.DIRECTORY);
	this.tokenizer = tokenizer;
	this.smoothedNgramClass = smoothedNgramClass;
	this.junkVariables = junkVariables;
}
 

@Override
public FullToken getTokenFromString(final String token) {
	if (token.equals(ITokenizer.SENTENCE_START)) {
		return new FullToken(ITokenizer.SENTENCE_START, SENTENCE_START);
	}

	if (token.equals(ITokenizer.SENTENCE_END)) {
		return new FullToken(ITokenizer.SENTENCE_END, SENTENCE_END);
	}
	return getTokenListFromCode(token.toCharArray()).get(1);
}
 

/**
 * 
 */
public BaseIdentifierRenamings(final ITokenizer tokenizer) {
	super();
	this.tokenizer = tokenizer;
	try {
		smoothedNgramClass = (Class<? extends AbstractNGramLM>) Class
				.forName(SettingsLoader.getStringSetting(
						"ngramSmootherClass",
						"codemining.lm.ngram.smoothing.StupidBackoff"));
	} catch (final ClassNotFoundException e) {
		LOGGER.severe(ExceptionUtils.getFullStackTrace(e));
		throw new IllegalArgumentException(e);
	}
}
 

public LeaveOneOutEvaluator(final File directory,
		final ITokenizer tokenizer,
		final Class<? extends AbstractNGramLM> smoother) {
	allFiles = FileUtils.listFiles(directory, tokenizer.getFileFilter(),
			DirectoryFileFilter.DIRECTORY);
	this.tokenizer = tokenizer;
	for (int i = 0; i < data.length; i++) {
		data[i] = new ResultObject();
	}
}
 

/**
 * @param args
 * @throws IllegalAccessException
 * @throws InstantiationException
 * @throws ClassNotFoundException
 */
public static void main(String[] args) throws InstantiationException,
		IllegalAccessException, ClassNotFoundException,
		SerializationException {
	if (args.length < 5) {
		System.err
				.println("Usage <folder> <tokenizerClass> <wrapperClass> variable|method <renamingClass> [<renamerConstrParams> ..]");
		return;
	}

	final File directory = new File(args[0]);

	final Class<? extends ITokenizer> tokenizerName = (Class<? extends ITokenizer>) Class
			.forName(args[1]);
	final ITokenizer tokenizer = tokenizerName.newInstance();

	final Class<? extends AbstractNGramLM> smoothedNgramClass = (Class<? extends AbstractNGramLM>) Class
			.forName(args[2]);

	final LeaveOneOutEvaluator eval = new LeaveOneOutEvaluator(directory,
			tokenizer, smoothedNgramClass);

	final IScopeExtractor scopeExtractor = ScopesTUI
			.getScopeExtractorByName(args[3]);

	eval.performEvaluation(scopeExtractor, args[4],
			args.length == 6 ? args[5] : null);
}
 

/**
 * @param tokenizer
 */
protected void testSample1Position(ITokenizer tokenizer) {
	final Map<Integer, String> toks = tokenizer
			.tokenListWithPos(CODE_SAMPLE1);
	for (int i = 0; i < TOKEN_POS_SAMPLE1.length; i++) {
		assertTrue(toks.containsKey(TOKEN_POS_SAMPLE1[i]));
		assertEquals(toks.get(TOKEN_POS_SAMPLE1[i]), TOKENS_SAMPLE1[i]);
	}
	assertEquals(toks.size(), TOKENS_SAMPLE1.length);
}
 

/**
 * 
 */
public SelectionSuggestionEval(final File directory,
		final ITokenizer codeTokenizer, final IScopeExtractor extractor) {
	tokenizer = codeTokenizer;
	allFiles = FileUtils.listFiles(directory, tokenizer.getFileFilter(),
			DirectoryFileFilter.DIRECTORY);
	scopeExtractor = extractor;
}
 

@Test
public void testTokenTypes() {
	final ITokenizer tokenizer = new JavascriptTokenizer();
	assertEquals(
			tokenizer.getTokenFromString("hello"),
			new ITokenizer.FullToken("hello", tokenizer.getIdentifierType()));
	assertEquals(
			tokenizer.getTokenFromString("{"),
			new ITokenizer.FullToken("{", Integer
					.toString(ITerminalSymbols.TokenNameLBRACE)));

}
 

@Test
public void testTokenTypes() {
	final ITokenizer tokenizer = new JavascriptTokenizer();
	assertEquals(
			tokenizer.getTokenFromString("hello"),
			new ITokenizer.FullToken("hello", tokenizer.getIdentifierType()));
	assertEquals(
			tokenizer.getTokenFromString("{"),
			new ITokenizer.FullToken("{", Integer
					.toString(ITerminalSymbols.TokenNameLBRACE)));

}
 

/**
 * @param tokenizer
 */
protected void testSample2(final ITokenizer tokenizer) {
	final List<String> tok = tokenizer.tokenListFromCode(CODE_SAMPLE2);

	for (int i = 0; i < TOKENS_SAMPLE2.length; i++) {
		assertEquals(tok.get(i), TOKENS_SAMPLE2[i]);
	}
}
 

/**
 * @param tokenizer
 */
protected void testSample1Position(final ITokenizer tokenizer) {
	final Map<Integer, String> toks = tokenizer
			.tokenListWithPos(CODE_SAMPLE1);
	for (int i = 0; i < TOKEN_POS_SAMPLE1.length; i++) {
		assertTrue(toks.containsKey(TOKEN_POS_SAMPLE1[i]));
		assertEquals(toks.get(TOKEN_POS_SAMPLE1[i]), TOKENS_SAMPLE1[i]);
	}
	assertEquals(toks.size(), TOKENS_SAMPLE1.length);
}
 

@Test
public void testTokenTypes() {
	ITokenizer tokenizer = new JavaTokenizer();
	assertEquals(
			tokenizer.getTokenFromString("hello"),
			new ITokenizer.FullToken("hello", tokenizer.getIdentifierType()));
	assertEquals(
			tokenizer.getTokenFromString("{"),
			new ITokenizer.FullToken("{", Integer
					.toString(ITerminalSymbols.TokenNameLBRACE)));

}
 

/**
 * @param args
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws IllegalAccessException
 * @throws InstantiationException
 */
public static void main(final String[] args) throws IOException,
		InstantiationException, IllegalAccessException,
		ClassNotFoundException {
	if (args.length != 2) {
		System.err.println("Usage <codeDir> <TokenizerClass>");
		return;
	}

	long tokenCount = 0;

	final ITokenizer tokenizer = TokenizerUtils.tokenizerForClass(args[1]);

	for (final File fi : FileUtils.listFiles(new File(args[0]),
			tokenizer.getFileFilter(), DirectoryFileFilter.DIRECTORY)) {
		try {
			final char[] code = FileUtils.readFileToString(fi)
					.toCharArray();
			tokenCount += tokenizer.tokenListFromCode(code).size() - 2; // Remove
																		// sentence
																		// start/end
		} catch (final IOException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	}

	System.out.println("Tokens: " + tokenCount);
}