org.antlr.v4.runtime.CodePointCharStream Java Examples

private static SchemaParser initParser(String value) {
  CodePointCharStream stream = CharStreams.fromString(value);
  UpperCaseCharStream upperCaseStream = new UpperCaseCharStream(stream);

  SchemaLexer lexer = new SchemaLexer(upperCaseStream);
  lexer.removeErrorListeners();
  lexer.addErrorListener(ErrorListener.INSTANCE);

  CommonTokenStream tokens = new CommonTokenStream(lexer);

  SchemaParser parser = new SchemaParser(tokens);
  parser.removeErrorListeners();
  parser.addErrorListener(ErrorListener.INSTANCE);

  return parser;
}
 

/**
 * This method will tokenize the program and get all the elements that are
 * variable names, variables types, methods names. Depending on the library
 * it will imported libraries names too.
 * @param program
 * @param language
 * @param keepOrder If false, then the program's tokens will be filtered using multiple threads
 * @return 
 */
public static Set<String> tokenize(String program, String language, boolean keepOrder)
{
	CodePointCharStream code = CharStreams.fromString(program);
	Set<String> filteresTokens;
	switch(language)
	{
		case "c": 
			filteresTokens=CTokens(code, keepOrder);
			break;
		case "java":
			filteresTokens=JavaTokens(code, keepOrder);
			break;
		case "javascript":
			filteresTokens=JavaScriptTokens(code, keepOrder);
			break;
		case "php":
			filteresTokens=PhpTokens(code, keepOrder);
			break;
		default:
			System.err.println("Not supported language");
			filteresTokens=new HashSet<String>(0);
			break;
	}
	return filteresTokens;
}
 

/**
 * This method will tokenize the program and get all the elements that are
 * variable names, variables types, methods names. Depending on the library
 * it will imported libraries names too.
 * @param program
 * @param language
 * @param keepOrder If false, then the program's tokens will be filtered using multiple threads
 * @return 
 */
public static List<String> tokenize(String program, String language, boolean keepOrder)
{
	CodePointCharStream code = CharStreams.fromString(program);
	List<String> filteresTokens;
	switch(language)
	{
		case "c": 
			filteresTokens=CTokens(code, keepOrder);
			break;
		case "java":
			filteresTokens=JavaTokens(code, keepOrder);
			break;
		case "javascript":
			filteresTokens=JavaScriptTokens(code, keepOrder);
			break;
		case "php":
			filteresTokens=PhpTokens(code, keepOrder);
			break;
		default:
			System.err.println("Not supported language");
			filteresTokens=new ArrayList<String>(0);
			break;
	}
	return filteresTokens;
}
 

private UserAgentContext parseUserAgent(MutableUserAgent userAgent) {
    String userAgentString = EvilManualUseragentStringHacks.fixIt(userAgent.getUserAgentString());

    CodePointCharStream input = CharStreams.fromString(userAgentString);
    UserAgentLexer lexer = new UserAgentLexer(input);

    CommonTokenStream tokens = new CommonTokenStream(lexer);

    UserAgentParser parser = new UserAgentParser(tokens);

    if (!verbose) {
        lexer.removeErrorListeners();
        parser.removeErrorListeners();
    }
    lexer.addErrorListener(userAgent);
    parser.addErrorListener(userAgent);

    return parser.userAgent();
}
 

private static Set<String> JavaTokens(CodePointCharStream code, boolean keepOrder)
{
	JavaLexer lexer = new JavaLexer(code);
	return filterTokens(lexer, javaTypes, keepOrder);
}
 

private static Set<String> CTokens(CodePointCharStream code, boolean keepOrder)
{
	CLexer lexer = new CLexer(code);
	return filterTokens(lexer, cTypes, keepOrder);
}
 

private static Set<String> PhpTokens(CodePointCharStream code, boolean keepOrder)
{
	PhpLexer lexer = new PhpLexer(code);
	return filterTokens(lexer, phpTypes, keepOrder);
}
 

private static Set<String> JavaScriptTokens(CodePointCharStream code, boolean keepOrder)
{
	JavaScriptLexer lexer = new JavaScriptLexer(code);
	return filterTokens(lexer, javaScriptTypes, keepOrder);
}
 

private static List<String> JavaTokens(CodePointCharStream code, boolean keepOrder)
{
	JavaLexer lexer = new JavaLexer(code);
	return filterTokens(lexer, javaTypes, keepOrder);
}
 

private static List<String> CTokens(CodePointCharStream code, boolean keepOrder)
{
	CLexer lexer = new CLexer(code);
	return filterTokens(lexer, cTypes, keepOrder);
}
 

private static List<String> PhpTokens(CodePointCharStream code, boolean keepOrder)
{
	PhpLexer lexer = new PhpLexer(code);
	return filterTokens(lexer, phpTypes, keepOrder);
}
 

private static List<String> JavaScriptTokens(CodePointCharStream code, boolean keepOrder)
{
	JavaScriptLexer lexer = new JavaScriptLexer(code);
	return filterTokens(lexer, javaScriptTypes, keepOrder);
}
 

public long initialize() {
    InitErrorListener errorListener = new InitErrorListener();

    CodePointCharStream input = CharStreams.fromString(this.matchExpression);
    UserAgentTreeWalkerLexer lexer = new UserAgentTreeWalkerLexer(input);

    lexer.addErrorListener(errorListener);

    CommonTokenStream tokens = new CommonTokenStream(lexer);
    UserAgentTreeWalkerParser parser = new UserAgentTreeWalkerParser(tokens);

    parser.addErrorListener(errorListener);

    ParserRuleContext requiredPattern = parseWalkerExpression(parser);

    // We couldn't ditch the double quotes around the fixed values in the parsing phase.
    // So we ditch them here. We simply walk the tree and modify some of the tokens.
    new UnQuoteValues().visit(requiredPattern);

    // Now we create an evaluator instance
    evaluator = new TreeExpressionEvaluator(requiredPattern, matcher, verbose);

    // Is a fixed value (i.e. no events will ever be fired)?
    String fixedValue = evaluator.getFixedValue();
    if (fixedValue != null) {
        setFixedValue(fixedValue);
        mustHaveMatches = false;
        matches = new MatchesList(0);
        return 0; // Not interested in any patterns
    }

    mustHaveMatches = evaluator.mustHaveMatches();

    int informs = calculateInformPath(this, "agent", requiredPattern);

    // If this is based on a variable we do not need any matches from the hashmap.
    if (mustHaveMatches && informs == 0) {
        mustHaveMatches = false;
    }

    int listSize = 0;
    if (informs > 0) {
        listSize = 1;
    }
    this.matches = new MatchesList(listSize);
    return informs;
}
 

public static DateTimeFormatter convert(String strfformat, ZoneId defaultZone) {
    CodePointCharStream input = CharStreams.fromString(strfformat);
    StrfTimeLexer lexer = new StrfTimeLexer(input);

    CommonTokenStream tokens = new CommonTokenStream(lexer);

    StrfTimeParser parser = new StrfTimeParser(tokens);

    lexer.removeErrorListeners();
    parser.removeErrorListeners();

    ParseTreeWalker walker = new ParseTreeWalker(); // create standard walker
    StrfTimeToDateTimeFormatter converter = new StrfTimeToDateTimeFormatter(strfformat, defaultZone);

    lexer.addErrorListener(converter);
    parser.addErrorListener(converter);

    StrfTimeParser.PatternContext pattern = parser.pattern();

    walker.walk(converter, pattern); // initiate walk of tree with listener

    if (converter.hasSyntaxError()) {
        return null;
    }

    return converter.build();
}