org.antlr.v4.runtime.CommonTokenStream Java Examples

@Override
public void parse(Snapshot snapshot, Task task, SourceModificationEvent event) throws ParseException {
  this.snapshot = snapshot;
  String text = snapshot.getText().toString();
  ANTLRInputStream input = new ANTLRInputStream(text);
  Lexer lexer = new EditorConfigLexer(input);
  lexer.removeErrorListeners();

  CommonTokenStream tokens = new CommonTokenStream(lexer);
  parser = new EditorConfigParser(tokens);
  parser.removeErrorListeners();
  syntaxErrors = new ArrayList<>();
  EditorConfigErrorListener errorListener = new EditorConfigErrorListener(syntaxErrors);
  parser.addErrorListener(errorListener);
  EditorConfigParser.FileContext root = parser.file();
  result = new ECParserResult(snapshot, parser, root);
}
 

public static TypedValue from(String value) {
    // Get our lexer
    CEXLLexer lexer = new CEXLLexer(CharStreams.fromString(value));

    // Get a list of matched tokens
    CommonTokenStream tokens = new CommonTokenStream(lexer);

    // Pass the tokens to the parser
    CEXLParser parser = new CEXLParser(tokens);

    // Specify our entry point
    final CEXLParser.ExpressionContext context = parser.expression();

    // Walk it and attach our listener
    ParseTreeWalker walker = new ParseTreeWalker();
    final CEXLTypeResolver resolver = new CEXLTypeResolver();
    walker.walk(resolver, context);

    return new TypedValue(resolver.getExpressionType(), value);
}
 

public static void main(String[] args) {
		String prop = "id=\"parrt\"\n";
		ANTLRInputStream input = new ANTLRInputStream(prop);
		PropertyFileLexer lexer = new PropertyFileLexer(input);
		CommonTokenStream tokens = new CommonTokenStream(lexer);
		PropertyFileParser parser = new PropertyFileParser(tokens);
		ParseTree tree = parser.file();
		System.out.println(tree.toStringTree(parser));
//		Trees.inspect(tree, parser);

		ParseTreeWalker walker = new ParseTreeWalker();
		PropertyFileLoader loader = new PropertyFileLoader();
		walker.walk(loader, tree);

		System.out.println(loader.props);
	}
 

public static void main(String[] args) throws IOException {
	ANTLRInputStream input = new ANTLRFileStream("colleges.csv");
	CSVLexer lexer = new CSVLexer(input);
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	CSVParser parser = new CSVParser(tokens);
	ParseTree tree = parser.file();
	System.out.println(tree.toStringTree(parser));

	ParseTreeWalker walker = new ParseTreeWalker();
	CSVLoader loader = new CSVLoader();
	walker.walk(loader, tree);

	for (List<String> row : loader.data) {
		System.out.println(row);
	}
}
 

public void scriptValidation(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
	String content=req.getParameter("content");
	if(StringUtils.isNotBlank(content)){
		ScriptType type=ScriptType.valueOf(req.getParameter("type"));
		ANTLRInputStream antlrInputStream=new ANTLRInputStream(content);
		RuleParserLexer lexer=new RuleParserLexer(antlrInputStream);
		CommonTokenStream steam=new CommonTokenStream(lexer);
		RuleParserParser parser=new RuleParserParser(steam);
		parser.removeErrorListeners();
		ScriptErrorListener errorListener=new ScriptErrorListener();
		parser.addErrorListener(errorListener);
		switch(type){
		case Script:
			parser.ruleSet();
			break;
		case DecisionNode:
			parser.condition();
			break;
		case ScriptNode:
			parser.actions();
			
		}
		List<ErrorInfo> infos=errorListener.getInfos();
		writeObjectToJson(resp, infos);
	}
}
 

public static void generateTreeFile(final File vb6InputFile, final File outputDirectory) throws IOException {
	final File outputFile = new File(outputDirectory + "/" + vb6InputFile.getName() + TREE_EXTENSION);

	final boolean createdNewFile = outputFile.createNewFile();

	if (createdNewFile || RENEW_TREE_FILE) {
		LOG.info("Creating tree file {}.", outputFile);

		final InputStream inputStream = new FileInputStream(vb6InputFile);
		final VisualBasic6Lexer lexer = new VisualBasic6Lexer(CharStreams.fromStream(inputStream));
		final CommonTokenStream tokens = new CommonTokenStream(lexer);
		final VisualBasic6Parser parser = new VisualBasic6Parser(tokens);
		final StartRuleContext startRule = parser.startRule();
		final String inputFileTree = TreeUtils.toStringTree(startRule, parser);

		final PrintWriter pWriter = new PrintWriter(new FileWriter(outputFile));

		pWriter.write(inputFileTree);
		pWriter.flush();
		pWriter.close();
	}
}
 

@VisibleForTesting
static AtlasDSLParser.QueryContext parse(String queryStr) throws AtlasBaseException {
    AtlasDSLParser.QueryContext ret;
    try {
        InputStream    stream           = new ByteArrayInputStream(queryStr.getBytes());
        AtlasDSLLexer  lexer            = new AtlasDSLLexer(CharStreams.fromStream(stream));
        Validator      validator        = new Validator();
        TokenStream    inputTokenStream = new CommonTokenStream(lexer);
        AtlasDSLParser parser           = new AtlasDSLParser(inputTokenStream);

        parser.removeErrorListeners();
        parser.addErrorListener(validator);

        // Validate the syntax of the query here
        ret = parser.query();
        if (!validator.isValid()) {
            LOG.error("Invalid DSL: {} Reason: {}", queryStr, validator.getErrorMsg());
            throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY, queryStr, validator.getErrorMsg());
        }

    } catch (IOException e) {
        throw new AtlasBaseException(e);
    }

    return ret;
}
 

public static void main(String[] args) {
	String code =
		"int x;\n" +
		"A b;\n";
	ANTLRInputStream input = new ANTLRInputStream(code);
	LangLexer lexer = new LangLexer(input);
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	LangParser parser = new LangParser(tokens);
	ParseTree tree = parser.file(); // start up

	System.out.println(tree.toStringTree(parser));

	ParseTreeWalker walker = new ParseTreeWalker();
	Gen listener = new Gen();
	walker.walk(listener, tree);

	System.out.println(listener.file.toString());
}
 

private Proto parseProto(String input) {
    CharStream stream = CharStreams.fromString(input);
    ProtoLexer lexer = new ProtoLexer(stream);
    lexer.removeErrorListeners();
    lexer.addErrorListener(TestUtils.ERROR_LISTENER);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    ProtoParser parser = new ProtoParser(tokenStream);
    parser.removeErrorListeners();
    parser.addErrorListener(TestUtils.ERROR_LISTENER);
    ProtoContext context = new ProtoContext("test.proto");
    ProtoParseListener protoParseListener = new ProtoParseListener(tokenStream, context);
    OptionParseListener optionParseListener = new OptionParseListener(tokenStream, context);
    parser.addParseListener(protoParseListener);
    parser.addParseListener(optionParseListener);
    parser.proto();
    return context.getProto();
}
 

@Override
public void parse() throws IOException {
	/* If file already exist, skip it */
	Entity fileEntity = entityRepo.getEntity(fileFullPath);
	if (fileEntity!=null && fileEntity instanceof FileEntity) {
		return;
	}
	/*parse file*/
       CharStream input = CharStreams.fromFileName(fileFullPath);
       Lexer lexer = new XMLLexer(input);
       CommonTokenStream tokens = new CommonTokenStream(lexer);
       XMLParser parser = new XMLParser(tokens);
       PomListener bridge = new PomListener(fileFullPath, entityRepo, includePaths,parseCreator,inferer);
    ParseTreeWalker walker = new ParseTreeWalker();
    walker.walk(bridge, parser.document());
	fileEntity = entityRepo.getEntity(fileFullPath);
	bridge.done();
	((FileEntity)fileEntity).cacheAllExpressions();
}
 

public static void main(String[] args) {
	String code =
		"int x;\n" +
		"A b;\n";
	ANTLRInputStream input = new ANTLRInputStream(code);
	LangLexer lexer = new LangLexer(input);
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	LangParser parser = new LangParser(tokens);
	ParseTree tree = parser.file(); // start up

	System.out.println(tree.toStringTree(parser));

	ParseTreeWalker walker = new ParseTreeWalker();
	Gen listener = new Gen();
	walker.walk(listener, tree);

	ST output = listener.file.getTemplate();
	System.out.println(output.render());
}
 

public static void main(String[] args) {
	String code =
		"int x;\n" +
		"A b;\n";
	ANTLRInputStream input = new ANTLRInputStream(code);
	LangLexer lexer = new LangLexer(input);
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	LangParser parser = new LangParser(tokens);
	ParseTree tree = parser.file(); // start up

	System.out.println(tree.toStringTree(parser));

	ParseTreeWalker walker = new ParseTreeWalker();
	Gen listener = new Gen();
	walker.walk(listener, tree);

	ST output = listener.file.getTemplate();
	System.out.println(output.render());
}
 

private FeatureParser(Feature feature, InputStream is) {
    this.feature = feature;
    CharStream stream;
    try {
        stream = CharStreams.fromStream(is, StandardCharsets.UTF_8);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    KarateLexer lexer = new KarateLexer(stream);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    KarateParser parser = new KarateParser(tokens);
    parser.addErrorListener(errorListener);
    RuleContext tree = parser.feature();
    if (logger.isTraceEnabled()) {
        logger.debug(tree.toStringTree(parser));
    }
    ParseTreeWalker walker = new ParseTreeWalker();
    walker.walk(this, tree);
    if (errorListener.isFail()) {
        String errorMessage = errorListener.getMessage();
        logger.error("not a valid feature file: {} - {}", feature.getResource().getRelativePath(), errorMessage);
        throw new RuntimeException(errorMessage);
    }
}
 

public static SiddhiApp parse(String siddhiApp) {
    CharStream input = CharStreams.fromString(siddhiApp);
    SiddhiQLLexer lexer = new SiddhiQLLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(SiddhiErrorListener.INSTANCE);

    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SiddhiQLParser parser = new SiddhiQLParser(tokens);
    //            parser.setErrorHandler(new BailErrorStrategy());
    parser.removeErrorListeners();
    parser.addErrorListener(SiddhiErrorListener.INSTANCE);
    ParseTree tree = parser.parse();

    SiddhiQLVisitor eval = new SiddhiQLBaseVisitorImpl();
    return (SiddhiApp) eval.visit(tree);
}
 

/**
 * Parses names to {@link CSLName} objects. Also handles strings
 * containing multiple names separated by {@code and}. The
 * method always returns at least one object, even if the given
 * names cannot be parsed. In this case the returned object just
 * contains a literal string.
 * @param names the names to parse
 * @return the {@link CSLName} objects (never {@code null} and never empty)
 */
public static CSLName[] parse(String names) {
    CharStream cs = CharStreams.fromString(names);
    InternalNameLexer lexer = new InternalNameLexer(cs);
    lexer.removeErrorListeners(); // do not output errors to console
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    InternalNameParser parser = new InternalNameParser(tokens);
    parser.removeErrorListeners(); // do not output errors to console
    NamesContext ctx = parser.names();
    if (ctx.result.isEmpty() || ctx.exception != null ||
            parser.getNumberOfSyntaxErrors() > 0) {
        // unparsable fall back to literal string
        return new CSLName[] { new CSLNameBuilder().literal(names).build() };
    }
    return ctx.result.toArray(new CSLName[0]);
}
 

private static StatementContext parseYangSource(final SourceIdentifier source, final InputStream stream)
        throws IOException, YangSyntaxErrorException {
    final YangStatementLexer lexer = new YangStatementLexer(CharStreams.fromStream(stream));
    final YangStatementParser parser = new YangStatementParser(new CommonTokenStream(lexer));
    // disconnect from console error output
    lexer.removeErrorListeners();
    parser.removeErrorListeners();

    final YangErrorListener errorListener = new YangErrorListener(source);
    lexer.addErrorListener(errorListener);
    parser.addErrorListener(errorListener);

    final StatementContext result = parser.statement();
    errorListener.validate();

    // Walk the resulting tree and replace each children with an immutable list, lowering memory requirements
    // and making sure the resulting tree will not get accidentally modified. An alternative would be to use
    // org.antlr.v4.runtime.Parser.TrimToSizeListener, but that does not make the tree immutable.
    ParseTreeWalker.DEFAULT.walk(MAKE_IMMUTABLE_LISTENER, result);

    return result;
}
 

public <T> Observable<MetricId<T>> parse(String tenantId, MetricType<T> metricType, String expression) {
    ANTLRInputStream input = new ANTLRInputStream(expression);
    TagQueryLexer tql = new TagQueryLexer(input);
    tql.removeErrorListeners();
    tql.addErrorListener(new ThrowingErrorListener());

    CommonTokenStream tokens = new CommonTokenStream(tql);

    TagQueryParser parser = new TagQueryParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(new ThrowingErrorListener());

    ParseTree parseTree = parser.tagquery();

    StorageTagQueryListener<T> listener = new StorageTagQueryListener<>(tenantId, metricType);
    ParseTreeWalker.DEFAULT.walk(listener, parseTree);

    return listener.getResult();
}
 

public static void main(String[] args) {
	String code =
		"int x;\n" +
		"A b;\n";
	ANTLRInputStream input = new ANTLRInputStream(code);
	LangLexer lexer = new LangLexer(input);
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	LangParser parser = new LangParser(tokens);
	ParseTree tree = parser.file(); // start up

	System.out.println(tree.toStringTree(parser));

	ParseTreeWalker walker = new ParseTreeWalker();
	Gen listener = new Gen();
	walker.walk(listener, tree);

	for (String decl : listener.decls) {
		System.out.println(decl);
	}
}
 

public static void main(String[] args) {
		String prop = "id=\"parrt\"\n";
		ANTLRInputStream input = new ANTLRInputStream(prop);
		PropertyFileLexer lexer = new PropertyFileLexer(input);
		CommonTokenStream tokens = new CommonTokenStream(lexer);
		PropertyFileParser parser = new PropertyFileParser(tokens);
		ParseTree tree = parser.file();
		System.out.println(tree.toStringTree(parser));
//		Trees.inspect(tree, parser);

		ParseTreeWalker walker = new ParseTreeWalker();
		PropertyFileLoader loader = new PropertyFileLoader();
		walker.walk(loader, tree);

		System.out.println(loader.props);
	}
 

public static FilterExpression of(String expression) {
    if (expression.equals("true")) {
        return ALWAYS_TRUE;
    }

    final ANTLRInputStream inputStream = new ANTLRInputStream(expression);
    final FilterLexer lexer = new FilterLexer(inputStream);
    lexer.getErrorListeners().clear();
    lexer.addErrorListener(ERROR_LISTENER);

    final CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    final FilterParser parser = new FilterParser(tokenStream);
    parser.getErrorListeners().clear();
    parser.addErrorListener(ERROR_LISTENER);

    return new FilterExpression(expression, parser);
}
 

public static InstructionContext parse(String instructionString) {
	CharStream is = CharStreams.fromString(instructionString); 
	LogInstructionLexer lexer = new LogInstructionLexer(is);
	lexer.removeErrorListeners();
	lexer.addErrorListener(new BaseErrorListener() {

		@Override
		public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line,
				int charPositionInLine, String msg, RecognitionException e) {
			throw new RuntimeException("Malformed log instruction");
		}
		
	});
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	LogInstructionParser parser = new LogInstructionParser(tokens);
	parser.removeErrorListeners();
	parser.setErrorHandler(new BailErrorStrategy());
	return parser.instruction();
}
 

private static Expr parseExpr(String sql) {
  ANTLRInputStream input = new ANTLRInputStream(sql);
  SQLLexer lexer = new SQLLexer(input);
  CommonTokenStream tokens = new CommonTokenStream(lexer);

  SQLParser parser = new SQLParser(tokens);
  parser.setErrorHandler(new BailErrorStrategy());
  parser.setBuildParseTree(true);

  SQLAnalyzer visitor = new SQLAnalyzer();
  SQLParser.Value_expressionContext context = parser.value_expression();
  return visitor.visitValue_expression(context);
}
 

public static UserMatch parse(@Nullable String userMatchString) {
	List<UserMatchCriteria> criterias = new ArrayList<>();
	List<UserMatchCriteria> exceptCriterias = new ArrayList<>();
	
	if (userMatchString != null) {
		CharStream is = CharStreams.fromString(userMatchString); 
		UserMatchLexer lexer = new UserMatchLexer(is);
		lexer.removeErrorListeners();
		lexer.addErrorListener(new BaseErrorListener() {

			@Override
			public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line,
					int charPositionInLine, String msg, RecognitionException e) {
				throw new RuntimeException("Malformed user match");
			}
			
		});
		CommonTokenStream tokens = new CommonTokenStream(lexer);
		UserMatchParser parser = new UserMatchParser(tokens);
		parser.removeErrorListeners();
		parser.setErrorHandler(new BailErrorStrategy());
		
		UserMatchContext userMatchContext = parser.userMatch();
		
		for (CriteriaContext criteriaContext: userMatchContext.criteria())
			criterias.add(getUserMatchCriteria(criteriaContext));
		
		for (ExceptCriteriaContext exceptCriteriaContext: userMatchContext.exceptCriteria()) {
			exceptCriterias.add(getUserMatchCriteria(exceptCriteriaContext.criteria()));
		}
	}
	
	return new UserMatch(criterias, exceptCriterias);
}
 

public static String convertWithAntlr(String query) {


        XPathLexer lexer = new XPathLexer(new ANTLRInputStream(query));
        XPathParser parser = new XPathParser(new CommonTokenStream(lexer));
        MainContext mainCtx = parser.main();
        StringBuilder output = new StringBuilder();
        writeTree(output, mainCtx, false);
        return output.toString();

    }
 

/**
 * {@inheritDoc}
 */
@Override
public ParseContext parse(String cql)
    throws ParseException
{
    LOG.info("start to parse cql : {}", cql);
    CQLErrorListener errorListener = new CQLErrorListener();
    
    CQLLexer lexer = new CQLLexer(new ANTLRIgnoreCaseStringStream(cql));
    lexer.removeErrorListeners();
    lexer.addErrorListener(errorListener);
    
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    CQLParser parser = new CQLParser(tokens);
    
    CQLErrorStrategy errorHandler = new CQLErrorStrategy();
    parser.setErrorHandler(errorHandler);
    
    parser.removeErrorListeners();
    parser.addErrorListener(errorListener);
    ParserRuleContext tree = parser.groupByList();
    
    if (errorListener.getRecException() != null)
    {
        errorListener.getRecException().setCql(cql);
        throw errorListener.getRecException();
    }
    
    LOG.info("Parse Completed");
    
    GroupbyClauseVisitor visitor = new GroupbyClauseVisitor();
    return visitor.visit(tree);
}
 

public static int getPrecedingNL(CommonTokenStream tokens, int i) {
	int precedingNL = 0;
	List<Token> previousWS = getPreviousWS(tokens, i);
	if ( previousWS!=null ) {
		for (Token ws : previousWS) {
			precedingNL += Tool.count(ws.getText(), '\n');
		}
	}
	return precedingNL;
}
 

public static TableDefinition parseTableDefinition(String tableDefinition) throws SiddhiParserException {
    CharStream input = CharStreams.fromString(tableDefinition);
    SiddhiQLLexer lexer = new SiddhiQLLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(SiddhiErrorListener.INSTANCE);

    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SiddhiQLParser parser = new SiddhiQLParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(SiddhiErrorListener.INSTANCE);
    ParseTree tree = parser.definition_table_final();

    SiddhiQLVisitor eval = new SiddhiQLBaseVisitorImpl();
    return (TableDefinition) eval.visit(tree);
}
 

private QuickstartCqlVisitor visitANTLRInputStream(ANTLRInputStream is) {
    cqlLexer lexer = new cqlLexer(is);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    cqlParser parser = new cqlParser(tokens);
    parser.setBuildParseTree(true);
    QuickstartCqlVisitor visitor = new QuickstartCqlVisitor();
    visitor.visit(parser.logic());

    return visitor;
}
 

@NotNull int parseQuery() {
  LOGGER.debug("Parsing query: {}", _queryString);
  PQL2Parser.OptionalClauseContext optionalClauseContext = null;
  PQL2Parser.WhereClauseContext whereClauseContext = null;
  if (_queryString == null) {
    return 0;
  }
  try {
    PQL2Lexer lexer = new PQL2Lexer(new ANTLRInputStream(_queryString));
    PQL2Parser parser = new PQL2Parser(new CommonTokenStream(lexer));
    ParseTree selectStatement = parser.root().statement().selectStatement();
    LOGGER.debug("selectStatement: {}", selectStatement.getText());

    for (int i = 0; i < selectStatement.getChildCount(); i++) {
      if (selectStatement.getChild(i) instanceof PQL2Parser.OptionalClauseContext) {
        optionalClauseContext = (PQL2Parser.OptionalClauseContext) selectStatement.getChild(i);
        LOGGER.debug("optionalClauseContext: {}", optionalClauseContext.getText());
        if (optionalClauseContext.getChild(0) instanceof PQL2Parser.WhereClauseContext) {
          whereClauseContext = (PQL2Parser.WhereClauseContext) optionalClauseContext.getChild(0);
          break;
        }
      }
    }
  } catch (Exception e) {
    return 0;
  }
  if (whereClauseContext == null) {
    return 0;
  }
  LOGGER.debug("whereClauseContext: {}", whereClauseContext.getText());

  return parsePredicateList(whereClauseContext.predicateList());
}
 

public RuleSet compile(String inputString) {
    ANTLRInputStream input = new ANTLRInputStream(inputString);
    RuleSetGrammarLexer lexer = new RuleSetGrammarLexer(input);
    TokenStream tokens = new CommonTokenStream(lexer);
    RuleSetGrammarParser parser = new RuleSetGrammarParser(tokens);

    TreeBuilder treeBuilder = new TreeBuilder();
    parser.addParseListener(treeBuilder);
    parser.setErrorHandler(new ExceptionThrowingErrorHandler());

    parser.rule_set();

    return treeBuilder.getRuleSet();
}