org.antlr.v4.runtime.UnbufferedTokenStream Java Examples

The following examples show how to use org.antlr.v4.runtime.UnbufferedTokenStream. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Pql2Compiler.java    From incubator-pinot with Apache License 2.0 6 votes vote down vote up 
public static AstNode buildAst(String expression) {
  CharStream charStream = new ANTLRInputStream(expression);
  PQL2Lexer lexer = new PQL2Lexer(charStream);
  lexer.setTokenFactory(new CommonTokenFactory(true));
  lexer.removeErrorListeners();
  lexer.addErrorListener(ERROR_LISTENER);
  TokenStream tokenStream = new UnbufferedTokenStream<CommonToken>(lexer);
  PQL2Parser parser = new PQL2Parser(tokenStream);
  parser.setErrorHandler(new BailErrorStrategy());
  parser.removeErrorListeners();
  parser.addErrorListener(ERROR_LISTENER);

  // Parse
  ParseTree parseTree = parser.root();

  ParseTreeWalker walker = new ParseTreeWalker();
  Pql2AstListener listener = new Pql2AstListener(expression);
  walker.walk(listener, parseTree);

  AstNode rootNode = listener.getRootNode();
  return rootNode;
}
Example #2
Source File: M2DocParser.java    From M2Doc with Eclipse Public License 1.0 5 votes vote down vote up 
/**
 * Parses while matching an AQL expression.
 * 
 * @param expression
 *            the expression to parse
 * @return the corresponding {@link AstResult}
 */
private AstResult parseWhileAqlExpression(String expression) {
    final IQueryBuilderEngine.AstResult result;

    if (expression != null && expression.length() > 0) {
        AstBuilderListener astBuilder = AQL56Compatibility.createAstBuilderListener(queryEnvironment);
        CharStream input = new UnbufferedCharStream(new StringReader(expression), expression.length());
        QueryLexer lexer = new QueryLexer(input);
        lexer.setTokenFactory(new CommonTokenFactory(true));
        lexer.removeErrorListeners();
        lexer.addErrorListener(astBuilder.getErrorListener());
        TokenStream tokens = new UnbufferedTokenStream<CommonToken>(lexer);
        QueryParser parser = new QueryParser(tokens);
        parser.addParseListener(astBuilder);
        parser.removeErrorListeners();
        parser.addErrorListener(astBuilder.getErrorListener());
        // parser.setTrace(true);
        parser.expression();
        result = astBuilder.getAstResult();
    } else {
        ErrorExpression errorExpression = (ErrorExpression) EcoreUtil
                .create(AstPackage.eINSTANCE.getErrorExpression());
        List<org.eclipse.acceleo.query.ast.Error> errors = new ArrayList<>(1);
        errors.add(errorExpression);
        final Map<Object, Integer> positions = new HashMap<>();
        if (expression != null) {
            positions.put(errorExpression, Integer.valueOf(0));
        }
        final BasicDiagnostic diagnostic = new BasicDiagnostic();
        diagnostic.add(new BasicDiagnostic(Diagnostic.ERROR, AstBuilderListener.PLUGIN_ID, 0,
                "null or empty string.", new Object[] {errorExpression }));
        result = new AstResult(errorExpression, positions, positions, errors, diagnostic);
    }

    return result;
}
Example #3
Source File: M2DocParser.java    From M2Doc with Eclipse Public License 1.0 5 votes vote down vote up 
/**
 * Parses while matching an AQL type literal.
 * 
 * @param expression
 *            the expression to parse
 * @return the corresponding {@link AstResult}
 */
protected AstResult parseWhileAqlTypeLiteral(String expression) {
    final IQueryBuilderEngine.AstResult result;

    if (expression != null && expression.length() > 0) {
        AstBuilderListener astBuilder = AQL56Compatibility.createAstBuilderListener(queryEnvironment);
        CharStream input = new UnbufferedCharStream(new StringReader(expression), expression.length());
        QueryLexer lexer = new QueryLexer(input);
        lexer.setTokenFactory(new CommonTokenFactory(true));
        lexer.removeErrorListeners();
        lexer.addErrorListener(astBuilder.getErrorListener());
        TokenStream tokens = new UnbufferedTokenStream<CommonToken>(lexer);
        QueryParser parser = new QueryParser(tokens);
        parser.addParseListener(astBuilder);
        parser.removeErrorListeners();
        parser.addErrorListener(astBuilder.getErrorListener());
        // parser.setTrace(true);
        parser.typeLiteral();
        result = astBuilder.getAstResult();
    } else {
        ErrorTypeLiteral errorTypeLiteral = (ErrorTypeLiteral) EcoreUtil
                .create(AstPackage.eINSTANCE.getErrorTypeLiteral());
        List<org.eclipse.acceleo.query.ast.Error> errs = new ArrayList<>(1);
        errs.add(errorTypeLiteral);
        final Map<Object, Integer> positions = new HashMap<>();
        if (expression != null) {
            positions.put(errorTypeLiteral, Integer.valueOf(0));
        }
        final BasicDiagnostic diagnostic = new BasicDiagnostic();
        diagnostic.add(new BasicDiagnostic(Diagnostic.ERROR, AstBuilderListener.PLUGIN_ID, 0,
                "missing type literal", new Object[] {errorTypeLiteral }));
        result = new AstResult(errorTypeLiteral, positions, positions, errs, diagnostic);
    }

    return result;
}
Example #4
Source File: TemplateCustomProperties.java    From M2Doc with Eclipse Public License 1.0 5 votes vote down vote up 
/**
 * Parses while matching an AQL expression.
 * 
 * @param queryEnvironment
 *            the {@link IReadOnlyQueryEnvironment}
 * @param type
 *            the type to parse
 * @return the corresponding {@link AstResult}
 */
private AstResult parseWhileAqlTypeLiteral(IReadOnlyQueryEnvironment queryEnvironment, String type) {
    final IQueryBuilderEngine.AstResult result;

    if (type != null && type.length() > 0) {
        AstBuilderListener astBuilder = AQL56Compatibility
                .createAstBuilderListener((IQueryEnvironment) queryEnvironment);
        CharStream input = new UnbufferedCharStream(new StringReader(type), type.length());
        QueryLexer lexer = new QueryLexer(input);
        lexer.setTokenFactory(new CommonTokenFactory(true));
        lexer.removeErrorListeners();
        lexer.addErrorListener(astBuilder.getErrorListener());
        TokenStream tokens = new UnbufferedTokenStream<CommonToken>(lexer);
        QueryParser parser = new QueryParser(tokens);
        parser.addParseListener(astBuilder);
        parser.removeErrorListeners();
        parser.addErrorListener(astBuilder.getErrorListener());
        // parser.setTrace(true);
        parser.typeLiteral();
        result = astBuilder.getAstResult();
    } else {
        ErrorTypeLiteral errorTypeLiteral = (ErrorTypeLiteral) EcoreUtil
                .create(AstPackage.eINSTANCE.getErrorTypeLiteral());
        List<org.eclipse.acceleo.query.ast.Error> errors = new ArrayList<>(1);
        errors.add(errorTypeLiteral);
        final Map<Object, Integer> positions = new HashMap<>();
        if (type != null) {
            positions.put(errorTypeLiteral, Integer.valueOf(0));
        }
        final BasicDiagnostic diagnostic = new BasicDiagnostic();
        diagnostic.add(new BasicDiagnostic(Diagnostic.ERROR, AstBuilderListener.PLUGIN_ID, 0, "null or empty type.",
                new Object[] {errorTypeLiteral }));
        result = new AstResult(errorTypeLiteral, positions, positions, errors, diagnostic);
    }

    return result;
}
Example #5
Source File: StringTemplate.java    From monsoon with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public static StringTemplate fromString(String pattern) {
    class DescriptiveErrorListener extends BaseErrorListener {
        public List<String> errors = new ArrayList<>();

        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
                                int line, int charPositionInLine,
                                String msg, RecognitionException e) {
            errors.add(String.format("%d:%d: %s", line, charPositionInLine, msg));
        }
    }

    final DescriptiveErrorListener error_listener = new DescriptiveErrorListener();

    final StringSubstitutionLexer lexer = new StringSubstitutionLexer(CharStreams.fromString(pattern));
    lexer.removeErrorListeners();
    lexer.addErrorListener(error_listener);
    final StringSubstitutionParser parser = new StringSubstitutionParser(new UnbufferedTokenStream(lexer));
    parser.removeErrorListeners();
    parser.addErrorListener(error_listener);

    parser.setErrorHandler(new BailErrorStrategy());
    final StringSubstitutionParser.ExprContext result = parser.expr();
    if (result.exception != null)
        throw new IllegalArgumentException("errors during parsing: " + pattern, result.exception);
    else if (!error_listener.errors.isEmpty())
        throw new IllegalArgumentException("syntax errors during parsing:\n" + String.join("\n", error_listener.errors.stream().map(s -> "  " + s).collect(Collectors.toList())));
    return result.s;
}
Example #6
Source File: CollectdTags.java    From monsoon with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
public static Map<String, Any2<String, Number>> parse(String pattern) {
    class DescriptiveErrorListener extends BaseErrorListener {
        public List<String> errors = new ArrayList<>();

        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
                                int line, int charPositionInLine,
                                String msg, RecognitionException e) {
            errors.add(String.format("%d:%d: %s", line, charPositionInLine, msg));
        }
    }

    final DescriptiveErrorListener error_listener = new DescriptiveErrorListener();

    final CollectdTagsLexer lexer = new CollectdTagsLexer(new ANTLRInputStream(pattern));
    lexer.removeErrorListeners();
    lexer.addErrorListener(error_listener);
    final CollectdTagsParser parser = new CollectdTagsParser(new UnbufferedTokenStream(lexer));
    parser.removeErrorListeners();
    parser.addErrorListener(error_listener);

    parser.setErrorHandler(new BailErrorStrategy());
    final CollectdTagsParser.ExprContext result = parser.expr();
    if (result.exception != null)
        throw new IllegalArgumentException("errors during parsing: " + pattern, result.exception);
    else if (!error_listener.errors.isEmpty())
        throw new IllegalArgumentException("syntax errors during parsing:\n" + String.join("\n", error_listener.errors.stream().map(s -> "  " + s).collect(Collectors.toList())));
    return result.result;
}
Example #7
Source File: IndexRQL.java    From indexr with Apache License 2.0 5 votes vote down vote up 
private static ParseTree parseSQL(String sql) {
    CharStream charStream = new ANTLRInputStream(sql);
    RQLLexer lexer = new RQLLexer(charStream);
    lexer.setTokenFactory(new CommonTokenFactory(true));
    TokenStream tokenStream = new UnbufferedTokenStream<CommonToken>(lexer);
    RQLParser parser = new RQLParser(tokenStream);
    parser.setErrorHandler(new BailErrorStrategy());
    return parser.root();
}
Example #8
Source File: CQL.java    From chronix.server with Apache License 2.0 5 votes vote down vote up 
/**
 * Parses the given Chronix Function parameter
 *
 * @param cf the Chronix Function parameter
 * @return the parse result containing the Chronix Functions
 * @throws CQLException if the given query string (value of cf) is invalid
 */
public CQLCFResult parseCF(String cf) throws CQLException {
    if (cf == null || cf.isEmpty()) {
        return new CQLCFResult();
    }
    this.errorListener.setQuery(cf);
    this.lexer.setInputStream(new ANTLRInputStream(cf));
    this.parser.setTokenStream(new UnbufferedTokenStream(lexer));

    return parseChronixFunctionParameter(this.parser.cqlcf());
}
Example #9
Source File: Pql2Compiler.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up 
public AstNode parseToAstNode(String expression) {
  CharStream charStream = new ANTLRInputStream(expression);
  PQL2Lexer lexer = new PQL2Lexer(charStream);
  lexer.setTokenFactory(new CommonTokenFactory(true));
  TokenStream tokenStream = new UnbufferedTokenStream<CommonToken>(lexer);
  PQL2Parser parser = new PQL2Parser(tokenStream);
  parser.setErrorHandler(new BailErrorStrategy());

  // Parse
  Pql2AstListener listener = new Pql2AstListener(expression);
  new ParseTreeWalker().walk(listener, parser.expression());
  return listener.getRootNode();
}