org.antlr.v4.runtime.TokenStream Java Examples

@Test
public void simpleCommentsWork() throws Exception {
    String queryString = "select * where { ?s ?p ?o }#?s ?p ?o }";
    String expectedQuery = "select * where { ?s ?p ?o }";
    Sparql11Parser parser = Query.getParser(queryString);
    TokenStream tokens = parser.getTokenStream();
    parser.addErrorListener(new BaseErrorListener() {
        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) {
            throw new IllegalStateException("failed to parse at line " + line + " due to " + msg, e);
        }
    });

    parser.query();
    assertEquals(expectedQuery, tokens.getText());
}
 

@VisibleForTesting
static AtlasDSLParser.QueryContext parse(String queryStr) throws AtlasBaseException {
    AtlasDSLParser.QueryContext ret;
    try {
        InputStream    stream           = new ByteArrayInputStream(queryStr.getBytes());
        AtlasDSLLexer  lexer            = new AtlasDSLLexer(CharStreams.fromStream(stream));
        Validator      validator        = new Validator();
        TokenStream    inputTokenStream = new CommonTokenStream(lexer);
        AtlasDSLParser parser           = new AtlasDSLParser(inputTokenStream);

        parser.removeErrorListeners();
        parser.addErrorListener(validator);

        // Validate the syntax of the query here
        ret = parser.query();
        if (!validator.isValid()) {
            LOG.error("Invalid DSL: {} Reason: {}", queryStr, validator.getErrorMsg());
            throw new AtlasBaseException(AtlasErrorCode.INVALID_DSL_QUERY, queryStr, validator.getErrorMsg());
        }

    } catch (IOException e) {
        throw new AtlasBaseException(e);
    }

    return ret;
}
 

/**
 * Parses and evaluates the given Stellar expression, {@code rule}.
 * @param rule The Stellar expression to parse and evaluate.
 * @return The Expression, which can be reevaluated without reparsing in different Contexts and Resolvers.
 */
public static StellarCompiler.Expression compile(final String rule) {
  if (rule == null || isEmpty(rule.trim())) {
    return null;
  }

  ANTLRInputStream input = new ANTLRInputStream(rule);
  StellarLexer lexer = new StellarLexer(input);
  lexer.removeErrorListeners();
  lexer.addErrorListener(new ErrorListener());
  TokenStream tokens = new CommonTokenStream(lexer);
  StellarParser parser = new StellarParser(tokens);

  StellarCompiler treeBuilder = new StellarCompiler(
      ArithmeticEvaluator.INSTANCE,
      NumberLiteralEvaluator.INSTANCE,
      ComparisonExpressionWithOperatorEvaluator.INSTANCE
  );
  parser.addParseListener(treeBuilder);
  parser.removeErrorListeners();
  parser.addErrorListener(new ErrorListener());
  parser.transformation();
  return treeBuilder.getExpression();
}
 

@NotNull
private String getText(TokenStream tokens, Interval interval)
{
    int start = interval.a;
    int stop = interval.b;
    if (start < 0 || stop < 0)
        return "";
    
    if (stop >= tokens.size())
        stop = tokens.size() - 1;
    
    StringBuilder buf = new StringBuilder();
    for (int i = start; i <= stop; i++)
    {
        Token t = tokens.get(i);
        if (t.getType() == Token.EOF)
            break;
        buf.append(t.getText());
        if (i != stop)
        {
            buf.append(" ");
        }
    }
    return buf.toString();
}
 

protected void reportNoViableAlternative(Parser recognizer, NoViableAltException e) {
  TokenStream tokens = recognizer.getInputStream();
  String input;
  if (tokens != null) {
    if (e.getStartToken().getType() == -1) {
      input = "<EOF>";
    } else {
      input = tokens.getText(e.getStartToken(), e.getOffendingToken());
    }
  } else {
    input = "<unknown input>";
  }

  String msg = "no viable alternative at input " + this.escapeWSAndQuote(input);
  recognizer.notifyErrorListeners(e.getOffendingToken(), msg, e);
}
 

@Override
protected void reportNoViableAlternative(Parser recognizer, NoViableAltException e) {
	// change error message from default implementation
	TokenStream tokens = recognizer.getInputStream();
	String input;
	if (tokens != null) {
		if (e.getStartToken().getType() == Token.EOF) {
			input = "the end";
		} else {
			input = escapeWSAndQuote(tokens.getText(e.getStartToken(), e.getOffendingToken()));
		}
	} else {
		input = escapeWSAndQuote("<unknown input>");
	}
	String msg = "inadmissible input at " + input;
	recognizer.notifyErrorListeners(e.getOffendingToken(), msg, e);
}
 

public String validateFormula(String formula, List<SimpleSelectionField> columns) throws ValidationException, JSONException {

		validateBrackets(formula);
		validateFields(formula, columns);

		formula = "select ".concat(formula);
		CharStream inputStream = CharStreams.fromString(formula);
		SQLiteLexer tokenSource = new SQLiteLexer(new CaseChangingCharStream(inputStream, true));
		TokenStream tokenStream = new CommonTokenStream(tokenSource);
		SQLiteParser sQLiteParser = new SQLiteParser(tokenStream);

		sQLiteParser.addErrorListener(ThrowingErrorListener.INSTANCE);
		try {
			ParseTree root = sQLiteParser.select_stmt();
			root.toStringTree();
		} catch (Exception e) {
			throw new ValidationException(e);
		}
		if (sQLiteParser.getNumberOfSyntaxErrors() > 0) {
			throw new ValidationException();

		}

		return VALIDATION_OK;
	}
 

/**
 "If an operator has whitespace on the right side only, it is treated as a
 postfix unary operator. As an example, the ++ operator in a++ b is treated
 as a postfix unary operator."
 "If an operator has no whitespace on the left but is followed immediately
 by a dot (.), it is treated as a postfix unary operator. As an example,
 the ++ operator in a++.b is treated as a postfix unary operator (a++ .b
 rather than a ++ .b)."
 */
public static boolean isPostfixOp(TokenStream tokens) {
	int stop = getLastOpTokenIndex(tokens);
	if ( stop==-1 ) return false;

	int start = tokens.index();
	Token prevToken = tokens.get(start-1); // includes hidden-channel tokens
	Token nextToken = tokens.get(stop+1);
	boolean prevIsWS = isLeftOperatorWS(prevToken);
	boolean nextIsWS = isRightOperatorWS(nextToken);
	boolean result =
		!prevIsWS && nextIsWS ||
		!prevIsWS && nextToken.getType()==SwiftParser.DOT;
	String text = tokens.getText(Interval.of(start, stop));
	//System.out.println("isPostfixOp: '"+prevToken+"','"+text+"','"+nextToken+"' is "+result);
	return result;
}
 

public static AstNode buildAst(String expression) {
  CharStream charStream = new ANTLRInputStream(expression);
  PQL2Lexer lexer = new PQL2Lexer(charStream);
  lexer.setTokenFactory(new CommonTokenFactory(true));
  lexer.removeErrorListeners();
  lexer.addErrorListener(ERROR_LISTENER);
  TokenStream tokenStream = new UnbufferedTokenStream<CommonToken>(lexer);
  PQL2Parser parser = new PQL2Parser(tokenStream);
  parser.setErrorHandler(new BailErrorStrategy());
  parser.removeErrorListeners();
  parser.addErrorListener(ERROR_LISTENER);

  // Parse
  ParseTree parseTree = parser.root();

  ParseTreeWalker walker = new ParseTreeWalker();
  Pql2AstListener listener = new Pql2AstListener(expression);
  walker.walk(listener, parseTree);

  AstNode rootNode = listener.getRootNode();
  return rootNode;
}
 

@Test
public void insensitiveToCase() throws Exception {
    String queryString = "select * WHERE { ?x ?Y ?z }";
    String queryNoSpaces = "select*WHERE{?x?Y?z}";
    Sparql11Parser parser = Query.getParser(queryString);
    TokenStream tokens = parser.getTokenStream();
    parser.addErrorListener(new BaseErrorListener() {
        @Override
        public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) {
            throw new IllegalStateException("failed to parse at line " + line + " due to " + msg, e);
        }
    });

    assertEquals(queryNoSpaces, parser.query().selectQuery().getText());
    assertEquals(queryString, tokens.getText());
}
 

private static Cql2ElmVisitor createElmTranslatorVisitor(TokenStream tokens, ParseTree tree) {
    CqlPreprocessorVisitor preprocessor = new CqlPreprocessorVisitor();
    preprocessor.visit(tree);
    ModelManager modelManager = new ModelManager();
    LibraryManager libraryManager = new LibraryManager(modelManager);
    LibraryBuilder libraryBuilder = new LibraryBuilder(modelManager, libraryManager, ucumService);
    Cql2ElmVisitor visitor = new Cql2ElmVisitor(libraryBuilder);
    visitor.setTokenStream(tokens);
    visitor.setLibraryInfo(preprocessor.getLibraryInfo());
    return visitor;
}
 

private static ParseTree parseSQL(String sql) {
    CharStream charStream = new ANTLRInputStream(sql);
    RQLLexer lexer = new RQLLexer(charStream);
    lexer.setTokenFactory(new CommonTokenFactory(true));
    TokenStream tokenStream = new UnbufferedTokenStream<CommonToken>(lexer);
    RQLParser parser = new RQLParser(tokenStream);
    parser.setErrorHandler(new BailErrorStrategy());
    return parser.root();
}
 

public static FieldAccessDescriptor parse(String expr) {
  CharStream charStream = CharStreams.fromString(expr);
  FieldSpecifierNotationLexer lexer = new FieldSpecifierNotationLexer(charStream);
  TokenStream tokens = new CommonTokenStream(lexer);
  FieldSpecifierNotationParser parser = new FieldSpecifierNotationParser(tokens);
  return new BuildFieldAccessDescriptor().visit(parser.dotExpression());
}
 

private Function parseFunctionDef(String code) {
       CharStream charStream = new ANTLRInputStream(code);
       RankPLLexer lexer = new RankPLLexer(charStream);
       TokenStream tokens = new CommonTokenStream(lexer);
       RankPLParser parser = new RankPLParser(tokens);

       ConcreteParser classVisitor = new ConcreteParser();
       FunctiondefContext ctx = parser.functiondef();
       Function res = (Function)classVisitor.visit(ctx);
       return res;
}
 

public static void main(String[] args) throws IOException {
		ANTLRInputStream input = new ANTLRFileStream(args[0]);
		SimpleLexer l = new SimpleLexer(input);
		TokenStream tokens = new CommonTokenStream(l);

		SimpleParser parser = new SimpleParser(tokens);
		ParserRuleContext tree = parser.file();
		System.out.println(tree.toStringTree(parser));
//		tree.inspect(parser);

		DefSymbolsAndScopes def = new DefSymbolsAndScopes();
		ParseTreeWalker walker = new ParseTreeWalker();
		walker.walk(def, tree);
	}
 

/**
 * Distinguish between method declaration and method call/constructor declaration
 */
public static boolean isInvalidMethodDeclaration(TokenStream ts) {
    int tokenType = ts.LT(1).getType();

    return (Identifier == tokenType || CapitalizedIdentifier == tokenType || StringLiteral == tokenType)
            && LPAREN == (ts.LT(2).getType());
}
 

protected String createNoViableAlternativeErrorMessage(Parser recognizer, NoViableAltException e) {
    TokenStream tokens = recognizer.getInputStream();
    String input;
    if (tokens != null) {
        if (e.getStartToken().getType() == Token.EOF) {
            input = "<EOF>";
        } else {
            input = charStream.getText(Interval.of(e.getStartToken().getStartIndex(), e.getOffendingToken().getStopIndex()));
        }
    } else {
        input = "<unknown input>";
    }

    return "Unexpected input: " + escapeWSAndQuote(input);
}
 

public GroovyLangParser(TokenStream input) {
    super(input);

    this.setInterpreter(new ParserATNSimulator(this, ParserAtnManager.INSTANCE.getATN()));

    if (GROOVY_PARSER_PROFILING_ENABLED) {
        this.setProfile(true);
    }
}
 

private AbstractExpression parseExpr(String code) {
       CharStream charStream = new ANTLRInputStream(code);
       RankPLLexer lexer = new RankPLLexer(charStream);
       TokenStream tokens = new CommonTokenStream(lexer);
       RankPLParser parser = new RankPLParser(tokens);

       ConcreteParser classVisitor = new ConcreteParser();
       ExpContext ctx = parser.exp();
       AbstractExpression res = (AbstractExpression)classVisitor.visit(ctx);
       return res;
}
 

public KuduSQLParser(TokenStream input)
{
  super(input);
  removeErrorListeners();
  kuduSQLExpressionErrorListener = new KuduSQLExpressionErrorListener();
  addErrorListener(kuduSQLExpressionErrorListener);
}
 

/**
 "If an operator has whitespace on the left side only, it is treated as a
 prefix unary operator. As an example, the ++ operator in a ++b is treated
 as a prefix unary operator."
*/
public static boolean isPrefixOp(TokenStream tokens) {
	int stop = getLastOpTokenIndex(tokens);
	if ( stop==-1 ) return false;

	int start = tokens.index();
	Token prevToken = tokens.get(start-1); // includes hidden-channel tokens
	Token nextToken = tokens.get(stop+1);
	boolean prevIsWS = isLeftOperatorWS(prevToken);
	boolean nextIsWS = isRightOperatorWS(nextToken);
	boolean result = prevIsWS && !nextIsWS;
	String text = tokens.getText(Interval.of(start, stop));
	//System.out.println("isPrefixOp: '"+prevToken+"','"+text+"','"+nextToken+"' is "+result);
	return result;
}
 

/**
 "If an operator has whitespace around both sides or around neither side,
 it is treated as a binary operator. As an example, the + operator in a+b
 and a + b is treated as a binary operator."
 */
public static boolean isBinaryOp(TokenStream tokens) {
	int stop = getLastOpTokenIndex(tokens);
	if ( stop==-1 ) return false;

	int start = tokens.index();
	Token prevToken = tokens.get(start-1); // includes hidden-channel tokens
	Token nextToken = tokens.get(stop+1);
	boolean prevIsWS = isLeftOperatorWS(prevToken);
	boolean nextIsWS = isRightOperatorWS(nextToken);
	boolean result = prevIsWS && nextIsWS || (!prevIsWS && !nextIsWS);
	String text = tokens.getText(Interval.of(start, stop));
	//System.out.println("isBinaryOp: '"+prevToken+"','"+text+"','"+nextToken+"' is "+result);
	return result;
}
 

private static TokenStream createTokenStream(String statement) {
  if (statement == null || isEmpty(statement.trim())) {
    return null;
  }
  statement = statement.trim();
  ANTLRInputStream input = new ANTLRInputStream(statement);
  WindowLexer lexer = new WindowLexer(input);
  lexer.removeErrorListeners();
  lexer.addErrorListener(new ErrorListener());
  TokenStream tokens = new CommonTokenStream(lexer);
  return tokens;
}
 

/** @since 4.5.1 */
public GrammarParserInterpreter createGrammarParserInterpreter(TokenStream tokenStream) {
	if (this.isLexer()) {
		throw new IllegalStateException("A parser interpreter can only be created for a parser or combined grammar.");
	}
	char[] serializedAtn = ATNSerializer.getSerializedAsChars(atn);
	ATN deserialized = new ATNDeserializer().deserialize(serializedAtn);
	return new GrammarParserInterpreter(this, deserialized, tokenStream);
}
 

@Override
		public void recover(Parser recognizer, RecognitionException e) {
			int errIndex = recognizer.getInputStream().index();
			if ( firstErrorTokenIndex == -1 ) {
				firstErrorTokenIndex = errIndex; // latch
			}
//			System.err.println("recover: error at " + errIndex);
			TokenStream input = recognizer.getInputStream();
			if ( input.index()<input.size()-1 ) { // don't consume() eof
				recognizer.consume(); // just kill this bad token and let it continue.
			}
		}
 

public static Token getTokenForCharIndex(TokenStream tokens, int charIndex) {
	for (int i=0; i<tokens.size(); i++) {
		Token t = tokens.get(i);
		if ( charIndex>=t.getStartIndex() && charIndex<=t.getStopIndex() ) {
			return t;
		}
	}
	return null;
}
 

/** Derive a new parser from an old one that has knowledge of the grammar.
 *  The Grammar object is used to correctly compute outer alternative
 *  numbers for parse tree nodes. A parser of the same type is created
 *  for subclasses of {@link ParserInterpreter}.
 */
public static ParserInterpreter deriveTempParserInterpreter(Grammar g, Parser originalParser, TokenStream tokens) {
	ParserInterpreter parser;
	if (originalParser instanceof ParserInterpreter) {
		Class<? extends ParserInterpreter> c = originalParser.getClass().asSubclass(ParserInterpreter.class);
		try {
			Constructor<? extends ParserInterpreter> ctor = c.getConstructor(Grammar.class, ATN.class, TokenStream.class);
			parser = ctor.newInstance(g, originalParser.getATN(), originalParser.getTokenStream());
		}
		catch (Exception e) {
			throw new IllegalArgumentException("can't create parser to match incoming "+originalParser.getClass().getSimpleName(), e);
		}
	}
	else { // must've been a generated parser
		char[] serializedAtn = ATNSerializer.getSerializedAsChars(originalParser.getATN());
		ATN deserialized = new ATNDeserializer().deserialize(serializedAtn);
		parser = new ParserInterpreter(originalParser.getGrammarFileName(),
									   originalParser.getVocabulary(),
									   Arrays.asList(originalParser.getRuleNames()),
									   deserialized,
									   tokens);
	}

	parser.setInputStream(tokens);

	// Make sure that we don't get any error messages from using this temporary parser
	parser.setErrorHandler(new BailErrorStrategy());
	parser.removeErrorListeners();
	parser.removeParseListeners();
	parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION);
	return parser;
}
 

protected void reportNoViableAlternative(@NotNull Parser recognizer, @NotNull NoViableAltException e)
{
	TokenStream tokens = recognizer.getInputStream();
	String input;
	if (tokens instanceof TokenStream)
	{
		if (e.getStartToken().getType() == Token.EOF)
			input = "<文件尾>";
		else
			input = tokens.getText(e.getStartToken(), e.getOffendingToken());
	}
	else
	{
		input = "<未知输入>";
	}
	BeetlException exception = null;
	if(keys.contains(e.getOffendingToken().getText())){
		 exception = new BeetlParserException(BeetlException.PARSER_VIABLE_ERROR,
					"不允许"+e.getOffendingToken().getText()+"关键出现在这里"+":"+escapeWSAndQuote(input), e);
	}else{
		exception = new BeetlParserException(BeetlException.PARSER_VIABLE_ERROR,
				escapeWSAndQuote(input), e);
	}
	//		String msg = "no viable alternative at input " + escapeWSAndQuote(input);

	exception.pushToken(this.getGrammarToken(e.getOffendingToken()));

	throw exception;
}
 

public ParserInterpreter createParserInterpreter(TokenStream tokenStream) {
	if (this.isLexer()) {
		throw new IllegalStateException("A parser interpreter can only be created for a parser or combined grammar.");
	}

	char[] serializedAtn = ATNSerializer.getSerializedAsChars(atn);
	ATN deserialized = new ATNDeserializer().deserialize(serializedAtn);
	return new ParserInterpreter(fileName, getVocabulary(), Arrays.asList(getRuleNames()), deserialized, tokenStream);
}
 

/** Return two booleans packed into lowest 2 bits for left (high) and right (low)
 *  whitespace.
 */
public static int getLeftRightWS(TokenStream tokens, ParserRuleContext ctx) {
	int left = ctx.start.getTokenIndex();
	int right = ctx.stop.getTokenIndex();
	Token prevToken = tokens.get(left-1); // includes hidden-channel tokens
	Token nextToken = tokens.get(right+1);
	boolean prevIsWS = isLeftOperatorWS(prevToken);
	boolean nextIsWS = isRightOperatorWS(nextToken);
	int b = (prevIsWS ? 1 : 0) << 1 | (nextIsWS ? 1 : 0) ;
	return b;
}