org.antlr.v4.runtime.CharStream Java Examples

@Test
public void docComment()
{
    final CharStream input = CharStreams.fromString(
            "/** doc comment */\n" +
            "/**\n" +
            " * doc comment\n" +
            " * multiline\n" +
            " */"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.DOC_COMMENT, ZserioLexer.DOC, "/** doc comment */");
    checkToken(lexer, ZserioLexer.DOC_COMMENT, ZserioLexer.DOC, "/**\n * doc comment\n * multiline\n */");

    checkEOF(lexer);
}
 

private Grammar processAntrlStream(CharStream inStream)
    {
        BNFLexer lexer = new BNFLexer(inStream);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        BNFParser parser = new BNFParser(tokens);
        // leaving the old listeners in gives a nice error messsage
        // parser.removeErrorListeners();
        // lexer.removeErrorListeners();
        lexer.addErrorListener(new FailingErrorListener());
        parser.addErrorListener(new FailingErrorListener());

        ParseTree tree = parser.rulelist();

        ParseTreeWalker walker = new ParseTreeWalker();
        BNFListener listener = new BNFListener(tokens);
        walker.walk(listener, tree);

        GrammarTop itemTree = listener.getTreeTop();
//        LOGGER.warn("bnf gave {}", itemTree.getStructure(""));
        // convert to openCypher grammar
        GrammarConverter converter = new GrammarConverter(itemTree);
        return converter.convert();
    }
 

@Test
public void octalLiteral()
{
    final CharStream input = CharStreams.fromString(
            "0101 0770 012 00 01234567"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "0101");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "0770");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "012");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "00");
    checkToken(lexer, ZserioLexer.OCTAL_LITERAL, "01234567");

    checkEOF(lexer);
}
 

private Enum parseEnumBlock(String input) {
    CharStream stream = CharStreams.fromString(input);
    ProtoLexer lexer = new ProtoLexer(stream);
    lexer.removeErrorListeners();
    lexer.addErrorListener(TestUtils.ERROR_LISTENER);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    ProtoParser parser = new ProtoParser(tokenStream);
    parser.setErrorHandler(new BailErrorStrategy());
    parser.removeErrorListeners();
    parser.addErrorListener(TestUtils.ERROR_LISTENER);
    ProtoContext context = new ProtoContext("test.proto");
    Proto proto = new Proto();
    context.push(proto);
    EnumParseListener enumParseListener = new EnumParseListener(tokenStream, context);
    OptionParseListener optionParseListener = new OptionParseListener(tokenStream, context);
    parser.addParseListener(enumParseListener);
    parser.addParseListener(optionParseListener);
    parser.enumBlock();
    return proto.getEnums().get(0);
}
 

@Test
public void invalidStringLiteral()
{
    final CharStream input = CharStreams.fromString(
            " \"invalid unicode escape character \\uBAD\" " +
            " \"invalid octal escape character \\09\" " +
            " \"invalid hexadecimal escape character \\xA\" "
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.INVALID_STRING_LITERAL, "\"invalid unicode escape character \\uBAD\"");
    checkToken(lexer, ZserioLexer.INVALID_STRING_LITERAL, "\"invalid octal escape character \\09\"");
    checkToken(lexer, ZserioLexer.INVALID_STRING_LITERAL, "\"invalid hexadecimal escape character \\xA\"");

    checkEOF(lexer);
}
 

@Override
public void parse() throws IOException {
       CharStream input = CharStreams.fromFileName(fileFullPath);
       Lexer lexer = new JavaLexer(input);
       lexer.setInterpreter(new LexerATNSimulator(lexer, lexer.getATN(), lexer.getInterpreter().decisionToDFA, new PredictionContextCache()));
       CommonTokenStream tokens = new CommonTokenStream(lexer);
       JavaParser parser = new JavaParser(tokens);
       ParserATNSimulator interpreter = new ParserATNSimulator(parser, parser.getATN(), parser.getInterpreter().decisionToDFA, new PredictionContextCache());
       parser.setInterpreter(interpreter);
       JavaListener bridge = new JavaListener(fileFullPath, entityRepo,inferer);
    ParseTreeWalker walker = new ParseTreeWalker();
    try {
    	walker.walk(bridge, parser.compilationUnit());
		Entity fileEntity = entityRepo.getEntity(fileFullPath);
		((FileEntity)fileEntity).cacheAllExpressions();
		interpreter.clearDFA();
		bridge.done();
    	
    }catch (Exception e) {
    	System.err.println("error encountered during parse..." );
    	e.printStackTrace();
    }
    
   }
 

public static Query parse(String input) {

        if (StringUtils.isEmpty(input)) {
            return Queries.empty();
        }
        //Lex (with Antlr's generated lexer)
        CharStream inputStream = CharStreams.fromString(input);
        EleQueryLexer lexer = new EleQueryLexer(inputStream);
        CommonTokenStream tokens = new CommonTokenStream(lexer);

        //Parse (with Antlr's generated parser)
        EleQueryParser parser = new EleQueryParser(tokens);
        ParseTree parseTree = parser.query();

        //Extract AST from the Antlr parse tree
        QueryListener listener = new QueryListener();
        ParseTreeWalker walker = new ParseTreeWalker();
        walker.walk(listener, parseTree);

        return listener.getQuery();
    }
 

public static ConcurnasParser miniParse(String text, String name, int startingLine, int startingColumn, LexParseErrorCapturer errors) {
	CharStream input = CharStreams.fromString(text, name);
	
	ConcurnasLexer lexer = new ConcurnasLexer(input);
	if(startingLine > -1) {
		lexer.setTokenFactory(new LocationRepointCommonTokenFactory(startingLine,startingColumn));
	}
	
	lexer.permitDollarPrefixRefName = true;
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	ConcurnasParser parser = new ConcurnasParser(tokens);
	
	parser.getInterpreter().setPredictionMode(PredictionMode.SLL); 
	
	parser.removeErrorListeners(); // remove ConsoleErrorListener
	parser.addErrorListener(errors); // add ours
	
	lexer.removeErrorListeners(); // remove ConsoleErrorListener
	lexer.addErrorListener(errors); // add ours
	
	return parser;
}
 

/** Parse a file and also traverse parse tree to set XML attributes Map
	 *  for any attrs rule refs.
	 */
	public Object parse(Artifact artifact, String fileName, boolean isRoot) throws IOException {
		CharStream input = CharStreams.fromFileName(fileName);
		BookishLexer lexer = new BookishLexer(input);
		CommonTokenStream tokens = new CommonTokenStream(lexer);
		BookishParser parser = new BookishParser(tokens);

		lexer.removeErrorListeners();
		lexer.addErrorListener(getErrorListener());
		parser.removeErrorListeners();
		parser.addErrorListener(getErrorListener());

		ParserRuleContext doctree;
		if ( isRoot ) doctree = parser.rootdocument();
		else doctree = parser.chapter();

		verifyAndGatherXMLAttributes(parser, doctree);

//		System.out.println();
//		System.out.println(doctree.toStringTree(Arrays.asList(BookishParser.ruleNames)));

		if ( isRoot ) {
			return new RootDocInfo(artifact, parser, doctree);
		}
		return new ChapDocInfo(artifact, parser, (BookishParser.ChapterContext)doctree);
	}
 

@Test
public void floatOrDoubleLiteral()
{
    final CharStream input = CharStreams.fromString(
            "0e1 0e-1 1e2f 9. 9.12 9.f 9.1f 9.e2 9.e2f 9.e-2f .0 .0e-1 .0f .1e3f"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "0e1");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "0e-1");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "1e2f");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "9.");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "9.12");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.f");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.1f");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, "9.e2");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.e2f");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, "9.e-2f");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, ".0");
    checkToken(lexer, ZserioLexer.DOUBLE_LITERAL, ".0e-1");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, ".0f");
    checkToken(lexer, ZserioLexer.FLOAT_LITERAL, ".1e3f");

    checkEOF(lexer);
}
 

private Message parseMessage(String input) {
    CharStream stream = CharStreams.fromString(input);
    ProtoLexer lexer = new ProtoLexer(stream);
    lexer.removeErrorListeners();
    lexer.addErrorListener(TestUtils.ERROR_LISTENER);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);
    ProtoParser parser = new ProtoParser(tokenStream);
    parser.removeErrorListeners();
    parser.addErrorListener(TestUtils.ERROR_LISTENER);
    ProtoContext context = new ProtoContext("test.proto");
    MessageParseListener messageParseListener = new MessageParseListener(tokenStream, context);
    OptionParseListener optionParseListener = new OptionParseListener(tokenStream, context);
    Proto proto = new Proto();
    context.push(proto);
    parser.addParseListener(messageParseListener);
    parser.addParseListener(optionParseListener);
    parser.messageBlock();
    return proto.getMessages().get(0);
}
 

private Grammar processAntrlStream(CharStream inStream)
    {
        Gee4Lexer lexer = new Gee4Lexer(inStream);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        Gee4Parser parser = new Gee4Parser(tokens);
        // leaving the old listeners in gives a nice error messsage
        // parser.removeErrorListeners();
        // lexer.removeErrorListeners();
        lexer.addErrorListener(new FailingErrorListener());
        parser.addErrorListener(new FailingErrorListener());

        ParseTree tree = parser.wholegrammar();

        ParseTreeWalker walker = new ParseTreeWalker();
        G4Listener listener = new G4Listener(tokens);
        walker.walk(listener, tree);
        
        GrammarTop itemTree = listener.getTreeTop();
//        LOGGER.warn("bnf gave {}", itemTree.getStructure(""));
        // convert to openCypher grammar
        GrammarConverter converter = new GrammarConverter(itemTree);
        return converter.convert();
    }
 

/**
 * Parses a given page or range of pages. If the given string cannot
 * be parsed, the method will return a page range with a literal string.
 * @param pages the page or range of pages
 * @return the parsed page or page range (never {@code null})
 */
public static PageRange parse(String pages) {
    CharStream cs = CharStreams.fromString(pages);
    InternalPageLexer lexer = new InternalPageLexer(cs);
    lexer.removeErrorListeners(); // do not output errors to console
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    InternalPageParser parser = new InternalPageParser(tokens);
    parser.removeErrorListeners(); // do not output errors to console
    PagesContext ctx = parser.pages();
    if (ctx.literal == null || ctx.literal.isEmpty() ||
            ctx.exception != null || parser.getNumberOfSyntaxErrors() > 0) {
        // unparsable fall back to literal string
        return new PageRange(pages, null, null, false);
    }
    return new PageRange(ctx.literal, ctx.pageFrom, ctx.numberOfPages,
            ctx.multiplePages);
}
 

/** {@inheritDoc} */
@Override public XGModelComposition parse(byte[] mdl) {
    try (ByteArrayInputStream bais = new ByteArrayInputStream(mdl)) {
        CharStream cStream = CharStreams.fromStream(bais);
        XGBoostModelLexer lexer = new XGBoostModelLexer(cStream);
        CommonTokenStream tokens = new CommonTokenStream(lexer);
        XGBoostModelParser parser = new XGBoostModelParser(tokens);

        XGModelVisitor visitor = new XGModelVisitor();

        return visitor.visit(parser.xgModel());
    }
    catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

static ParsedStrategy parseWithAntlr(final CharStream s) {
    final NaturalLanguageStrategyLexer l = new NaturalLanguageStrategyLexer(s);
    l.removeErrorListeners(); // no sysout
    final CommonTokenStream ts = new CommonTokenStream(l);
    final NaturalLanguageStrategyParser p = new NaturalLanguageStrategyParser(ts);
    p.removeErrorListeners(); // turn sysout into an exception
    p.addErrorListener(ERROR_LISTENER);
    return p.primaryExpression().result;
}
 

public static Interpolative parse(String interpolativeString) {
	CharStream is = CharStreams.fromString(interpolativeString); 
	InterpolativeLexer lexer = new InterpolativeLexer(is);
	lexer.removeErrorListeners();
	lexer.addErrorListener(new BaseErrorListener() {

		@Override
		public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line,
				int charPositionInLine, String msg, RecognitionException e) {
			throw new RuntimeException("Malformed interpolative: " + interpolativeString);
		}
		
	});
	CommonTokenStream tokens = new CommonTokenStream(lexer);
	InterpolativeParser parser = new InterpolativeParser(tokens);
	parser.removeErrorListeners();
	parser.setErrorHandler(new BailErrorStrategy());
	
	List<Segment> segments = new ArrayList<>();
	for (SegmentContext segment: parser.interpolative().segment()) {
		if (segment.Literal() != null) 
			segments.add(new Segment(Type.LITERAL, segment.Literal().getText()));
		else
			segments.add(new Segment(Type.VARIABLE, FenceAware.unfence(segment.Variable().getText())));
	}
	return new Interpolative(segments);
}
 

@Nullable
@Override
public CharStream read(String contentOrName) {
  if (!contentReaded) {
    contentReaded = true;
    return CharStreams.fromString(contentOrName);
  }

  return importReader.read(contentOrName);
}
 

private JaybirdSqlParser createParser(String testString) {
    CharStream stream = CharStreams.fromString(testString);
    
    JaybirdSqlLexer lexer = new JaybirdSqlLexer(stream);
    CommonTokenStream tokenStream = new CommonTokenStream(lexer);

    return new JaybirdSqlParser(tokenStream);
}
 

@Override
public CommonToken create(Pair<TokenSource, CharStream> source, int type, String text,
						  int channel, int start, int stop,
						  int line, int charPositionInLine)
{
	return super.create( source,  type,  text, channel,  start,  stop, /*line +*/ lineoffset,  charPositionInLine + coloffset);
}
 

static String getString(TerminalNode rule, Decoder decoder) {
    Token token = rule.getSymbol();
    CharStream cs = token.getInputStream();
    int pos = cs.index();
    cs.seek(token.getStartIndex() + 1);
    int len = token.getStopIndex() - token.getStartIndex() - 1;
    IntSupplier supplier = new IntSupplier() {
        int i = 0;
        
        @Override
        public int getAsInt() {
            if (i >= len) {
                return -1;
            }
            int ch = cs.LA(i + 1);
            if (ch == '\\') {
                i++;
                ch = cs.LA(i + 1);
                if (ch == '0') {
                    ch = 0;
                }
                else if (ch == 'n') {
                    ch = '\n';
                }
                else if (ch == 'r') {
                    ch = '\r';
                }
                else if (ch == 'Z') {
                    ch = '\032';
                }
            }
            i++;
            return ch;
        }
    };
    String result = decoder.toString(supplier);
    cs.seek(pos);
    return result;
}
 

static FishParser.ScriptContext parse(CharStream cs) {
    FishLexer lexer = new FishLexer(cs);
    CommonTokenStream tokens = new CommonTokenStream(lexer);
    tokens.setTokenSource(lexer);
    FishParser parser = new FishParser(tokens);
    parser.setErrorHandler(new BailErrorStrategy());
    FishParser.ScriptContext script = parser.script();
    return script;
}
 

public static FieldAccessDescriptor parse(String expr) {
  CharStream charStream = CharStreams.fromString(expr);
  FieldSpecifierNotationLexer lexer = new FieldSpecifierNotationLexer(charStream);
  TokenStream tokens = new CommonTokenStream(lexer);
  FieldSpecifierNotationParser parser = new FieldSpecifierNotationParser(tokens);
  return new BuildFieldAccessDescriptor().visit(parser.dotExpression());
}
 

@Nullable
@Override
public CharStream read(String name) {
    for (FileReader delegate : delegateList) {
        CharStream result = delegate.read(name);
        if (result != null) {
            return result;
        }
    }
    return null;
}
 

protected static Object evalProgram(String program) {
       CharStream charStream = new ANTLRInputStream(program);
       RankPLLexer lexer = new RankPLLexer(charStream);
       TokenStream tokens = new CommonTokenStream(lexer);
       RankPLParser parser = new RankPLParser(tokens);

       ConcreteParser classVisitor = new ConcreteParser();
       ExpContext ctx = parser.exp();
       AbstractExpression res = (AbstractExpression)classVisitor.visit(ctx);
       
       
       return res;
}
 

public Operator parse(String sql, ZoneId zoneId) throws ParseCancellationException {
  LogicalGenerator logicalGenerator = new LogicalGenerator(zoneId);
  CharStream charStream1 = CharStreams.fromString(sql);
  SqlBaseLexer lexer1 = new SqlBaseLexer(charStream1);
  CommonTokenStream tokens1 = new CommonTokenStream(lexer1);
  SqlBaseParser parser1 = new SqlBaseParser(tokens1);
  parser1.getInterpreter().setPredictionMode(PredictionMode.SLL);
  parser1.removeErrorListeners();
  parser1.addErrorListener(LogicalGeneratorError.INSTANCE);
  ParseTree tree;
  try {
    tree = parser1.singleStatement();  // STAGE 1
  }
  catch (Exception ex) {
    CharStream charStream2 = CharStreams.fromString(sql);
    SqlBaseLexer lexer2 = new SqlBaseLexer(charStream2);
    CommonTokenStream tokens2 = new CommonTokenStream(lexer2);
    SqlBaseParser parser2 = new SqlBaseParser(tokens2);
    parser2.getInterpreter().setPredictionMode(PredictionMode.LL);
    parser2.removeErrorListeners();
    parser2.addErrorListener(LogicalGeneratorError.INSTANCE);
    tree = parser2.singleStatement();  // STAGE 2
    // if we parse ok, it's LL not SLL
  }
  walker.walk(logicalGenerator, tree);
  return logicalGenerator.getLogicalPlan();
}
 

@Nullable
@Override
public CharStream read(String name) {
    try {
        InputStream resource = readResource(name);
        if (resource != null) {
            return CharStreams.fromStream(resource);
        }
    } catch (Exception e) {
        LOGGER.error("Could not read {}", name, e);
    }
    return null;
}
 

@Test
public void testRead() throws Exception {
    LocalFileReader reader = new LocalFileReader(tempDirectory1, tempDirectory2);
    CharStream a = reader.read("1.proto");
    CharStream b = reader.read("2.proto");
    CharStream c = reader.read("3.proto");
    assertNotNull(a);
    assertNotNull(b);
    assertNull(c);
    assertEquals("1", a.getText(Interval.of(0, 1)));
    assertEquals("2", b.getText(Interval.of(0, 1)));
}
 

@Test
public void testRead() throws Exception {
    ClasspathFileReader reader = new ClasspathFileReader();
    CharStream a = reader.read("protostuff_unittest/messages_sample.proto");
    CharStream b = reader.read("this_file_does_not_exist.proto");
    assertNotNull(a);
    assertNull(b);
}
 

public static Expression parseExpression(String expression) {
    CharStream input = CharStreams.fromString(expression);
    SiddhiQLLexer lexer = new SiddhiQLLexer(input);
    lexer.removeErrorListeners();
    lexer.addErrorListener(SiddhiErrorListener.INSTANCE);

    CommonTokenStream tokens = new CommonTokenStream(lexer);
    SiddhiQLParser parser = new SiddhiQLParser(tokens);
    parser.removeErrorListeners();
    parser.addErrorListener(SiddhiErrorListener.INSTANCE);
    ParseTree tree = parser.expression();

    SiddhiQLVisitor eval = new SiddhiQLBaseVisitorImpl();
    return (Expression) eval.visit(tree);
}
 

public ConcBuilder createBuilder() {
	ErrorCap errors = new ErrorCap();
	ConcBuilder builder;
	try {
		CharStream input = CharStreams.fromString(inputString, "cmd");
		
		ConcLexer lexer = new ConcLexer(input);
		CommonTokenStream tokens = new CommonTokenStream(lexer);
		ConcParser parser = new ConcParser(tokens);
		
		parser.getInterpreter().setPredictionMode(PredictionMode.SLL); 
		
		parser.removeErrorListeners(); // remove ConsoleErrorListener
		parser.addErrorListener(errors); // add ours
		
		lexer.removeErrorListeners(); // remove ConsoleErrorListener
		lexer.addErrorListener(errors); // add ours
		
		builder = new ConcBuilder();
		parser.conc().accept(builder);
		
	}catch(Throwable e) {
		return null;
	}
	
	if(errors.hasErrors) {
		return null;
	}
	
	return builder;
}