org.antlr.v4.runtime.Lexer Java Examples

public CSSTokenRecovery(Lexer lexer,
                            CharStream input,
                            CSSLexerState ls,
                            Logger log) {
        this.lexer = lexer;
        this.input = input;
//        this.state = state;
        this.ls = ls;
        this.log = log;
        this.expectedToken = new Stack<Integer>();
        this.eof = false;
        lexerTypeMapper = CSSToken.createDefaultTypeMapper(lexer.getClass());
        typeMapper = new CSSToken.TypeMapper(CSSTokenRecovery.class, lexer.getClass(),
                "APOS", "QUOT", "RPAREN", "RCURLY", "IMPORT",
                "CHARSET", "STRING", "INVALID_STRING", "RBRACKET");
    }
 

/**
 * Consume all tokens a whole line at a time until the next token is one expected by the current
 * rule. Each line (as delimited by supplied separator token) starting from the current line up to
 * the last line consumed is placed in an {@link ErrorNode} and inserted as a child of the current
 * rule.
 *
 * @param recognizer The {@link Parser} to whom to delegate creation of each {@link ErrorNode}
 */
private void consumeBlocksUntilWanted(Parser recognizer) {
  IntervalSet expecting = recognizer.getExpectedTokens();
  IntervalSet whatFollowsLoopIterationOrRule = expecting.or(getErrorRecoverySet(recognizer));

  int nextToken;
  do {
    // Eat tokens until we are at the end of the line
    consumeUntilEndOfLine(recognizer);

    // Get the line number and separator text from the separator token
    Token separatorToken = recognizer.getCurrentToken();

    // Insert the current line as an {@link ErrorNode} as a child of the current rule
    createErrorNode(recognizer, recognizer.getContext(), separatorToken);

    // Eat the separator token
    recognizer.consume();

    nextToken = recognizer.getInputStream().LA(1);
  } while (!whatFollowsLoopIterationOrRule.contains(nextToken) && nextToken != Lexer.EOF);
}
 

@Override
public void parse() throws IOException {
       CharStream input = CharStreams.fromFileName(fileFullPath);
       Lexer lexer = new JavaLexer(input);
       lexer.setInterpreter(new LexerATNSimulator(lexer, lexer.getATN(), lexer.getInterpreter().decisionToDFA, new PredictionContextCache()));
       CommonTokenStream tokens = new CommonTokenStream(lexer);
       JavaParser parser = new JavaParser(tokens);
       ParserATNSimulator interpreter = new ParserATNSimulator(parser, parser.getATN(), parser.getInterpreter().decisionToDFA, new PredictionContextCache());
       parser.setInterpreter(interpreter);
       JavaListener bridge = new JavaListener(fileFullPath, entityRepo,inferer);
    ParseTreeWalker walker = new ParseTreeWalker();
    try {
    	walker.walk(bridge, parser.compilationUnit());
		Entity fileEntity = entityRepo.getEntity(fileFullPath);
		((FileEntity)fileEntity).cacheAllExpressions();
		interpreter.clearDFA();
		bridge.done();
    	
    }catch (Exception e) {
    	System.err.println("error encountered during parse..." );
    	e.printStackTrace();
    }
    
   }
 

@Override
public void parse() throws IOException {
	/* If file already exist, skip it */
	Entity fileEntity = entityRepo.getEntity(fileFullPath);
	if (fileEntity!=null && fileEntity instanceof FileEntity) {
		return;
	}
	/*parse file*/
       CharStream input = CharStreams.fromFileName(fileFullPath);
       Lexer lexer = new XMLLexer(input);
       CommonTokenStream tokens = new CommonTokenStream(lexer);
       XMLParser parser = new XMLParser(tokens);
       PomListener bridge = new PomListener(fileFullPath, entityRepo, includePaths,parseCreator,inferer);
    ParseTreeWalker walker = new ParseTreeWalker();
    walker.walk(bridge, parser.document());
	fileEntity = entityRepo.getEntity(fileFullPath);
	bridge.done();
	((FileEntity)fileEntity).cacheAllExpressions();
}
 

/** Return a string representing the escaped char for code c.  E.g., If c
 *  has value 0x100, you will get "\u0100".  ASCII gets the usual
 *  char (non-hex) representation.  Control characters are spit out
 *  as unicode.  While this is specially set up for returning Java strings,
 *  it can be used by any language target that has the same syntax. :)
 */
public static String getANTLRCharLiteralForChar(int c) {
	if ( c< Lexer.MIN_CHAR_VALUE ) {
		return "'<INVALID>'";
	}
	if ( c<ANTLRLiteralCharValueEscape.length && ANTLRLiteralCharValueEscape[c]!=null ) {
		return '\''+ANTLRLiteralCharValueEscape[c]+'\'';
	}
	if ( Character.UnicodeBlock.of((char)c)==Character.UnicodeBlock.BASIC_LATIN &&
		 !Character.isISOControl((char)c) ) {
		if ( c=='\\' ) {
			return "'\\\\'";
		}
		if ( c=='\'') {
			return "'\\''";
		}
		return '\''+Character.toString((char)c)+'\'';
	}
	// turn on the bit above max "\uFFFF" value so that we pad with zeros
	// then only take last 4 digits
	String hex = Integer.toHexString(c|0x10000).toUpperCase().substring(1,5);
	String unicodeStr = "'\\u"+hex+"'";
	return unicodeStr;
}
 

public ANTLRAssistBehavior(Class<? extends Parser> parserClass, Class<? extends Lexer> lexerClass, 
		String grammarFiles[], String tokenFile, String ruleName, boolean findAllPaths) {
	this.lexerClass = lexerClass;
	this.parserClass = parserClass;
	
	codeAssist = new CodeAssist(lexerClass, grammarFiles, tokenFile, findAllPaths) {

		@Override
		protected List<InputSuggestion> suggest(TerminalExpect terminalExpect) {
			return ANTLRAssistBehavior.this.suggest(terminalExpect);
		}

		@Override
		protected List<String> getHints(TerminalExpect terminalExpect) {
			return ANTLRAssistBehavior.this.getHints(terminalExpect);
		}

		@Override
		protected Optional<String> describe(TerminalExpect terminalExpect, String suggestedLiteral) {
			return ANTLRAssistBehavior.this.describe(terminalExpect, suggestedLiteral);
		}

	};
	this.ruleName = ruleName;
}
 

/**
 * Gets the name by which a token can be referenced in the generated code.
 * For tokens defined in a {@code tokens{}} block or via a lexer rule, this
 * is the declared name of the token. For token types generated by the use
 * of a string literal within a parser rule of a combined grammar, this is
 * the automatically generated token type which includes the
 * {@link #AUTO_GENERATED_TOKEN_NAME_PREFIX} prefix. For types which are not
 * associated with a defined token, this method returns
 * {@link #INVALID_TOKEN_NAME}.
 *
 * @param ttype The token type.
 * @return The name of the token with the specified type.
 */

public String getTokenName(int ttype) {
	// inside any target's char range and is lexer grammar?
	if ( isLexer() &&
		 ttype >= Lexer.MIN_CHAR_VALUE && ttype <= Lexer.MAX_CHAR_VALUE )
	{
		return CharSupport.getANTLRCharLiteralForChar(ttype);
	}

	if ( ttype==Token.EOF ) {
		return "EOF";
	}

	if (ttype >= 0 && ttype < typeToTokenList.size() && typeToTokenList.get(ttype) != null) {
		return typeToTokenList.get(ttype);
	}

	return INVALID_TOKEN_NAME;
}
 

@Test
public void blockComment()
{
    final CharStream input = CharStreams.fromString(
            "/* block comment */\n" +
            "/* block comment\n" +
            "   multiline */"
    );

    ZserioLexer lexer = new ZserioLexer(input);

    checkToken(lexer, ZserioLexer.BLOCK_COMMENT, Lexer.HIDDEN, "/* block comment */");
    checkToken(lexer, ZserioLexer.BLOCK_COMMENT, Lexer.HIDDEN, "/* block comment\n   multiline */");

    checkEOF(lexer);
}
 

@Override
public void parse(Snapshot snapshot, Task task, SourceModificationEvent event) throws ParseException {
  this.snapshot = snapshot;
  String text = snapshot.getText().toString();
  ANTLRInputStream input = new ANTLRInputStream(text);
  Lexer lexer = new EditorConfigLexer(input);
  lexer.removeErrorListeners();

  CommonTokenStream tokens = new CommonTokenStream(lexer);
  parser = new EditorConfigParser(tokens);
  parser.removeErrorListeners();
  syntaxErrors = new ArrayList<>();
  EditorConfigErrorListener errorListener = new EditorConfigErrorListener(syntaxErrors);
  parser.addErrorListener(errorListener);
  EditorConfigParser.FileContext root = parser.file();
  result = new ECParserResult(snapshot, parser, root);
}
 

public LangDescriptor(String name,
                      String corpusDir,
                      String fileRegex,
                      Class<? extends Lexer> lexerClass,
                      Class<? extends Parser> parserClass,
                      String startRuleName,
                      int indentSize,
                      int singleLineCommentType)
{
	this.name = name;
	this.corpusDir = corpusDir;
	this.fileRegex = fileRegex;
	this.lexerClass = lexerClass;
	this.parserClass = parserClass;
	this.startRuleName = startRuleName;
	this.indentSize = indentSize;
	this.singleLineCommentType = singleLineCommentType;
}
 

@Override
public void visitErrorNode(ErrorNode ctx) {
  String nodeText = BatfishCombinedParser.escape(ctx.getText());
  // _sb.append("\n");
  _ptSentences.getSentences().add("");
  for (int i = 0; i < _indent; i++) {
    _ptSentences.appendToLastSentence("  ");
  }
  int tokenType = ctx.getSymbol().getType();
  String tokenName;
  if (tokenType == Lexer.EOF) {
    tokenName = "EOF";
    _ptSentences.appendToLastSentence(tokenName + ":" + nodeText);
  } else if (tokenType == BatfishLexer.UNRECOGNIZED_LINE_TOKEN) {
    _ptSentences.appendToLastSentence("<UnrecognizedLine>:'" + nodeText + "'");
  } else {
    tokenName = _vocabulary.getSymbolicName(tokenType);
    _ptSentences.appendToLastSentence("<ErrorNode>:" + tokenName + ":'" + nodeText + "'");
  }
}
 

/**
 * Create an error node with the text of the current line and insert it into parse tree
 *
 * @param recognizer The recognizer with which to create the error node
 * @param separator The token that ends the unrecognized link. This is also used to determine the
 *     index of the line to return in error messages.
 * @return The token contained in the error node
 */
private Token createErrorNode(Parser recognizer, ParserRuleContext ctx, Token separator) {
  if (_recoveredAtEof) {
    _recoveredAtEof = false;
    throw new BatfishRecognitionException(recognizer, recognizer.getInputStream(), ctx);
  }
  if (separator.getType() == Lexer.EOF) {
    _recoveredAtEof = true;
  }
  String lineText = _lines[separator.getLine() - 1] + separator.getText();
  Token lineToken =
      new UnrecognizedLineToken(lineText, separator.getLine(), _parserStateAtRecovery);
  ErrorNode errorNode = recognizer.createErrorNode(ctx, lineToken);
  ctx.addErrorNode(errorNode);
  return lineToken;
}
 

public GUIController(List<TokenPositionAnalysis> analysisPerToken,
                     InputDocument testDoc,
                     String formattedText,
                     Class<? extends Lexer> lexerClass)
{
	this.analysisPerToken = analysisPerToken;
	this.formattedText = formattedText;
	this.lexerClass = lexerClass;
	this.testDoc = testDoc;
	this.scope = new BuffScope();
}
 

/**
 * Test with main grammar
 * @throws java.io.IOException
 */
@Test
public void testMainStrGram() throws IOException {
  Lexer lexer = new bnfLexer(new ANTLRInputStream(getClass().getResourceAsStream("/main.bnf")));        
  CommonTokenStream tokens = new CommonTokenStream(lexer);
  bnfParser grammarparser = new bnfParser(tokens);
  ParserRuleContext tree = grammarparser.rulelist();
  GeneratorVisitor extractor = new GeneratorVisitor(100,2,1,10,true);
  extractor.visit(tree);
  List<String> generatedTests = extractor.getTests();
  Assert.assertEquals(100, generatedTests.size());
}
 

/** What is the max char value possible for this grammar's target?  Use
	 *  unicode max if no target defined.
	 */
	public int getMaxCharValue() {
		return org.antlr.v4.runtime.Lexer.MAX_CHAR_VALUE;
//		if ( generator!=null ) {
//			return generator.target.getMaxCharValue(generator);
//		}
//		else {
//			return Label.MAX_CHAR_VALUE;
//		}
	}
 

/**
 * Test with course codes grammar
 * @throws java.io.IOException
 */
@Test
public void testCourseCodeGram() throws IOException {
  Lexer lexer = new bnfLexer(new ANTLRInputStream(getClass().getResourceAsStream("/coursecodes.bnf")));        
  CommonTokenStream tokens = new CommonTokenStream(lexer);
  bnfParser grammarparser = new bnfParser(tokens);
  ParserRuleContext tree = grammarparser.rulelist();
  GeneratorVisitor extractor = new GeneratorVisitor();
  extractor.visit(tree);
  List<String> generatedTests = extractor.getTests();
  Assert.assertEquals(10, generatedTests.size());
}
 

/**
 * Get a {@code ANTLRLexerState} instance representing the current state
 * of the specified lexer.
 *
 * @param lexer The lexer.
 * @return A {@code ANTLRLexerState} instance containing the current state of the lexer.
 */
protected ANTLRLexerState getLexerState(Lexer lexer) {
	if (lexer._modeStack.isEmpty()) {
		return new ANTLRLexerState(lexer._mode, null);
	}

	return new ANTLRLexerState(lexer._mode, lexer._modeStack);
}
 

public void apply(@NotNull Lexer lexer) {
	lexer._mode = getMode();
	lexer._modeStack.clear();
	if (getModeStack() != null) {
		lexer._modeStack.addAll(getModeStack());
	}
}
 

@SuppressWarnings("ObjectEquality")
public void configureLexer(Lexer lexer)
{
    requireNonNull(lexer, "lexer is null");
    // Intentional identity equals comparison
    checkArgument(atn == lexer.getATN(), "Lexer ATN mismatch: expected %s, found %s", atn, lexer.getATN());
    lexer.setInterpreter(new LexerATNSimulator(lexer, atn, decisionToDFA, predictionContextCache));
}
 

/**
 * Test with JSON grammar
 * @throws java.io.IOException
 */
@Test
public void testJSONGram() throws IOException {
  Lexer lexer = new bnfLexer(new ANTLRInputStream(getClass().getResourceAsStream("/json.bnf")));
  CommonTokenStream tokens = new CommonTokenStream(lexer);
  bnfParser grammarparser = new bnfParser(tokens);
  ParserRuleContext tree = grammarparser.rulelist();
  GeneratorVisitor extractor = new GeneratorVisitor(100,2,10,100,true);
  extractor.visit(tree);
  List<String> generatedTests = extractor.getTests();
  Assert.assertEquals(100, generatedTests.size());
}
 

@Override
public void exitS_groups_named(S_groups_namedContext ctx) {
  String groupName = unquote(ctx.name.getText());
  HierarchyTree tree = _hierarchy.getTree(groupName);
  if (tree == null) {
    tree = _hierarchy.newTree(groupName);
  }
  StatementContext statement = ctx.s_groups_tail().statement();
  if (statement == null) {
    return;
  }
  Interval interval = ctx.s_groups_tail().getSourceInterval();
  List<Token> unfilteredTokens = _combinedParser.getTokens().getTokens(interval.a, interval.b);
  HierarchyPath path = new HierarchyPath();
  for (Token currentToken : unfilteredTokens) {
    if (currentToken.getChannel() != Lexer.HIDDEN) {
      String text = currentToken.getText();
      int line = currentToken.getLine();
      if (currentToken.getType() == FlatJuniperLexer.WILDCARD) {
        path.addWildcardNode(text, line);
      } else {
        path.addNode(text, line);
      }
    }
  }
  path.setStatement(statement);
  tree.addPath(path, _currentSetLine, null);
}
 

public static CodeBuffTokenStream tokenize(String doc, Class<? extends Lexer> lexerClass)
	throws Exception {
	ANTLRInputStream input = new ANTLRInputStream(doc);
	Lexer lexer = getLexer(lexerClass, input);

	CodeBuffTokenStream tokens = new CodeBuffTokenStream(lexer);
	tokens.fill();
	return tokens;
}
 

/** Compute a document difference metric 0-1.0 between two documents that
	 *  are identical other than (likely) the whitespace and comments.
	 *
	 *  1.0 means the docs are maximally different and 0 means docs are identical.
	 *
	 *  The Levenshtein distance between the docs counts only
	 *  whitespace diffs as the non-WS content is identical.
	 *  Levenshtein distance is bounded by 0..max(len(doc1),len(doc2)) so
	 *  we normalize the distance by dividing by max WS count.
	 *
	 *  TODO: can we simplify this to a simple walk with two
	 *  cursors through the original vs formatted counting
	 *  mismatched whitespace? real text are like anchors.
	 */
	public static double docDiff(String original,
	                             String formatted,
	                             Class<? extends Lexer> lexerClass)
		throws Exception
	{
		// Grammar must strip all but real tokens and whitespace (and put that on hidden channel)
		CodeBuffTokenStream original_tokens = Tool.tokenize(original, lexerClass);
//		String s = original_tokens.getText();
		CodeBuffTokenStream formatted_tokens = Tool.tokenize(formatted, lexerClass);
//		String t = formatted_tokens.getText();

		// walk token streams and examine whitespace in between tokens
		int i = -1;
		int ws_distance = 0;
		int original_ws = 0;
		int formatted_ws = 0;
		while ( true ) {
			Token ot = original_tokens.LT(i); // TODO: FIX THIS! can't use LT()
			if ( ot==null || ot.getType()==Token.EOF ) break;
			List<Token> ows = original_tokens.getHiddenTokensToLeft(ot.getTokenIndex());
			original_ws += tokenText(ows).length();

			Token ft = formatted_tokens.LT(i); // TODO: FIX THIS! can't use LT()
			if ( ft==null || ft.getType()==Token.EOF ) break;
			List<Token> fws = formatted_tokens.getHiddenTokensToLeft(ft.getTokenIndex());
			formatted_ws += tokenText(fws).length();

			ws_distance += whitespaceEditDistance(tokenText(ows), tokenText(fws));
			i++;
		}
		// it's probably ok to ignore ws diffs after last real token

		int max_ws = Math.max(original_ws, formatted_ws);
		double normalized_ws_distance = ((float) ws_distance)/max_ws;
		return normalized_ws_distance;
	}
 

@Override
public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line,
                        int charPositionInLine, String msg, RecognitionException e) {

    String position = "line " + line + ", pos " + charPositionInLine;
    String charText = "";
    String hint = "";
    if (recognizer != null && recognizer instanceof Lexer) {
        Lexer lexer = (Lexer) recognizer;
        String fullText = lexer.getInputStream().toString();
        charText = String.valueOf(fullText.charAt(lexer.getCharIndex()));
        hint = AntlrUtils.underlineError(fullText, charText, line, charPositionInLine);
    }
    throw new LexicalErrorException(position + " near " + charText + " : " + msg + "\n" + hint, e);
}
 

/**
 * Returns {@code true} iff on the current index of the parser's
 * token stream a token exists on the {@code HIDDEN} channel which
 * either is a line terminator, or is a multi line comment that
 * contains a line terminator.
 *
 * @return {@code true} iff on the current index of the parser's
 * token stream a token exists on the {@code HIDDEN} channel which
 * either is a line terminator, or is a multi line comment that
 * contains a line terminator.
 */
protected boolean lineTerminatorAhead() {

    // Get the token ahead of the current index.
    int possibleIndexEosToken = this.getCurrentToken().getTokenIndex() - 1;
    Token ahead = _input.get(possibleIndexEosToken);

    if (ahead.getChannel() != Lexer.HIDDEN) {
        // We're only interested in tokens on the HIDDEN channel.
        return false;
    }

    if (ahead.getType() == com.uddernetworks.mspaint.code.lexer.javascript.JavaScriptParser.LineTerminator) {
        // There is definitely a line terminator ahead.
        return true;
    }

    if (ahead.getType() == com.uddernetworks.mspaint.code.lexer.javascript.JavaScriptParser.WhiteSpaces) {
        // Get the token ahead of the current whitespaces.
        possibleIndexEosToken = this.getCurrentToken().getTokenIndex() - 2;
        ahead = _input.get(possibleIndexEosToken);
    }

    // Get the token's text and type.
    String text = ahead.getText();
    int type = ahead.getType();

    // Check if the token is, or contains a line terminator.
    return (type == com.uddernetworks.mspaint.code.lexer.javascript.JavaScriptParser.MultiLineComment && (text.contains("\r") || text.contains("\n"))) ||
            (type == com.uddernetworks.mspaint.code.lexer.javascript.JavaScriptParser.LineTerminator);
}
 

private String printToken(Token token) {
  int modeAsInt = _combinedParser.getTokenMode(token);
  String mode = _combinedParser.getLexer().getModeNames()[modeAsInt];
  String rawTokenText = token.getText();
  String tokenText = BatfishCombinedParser.escape(rawTokenText);
  int tokenType = token.getType();
  String channel = token.getChannel() == Lexer.HIDDEN ? "(HIDDEN) " : "";
  String tokenName;
  int line = token.getLine();
  int col = token.getCharPositionInLine();
  if (tokenType == -1) {
    tokenName = "EOF";
  } else {
    tokenName = _combinedParser.getParser().getVocabulary().getSymbolicName(tokenType);
    tokenText = "'" + tokenText + "'";
  }
  return " line "
      + line
      + ":"
      + col
      + " "
      + channel
      + " "
      + tokenName
      + ":"
      + tokenText
      + (!"DEFAULT_MODE".equals(mode) ? "  <== mode:" + mode : "");
}
 

@Test
public void create_succeeds() {
    LexerAndParserFactory factory = new ReflectionLexerAndParserFactory(TestGrammarLexer.class,
            TestGrammarParser.class);
    Lexer createdLexer = factory.createLexer(null);
    Parser createdParser = factory.createParser(null);
    assertThat(createdLexer, instanceOf(TestGrammarLexer.class));
    assertThat(createdParser, instanceOf(TestGrammarParser.class));
}
 

/**
 * Sets the internal lexer and the parser.
 * @param lexer  the lexer
 * @param parser the parser
 */
void setLexerAndParser(final Lexer lexer, final ParserWithFormula parser) {
  this.lexer = lexer;
  this.parser = parser;
  this.parser.setFormulaFactory(this.f);
  this.lexer.removeErrorListeners();
  this.parser.removeErrorListeners();
  this.parser.setErrorHandler(new BailErrorStrategy());
}
 

/**
 * Wrap current unmatchable char up to next char in provided separator chars in a {@link
 * BatfishLexer#UNMATCHABLE_TOKEN} and emit it.
 */
public void recover() {
  // Always recover in the default mode -- otherwise, the parser can get stuck in an infinite
  // loop, e.g. if separator is not valid in the current mode.
  _lexer._mode = Lexer.DEFAULT_MODE;

  int tokenStartMarker = _lexer._input.mark();
  try {
    _lexer._token = null;
    _lexer._channel = Token.DEFAULT_CHANNEL;
    _lexer._tokenStartCharIndex = _lexer._input.index();
    _lexer._tokenStartCharPositionInLine = _lexer.getInterpreter().getCharPositionInLine();
    _lexer._tokenStartLine = _lexer.getInterpreter().getLine();
    _lexer._text = null;
    _lexer._type = BatfishLexer.UNMATCHABLE_TOKEN;
    for (int nextChar = _lexer._input.LA(1);
        !_separatorChars.contains(nextChar);
        nextChar = _lexer._input.LA(1)) {
      if (nextChar == IntStream.EOF) {
        _lexer._hitEOF = true;
        _lexer.emitEOF();
        return;
      }
      _lexer.getInterpreter().consume(_lexer._input);
    }
    _lexer.emit();
  } finally {
    // make sure we release marker after match or
    // unbuffered char stream will keep buffering
    _lexer._input.release(tokenStartMarker);
  }
}
 

public static List<Token> getRealTokens(CommonTokenStream tokens) {
	List<Token> real = new ArrayList<>();
	for (int i=0; i<tokens.size(); i++) {
		Token t = tokens.get(i);
		if ( t.getType()!=Token.EOF &&
			t.getChannel()==Lexer.DEFAULT_TOKEN_CHANNEL )
		{
			real.add(t);
		}
	}
	return real;
}