Java Code Examples for org.antlr.v4.runtime.Token#getLine()

public static void print(final ParseTree node, final int level, CommonTokenStream stream) {
	final Interval sourceInterval = node.getSourceInterval();

	final Token firstToken = stream.get(sourceInterval.a);

	int line = firstToken.getLine();
	int charStart = firstToken.getCharPositionInLine();

	int endLine = line;
	int endChar = charStart + firstToken.getText().length();

	String data = "@(" + line + ":" + charStart + "," + endLine + ":" + endChar + ") with text: "
			+ firstToken.getText();
	final int tmp = level + 1;
	final StringBuilder sb = new StringBuilder();
	sb.append(StringUtils.repeat("\t", level));
	sb.append(node.getClass().getSimpleName() + ": " + data + " :" + node.getText());
	System.out.println(sb.toString());
	final int n = node.getChildCount();
	for (int i = 0; i < n; i++) {

		final ParseTree c = node.getChild(i);
		print(c, tmp, stream);

	}
}
 

@Override
public void visitErrorNode(ErrorNode errorNode) {
  Token token = errorNode.getSymbol();
  int line = token.getLine();
  String lineText = errorNode.getText().replace("\n", "").replace("\r", "").trim();

  if (token instanceof UnrecognizedLineToken) {
    UnrecognizedLineToken unrecToken = (UnrecognizedLineToken) token;
    ParseWarning warning =
        new ParseWarning(
            line, lineText, unrecToken.getParserContext(), "This syntax is unrecognized");
    unrecognized(warning, null);
  } else {
    String msg = String.format("Unrecognized Line: %d: %s", line, lineText);
    _w.redFlag(msg + " Subsequent lines may not be processed correctly");
  }
}
 

public SyntaxError(GyroCharStream stream, String message, Token token) {
    this.stream = stream;
    this.message = message;
    this.line = token.getLine() - 1;
    this.startColumn = token.getCharPositionInLine();

    int start = token.getStartIndex();
    int stop = token.getStopIndex();

    if (start >= 0 && stop >= 0 && stop > start) {
        this.stopColumn = this.startColumn + stop - start;

    } else {
        this.stopColumn = this.startColumn;
    }
}
 

/**
 * Constructor from grammar token.
 *
 * @param token Grammar token to localize AST node in the sources.
 */
public AstLocation(Token token)
{
    if (token == null)
    {
        fileName = null;
        line = 0;
        column = 0;
    }
    else
    {
        fileName = token.getInputStream().getSourceName();
        line = token.getLine();
        column = token.getCharPositionInLine() + 1;
    }
}
 

@Override
public void visitErrorNode(ErrorNode errorNode) {
  Token token = errorNode.getSymbol();
  int line = token.getLine();
  String lineText = errorNode.getText().replace("\n", "").replace("\r", "").trim();
  _config.setUnrecognized(true);

  if (token instanceof UnrecognizedLineToken) {
    UnrecognizedLineToken unrecToken = (UnrecognizedLineToken) token;
    _w.getParseWarnings()
        .add(
            new ParseWarning(
                line, lineText, unrecToken.getParserContext(), "This syntax is unrecognized"));
  } else {
    String msg = String.format("Unrecognized Line: %d: %s", line, lineText);
    _w.redFlag(msg + " SUBSEQUENT LINES MAY NOT BE PROCESSED CORRECTLY");
  }
}
 

@Override
public void exitColonIdentifier(SqlBaseParser.ColonIdentifierContext context) {
    Token token = context.COLON_IDENT().getSymbol();
    throw new ParsingException(
        "identifiers must not contain ':'",
        null,
        token.getLine(),
        token.getCharPositionInLine());
}
 

@Override
public void exitDigitIdentifier(SqlBaseParser.DigitIdentifierContext context)
{
    Token token = context.DIGIT_IDENTIFIER().getSymbol();
    throw new ParsingException(
            "identifiers must not start with a digit; surround the identifier with double quotes",
            null,
            token.getLine(),
            token.getCharPositionInLine());
}
 

@Override
public void exitBackQuotedIdentifier(SqlBaseParser.BackQuotedIdentifierContext context)
{
    Token token = context.BACKQUOTED_IDENTIFIER().getSymbol();
    throw new ParsingException(
            "backquoted identifiers are not supported; use double quotes to quote identifiers",
            null,
            token.getLine(),
            token.getCharPositionInLine());
}
 

/**
 * Constructor from ANTLR4 token, expects that the input is parsed from a token
 * (e.g. documentation comment).
 *
 * @param token Token which text content is being parsed.
 */
public TokenParseErrorListener(Token token)
{
    sourceName = token.getInputStream().getSourceName();
    line = token.getLine();
    charPositionInLine = token.getCharPositionInLine();
}
 

private ParseException toParseException(final RecognitionException e) {
    final Token token = e.getOffendingToken();

    if (token.getType() == HeroicQueryLexer.UnterminatedQutoedString) {
        return new ParseException(String.format("unterminated string: %s", token.getText()),
            null, token.getLine(), token.getCharPositionInLine());
    }

    return new ParseException("unexpected token: " + token.getText(), null, token.getLine(),
        token.getCharPositionInLine());
}
 

public static OffsetRange getOffsetRange(Token start, Token stop) {
    OffsetRange offset = new OffsetRange();
    offset.startOffset = start.getStartIndex();
    offset.stopOffset = stop.getStopIndex();
    offset.startLine = start.getLine();
    offset.startLineColumn = start.getCharPositionInLine();
    offset.stopLine = stop.getLine();
    offset.stopLineColumn = stop.getCharPositionInLine();
    return offset;
}
 

/**
 * Ensures the ANTLR parser will throw an exception after the first error
 *
 * @param recognizer the parser being used
 * @return no actual return value
 * @throws RecognitionException not used as a ParseException wrapped in a RuntimeException is thrown instead
 */
@Override
public Token recoverInline(Parser recognizer) throws RecognitionException {
  Token token = recognizer.getCurrentToken();
  String message = "unexpected token " + getTokenErrorDisplay(token) +
      " on line (" + token.getLine() + ") position (" + token.getCharPositionInLine() + ")" +
      " was expecting one of " + recognizer.getExpectedTokens().toString(recognizer.getVocabulary());
  ParseException parseException = new ParseException(message, token.getStartIndex());
  throw new RuntimeException(parseException);
}
 

private Location toLocation(Scope scope, ParseTree node) {
    Token start;
    if (node instanceof ParserRuleContext) {
        start = ((ParserRuleContext) node).start;
    } else if (node instanceof TerminalNode) {
        start = ((TerminalNode) node).getSymbol();
    } else {
        throw new ProgramCompileException("Location is not available for type " + node.getClass());
    }
    Location location = new Location(scope != null ? scope.programName : "<string>", start.getLine(), start.getCharPositionInLine());
    return location;
}
 

private Point getStartPoint(Token token) {
    return new Point(token.getLine(), token.getCharPositionInLine());
}
 

XmlTerminal( Token t, XmlPart parent )
{
  super( parent, t.getStartIndex(), t.getStopIndex() - t.getStartIndex() + 1, t.getLine() );
  _text = t.getText();
}
 

public GrammarToken getBTToken(Token t)
{
	org.beetl.core.statement.GrammarToken token = new org.beetl.core.statement.GrammarToken(t.getText(),
			t.getLine(), t.getCharPositionInLine());
	return token;
}
 

private static NodeLocation getLocation(Token token) {
    requireNonNull(token, "token is null");
    return new NodeLocation(token.getLine(), token.getCharPositionInLine());
}
 

public void setProfilerData(PreviewState previewState, long parseTime_ns) {
	this.previewState = previewState;
	Parser parser = previewState.parsingResult.parser;
	ParseInfo parseInfo = parser.getParseInfo();
	updateTableModelPerExpertCheckBox(parseInfo);
	double parseTimeMS = parseTime_ns/(1000.0*1000.0);
	// microsecond decimal precision
	NumberFormat formatter = new DecimalFormat("#.###");
	parseTimeField.setText(formatter.format(parseTimeMS));
	double predTimeMS = parseInfo.getTotalTimeInPrediction()/(1000.0*1000.0);
	predictionTimeField.setText(
		String.format("%s = %3.2f%%", formatter.format(predTimeMS), 100*(predTimeMS)/parseTimeMS)
	                           );
	TokenStream tokens = parser.getInputStream();
	int numTokens = tokens.size();
	Token lastToken = tokens.get(numTokens-1);
	int numChar = lastToken.getStopIndex();
	int numLines = lastToken.getLine();
	if ( lastToken.getType()==Token.EOF ) {
		if ( numTokens<=1 ) {
			numLines = 0;
		}
		else {
			Token secondToLastToken = tokens.get(numTokens-2);
			numLines = secondToLastToken.getLine();
		}
	}
	inputSizeField.setText(String.format("%d char, %d lines",
	                                     numChar,
	                                     numLines));
	numTokensField.setText(String.valueOf(numTokens));
	double look =
		parseInfo.getTotalSLLLookaheadOps()+
			parseInfo.getTotalLLLookaheadOps();
	lookaheadBurdenField.setText(
		String.format("%d/%d = %3.2f", (long) look, numTokens, look/numTokens)
	                            );
	double atnLook = parseInfo.getTotalATNLookaheadOps();
	cacheMissRateField.setText(
		String.format("%d/%d = %3.2f%%", (long) atnLook, (long) look, atnLook*100.0/look)
	                          );
}
 

/** Get the token type and tree ancestor features. These are computed
 *  the same for both training and formatting.
 */
public static int[] getContextFeatures(Corpus corpus,
                                       Map<Token, TerminalNode> tokenToNodeMap,
                                       InputDocument doc,
                                       int i)
{
	int[] features = new int[NUM_FEATURES];
	CodeBuffTokenStream tokens = doc.tokens;
	TerminalNode node = tokenToNodeMap.get(tokens.get(i));
	if ( node==null ) {
		System.err.println("### No node associated with token "+tokens.get(i));
		return features;
	}
	Token curToken = node.getSymbol();

	// Get context information for previous token
	Token prevToken = tokens.getPreviousRealToken(i);
	TerminalNode prevNode = tokenToNodeMap.get(prevToken);

	ParserRuleContext prevEarliestRightAncestor = earliestAncestorEndingWithToken(prevNode);
	int prevEarliestAncestorRuleIndex = prevEarliestRightAncestor.getRuleIndex();
	int prevEarliestAncestorRuleAltNum = prevEarliestRightAncestor.getAltNumber();

	// Get context information for current token
	ParserRuleContext earliestLeftAncestor = earliestAncestorStartingWithToken(node);

	ParserRuleContext earliestLeftAncestorParent  =
		earliestLeftAncestor!=null ? earliestLeftAncestor.getParent() : null;

	ParserRuleContext earliestLeftAncestorParent2 =
		earliestLeftAncestorParent!=null ? earliestLeftAncestorParent.getParent() : null;

	ParserRuleContext earliestLeftAncestorParent3 =
		earliestLeftAncestorParent2!=null ? earliestLeftAncestorParent2.getParent() : null;

	ParserRuleContext earliestLeftAncestorParent4 =
		earliestLeftAncestorParent3!=null ? earliestLeftAncestorParent3.getParent() : null;

	ParserRuleContext earliestLeftAncestorParent5 =
		earliestLeftAncestorParent4!=null ? earliestLeftAncestorParent4.getParent() : null;

	features[INDEX_PREV_TYPE]                     = prevToken.getType();
	features[INDEX_PREV_EARLIEST_RIGHT_ANCESTOR]  = rulealt(prevEarliestAncestorRuleIndex,prevEarliestAncestorRuleAltNum);
	features[INDEX_CUR_TOKEN_TYPE]                = curToken.getType();
	features[INDEX_CUR_TOKEN_CHILD_INDEX]         = getChildIndexOrListMembership(node);
	features[INDEX_EARLIEST_LEFT_ANCESTOR]        = rulealt(earliestLeftAncestor);
	features[INDEX_ANCESTORS_CHILD_INDEX]         = getChildIndexOrListMembership(earliestLeftAncestor);
	features[INDEX_ANCESTORS_PARENT_RULE]         = earliestLeftAncestorParent!=null ? rulealt(earliestLeftAncestorParent) : -1;
	features[INDEX_ANCESTORS_PARENT_CHILD_INDEX]  = getChildIndexOrListMembership(earliestLeftAncestorParent);
	features[INDEX_ANCESTORS_PARENT2_RULE]        = earliestLeftAncestorParent2!=null ? rulealt(earliestLeftAncestorParent2) : -1;
	features[INDEX_ANCESTORS_PARENT2_CHILD_INDEX] = getChildIndexOrListMembership(earliestLeftAncestorParent2);
	features[INDEX_ANCESTORS_PARENT3_RULE]        = earliestLeftAncestorParent3!=null ? rulealt(earliestLeftAncestorParent3) : -1;
	features[INDEX_ANCESTORS_PARENT3_CHILD_INDEX] = getChildIndexOrListMembership(earliestLeftAncestorParent3);
	features[INDEX_ANCESTORS_PARENT4_RULE]        = earliestLeftAncestorParent4!=null ? rulealt(earliestLeftAncestorParent4) : -1;
	features[INDEX_ANCESTORS_PARENT4_CHILD_INDEX] = getChildIndexOrListMembership(earliestLeftAncestorParent4);
	features[INDEX_ANCESTORS_PARENT5_RULE]        = earliestLeftAncestorParent5!=null ? rulealt(earliestLeftAncestorParent5) : -1;
	features[INDEX_ANCESTORS_PARENT5_CHILD_INDEX] = getChildIndexOrListMembership(earliestLeftAncestorParent5);

	features[INDEX_MATCHING_TOKEN_STARTS_LINE] = getMatchingSymbolStartsLine(corpus, doc, node);
	features[INDEX_MATCHING_TOKEN_ENDS_LINE]   = getMatchingSymbolEndsLine(corpus, doc, node);

	features[INDEX_INFO_FILE]    = 0; // dummy; _toString() dumps filename w/o this value; placeholder for col in printout
	features[INDEX_INFO_LINE]    = curToken.getLine();
	features[INDEX_INFO_CHARPOS] = curToken.getCharPositionInLine();

	return features;
}
 

public static NodeLocation getLocation(Token token)
{
    requireNonNull(token, "token is null");
    return new NodeLocation(token.getLine(), token.getCharPositionInLine());
}