Java Code Examples for org.antlr.v4.runtime.Token#getStopIndex()

@Override
public void exitNonReserved(SqlBaseParser.NonReservedContext context)
{
    // we can't modify the tree during rule enter/exit event handling unless we're dealing with a terminal.
    // Otherwise, ANTLR gets confused an fires spurious notifications.
    if (!(context.getChild(0) instanceof TerminalNode)) {
        int rule = ((ParserRuleContext) context.getChild(0)).getRuleIndex();
        throw new AssertionError("nonReserved can only contain tokens. Found nested rule: " + ruleNames.get(rule));
    }

    // replace nonReserved words with IDENT tokens
    context.getParent().removeLastChild();

    Token token = (Token) context.getChild(0).getPayload();
    Token newToken = new CommonToken(
            new Pair<>(token.getTokenSource(), token.getInputStream()),
            SqlBaseLexer.IDENTIFIER,
            token.getChannel(),
            token.getStartIndex(),
            token.getStopIndex());

    context.getParent().addChild(parser.createTerminalNode(context.getParent(), newToken));
}
 

public SyntaxError(GyroCharStream stream, String message, Token token) {
    this.stream = stream;
    this.message = message;
    this.line = token.getLine() - 1;
    this.startColumn = token.getCharPositionInLine();

    int start = token.getStartIndex();
    int stop = token.getStopIndex();

    if (start >= 0 && stop >= 0 && stop > start) {
        this.stopColumn = this.startColumn + stop - start;

    } else {
        this.stopColumn = this.startColumn;
    }
}
 

public Location(ParserRuleContext parserRuleContext) {
  Token start = parserRuleContext.getStart();
  this.start = start.getStartIndex();
  Token stop = parserRuleContext.getStop();
  end = stop.getStopIndex();
  line = start.getLine();
}
 

public static OffsetRange getOffsetRange(Token start, Token stop) {
    OffsetRange offset = new OffsetRange();
    offset.startOffset = start.getStartIndex();
    offset.stopOffset = stop.getStopIndex();
    offset.startLine = start.getLine();
    offset.startLineColumn = start.getCharPositionInLine();
    offset.stopLine = stop.getLine();
    offset.stopLineColumn = stop.getCharPositionInLine();
    return offset;
}
 

static String getString(TerminalNode rule, Decoder decoder) {
    Token token = rule.getSymbol();
    CharStream cs = token.getInputStream();
    int pos = cs.index();
    cs.seek(token.getStartIndex() + 1);
    int len = token.getStopIndex() - token.getStartIndex() - 1;
    IntSupplier supplier = new IntSupplier() {
        int i = 0;
        
        @Override
        public int getAsInt() {
            if (i >= len) {
                return -1;
            }
            int ch = cs.LA(i + 1);
            if (ch == '\\') {
                i++;
                ch = cs.LA(i + 1);
                if (ch == '0') {
                    ch = 0;
                }
                else if (ch == 'n') {
                    ch = '\n';
                }
                else if (ch == 'r') {
                    ch = '\r';
                }
                else if (ch == 'Z') {
                    ch = '\032';
                }
            }
            i++;
            return ch;
        }
    };
    String result = decoder.toString(supplier);
    cs.seek(pos);
    return result;
}
 

private static byte[] getBytes(TerminalNode rule) {
    Token token = rule.getSymbol();
    byte[] bytes = new byte[token.getStopIndex() - token.getStartIndex() - 1];
    CharStream cs = token.getInputStream();
    int pos = cs.index();
    cs.seek(token.getStartIndex() + 1);
    int j = 0;
    for (int i = 0; i < bytes.length; i++) {
        int ch = cs.LA(i + 1);
        if (ch == '\\') {
            i++;
            ch = cs.LA(i + 1);
            if (ch == '0') {
                ch = 0;
            }
            else if (ch == 'n') {
                ch = '\n';
            }
            else if (ch == 'r') {
                ch = '\r';
            }
            else if (ch == 'Z') {
                ch = '\032';
            }
        }
        bytes[j] = (byte) ch;
        j++;
    }
    cs.seek(pos);
    if (j != bytes.length) {
        // esacpe characters
        byte[] old = bytes;
        bytes = new byte[j];
        System.arraycopy(old, 0, bytes, 0, j);
    }
    return bytes;
}
 

private ModifyColumnDefinitionSegment extractModifyColumnDefinition(final Token start, final Token stop, 
                                                                    final ColumnDefinitionContext columnDefinition, final FirstOrAfterColumnContext firstOrAfterColumn) {
    ModifyColumnDefinitionSegment result = new ModifyColumnDefinitionSegment(start.getStartIndex(), stop.getStopIndex(),
            (ColumnDefinitionSegment) visit(columnDefinition));
    if (null != firstOrAfterColumn) {
        result.setColumnPosition(getColumnPositionSegment(result.getColumnDefinition(), (ColumnPositionSegment) visit(firstOrAfterColumn)));
    }
    return result;
}
 

/** By default ANTLR makes the start/stop -1/-1 for invalid tokens
 *  which is reasonable but here we want to highlight the
 *  current position indicating that is where we lack a token.
 *  if no input, highlight at position 0.
 */
protected Token getMissingSymbol(Parser recognizer) {
	Token missingSymbol = super.getMissingSymbol(recognizer);
	// alter the default missing symbol.
	if ( missingSymbol instanceof CommonToken) {
		int start, stop;
		Token current = recognizer.getCurrentToken();
		start = current.getStartIndex();
		stop = current.getStopIndex();
		((CommonToken) missingSymbol).setStartIndex(start);
		((CommonToken) missingSymbol).setStopIndex(stop);
	}
	return missingSymbol;
}
 

private boolean inBounds(ParserRuleContext ctx, int offset) {
	Token start = ctx.getStart();
	Token stop = ctx.getStop();
	if ( start!=null && stop!=null ) {
		return start.getStartIndex()<=offset && stop.getStopIndex()>=offset;
	}
	return false;
}
 

public static Token getTokenForCharIndex(TokenStream tokens, int charIndex) {
	for (int i=0; i<tokens.size(); i++) {
		Token t = tokens.get(i);
		if ( charIndex>=t.getStartIndex() && charIndex<=t.getStopIndex() ) {
			return t;
		}
	}
	return null;
}
 

@Override
public AttributedString highlight(LineReader reader, String buffer) {
	
	LexParseErrorCapturer errs = new LexParseErrorCapturer("");
	
	CharStream input = CharStreams.fromString(buffer, "");
	ConcurnasLexer lexer = new ConcurnasLexer(input);
	lexer.removeErrorListeners();
	lexer.addErrorListener(errs);
	
	List<? extends Token> toks = lexer.getAllTokens();
	if(toks.isEmpty() || !errs.errors.isEmpty()) {
		return AttributedString.fromAnsi(buffer);//do nothing
	}
	
	int cursor = 0;
	
	StringBuilder sb = new StringBuilder();
	for(Token tok : toks) {
		int startIdx = tok.getStartIndex();
		while(cursor++ < startIdx-1) {
			sb.append(" ");//add whitespace padding
		}
		
		String tokenName = ConcurnasLexer.VOCABULARY.getDisplayName(tok.getType());
		
		int color = Color_Default;
		if(null != tokenName && tokenToColor.containsKey(tokenName)) {
			color = tokenToColor.get(tokenName);
		}

		boolean escaped = buffer.charAt(startIdx) == '\\';
		
		sb.append(String.format("\u001b[38;5;%sm%s", color, (escaped?"\\" : "") + tok.getText()));
		cursor = tok.getStopIndex();
	}
	
	int len = buffer.length();
	while(cursor++ < len-1) {
		sb.append(" ");//trailing spaces
	}
	return AttributedString.fromAnsi(sb.toString());
}
 

private int getEndOffset(Token token) {
    return token.getStopIndex() + 1;
}
 

XmlTerminal( Token t, XmlPart parent )
{
  super( parent, t.getStartIndex(), t.getStopIndex() - t.getStartIndex() + 1, t.getLine() );
  _text = t.getText();
}
 

@Override
public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine,
		String msg, RecognitionException e) {
	super.syntaxError(recognizer, offendingSymbol, line, charPositionInLine, msg, e);

	this.line = line;

	if (recognizer.getInputStream() instanceof CommonTokenStream) {
		StringBuilder errorBuilder = new StringBuilder(msg);

		CommonTokenStream tokenStream = (CommonTokenStream) recognizer.getInputStream();
		errorBuilder.append("\n");

		String expression = tokenStream.getTokenSource().getInputStream().toString();
		String[] split = expression.split("(\r\n|\n)");
		String error = null;
		if (split.length > 0) {
			if (line - 1 >= 0 && line - 1 < split.length) {
				error = split[line - 1].replace("\t", " ");
			} else {
				error = split[split.length - 1].replace("\t", " ");
			}
			errorBuilder.append(error);
			errorBuilder.append("\n");
		} else {
			errorBuilder.append(expression);
			errorBuilder.append("\n");
		}

		for (int i = 0; i < charPositionInLine; i++) {
			errorBuilder.append(" ");
		}

		if (offendingSymbol instanceof Token) {
			Token token = (Token) offendingSymbol;
			int startIndex = token.getStartIndex();
			int stopIndex = token.getStopIndex();
			if (startIndex >= 0 && stopIndex >= 0 && startIndex <= stopIndex) {
				for (int i = token.getStartIndex(); i <= token.getStopIndex(); i++) {
					errorBuilder.append("^");
				}
			} else {
				errorBuilder.append("^");
			}
		} else {
			errorBuilder.append("^");
		}

		errorMessage = errorBuilder.toString();
	} else {
		errorMessage = msg;
	}
}
 

public void setProfilerData(PreviewState previewState, long parseTime_ns) {
	this.previewState = previewState;
	Parser parser = previewState.parsingResult.parser;
	ParseInfo parseInfo = parser.getParseInfo();
	updateTableModelPerExpertCheckBox(parseInfo);
	double parseTimeMS = parseTime_ns/(1000.0*1000.0);
	// microsecond decimal precision
	NumberFormat formatter = new DecimalFormat("#.###");
	parseTimeField.setText(formatter.format(parseTimeMS));
	double predTimeMS = parseInfo.getTotalTimeInPrediction()/(1000.0*1000.0);
	predictionTimeField.setText(
		String.format("%s = %3.2f%%", formatter.format(predTimeMS), 100*(predTimeMS)/parseTimeMS)
	                           );
	TokenStream tokens = parser.getInputStream();
	int numTokens = tokens.size();
	Token lastToken = tokens.get(numTokens-1);
	int numChar = lastToken.getStopIndex();
	int numLines = lastToken.getLine();
	if ( lastToken.getType()==Token.EOF ) {
		if ( numTokens<=1 ) {
			numLines = 0;
		}
		else {
			Token secondToLastToken = tokens.get(numTokens-2);
			numLines = secondToLastToken.getLine();
		}
	}
	inputSizeField.setText(String.format("%d char, %d lines",
	                                     numChar,
	                                     numLines));
	numTokensField.setText(String.valueOf(numTokens));
	double look =
		parseInfo.getTotalSLLLookaheadOps()+
			parseInfo.getTotalLLLookaheadOps();
	lookaheadBurdenField.setText(
		String.format("%d/%d = %3.2f", (long) look, numTokens, look/numTokens)
	                            );
	double atnLook = parseInfo.getTotalATNLookaheadOps();
	cacheMissRateField.setText(
		String.format("%d/%d = %3.2f%%", (long) atnLook, (long) look, atnLook*100.0/look)
	                          );
}