Java Code Examples for org.antlr.v4.runtime.Token#getType()

@Override
public Token consume() {
  Token o = getCurrentToken();
  if (o.getType() != EOF) {
    getInputStream().consume();
    _lastConsumedToken = o.getType();
  }
  boolean hasListener = _parseListeners != null && !_parseListeners.isEmpty();
  if ((_buildParseTrees || hasListener) && !_errHandler.inErrorRecoveryMode(this)) {
    TerminalNode node = _ctx.addChild(createTerminalNode(_ctx, o));
    if (_parseListeners != null) {
      for (ParseTreeListener listener : _parseListeners) {
        listener.visitTerminal(node);
      }
    }
  }
  return o;
}
 

private static ComparisonExpression.Type getComparisonOperator(Token symbol) {
  switch (symbol.getType()) {
    case SqlBaseLexer.EQ:
      return ComparisonExpression.Type.EQUAL;
    case SqlBaseLexer.NEQ:
      return ComparisonExpression.Type.NOT_EQUAL;
    case SqlBaseLexer.LT:
      return ComparisonExpression.Type.LESS_THAN;
    case SqlBaseLexer.LTE:
      return ComparisonExpression.Type.LESS_THAN_OR_EQUAL;
    case SqlBaseLexer.GT:
      return ComparisonExpression.Type.GREATER_THAN;
    case SqlBaseLexer.GTE:
      return ComparisonExpression.Type.GREATER_THAN_OR_EQUAL;
    default:
      throw new IllegalArgumentException("Unsupported operator: " + symbol.getText());
  }
}
 

private Expr<?> createMultiplicativeSubExpr(final Expr<?> leftOp, final Expr<?> rightOp, final Token oper) {
	switch (oper.getType()) {

		case StsDslParser.MUL:
			return createMulExpr(leftOp, rightOp);

		case StsDslParser.DIV:
			return createDivExpr(leftOp, rightOp);

		case StsDslParser.MOD:
			return createModExpr(leftOp, rightOp);

		case StsDslParser.REM:
			return createRemExpr(leftOp, rightOp);

		default:
			throw new AssertionError();
	}
}
 

private static IntervalLiteral.IntervalField getIntervalFieldType(Token token) {
    switch (token.getType()) {
        case SqlBaseLexer.YEAR:
            return IntervalLiteral.IntervalField.YEAR;
        case SqlBaseLexer.MONTH:
            return IntervalLiteral.IntervalField.MONTH;
        case SqlBaseLexer.DAY:
            return IntervalLiteral.IntervalField.DAY;
        case SqlBaseLexer.HOUR:
            return IntervalLiteral.IntervalField.HOUR;
        case SqlBaseLexer.MINUTE:
            return IntervalLiteral.IntervalField.MINUTE;
        case SqlBaseLexer.SECOND:
            return IntervalLiteral.IntervalField.SECOND;
        default:
            throw new IllegalArgumentException("Unsupported interval field: " + token.getText());
    }
}
 

private static ArithmeticBinaryExpression.Type getArithmeticBinaryOperator(Token operator) {
  switch (operator.getType()) {
    case SqlBaseLexer.PLUS:
      return ArithmeticBinaryExpression.Type.ADD;
    case SqlBaseLexer.MINUS:
      return ArithmeticBinaryExpression.Type.SUBTRACT;
    case SqlBaseLexer.ASTERISK:
      return ArithmeticBinaryExpression.Type.MULTIPLY;
    case SqlBaseLexer.SLASH:
      return ArithmeticBinaryExpression.Type.DIVIDE;
    case SqlBaseLexer.PERCENT:
      return ArithmeticBinaryExpression.Type.MODULUS;
    default:
      throw new UnsupportedOperationException("Unsupported operator: " + operator.getText());
  }
}
 

private static ComparisonExpression.Operator getComparisonOperator(Token symbol)
{
    switch (symbol.getType()) {
        case SqlBaseLexer.EQ:
            return ComparisonExpression.Operator.EQUAL;
        case SqlBaseLexer.NEQ:
            return ComparisonExpression.Operator.NOT_EQUAL;
        case SqlBaseLexer.LT:
            return ComparisonExpression.Operator.LESS_THAN;
        case SqlBaseLexer.LTE:
            return ComparisonExpression.Operator.LESS_THAN_OR_EQUAL;
        case SqlBaseLexer.GT:
            return ComparisonExpression.Operator.GREATER_THAN;
        case SqlBaseLexer.GTE:
            return ComparisonExpression.Operator.GREATER_THAN_OR_EQUAL;
    }

    throw new IllegalArgumentException("Unsupported operator: " + symbol.getText());
}
 

public static String squeezeStatement(String sql)
{
    TokenSource tokens = getLexer(sql, ImmutableSet.of());
    StringBuilder sb = new StringBuilder();
    while (true) {
        Token token = tokens.nextToken();
        if (token.getType() == Token.EOF) {
            break;
        }
        if (token.getType() == SqlBaseLexer.WS) {
            sb.append(' ');
        }
        else {
            sb.append(token.getText());
        }
    }
    return sb.toString().trim();
}
 

@Test
public void compareWithSonarInputFile() throws IOException {
	String s = "select " + "*" + "from dbo.test\r\nselect 1";
	File ff = folder.newFile("test.sql");
	FileUtils.write(ff, s);
	DefaultInputFile f = new TestInputFileBuilder("", "test.sql").setLanguage(TSQLLanguage.KEY).initMetadata(s)
			.build();
	AntlrContext result = AntlrUtils.getRequest(s);

	for (Token t : result.getStream().getTokens()) {
		if (t.getType() == Token.EOF) {
			continue;
		}
		int[] start = result.getLineAndColumn(t.getStartIndex());
		int[] end = result.getLineAndColumn(t.getStopIndex());
		TextPointer p1 = f.newPointer(t.getStartIndex());
		TextPointer p2 = f.newPointer(t.getStopIndex());
		Assert.assertNotNull(start);
		Assert.assertNotNull(end);
		Assert.assertEquals(p1.line(), start[0]);
		Assert.assertEquals(p1.lineOffset(), start[1]);
		Assert.assertEquals(p2.line(), end[0]);
		Assert.assertEquals(p2.lineOffset(), end[1]);
	}
}
 

public static int getMatchingSymbolEndsLine(Corpus corpus,
	                                            InputDocument doc,
	                                            TerminalNode node)
	{
		TerminalNode matchingLeftNode = getMatchingLeftSymbol(corpus, doc, node);
		if ( matchingLeftNode != null ) {
			Token matchingLeftToken = matchingLeftNode.getSymbol();
			int i = matchingLeftToken.getTokenIndex();
			Token tokenAfterMatchingToken = doc.tokens.getNextRealToken(i);
//			System.out.printf("doc=%s node=%s, pair=%s, after=%s\n",
//			                  new File(doc.fileName).getName(), node.getSymbol(), matchingLeftToken, tokenAfterMatchingToken);
			if ( tokenAfterMatchingToken!=null ) {
				if ( tokenAfterMatchingToken.getType()==Token.EOF ) {
					return 1;
				}
				return tokenAfterMatchingToken.getLine()>matchingLeftToken.getLine() ? 1 : 0;
			}
		}
		return NOT_PAIR;
	}
 

@Override
public final void emit(Token token) {
  super.emit(token);
  if (token.getChannel() != HIDDEN) {
    _lastTokenType = token.getType();
  }
}
 

@Override
public Code visitConstant(ConstantContext ctx) {
    Token token = ((TerminalNode) ctx.getChild(0)).getSymbol();
    String text = token.getText();
    switch (token.getType()) {
    case JetTemplateParser.STRING_DOUBLE:
        return new SegmentCode(String.class, text, ctx);
    case JetTemplateParser.STRING_SINGLE:
        text = StringEscapeUtils.asCanonicalJavaString(text);
        return new SegmentCode(String.class, text, ctx);
    case JetTemplateParser.INTEGER:
    case JetTemplateParser.INTEGER_HEX:
    case JetTemplateParser.FLOATING_POINT:
        Class<?> klass;
        if (text.endsWith("l") || text.endsWith("L")) {
            klass = Long.TYPE;
        } else if (text.endsWith("f") || text.endsWith("F")) {
            klass = Float.TYPE;
        } else if (text.endsWith("d") || text.endsWith("D")) {
            klass = Double.TYPE;
        } else if (token.getType() == JetTemplateParser.FLOATING_POINT) {
            klass = Double.TYPE; // 浮点数默认是double
        } else {
            klass = Integer.TYPE;
        }
        return new SegmentCode(klass, text, ctx);
    case JetTemplateParser.KEYWORD_TRUE:
        return new SegmentCode(Boolean.TYPE, text, ctx);
    case JetTemplateParser.KEYWORD_FALSE:
        return new SegmentCode(Boolean.TYPE, text, ctx);
    case JetTemplateParser.KEYWORD_NULL:
        return new SegmentCode(TypedKlass.NULL, text, ctx);
    default:
        throw reportError("Unexpected token type :" + token.getType(), ctx);
    }
}
 

private static IntervalLiteral.Sign getIntervalSign(Token token) {
    switch (token.getType()) {
        case SqlBaseLexer.MINUS:
            return IntervalLiteral.Sign.MINUS;
        case SqlBaseLexer.PLUS:
            return IntervalLiteral.Sign.PLUS;
        default:
            throw new IllegalArgumentException("Unsupported sign: " + token.getText());
    }
}
 

private TrimMode getTrimMode(Token type) {
    if (type == null) {
        return TrimMode.BOTH;
    }
    switch (type.getType()) {
        case SqlBaseLexer.BOTH:
            return TrimMode.BOTH;
        case SqlBaseLexer.LEADING:
            return TrimMode.LEADING;
        case SqlBaseLexer.TRAILING:
            return TrimMode.TRAILING;
        default:
            throw new UnsupportedOperationException("Unsupported trim mode: " + type.getText());
    }
}
 

private static WindowFrame.Type getFrameType(Token type)
{
    switch (type.getType()) {
        case SqlBaseLexer.RANGE:
            return WindowFrame.Type.RANGE;
        case SqlBaseLexer.ROWS:
            return WindowFrame.Type.ROWS;
    }

    throw new IllegalArgumentException("Unsupported frame type: " + type.getText());
}
 

private static String getTokenName(Token token) {
  int tokenType = token.getType();
  switch (tokenType) {
    case TomlLexer.NewLine:
      return "end of line";
    case TomlLexer.EOF:
      return "end of input";
    default:
      return "'" + Toml.tomlEscape(token.getText()) + '\'';
  }
}
 

private Expr<?> createAdditiveSubExpr(final Expr<?> leftOp, final Expr<?> rightOp, final Token oper) {
	switch (oper.getType()) {

		case StsDslParser.PLUS:
			return createAddExpr(leftOp, rightOp);

		case StsDslParser.MINUS:
			return createSubExpr(leftOp, rightOp);

		default:
			throw new AssertionError();
	}
}
 

public static List<Integer> viableLeftTokenTypes(ParserRuleContext node,
                                                 Token curToken,
                                                 List<Pair<Integer,Integer>> pairs)
{
	List<Integer> newPairs = new ArrayList<>();
	for (Pair<Integer, Integer> p : pairs) {
		if ( p.b==curToken.getType() && !node.getTokens(p.a).isEmpty() ) {
			newPairs.add(p.a);
		}
	}
	return newPairs;
}
 

private static QuantifiedComparisonExpression.Quantifier getComparisonQuantifier(Token symbol)
{
    switch (symbol.getType()) {
        case SqlBaseLexer.ALL:
            return QuantifiedComparisonExpression.Quantifier.ALL;
        case SqlBaseLexer.ANY:
            return QuantifiedComparisonExpression.Quantifier.ANY;
        case SqlBaseLexer.SOME:
            return QuantifiedComparisonExpression.Quantifier.SOME;
    }

    throw new IllegalArgumentException("Unsupported quantifier: " + symbol.getText());
}
 

@Override
public final void emit(Token token) {
  super.emit(token);
  if (token.getChannel() != HIDDEN) {
    _secondToLastTokenType = _lastTokenType;
    _lastTokenType = token.getType();
  }
}
 

/**
 * Returns {@code true} iff on the current index of the parser's
 * token stream a token of the given {@code type} exists on the
 * {@code HIDDEN} channel.
 *
 * @param type the type of the token on the {@code HIDDEN} channel
 *             to check.
 * @return {@code true} iff on the current index of the parser's
 * token stream a token of the given {@code type} exists on the
 * {@code HIDDEN} channel.
 */
private boolean here(final int type) {

    // Get the token ahead of the current index.
    int possibleIndexEosToken = this.getCurrentToken().getTokenIndex() - 1;
    Token ahead = _input.get(possibleIndexEosToken);

    // Check if the token resides on the HIDDEN channel and if it's of the
    // provided type.
    return (ahead.getChannel() == Lexer.HIDDEN) && (ahead.getType() == type);
}