Java Code Examples for org.antlr.runtime.Token#getType()

The following examples show how to use org.antlr.runtime.Token#getType() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 6 votes vote down vote up 
/** Create singleton template for use with dictionary values. */

    public ST createSingleton(Token templateToken) {
        String template;
        if ( templateToken.getType()==GroupParser.BIGSTRING || templateToken.getType()==GroupParser.BIGSTRING_NO_NL ) {
            template = Misc.strip(templateToken.getText(), 2);
        }
        else {
            template = Misc.strip(templateToken.getText(), 1);
        }

        CompiledST impl = compile(getFileName(), null, null, template, templateToken);
        ST st = createStringTemplateInternally(impl);
        st.groupThatCreatedThisInstance = this;
        st.impl.hasFormalArgs = false;
        st.impl.name = ST.UNKNOWN_NAME;
        st.impl.defineImplicitlyDefinedTemplates(this);
        return st;
    }
Example 2
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 6 votes vote down vote up 
/** Create singleton template for use with dictionary values. */

    public ST createSingleton(Token templateToken) {
        String template;
        if ( templateToken.getType()==GroupParser.BIGSTRING || templateToken.getType()==GroupParser.BIGSTRING_NO_NL ) {
            template = Misc.strip(templateToken.getText(), 2);
        }
        else {
            template = Misc.strip(templateToken.getText(), 1);
        }

        CompiledST impl = compile(getFileName(), null, null, template, templateToken);
        ST st = createStringTemplateInternally(impl);
        st.groupThatCreatedThisInstance = this;
        st.impl.hasFormalArgs = false;
        st.impl.name = ST.UNKNOWN_NAME;
        st.impl.defineImplicitlyDefinedTemplates(this);
        return st;
    }
Example 3
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 6 votes vote down vote up 
/** Create singleton template for use with dictionary values. */

    public ST createSingleton(Token templateToken) {
        String template;
        if ( templateToken.getType()== GroupParser.BIGSTRING || templateToken.getType()== GroupParser.BIGSTRING_NO_NL ) {
            template = Misc.strip(templateToken.getText(), 2);
        }
        else {
            template = Misc.strip(templateToken.getText(), 1);
        }

        CompiledST impl = compile(getFileName(), null, null, template, templateToken);
        ST st = createStringTemplateInternally(impl);
        st.groupThatCreatedThisInstance = this;
        st.impl.hasFormalArgs = false;
        st.impl.name = ST.UNKNOWN_NAME;
        st.impl.defineImplicitlyDefinedTemplates(this);
        return st;
    }
Example 4
Source File: XtendDocumentTokenSource.java    From xtext-xtend with Eclipse Public License 2.0 6 votes vote down vote up 
@Override
protected TokenSource createTokenSource(String string) {
	final FlexTokenSource delegate = flexerFactory.createTokenSource(new StringReader(string));
	return new TokenSource() {

		@Override
		public Token nextToken() {
			Token token = delegate.nextToken();
			if(token.getType() == multilineTokenType) {
				String text = token.getText();
				if(text.startsWith("/**") && !text.startsWith("/***")){
					token.setType(JAVA_DOC_TOKEN_TYPE);
				}
			}
			return token;
		}

		@Override
		public String getSourceName() {
			return delegate.getSourceName();
		}
	};
}
Example 5
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 6 votes vote down vote up 
/** Create singleton template for use with dictionary values. */
public ST createSingleton(Token templateToken) {
	String template;
	if ( templateToken.getType()==GroupParser.BIGSTRING || templateToken.getType()==GroupParser.BIGSTRING_NO_NL ) {
		template = Misc.strip(templateToken.getText(),2);
	}
	else {
		template = Misc.strip(templateToken.getText(),1);
	}
	CompiledST impl = compile(getFileName(), null, null, template, templateToken);
	ST st = createStringTemplateInternally(impl);
	st.groupThatCreatedThisInstance = this;
	st.impl.hasFormalArgs = false;
	st.impl.name = ST.UNKNOWN_NAME;
	st.impl.defineImplicitlyDefinedTemplates(this);
	return st;
}
Example 6
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 6 votes vote down vote up 
/** Create singleton template for use with dictionary values. */

    public ST createSingleton(Token templateToken) {
        String template;
        if ( templateToken.getType()==GroupParser.BIGSTRING || templateToken.getType()==GroupParser.BIGSTRING_NO_NL ) {
            template = Misc.strip(templateToken.getText(), 2);
        }
        else {
            template = Misc.strip(templateToken.getText(), 1);
        }

        CompiledST impl = compile(getFileName(), null, null, template, templateToken);
        ST st = createStringTemplateInternally(impl);
        st.groupThatCreatedThisInstance = this;
        st.impl.hasFormalArgs = false;
        st.impl.name = ST.UNKNOWN_NAME;
        st.impl.defineImplicitlyDefinedTemplates(this);
        return st;
    }
Example 7
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 6 votes vote down vote up 
/** Create singleton template for use with dictionary values. */

    public ST createSingleton(Token templateToken) {
        String template;
        if ( templateToken.getType()== GroupParser.BIGSTRING || templateToken.getType()== GroupParser.BIGSTRING_NO_NL ) {
            template = Misc.strip(templateToken.getText(), 2);
        }
        else {
            template = Misc.strip(templateToken.getText(), 1);
        }

        CompiledST impl = compile(getFileName(), null, null, template, templateToken);
        ST st = createStringTemplateInternally(impl);
        st.groupThatCreatedThisInstance = this;
        st.impl.hasFormalArgs = false;
        st.impl.name = ST.UNKNOWN_NAME;
        st.impl.defineImplicitlyDefinedTemplates(this);
        return st;
    }
Example 8
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 6 votes vote down vote up 
/** Create singleton template for use with dictionary values. */

    public ST createSingleton(Token templateToken) {
        String template;
        if ( templateToken.getType()== GroupParser.BIGSTRING ||
             templateToken.getType()== GroupParser.BIGSTRING_NO_NL ) {
            template = Misc.strip(templateToken.getText(), 2);
        }
        else {
            template = Misc.strip(templateToken.getText(), 1);
        }

        CompiledST impl = compile(getFileName(), null, null, template, templateToken);
        ST st = createStringTemplateInternally(impl);
        st.groupThatCreatedThisInstance = this;
        st.impl.hasFormalArgs = false;
        st.impl.name = ST.UNKNOWN_NAME;
        st.impl.defineImplicitlyDefinedTemplates(this);
        return st;
    }
Example 9
Source File: SemicolonInjectionHelper.java    From n4js with Eclipse Public License 1.0 6 votes vote down vote up 
/**
 * Returns true if the given token is considered to be semicolon equivalent.
 */
static boolean isSemicolonEquivalent(Token lt) {
	if (lt.getType() == InternalN4JSParser.RULE_EOL) {
		return true;
	}
	if (lt.getType() == InternalN4JSParser.RULE_ML_COMMENT) {
		String tokenText = lt.getText();
		for (int i = 2; i < tokenText.length() - 2; i++) {
			char c = tokenText.charAt(i);
			if (c == '\n' || c == '\r') {
				return true;
			}
		}
	}
	return false;
}
Example 10
Source File: STGroup.java    From codebuff with BSD 2-Clause "Simplified" License 6 votes vote down vote up 
/** Create singleton template for use with dictionary values. */

    public ST createSingleton(Token templateToken) {
        String template;
        if ( templateToken.getType()== GroupParser.BIGSTRING || templateToken.getType()== GroupParser.BIGSTRING_NO_NL ) {
            template = Misc.strip(templateToken.getText(), 2);
        }
        else {
            template = Misc.strip(templateToken.getText(), 1);
        }

        CompiledST impl = compile(getFileName(), null, null, template, templateToken);
        ST st = createStringTemplateInternally(impl);
        st.groupThatCreatedThisInstance = this;
        st.impl.hasFormalArgs = false;
        st.impl.name = ST.UNKNOWN_NAME;
        st.impl.defineImplicitlyDefinedTemplates(this);
        return st;
    }
Example 11
Source File: CFBinaryExpression.java    From openbd-core with GNU General Public License v3.0 5 votes vote down vote up 
public CFBinaryExpression( Token t, CFExpression left, CFExpression right ) {
	super(t);
	_kind = t.getType();
	operatorImage = t.getText();
	if ( _kind == CFMLLexer.ANDOPERATOR ) {
		_kind = CFMLLexer.AND;
	} else if ( _kind == CFMLLexer.OROPERATOR ) {
		_kind = CFMLLexer.OR;
	} else if ( _kind == CFMLLexer.MODOPERATOR ) {
		_kind = CFMLLexer.MOD;
	}
	_left = left;
	_right = right;
}
Example 12
Source File: TokenSequencePreservingPartialParsingHelper.java    From xtext-extras with Eclipse Public License 2.0 5 votes vote down vote up 
protected boolean isSameTokenSequence(TokenSource originalSource, TokenSource newSource, int expectedLength) {
	Token token = originalSource.nextToken();
	int newLength = 0;
	while(Token.EOF != token.getType()) {
		Token newToken = newSource.nextToken();
		if (token.getType() != newToken.getType()) {
			return false;
		}
		newLength += TokenTool.getLength(newToken);
		token = originalSource.nextToken();
	}
	return newLength == expectedLength;
}
Example 13
Source File: ProguardLexer.java    From NBANDROID-V2 with Apache License 2.0 5 votes vote down vote up 
@Override
public org.netbeans.api.lexer.Token<ProguardTokenId> nextToken() {
    Token token = oracleLexer.nextToken();
    if (token.getType() != ProguardLexerAntlr.EOF) {
        ProguardTokenId tokenId = ProguardLanguageHierarchy.getToken(token.getType());
        return info.tokenFactory().createToken(tokenId);
    }
    return null;
}
Example 14
Source File: IndentTokenSource.java    From xtext-core with Eclipse Public License 2.0 5 votes vote down vote up 
@Override
protected void doSplitToken(Token token, ITokenAcceptor result) {
	int nextLevel = (token.getType() == Token.EOF) ? 0 : countTabs(token.getText());
	while (this.level > nextLevel) {
		this.level--;
		result.accept(new CommonToken(InternalIndentationAwareTestLanguageLexer.RULE_END, ""));
	}
	result.accept(token);
	while (this.level < nextLevel) {
		this.level++;
		result.accept(new CommonToken(InternalIndentationAwareTestLanguageLexer.RULE_BEGIN, ""));
	}
}
Example 15
Source File: AbstractInternalAntlrParser.java    From xtext-core with Eclipse Public License 2.0 5 votes vote down vote up 
private ILeafNode createLeafNode(Token token, EObject grammarElement) {
	boolean isHidden = token.getChannel() == HIDDEN;
	SyntaxErrorMessage error = null;
	if (!isHidden) {
		if (currentError != null) {
			error = currentError;
			currentError = null;
		}
	}
	if (token.getType() == Token.INVALID_TOKEN_TYPE) {
		if (error == null) {
			String lexerErrorMessage = ((XtextTokenStream) input).getLexerErrorMessage(token);
			LexerErrorContext errorContext = new LexerErrorContext(lexerErrorMessage);
			error = syntaxErrorProvider.getSyntaxErrorMessage(errorContext);
		}
	}
	if (grammarElement == null) {
		String ruleName = antlrTypeToLexerName.get(token.getType());
		grammarElement = allRules.get(ruleName);
	}
	CommonToken commonToken = (CommonToken) token;
	if (error != null)
		hadErrors = true;
	return nodeBuilder.newLeafNode(
			commonToken.getStartIndex(), 
			commonToken.getStopIndex() - commonToken.getStartIndex() + 1, 
			grammarElement, 
			isHidden, 
			error, 
			currentNode);
}
Example 16
Source File: ObservableXtextTokenStream.java    From xtext-core with Eclipse Public License 2.0 5 votes vote down vote up 
@Override
public int LA(int i) {
	Token lookaheadToken = LT(i);
	int result = lookaheadToken.getType();
	if (result == Token.EOF && getListener() != null)
		getListener().announceEof(i);
	return result;
}
Example 17
Source File: LazyTokenStream.java    From n4js with Eclipse Public License 1.0 4 votes vote down vote up 
/**
 * Fills the buffer but stops on a div or div-equals token.
 */
@SuppressWarnings("unchecked")
@Override
protected void fillBuffer() {
	int oldP = p;
	int index = tokens.size();
	Token t = tokenSource.nextToken();
	while (t != null && t.getType() != CharStream.EOF) {
		// stop on div, div-equal and right curly brace tokens.
		int type = t.getType();
		if (type == InternalN4JSLexer.Solidus || type == InternalN4JSLexer.SolidusEqualsSign
				|| type == InternalN4JSLexer.RightCurlyBracket) {
			t.setTokenIndex(index);
			tokens.add(t);
			index++;
			break;
		}
		boolean discard = false;
		// is there a channel override for token type?
		if (channelOverrideMap != null) {
			Integer channelI = (Integer) channelOverrideMap.get(Integer.valueOf(type));
			if (channelI != null) {
				t.setChannel(channelI.intValue());
			}
		}
		if (discardSet != null &&
				discardSet.contains(Integer.valueOf(type))) {
			discard = true;
		} else if (discardOffChannelTokens && t.getChannel() != this.channel) {
			discard = true;
		}
		if (!discard) {
			t.setTokenIndex(index);
			tokens.add(t);
			index++;
		}
		t = tokenSource.nextToken();
	}
	// leave p pointing at first token on channel
	p = oldP == -1 ? 0 : oldP;
	p = skipOffTokenChannels(p);
}
Example 18
Source File: BtrPlaceTreeAdaptor.java    From scheduler with GNU Lesser General Public License v3.0 4 votes vote down vote up 
@Override
public Object create(Token payload) {
    if (payload == null) {
        return new BtrPlaceTree(null, errors);
    }
    switch (payload.getType()) {
        case ANTLRBtrplaceSL2Lexer.RANGE:
            return new Range(payload, errors);
        case ANTLRBtrplaceSL2Lexer.ENUM_VAR:
            return new EnumVar(payload, symbols, errors);
        case ANTLRBtrplaceSL2Lexer.ENUM_FQDN:
            return new EnumElement(payload, srvNodes, srvVMs, script, BtrpOperand.Type.NODE, errors);
        case ANTLRBtrplaceSL2Lexer.ENUM_ID:
            return new EnumElement(payload, srvNodes, srvVMs, script, BtrpOperand.Type.VM, errors);
        case ANTLRBtrplaceSL2Lexer.AND:
            return new BooleanBinaryOperation(payload, true, errors);
        case ANTLRBtrplaceSL2Lexer.OR:
            return new BooleanBinaryOperation(payload, false, errors);
        case ANTLRBtrplaceSL2Lexer.BLOCK:
            return new BlockStatement(payload, errors);
        case ANTLRBtrplaceSL2Lexer.FLOAT:
        case ANTLRBtrplaceSL2Lexer.OCTAL:
        case ANTLRBtrplaceSL2Lexer.DECIMAL:
        case ANTLRBtrplaceSL2Lexer.HEXA:
            return new NumberTree(payload, errors);
        case ANTLRBtrplaceSL2Lexer.STRING:
            return new StringTree(payload, errors);
        case ANTLRBtrplaceSL2Lexer.PLUS:
            return new AddOperator(payload, errors);
        case ANTLRBtrplaceSL2Lexer.MINUS:
            return new MinusOperator(payload, errors);
        case ANTLRBtrplaceSL2Lexer.TIMES:
            return new TimesOperator(payload, errors);
        case ANTLRBtrplaceSL2Lexer.POWER:
            return new PowerOperator(payload, errors);
        case ANTLRBtrplaceSL2Lexer.EQUALS:
            return new AssignmentStatement(payload, errors, symbols);
        case ANTLRBtrplaceSL2Lexer.PLUS_EQUALS:
            return new SelfAssignmentStatement(SelfAssignmentStatement.Type.PLUS_EQUALS, payload, errors, symbols);
        case ANTLRBtrplaceSL2Lexer.MINUS_EQUALS:
            return new SelfAssignmentStatement(SelfAssignmentStatement.Type.MINUS_EQUALS, payload, errors, symbols);
        case ANTLRBtrplaceSL2Lexer.DIV_EQUALS:
            return new SelfAssignmentStatement(SelfAssignmentStatement.Type.DIV_EQUALS, payload, errors, symbols);
        case ANTLRBtrplaceSL2Lexer.TIMES_EQUALS:
            return new SelfAssignmentStatement(SelfAssignmentStatement.Type.TIMES_EQUALS, payload, errors, symbols);
        case ANTLRBtrplaceSL2Lexer.REMAINDER_EQUALS:
            return new SelfAssignmentStatement(SelfAssignmentStatement.Type.REMAINDER_EQUALS, payload, errors, symbols);
        case ANTLRBtrplaceSL2Lexer.VARIABLE:
            return new VariableTree(payload, errors, symbols);
        case ANTLRBtrplaceSL2Lexer.DIV:
            return new DivideOperator(payload, errors);
        case ANTLRBtrplaceSL2Lexer.REMAINDER:
            return new RemainderOperator(payload, errors);
        case ANTLRBtrplaceSL2Lexer.EQ:
            return new EqComparisonOperator(payload, false, errors);
        case ANTLRBtrplaceSL2Lexer.NOT:
            return new NotOperator(payload, errors);
        case ANTLRBtrplaceSL2Lexer.NEQ:
            return new EqComparisonOperator(payload, true, errors);
        case ANTLRBtrplaceSL2Lexer.GT:
            return new StrictComparisonOperator(payload, false, errors);
        case ANTLRBtrplaceSL2Lexer.LT:
            return new StrictComparisonOperator(payload, true, errors);
        case ANTLRBtrplaceSL2Lexer.LEQ:
            return new NonStrictComparisonOperator(payload, true, errors);
        case ANTLRBtrplaceSL2Lexer.GEQ:
            return new NonStrictComparisonOperator(payload, false, errors);
        case ANTLRBtrplaceSL2Lexer.IF:
            return new IfStatement(payload, symbols, errors);
        case ANTLRBtrplaceSL2Lexer.FOR:
            return new ForStatement(payload, symbols, errors);
        case ANTLRBtrplaceSL2Lexer.IDENTIFIER:
            return new ElementTree(payload, srvNodes, srvVMs, script, errors);
        case ANTLRBtrplaceSL2Lexer.NODE_NAME:
            return new ElementTree(payload, srvNodes, srvVMs, script, errors);
        case ANTLRBtrplaceSL2Lexer.EXPLODED_SET:
            return new ExplodedSetTree(payload, errors);
        case ANTLRBtrplaceSL2Lexer.CARDINALITY:
            return new CardinalityOperator(payload, errors);
        case ANTLRBtrplaceSL2Lexer.CONSTRAINTIDENTIFIER:
            return new ConstraintStatement(payload, script, catalog, errors);
        case ANTLRBtrplaceSL2Lexer.TYPE_DEFINITION:
            return new TemplateAssignment(payload, script, tpls, symbols, model, srvNodes, srvVMs, errors);
        case ANTLRBtrplaceSL2Lexer.EXPORT:
            return new ExportStatement(payload, script, errors);
        case ANTLRBtrplaceSL2Lexer.USE:
            return new ImportStatement(payload, includes, symbols, script, errors);
        case ANTLRBtrplaceSL2Lexer.NAMESPACE:
            return new NameSpaceStatement(payload, script, symbols, errors);
        case ANTLRBtrplaceSL2Lexer.TEMPLATE_OPTION:
            return new TemplateOptionTree(payload, errors);
        case ANTLRBtrplaceSL2Lexer.EOF:
            return new ErrorTree(payload, null);
        case ANTLRBtrplaceSL2Lexer.DISCRETE:
            return new DiscreteToken(payload);
        case ANTLRBtrplaceSL2Lexer.BLANK:
        default:
            return new BtrPlaceTree(payload, errors);
    }
}
Example 19
Source File: AbstractIndentationTokenSource.java    From xtext-core with Eclipse Public License 2.0 4 votes vote down vote up 
@Override
protected boolean shouldSplitToken(Token token) {
	return token.getType() == Token.EOF || shouldSplitTokenImpl(token);
}
Example 20
Source File: DRL6Parser.java    From kogito-runtimes with Apache License 2.0 4 votes vote down vote up 
private boolean parseEvalExpression(EvalDescrBuilder<?> eval) throws RecognitionException {
    match(input,
            DRL6Lexer.LEFT_PAREN,
            null,
            null,
            DroolsEditorType.SYMBOL);
    if (state.failed)
        return false;

    if (state.backtracking == 0) {
        helper.emit(Location.LOCATION_LHS_INSIDE_EVAL);
    }

    int idx = input.index();
    final String expr;
    try {
        expr = conditionalExpression();
    } catch (RecognitionException e) {
        final Token tempToken = helper.getLastTokenOnList(helper.getEditorInterface().getLast().getContent());
        if (tempToken != null) {
            for (int i = tempToken.getTokenIndex() + 1; i < input.size(); i++) {
                final Token token = input.get(i);
                if (token.getType() == DRL6Lexer.EOF) {
                    break;
                }
                helper.emit(token, DroolsEditorType.CODE_CHUNK);
            }
        }

        throw e;
    }

    if (state.backtracking == 0) {
        eval.constraint(expr);
    }

    match(input,
            DRL6Lexer.RIGHT_PAREN,
            null,
            null,
            DroolsEditorType.SYMBOL);
    if (state.failed)
        return false;

    helper.emit(Location.LOCATION_LHS_BEGIN_OF_CONDITION);
    return true;
}