org.antlr.runtime.Token Java Examples

public void testList2() throws Exception {
	// Add child ^(nil 101 102 103) to root 5
	// should pull 101 102 103 directly to become 5's child list
	CommonTree root = new CommonTree(new CommonToken(5));

	// child tree
	CommonTree r0 = new CommonTree((Token)null);
	CommonTree c0, c1, c2;
	r0.addChild(c0=new CommonTree(new CommonToken(101)));
	r0.addChild(c1=new CommonTree(new CommonToken(102)));
	r0.addChild(c2=new CommonTree(new CommonToken(103)));

	root.addChild(r0);

	assertNull(root.parent);
	assertEquals(-1, root.childIndex);
	// check children of root all point at root
	assertEquals(root, c0.parent);
	assertEquals(0, c0.childIndex);
	assertEquals(root, c0.parent);
	assertEquals(1, c1.childIndex);
	assertEquals(root, c0.parent);
	assertEquals(2, c2.childIndex);
}
 

@Override
protected boolean isSameTokenSequence(String originalText, String newText, int expectedLength) {
	try {
		InternalFlexer originalSequence = flexerFactory.createFlexer(new StringReader(originalText));
		InternalFlexer newSequence = flexerFactory.createFlexer(new StringReader(newText));
		int token = originalSequence.advance();
		int newLength = 0;
		while(token != Token.EOF) {
			if (token != newSequence.advance()) {
				return false;
			}
			newLength += newSequence.getTokenLength();
			token = originalSequence.advance();
		}
		return newLength == expectedLength;
	} catch(IOException e) {
		throw new RuntimeException(e);
	}
}
 

private ObservableXtextTokenStream toTokenStream(FollowElement element,
		ITokenDefProvider tokenDefProvider) {
	Iterator<LookAheadTerminal> iter = element.getLookAheadTerminals().iterator();
	return new ObservableXtextTokenStream(new TokenSource() {
		@Override
		public Token nextToken() {
			if (iter.hasNext()) {
				LookAheadTerminal lookAhead = iter.next();
				return lookAhead.getToken();
			}
			return Token.EOF_TOKEN;
		}

		@Override
		public String getSourceName() {
			return "LookAheadTerminalTokenSource";
		}
	}, tokenDefProvider);
}
 

public CompiledST defineRegion(String enclosingTemplateName, Token regionT, String template, Token templateToken) {
    String name = regionT.getText();
    template = Misc.trimOneStartingNewline(template);
    template = Misc.trimOneTrailingNewline(template);
    CompiledST code = compile(getFileName(), enclosingTemplateName, null, template, templateToken);
    String mangled = getMangledRegionName(enclosingTemplateName, name);
    if ( lookupTemplate(mangled) ==null ) {
        errMgr.compileTimeError(ErrorType.NO_SUCH_REGION,
                                templateToken,
                                regionT,
                                enclosingTemplateName,
                                name);
        return new CompiledST();
    }
    code.name = mangled;
    code.isRegion = true;
    code.regionDefType = ST.RegionType.EXPLICIT;
    code.templateDefStartToken = regionT;
    rawDefineTemplate(mangled, code, regionT);
    code.defineArgDefaultValueTemplates(this);
    code.defineImplicitlyDefinedTemplates(this);
    return code;
}
 

Token COMMENT() {
    match('!');
    while ( !(c=='!' && input.LA(2)==delimiterStopChar) ) {
        if ( c==EOF ) {
            RecognitionException re = new MismatchedTokenException((int)'!', input);
            re.line = input.getLine();
            re.charPositionInLine = input.getCharPositionInLine();
            errMgr.lexerError(input.getSourceName(),
                              "Nonterminated comment starting at " +
                              startLine +":"+startCharPositionInLine +
                              ": '!" +
                              delimiterStopChar +
                              "' missing", templateToken, re);
            break;
        }
        consume();
    }
    consume();
    consume(); // grab !>
    return newToken(COMMENT);
}
 

public CompiledST defineRegion(String enclosingTemplateName, Token regionT, String template, Token templateToken) {
    String name = regionT.getText();
    template = Misc.trimOneStartingNewline(template);
    template = Misc.trimOneTrailingNewline(template);
    CompiledST code = compile(getFileName(), enclosingTemplateName, null, template, templateToken);
    String mangled = getMangledRegionName(enclosingTemplateName, name);
    if ( lookupTemplate(mangled)==null ) {
        errMgr.compileTimeError(ErrorType.NO_SUCH_REGION, templateToken, regionT, enclosingTemplateName, name);
        return new CompiledST();
    }
    code.name = mangled;
    code.isRegion = true;
    code.regionDefType = ST.RegionType.EXPLICIT;
    code.templateDefStartToken = regionT;
    rawDefineTemplate(mangled, code, regionT);
    code.defineArgDefaultValueTemplates(this);
    code.defineImplicitlyDefinedTemplates(this);
    return code;
}
 

public CompiledST defineRegion(String enclosingTemplateName, Token regionT, String template, Token templateToken) {
    String name = regionT.getText();
    template = Misc.trimOneStartingNewline(template);
    template = Misc.trimOneTrailingNewline(template);
    CompiledST code = compile(getFileName(), enclosingTemplateName, null, template, templateToken);
    String mangled = getMangledRegionName(enclosingTemplateName, name);
    if ( lookupTemplate(mangled)==null ) {
        errMgr.compileTimeError(ErrorType.NO_SUCH_REGION, templateToken, regionT, enclosingTemplateName, name);
        return new CompiledST();
    }
    code.name = mangled;
    code.isRegion = true;
    code.regionDefType = ST.RegionType.EXPLICIT;
    code.templateDefStartToken = regionT;
    rawDefineTemplate(mangled, code, regionT);
    code.defineArgDefaultValueTemplates(this);
    code.defineImplicitlyDefinedTemplates(this);
    return code;
}
 

public CompiledST defineTemplate(String fullyQualifiedTemplateName,
                                 Token nameT,
                                 List<FormalArgument> args,
                                 String template, Token templateToken) {
    if ( verbose ) System.out.println("defineTemplate("+fullyQualifiedTemplateName+")");
    if ( fullyQualifiedTemplateName==null || fullyQualifiedTemplateName.length()==0 ) {
        throw new IllegalArgumentException("empty template name");
    }
    if ( fullyQualifiedTemplateName.indexOf('.')>=0 ) {
        throw new IllegalArgumentException("cannot have '.' in template names");
    }
    template = Misc.trimOneStartingNewline(template);
    template = Misc.trimOneTrailingNewline(template);
    // compile, passing in templateName as enclosing name for any embedded regions
    CompiledST code = compile(getFileName(),
                              fullyQualifiedTemplateName,
                              args,
                              template,
                              templateToken);
    code.name = fullyQualifiedTemplateName;
    rawDefineTemplate(fullyQualifiedTemplateName, code, nameT);
    code.defineArgDefaultValueTemplates(this);
    code.defineImplicitlyDefinedTemplates(this); // define any anonymous subtemplates
    return code;
}
 

public void rawDefineTemplate(String name, CompiledST code, Token defT) {
    CompiledST prev = rawGetTemplate(name);
    if ( prev!=null ) {
        if ( !prev.isRegion ) {
            errMgr.compileTimeError(ErrorType.TEMPLATE_REDEFINITION, null, defT);
            return;
        }
        else {
            if ( code.regionDefType!= ST.RegionType.IMPLICIT && prev.regionDefType== ST.RegionType.EMBEDDED ) {
                errMgr.compileTimeError(ErrorType.EMBEDDED_REGION_REDEFINITION, null, defT, getUnMangledTemplateName(name));
                return;
            }
            else if ( code.regionDefType== ST.RegionType.IMPLICIT || prev.regionDefType== ST.RegionType.EXPLICIT ) {
                errMgr.compileTimeError(ErrorType.REGION_REDEFINITION,
                                        null,
                                        defT,
                                        getUnMangledTemplateName(name));
                return;
            }
        }
    }
    code.nativeGroup = this;
    code.templateDefStartToken = defT;
    templates.put(name, code);
}
 

public CompiledST defineRegion(String enclosingTemplateName, Token regionT, String template, Token templateToken) {
    String name = regionT.getText();
    template = Misc.trimOneStartingNewline(template);
    template = Misc.trimOneTrailingNewline(template);
    CompiledST code = compile(getFileName(), enclosingTemplateName, null, template, templateToken);
    String mangled = getMangledRegionName(enclosingTemplateName, name);
    if ( lookupTemplate(mangled)==null ) {
        errMgr.compileTimeError(ErrorType.NO_SUCH_REGION, templateToken, regionT, enclosingTemplateName, name);
        return new CompiledST();
    }
    code.name = mangled;
    code.isRegion = true;
    code.regionDefType = ST.RegionType.EXPLICIT;
    code.templateDefStartToken = regionT;
    rawDefineTemplate(mangled, code, regionT);
    code.defineArgDefaultValueTemplates(this);
    code.defineImplicitlyDefinedTemplates(this);
    return code;
}
 

Token COMMENT() {
    match('!');
    while ( !(c=='!' && input.LA(2)==delimiterStopChar) ) {
        if ( c==EOF ) {
            RecognitionException re = new MismatchedTokenException((int)'!', input);
            re.line = input.getLine();
            re.charPositionInLine = input.getCharPositionInLine();
            errMgr.lexerError(input.getSourceName(), "Nonterminated comment starting at "+startLine+":"+startCharPositionInLine+": '!"+delimiterStopChar+"' missing", templateToken, re);
            break;
        }
        consume();
    }
    consume();
    consume(); // grab !>
    return newToken(COMMENT);
}
 

public void testList() throws Exception {
	// ^(nil 101 102 103)
	CommonTree r0 = new CommonTree((Token)null);
	CommonTree c0, c1, c2;
	r0.addChild(c0=new CommonTree(new CommonToken(101)));
	r0.addChild(c1=new CommonTree(new CommonToken(102)));
	r0.addChild(c2=new CommonTree(new CommonToken(103)));

	assertNull(r0.parent);
	assertEquals(-1, r0.childIndex);
	assertEquals(r0, c0.parent);
	assertEquals(0, c0.childIndex);
	assertEquals(r0, c1.parent);
	assertEquals(1, c1.childIndex);		
	assertEquals(r0, c2.parent);
	assertEquals(2, c2.childIndex);
}
 

@Override
	protected void createContextsForLastCompleteNode(EObject previousModel, boolean strict){
		String currentNodePrefix = getPrefix(currentNode);
		if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) {
			InternalFlexer flexer = flexerFactory.createFlexer(new StringReader(currentNodePrefix));
			try {
				int token = flexer.advance();
				if (token == Token.EOF) { // error case - nothing could be parsed
					return;
				}
				while(token != Token.EOF) {
					if (isErrorToken(token))
						return;
					token = flexer.advance();
				}
			} catch (IOException e) {
				log.error(e.getMessage(), e);
				return;
			}
		}
		String prefix = "";
		Collection<FollowElement> followElements = parseFollowElements(completionOffset, strict);
//		String completeInput = viewer.getDocument().get(0, completionOffset);
//		Collection<FollowElement> followElements = parser.getFollowElements(completeInput, strict);
		doCreateContexts(lastCompleteNode, currentNode, prefix, previousModel, followElements);
	}
 

public static List<ActionChunk> translateActionChunk(OutputModelFactory factory,
													 RuleFunction rf,
													 String action,
													 ActionAST node)
{
	Token tokenWithinAction = node.token;
	ActionTranslator translator = new ActionTranslator(factory, node);
	translator.rf = rf;
       factory.getGrammar().tool.log("action-translator", "translate " + action);
	String altLabel = node.getAltLabel();
	if ( rf!=null ) {
	    translator.nodeContext = rf.ruleCtx;
        if ( altLabel!=null ) translator.nodeContext = rf.altLabelCtxs.get(altLabel);
	}
	ANTLRStringStream in = new ANTLRStringStream(action);
	in.setLine(tokenWithinAction.getLine());
	in.setCharPositionInLine(tokenWithinAction.getCharPositionInLine());
	ActionSplitter trigger = new ActionSplitter(in, translator);
	// forces eval, triggers listener methods
	trigger.getActionTokens();
	return translator.chunks;
}
 

static XTree getParseTree(CharStream sparqlFile)
		throws RecognitionException {
	IbmSparqlLexer lex = new IbmSparqlLexer(sparqlFile);
	CommonTokenStream tokens = new CommonTokenStream(lex);
	IbmSparqlParser parser = new IbmSparqlParser(tokens);
	parser.setTreeAdaptor(new CommonTreeAdaptor() {
		@Override
		public Object create(Token t) {
			return new XTree(t);
		}
	});
	IbmSparqlParser.queryUnit_return ret = parser.queryUnit();
	XTree ast = (XTree) ret.getTree();
	return ast;
}
 

public PigParserNode(Token t, String fileName, int lineOffset) {
    super(t);
    if (t != null && lineOffset > 0) {
        t.setLine(t.getLine() + lineOffset);
    }
    this.fileName = fileName;
}
 

/**
 * defaultConsequence := THEN chunk
 * @param rule
 */
public void defaultConsequence(RuleDescrBuilder rule) {
    try {
        int first = input.index();
        Token t = match(input,
                DRL6Lexer.ID,
                DroolsSoftKeywords.THEN,
                null,
                DroolsEditorType.KEYWORD);
        if (state.failed)
            return;

        if (state.backtracking == 0) {
            rule.getDescr().setConsequenceLocation(t.getLine(),
                    t.getCharPositionInLine());
            helper.emit(Location.LOCATION_RHS);
        }

        String chunk = getConsequenceCode(first);

        // remove the "then" keyword and any subsequent spaces and line breaks
        // keep indentation of 1st non-blank line
        chunk = chunk.replaceFirst("^then\\s*\\r?\\n?",
                "");
        rule.rhs(chunk);

    } catch (RecognitionException re) {
        reportError(re);
    }
}
 

@Override
public void importTemplates(Token fileNameToken) {
    String msg = "import illegal in group files embedded in STGroupDirs; "+"import " +
                 fileNameToken.getText() +
                 " in STGroupDir " +
                 this.getName();
    throw new UnsupportedOperationException(msg);
}
 

public STLexer(ErrorManager errMgr, CharStream input, Token templateToken, char delimiterStartChar, char delimiterStopChar) {
    this.errMgr = errMgr;
    this.input = input;
    c = (char)input.LA(1); // prime lookahead
    this.templateToken = templateToken;
    this.delimiterStartChar = delimiterStartChar;
    this.delimiterStopChar = delimiterStopChar;
}
 

/**
 * Constructor that preserves the char offset
 *
 * @param oldToken
 */
public DroolsToken(Token oldToken) {
    super(oldToken);
    if (null != oldToken
            && (oldToken.getClass().equals(CommonToken.class) || oldToken
                    .getClass().equals(DroolsToken.class))) {
        start = ((CommonToken) oldToken).getStartIndex();
        stop = ((CommonToken) oldToken).getStopIndex();
    }
}
 

Token mTEXT() {
boolean modifiedText = false;
      StringBuilder buf = new StringBuilder();
      while ( c != EOF && c != delimiterStartChar ) {
	if ( c=='\r' || c=='\n') break;
	if ( c=='}' && subtemplateDepth>0 ) break;
          if ( c=='\\' ) {
              if ( input.LA(2)=='\\' ) { // convert \\ to \
                  consume(); consume(); buf.append('\\');
                  modifiedText = true;
                  continue;
              }
              if ( input.LA(2)==delimiterStartChar ||
			 input.LA(2)=='}' )
		{
                  modifiedText = true;
                  consume(); // toss out \ char
                  buf.append(c); consume();
              }
              else {
                  buf.append(c);
                  consume();
              }
              continue;
          }
          buf.append(c);
          consume();
      }
      if ( modifiedText )	return newToken(TEXT, buf.toString());
      else return newToken(TEXT);
  }
 

protected void assertLexing(String input, Pair<String,String>... expectedTokens) {
	Lexer lexer = new InternalXtendLexer(null);
	CharStream stream = new ANTLRStringStream(input);
	lexer.setCharStream(stream);
	XtextTokenStream tokenStream = new XtextTokenStream(lexer, tokenDefProvider);
	List<?> tokens = tokenStream.getTokens();
	assertEquals(input + " / " + tokens, expectedTokens.length, tokens.size());
	for(int i = 0;i < tokens.size(); i++) {
		Token token = (Token) tokens.get(i);
		assertEquals(token.toString(), expectedTokens[i].getFirst(), token.getText());
		final String expected = expectedTokens[i].getSecond();
		String actual = tokenDefProvider.getTokenDefMap().get(token.getType());
		assertEquals("expected "+expected+" but was "+actual, expected, actual);
	}
}
 

public static CompiledST defineBlankRegion(CompiledST outermostImpl, Token nameToken) {
    String outermostTemplateName = outermostImpl.name;
    String mangled = STGroup.getMangledRegionName(outermostTemplateName, nameToken.getText());
    CompiledST blank = new CompiledST();
    blank.isRegion = true;
    blank.templateDefStartToken = nameToken;
    blank.regionDefType = ST.RegionType.IMPLICIT;
    blank.name = mangled;
    outermostImpl.addImplicitlyDefinedTemplate(blank);
    return blank;
}
 

public Token newToken(int ttype, String text, int pos) {
    STToken t = new STToken(ttype, text);
    t.setStartIndex(startCharIndex);
    t.setStopIndex(input.index() -1);
    t.setLine(input.getLine());
    t.setCharPositionInLine(pos);
    return t;
}
 

Token mTEXT() {
    boolean modifiedText = false;
    StringBuilder buf = new StringBuilder();
    while ( c != EOF && c != delimiterStartChar ) {
        if ( c=='\r' || c=='\n' ) break;
        if ( c=='}' && subtemplateDepth>0) break;
        if ( c=='\\' ) {
            if ( input.LA(2)=='\\' ) { // convert \\ to \
                consume();
                consume();
                buf.append('\\');
                modifiedText = true;
                continue;
            }

            if ( input.LA(2)==delimiterStartChar || input.LA(2)=='}' ) {
                modifiedText = true;
                consume(); // toss out \ char
                buf.append(c);
                consume();
            }
            else {
                buf.append(c);
                consume();
            }
            continue;
        }
        buf.append(c);
        consume();
    }

    if ( modifiedText ) return newToken(TEXT, buf.toString());
    else return newToken(TEXT);
}
 

protected Token outside() {
    if ( input.getCharPositionInLine()==0 &&(c==' ' || c=='\t') ) {
        while ( c==' ' || c=='\t' ) consume(); // scarf indent
        if ( c!= EOF ) return newToken(INDENT);
        return newToken(TEXT);
    }

    if ( c==delimiterStartChar ) {
        consume();
        if ( c=='!' ) return COMMENT();
        if ( c=='\\' ) return ESCAPE(); // <\\> <\uFFFF> <\n> etc...
        scanningInsideExpr = true;
        return newToken(LDELIM);
    }

    if ( c=='\r' ) {
        consume();
        consume();
        return newToken(NEWLINE);
    } // \r\n -> \n

    if ( c=='\n' ) {
        consume();
        return newToken(NEWLINE);
    }

    if ( c=='}' && subtemplateDepth>0) {
        scanningInsideExpr = true;
        subtemplateDepth--;
        consume();
        return newTokenFromPreviousChar(RCURLY);
    }
    return mTEXT();
}
 

public void compileTimeError(ErrorType error, Token templateToken, Token t) {
    CharStream input = t.getInputStream();
    String srcName = null;
    if ( input!=null ) {
        srcName = input.getSourceName();
        if ( srcName!=null ) srcName = Misc.getFileName(srcName);
    }
    listener.compileTimeError(new STCompiletimeMessage(error, srcName, templateToken, t, null, t.getText()));
}
 

/** 
 *  Match current input symbol against ttype and optionally
 *  check the text of the token against text.  Attempt
 *  single token insertion or deletion error recovery.  If
 *  that fails, throw MismatchedTokenException.
 */
private Token match( TokenStream input,
                     int ttype,
                     String text,
                     int[] follow,
                     DroolsEditorType etype ) throws RecognitionException {
    Token matchedSymbol = null;
    matchedSymbol = input.LT( 1 );
    if ( input.LA( 1 ) == ttype && (text == null || text.equals( matchedSymbol.getText() )) ) {
        input.consume();
        state.errorRecovery = false;
        state.failed = false;
        helper.emit( matchedSymbol,
                     etype );
        return matchedSymbol;
    }
    if ( state.backtracking > 0 ) {
        state.failed = true;
        return matchedSymbol;
    }
    matchedSymbol = recoverFromMismatchedToken( input,
                                                ttype,
                                                text,
                                                follow );
    helper.emit( matchedSymbol,
                 etype );
    return matchedSymbol;
}
 

public STCompiletimeMessage(ErrorType error,
                            String srcName,
                            Token templateToken,
                            Token t,
                            Throwable cause, Object arg) {
    this(error, srcName, templateToken, t, cause, arg, null);
}
 

@SuppressWarnings("unchecked")
@Override
protected void assertLexing(String input, Pair<String,String>... expectedTokens) {
	FlexTokenSource tokenSource = new FlexerFactory().createTokenSource(new StringReader(input));
	XtextTokenStream tokenStream = new XtextTokenStream(tokenSource, getTokenDefProvider());
	List<?> tokens = tokenStream.getTokens();
	assertEquals(input + " / " + tokens, expectedTokens.length, tokens.size());
	for(int i = 0;i < tokens.size(); i++) {
		Token token = (Token) tokens.get(i);
		assertEquals(token.toString(), expectedTokens[i].getFirst(), token.getText());
		final String expected = expectedTokens[i].getSecond();
		String actual = getTokenDefProvider().getTokenDefMap().get(token.getType());
		assertEquals("expected "+expected+" but was "+actual, expected, actual);
	}
}