org.antlr.runtime.CommonToken Java Examples

public void testDupTree() throws Exception {
	// ^(101 ^(102 103 ^(106 107) ) 104 105)
	CommonTree r0 = new CommonTree(new CommonToken(101));
	CommonTree r1 = new CommonTree(new CommonToken(102));
	r0.addChild(r1);
	r1.addChild(new CommonTree(new CommonToken(103)));
	Tree r2 = new CommonTree(new CommonToken(106));
	r2.addChild(new CommonTree(new CommonToken(107)));
	r1.addChild(r2);
	r0.addChild(new CommonTree(new CommonToken(104)));
	r0.addChild(new CommonTree(new CommonToken(105)));

	CommonTree dup = (CommonTree)(new CommonTreeAdaptor()).dupTree(r0);

	assertNull(dup.parent);
	assertEquals(-1, dup.childIndex);
	dup.sanityCheckParentAndChildIndexes();
}
 

public void testList2() throws Exception {
	// Add child ^(nil 101 102 103) to root 5
	// should pull 101 102 103 directly to become 5's child list
	CommonTree root = new CommonTree(new CommonToken(5));

	// child tree
	CommonTree r0 = new CommonTree((Token)null);
	CommonTree c0, c1, c2;
	r0.addChild(c0=new CommonTree(new CommonToken(101)));
	r0.addChild(c1=new CommonTree(new CommonToken(102)));
	r0.addChild(c2=new CommonTree(new CommonToken(103)));

	root.addChild(r0);

	assertNull(root.parent);
	assertEquals(-1, root.childIndex);
	// check children of root all point at root
	assertEquals(root, c0.parent);
	assertEquals(0, c0.childIndex);
	assertEquals(root, c0.parent);
	assertEquals(1, c1.childIndex);
	assertEquals(root, c0.parent);
	assertEquals(2, c2.childIndex);
}
 

public void setCursorToGrammarElement(Project project, PreviewState previewState, int offset) {
	Token tokenUnderCursor = ParsingUtils.getTokenUnderCursor(previewState, offset);
	if ( tokenUnderCursor==null ) {
		return;
	}

	PreviewParser parser = (PreviewParser) previewState.parsingResult.parser;
	Integer atnState = parser.inputTokenToStateMap.get(tokenUnderCursor);
	if ( atnState==null ) { // likely an error token
		//LOG.error("no ATN state for input token " + tokenUnderCursor);
		return;
	}

	Interval region = previewState.g.getStateToGrammarRegion(atnState);
	CommonToken token =
		(CommonToken) previewState.g.tokenStream.get(region.a);
	jumpToGrammarPosition(project, token.getStartIndex());
}
 

/** for testing */

    public CompiledST defineTemplate(String templateName, String template) {
        if ( templateName.charAt(0) !='/' ) templateName = "/"+templateName;
        try {
            CompiledST impl = defineTemplate(templateName,
                                             new CommonToken(GroupParser.ID, templateName),
                                             null,
                                             template,
                                             null);
            return impl;
        }
        catch (STException se) {
            // we have reported the error; the exception just blasts us
            // out of parsing this template
        }
        return null;
    }
 

@Override
protected void announce(Token start, Token stop, AbstractElement element) {
	if (start != null && start != Token.EOF_TOKEN) {
		if (start == stop) {
			announce(start, element);
		} else {
			CommonToken castedStart = (CommonToken) start;
			if (stop == null) { // possible error condition
				if (start.getTokenIndex() == state.lastErrorIndex) {
					return;
				}
			}
			CommonToken castedEnd = (CommonToken) stop;
			Integer newType = rewriter.rewrite(castedStart, element);
			if (newType != null && castedEnd != null && castedEnd != Token.EOF_TOKEN) {
				LazyTokenStream castedInput = (LazyTokenStream) this.input;
				for (int i = castedStart.getTokenIndex() + 1; i < castedEnd.getTokenIndex(); i++) {
					Token token = castedInput.get(i);
					if (token.getChannel() != Token.HIDDEN_CHANNEL)
						token.setType(newType);
				}
				castedEnd.setType(newType);
			}
		}
	}
}
 

public void setCursorToGrammarRule(Project project, PreviewState previewState, int offset) {
	Token tokenUnderCursor = ParsingUtils.getTokenUnderCursor(previewState, offset);
	if ( tokenUnderCursor==null ) {
		return;
	}

	ParseTree tree = previewState.parsingResult.tree;
	TerminalNode nodeWithToken =
		(TerminalNode) ParsingUtils.getParseTreeNodeWithToken(tree, tokenUnderCursor);
	if ( nodeWithToken==null ) {
		// hidden token
		return;
	}

	ParserRuleContext parent = (ParserRuleContext) nodeWithToken.getParent();
	int ruleIndex = parent.getRuleIndex();
	Rule rule = previewState.g.getRule(ruleIndex);
	GrammarAST ruleNameNode = (GrammarAST) rule.ast.getChild(0);
	int start = ((CommonToken) ruleNameNode.getToken()).getStartIndex();

	jumpToGrammarPosition(project, start);
}
 

/** for testing */

    public CompiledST defineTemplate(String templateName, String template) {
        if ( templateName.charAt(0) !='/' ) templateName = "/"+templateName;
        try {
            CompiledST impl = defineTemplate(templateName,
                                             new CommonToken(GroupParser.ID, templateName),
                                             null,
                                             template,
                                             null);
            return impl;
        }
        catch (STException se) {
            // we have reported the error; the exception just blasts us
            // out of parsing this template
        }
        return null;
    }
 

/**
 * Create a new token from the given prototype. Any argument besides the prototype is optional and
 * will be ignored if its value is <code>null</code>.
 */
protected CommonToken createToken(CommonToken prototype, String text, 
		Integer charPosInLine, Integer channel, Integer start, Integer stop, Integer type) {
	if (prototype == null)
		throw new IllegalArgumentException("Prototype may not be null.");
	CommonToken result = new CommonToken(prototype);
	if (text != null)
		result.setText(text);
	if (charPosInLine != null)
		result.setCharPositionInLine(charPosInLine.intValue());
	if (channel != null)
		result.setChannel(channel.intValue());
	if (start != null)
		result.setStartIndex(start.intValue());
	if (stop != null)
		result.setStopIndex(stop.intValue());
	if (type != null)
		result.setType(type.intValue());
	return result;
}
 

public void testReplaceOneWithTwoAtLeft() throws Exception {
	CommonTree t = new CommonTree(new CommonToken(99, "a"));
	t.addChild(new CommonTree(new CommonToken(99, "b")));
	t.addChild(new CommonTree(new CommonToken(99, "c")));
	t.addChild(new CommonTree(new CommonToken(99, "d")));

	CommonTree newChildren = (CommonTree)adaptor.nil();
	newChildren.addChild(new CommonTree(new CommonToken(99,"x")));
	newChildren.addChild(new CommonTree(new CommonToken(99,"y")));

	t.replaceChildren(0, 0, newChildren);
	String expecting = "(a x y c d)";
	assertEquals(expecting, t.toStringTree());
	t.sanityCheckParentAndChildIndexes();
}
 

@Test public void testMergeFirstToken() {
	CommonToken commonToken = new CommonToken(TOKEN_TYPE, "text");
	commonToken.setStartIndex(4);
	AntlrDatatypeRuleToken token = new AntlrDatatypeRuleToken();
	token.merge(commonToken);
	assertEquals("text", token.getText());
	assertEquals(8, token.getExpectedOffset());
}
 

@Override
public CompiledST loadTemplateFile(String prefix, String unqualifiedFileName, CharStream templateStream) {
    String template = templateStream.substring(0, templateStream.size()- 1);
    String templateName = Misc.getFileNameNoSuffix(unqualifiedFileName);
    String fullyQualifiedTemplateName = prefix+templateName;
    CompiledST impl = new Compiler(this).compile(fullyQualifiedTemplateName, template);
    CommonToken nameT = new CommonToken(STLexer.SEMI); // Seems like a hack, best I could come up with.
    nameT.setInputStream(templateStream);
    rawDefineTemplate(fullyQualifiedTemplateName, impl, nameT);
    impl.defineImplicitlyDefinedTemplates(this);
    return impl;
}
 

/** for testing */

    public CompiledST defineTemplate(String templateName, String template) {
        if ( templateName.charAt(0)!='/' ) templateName = "/"+templateName;
        try {
            CompiledST impl = defineTemplate(templateName, new CommonToken(GroupParser.ID, templateName), null, template, null);
            return impl;
        }
        catch (STException se) {
            // we have reported the error; the exception just blasts us
            // out of parsing this template
        }
        return null;
    }
 

/** for testing */

    public CompiledST defineTemplate(String templateName, String template) {
        if ( templateName.charAt(0)!='/' ) templateName = "/"+templateName;
        try {
            CompiledST impl = defineTemplate(templateName, new CommonToken(GroupParser.ID, templateName), null, template, null);
            return impl;
        }
        catch (STException se) {
            // we have reported the error; the exception just blasts us
            // out of parsing this template
        }
        return null;
    }
 

/**
 * see BUG 234135: Comments on EOF not detected
 * https://bugs.eclipse.org/bugs/show_bug.cgi?id=234135
 */
@Test public void testSlCommentEOF() {
	String model = "a\n//sl comment";
	InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
	lexer.setCharStream(new ANTLRStringStream(model));
	CommonTokenStream stream = new CommonTokenStream(lexer);
	Object eofLineComment = stream.getTokens().get(2);
	assertTrue(eofLineComment instanceof CommonToken);
	CommonToken commonToken = (CommonToken) eofLineComment;
	int type = commonToken.getType();
	assertEquals(InternalSimpleExpressionsTestLanguageLexer.RULE_SL_COMMENT, type);
}
 

public void testBecomeRoot6() throws Exception {
	// emulates construction of ^(5 6)
	CommonTree root_0 = (CommonTree)adaptor.nil();
	CommonTree root_1 = (CommonTree)adaptor.nil();
	root_1 = (CommonTree)adaptor.becomeRoot(new CommonTree(new CommonToken(5)), root_1);

	adaptor.addChild(root_1, new CommonTree(new CommonToken(6)));

	adaptor.addChild(root_0, root_1);

	root_0.sanityCheckParentAndChildIndexes();
}
 

/** for testing */

    public CompiledST defineTemplate(String templateName, String template) {
        if ( templateName.charAt(0)!='/' ) templateName = "/"+templateName;
        try {
            CompiledST impl = defineTemplate(templateName, new CommonToken(GroupParser.ID, templateName), null, template, null);
            return impl;
        }
        catch (STException se) {
            // we have reported the error; the exception just blasts us
            // out of parsing this template
        }
        return null;
    }
 

public void testReplaceTwoWithOneAtRight() throws Exception {
	CommonTree t = new CommonTree(new CommonToken(99, "a"));
	t.addChild(new CommonTree(new CommonToken(99, "b")));
	t.addChild(new CommonTree(new CommonToken(99, "c")));
	t.addChild(new CommonTree(new CommonToken(99, "d")));

	CommonTree newChild = new CommonTree(new CommonToken(99,"x"));

	t.replaceChildren(1, 2, newChild);
	String expecting = "(a b x)";
	assertEquals(expecting, t.toStringTree());
	t.sanityCheckParentAndChildIndexes();
}
 

/** for testing */

    public CompiledST defineTemplate(String name, String argsS, String template) {
        if ( name.charAt(0) !='/' ) name = "/"+name;
        String[] args = argsS.split(",");
        List<FormalArgument> a = new ArrayList<FormalArgument>();
        for (String arg : args) {
            a.add(
                new FormalArgument(arg)
            );
        }
        return defineTemplate(name, new CommonToken(GroupParser.ID, name), a, template, null);
    }
 

public void testMarkRewindEntire() throws Exception {
	// ^(101 ^(102 103 ^(106 107) ) 104 105)
	// stream has 7 real + 6 nav nodes
	// Sequence of types: 101 DN 102 DN 103 106 DN 107 UP UP 104 105 UP EOF
	Tree r0 = new CommonTree(new CommonToken(101));
	Tree r1 = new CommonTree(new CommonToken(102));
	r0.addChild(r1);
	r1.addChild(new CommonTree(new CommonToken(103)));
	Tree r2 = new CommonTree(new CommonToken(106));
	r2.addChild(new CommonTree(new CommonToken(107)));
	r1.addChild(r2);
	r0.addChild(new CommonTree(new CommonToken(104)));
	r0.addChild(new CommonTree(new CommonToken(105)));

	CommonTreeNodeStream stream = new CommonTreeNodeStream(r0);
	int m = stream.mark(); // MARK
	for (int k=1; k<=13; k++) { // consume til end
		stream.LT(1);
		stream.consume();
	}
	assertEquals(Token.EOF, ((Tree)stream.LT(1)).getType());
	assertEquals(Token.UP, ((Tree)stream.LT(-1)).getType());
	stream.rewind(m);      // REWIND

	// consume til end again :)
	for (int k=1; k<=13; k++) { // consume til end
		stream.LT(1);
		stream.consume();
	}
	assertEquals(Token.EOF, ((Tree)stream.LT(1)).getType());
	assertEquals(Token.UP, ((Tree)stream.LT(-1)).getType());
}
 

/** for testing */

    public CompiledST defineTemplate(String name, String argsS, String template) {
        if ( name.charAt(0)!='/' ) name = "/"+name;
        String[] args = argsS.split(",");
        List<FormalArgument> a = new ArrayList<FormalArgument>();
        for (String arg : args) {
            a.add(new FormalArgument(arg));
        }
        return defineTemplate(name, new CommonToken(GroupParser.ID, name), a, template, null);
    }
 

@Override
public Token nextToken() {
	while (true) {
		this.state.token = null;
		this.state.channel = Token.DEFAULT_CHANNEL;
		this.state.tokenStartCharIndex = input.index();
		this.state.tokenStartCharPositionInLine = input
				.getCharPositionInLine();
		this.state.tokenStartLine = input.getLine();
		this.state.text = null;
		if (input.LA(1) == CharStream.EOF) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if (this.state.token == null) {
				emit();
			} else if (this.state.token == Token.SKIP_TOKEN) {
				continue;
			}
			return this.state.token;
		} catch (RecognitionException re) {
			reportError(re);
			if (re instanceof NoViableAltException
					|| re instanceof FailedPredicateException) {
				recover(re);
			}
			// create token that holds mismatched char
			Token t = new CommonToken(input, Token.INVALID_TOKEN_TYPE,
					Token.HIDDEN_CHANNEL, this.state.tokenStartCharIndex,
					getCharIndex() - 1);
			t.setLine(this.state.tokenStartLine);
			t.setCharPositionInLine(
					this.state.tokenStartCharPositionInLine);
			tokenErrorMap.put(t, getErrorMessage(re, this.getTokenNames()));
			emit(t);
			return this.state.token;
		}
	}
}
 

SemanticException(IntStream input, Token token, String errorMessage, Object... messageArguments) {
    super();
    this.input = input;
    this.token = token;
    this.index = ((CommonToken)token).getStartIndex();
    this.line = token.getLine();
 this.charPositionInLine = token.getCharPositionInLine();
    this.errorMessage = String.format(errorMessage, messageArguments);
}
 

SemanticException(IntStream input, Token token, String errorMessage, Object... messageArguments) {
    super();
    this.input = input;
    this.token = token;
    this.index = ((CommonToken)token).getStartIndex();
    this.line = token.getLine();
 this.charPositionInLine = token.getCharPositionInLine();
    this.errorMessage = String.format(errorMessage, messageArguments);
}
 

public void testBecomeRoot() throws Exception {
	// 5 becomes new root of ^(nil 101 102 103)
	CommonTree newRoot = new CommonTree(new CommonToken(5));

	CommonTree oldRoot = new CommonTree((Token)null);
	oldRoot.addChild(new CommonTree(new CommonToken(101)));
	oldRoot.addChild(new CommonTree(new CommonToken(102)));
	oldRoot.addChild(new CommonTree(new CommonToken(103)));

	TreeAdaptor adaptor = new CommonTreeAdaptor();
	adaptor.becomeRoot(newRoot, oldRoot);
	newRoot.sanityCheckParentAndChildIndexes();
}
 

public void testReplaceAllWithOne() throws Exception {
	CommonTree t = new CommonTree(new CommonToken(99, "a"));
	t.addChild(new CommonTree(new CommonToken(99, "b")));
	t.addChild(new CommonTree(new CommonToken(99, "c")));
	t.addChild(new CommonTree(new CommonToken(99, "d")));

	CommonTree newChild = new CommonTree(new CommonToken(99,"x"));

	t.replaceChildren(0, 2, newChild);
	String expecting = "(a x)";
	assertEquals(expecting, t.toStringTree());
	t.sanityCheckParentAndChildIndexes();
}
 

public void testAoverB() throws Exception {
	Tree t = new CommonTree(new CommonToken(101));
	t.addChild(new CommonTree(new CommonToken(102)));

	TreeNodeStream stream = newStream(t);
	String expecting = " 101 102";
	String found = toNodesOnlyString(stream);
	assertEquals(expecting, found);

	expecting = " 101 2 102 3";
	found = stream.toString();
	assertEquals(expecting, found);
}
 

@Override
public void consumeToken(Token token) {
    if (backtracking > 0 || resync) {
        return;
    }

    if (debug_tokens) {
        CommonToken ct = (CommonToken) token;
        int[] ctr = CommonTokenUtil.getCommonTokenOffsetRange(ct);
        System.out.println(token + "(" + ctr[0] + "-" + ctr[1] + ")");
    }

    //ignore the closing EOF token, we do not want it
    //it the parse tree
    if (token.getType() == Css3Lexer.EOF) {
        return;
    }

    //also ignore error tokens - they are added as children of ErrorNode-s in the recognitionException(...) method
    if (token.getType() == Token.INVALID_TOKEN_TYPE) {
        return;
    }

    lastConsumedToken = (CommonToken) token;

    RuleNode ruleNode = callStack.peek();
    TokenNode elementNode = new TokenNode(source, (CommonToken) token);
    elementNode.hiddenTokens = this.hiddenTokens;
    hiddenTokens.clear();
    ruleNode.addChild(elementNode);

    updateFirstTokens(ruleNode, lastConsumedToken);
}
 

/** for testing */

    public CompiledST defineTemplate(String templateName, String template) {
        if ( templateName.charAt(0)!='/' ) templateName = "/"+templateName;
        try {
            CompiledST impl = defineTemplate(templateName, new CommonToken(GroupParser.ID, templateName), null, template, null);
            return impl;
        }
        catch (STException se) {
            // we have reported the error; the exception just blasts us
            // out of parsing this template
        }
        return null;
    }
 

public void testBecomeRoot5() throws Exception {
	// ^(nil 5) becomes new root of ^(101 102 103)
	CommonTree newRoot = new CommonTree((Token)null);
	newRoot.addChild(new CommonTree(new CommonToken(5)));

	CommonTree oldRoot = new CommonTree(new CommonToken(101));
	oldRoot.addChild(new CommonTree(new CommonToken(102)));
	oldRoot.addChild(new CommonTree(new CommonToken(103)));

	TreeAdaptor adaptor = new CommonTreeAdaptor();
	adaptor.becomeRoot(newRoot, oldRoot);
	newRoot.sanityCheckParentAndChildIndexes();
}
 

@Test public void testLexerError_04() throws Exception {
	String model = "a 'incomplete string with bad escape sequence \\";
	InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
	lexer.setCharStream(new ANTLRStringStream(model));
	CommonTokenStream stream = new CommonTokenStream(lexer);
	@SuppressWarnings("unchecked")
	List<CommonToken> tokens = stream.getTokens();
	assertEquals(tokens.toString(), 3, tokens.size());
	assertEquals("a", tokens.get(0).getText());
	assertEquals(" ", tokens.get(1).getText());
	assertEquals("'incomplete string with bad escape sequence \\", tokens.get(2).getText());
	assertEquals(0, tokens.get(2).getType());
}