Java Code Examples for org.antlr.runtime.Token#EOF

/** Consume tokens until one matches the given token set */
    @Override
    public void consumeUntil(IntStream i, BitSet set) {
//        System.out.println("consumeUntil(" + set.toString(getTokenNames()) + ")");
        Token ttype;
        List<Token> skipped = new ArrayList<>();
        beginResync();
        try {
            while ((ttype = input.LT(1)) != null && ttype.getType() != Token.EOF && !set.member(ttype.getType())) {
//            System.out.println("consume during recover LA(1)=" + getTokenNames()[input.LA(1)]);
                input.consume();
                skipped.add(ttype);
            }
        } finally {
            endResync();
        }
        ((NbParseTreeBuilder) dbg).consumeSkippedTokens(skipped);
    }
 

@Test
public void TestLexer() throws IOException {
    CharStream input = new QueryParserFileStream( "test/org/apache/pig/parser/TestLexer.pig" );
    QueryLexer lexer = new QueryLexer( input );
    int tokenCount = 0;
    Token token;
    while( ( token = lexer.nextToken() ).getType() != Token.EOF ) {
        if( token.getChannel() == Token.HIDDEN_CHANNEL )
            continue;
        tokenCount++;
        if( token.getText().equals( ";" ) ) {
            System.out.println( token.getText() );
        } else {
            System.out.print( token.getText() + "(" + token.getType() + ") " );
        }
    }

    // While we can check more conditions, such as type of each token, for now I think the following
    // is enough. If the token type is wrong, it will be most likely caught by the parser.
    Assert.assertEquals( 455, tokenCount );
    Assert.assertEquals( 0, lexer.getNumberOfSyntaxErrors() );
}
 

@Test
public void test2() throws IOException {
    String query = "A = load 'input' using PigStorage(';');" +
                   "B = foreach ^ A generate string.concatsep( ';', $1, $2 );";
    CharStream input = new QueryParserStringStream( query, null );
    QueryLexer lexer = new QueryLexer( input );
    Token token;
    try {
        while( ( token = lexer.nextToken() ).getType() != Token.EOF ) {
            if( token.getChannel() == Token.HIDDEN_CHANNEL )
                continue;
            if( token.getText().equals( ";" ) ) {
                System.out.println( token.getText() );
            } else {
                System.out.print( token.getText() + "(" + token.getType() + ") " );
            }
        }
    } catch(Exception ex) {
        Assert.assertTrue( ex.getMessage().contains( "Unexpected character" ) );
        return;
    }
    Assert.fail( "Query should fail." );
}
 

@Override
public boolean existsConflict(String lastCompleteText, String proposal, ContentAssistContext context) {
	initTokenSources(lastCompleteText, proposal, context);
	try {
		if (!equalTokenSequence(lastCompleteFlexer, combinedFlexer))
			return true;
		if (!equalTokenSequence(proposalFlexer, combinedFlexer))
			return true;
		int lastToken = proposalFlexer.advance();
		if (lastToken != Token.EOF)
			return true;
		return false;
	} catch(IOException e) {
		throw new RuntimeException(e);
	}
}
 

@Override
	protected void createContextsForLastCompleteNode(EObject previousModel, boolean strict){
		String currentNodePrefix = getPrefix(currentNode);
		if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) {
			InternalFlexer flexer = flexerFactory.createFlexer(new StringReader(currentNodePrefix));
			try {
				int token = flexer.advance();
				if (token == Token.EOF) { // error case - nothing could be parsed
					return;
				}
				while(token != Token.EOF) {
					if (isErrorToken(token))
						return;
					token = flexer.advance();
				}
			} catch (IOException e) {
				log.error(e.getMessage(), e);
				return;
			}
		}
		String prefix = "";
		Collection<FollowElement> followElements = parseFollowElements(completionOffset, strict);
//		String completeInput = viewer.getDocument().get(0, completionOffset);
//		Collection<FollowElement> followElements = parser.getFollowElements(completeInput, strict);
		doCreateContexts(lastCompleteNode, currentNode, prefix, previousModel, followElements);
	}
 

@Override
public ITextRegion getTokenRegion(String text, ITextRegion region) {
	try {
		InternalFlexer flexer = flexerFactory.createFlexer(new StringReader(text));
		int token = flexer.advance();
		int prevStart = 0;
		int nextStart = flexer.getTokenLength();
		
		final int regionStartOffset = region.getOffset();
		final int regionEnd = regionStartOffset + region.getLength();
		while (token != Token.EOF && nextStart <= regionStartOffset) {
			prevStart = nextStart;
			token = flexer.advance();
			nextStart += flexer.getTokenLength();
		}
		while (token != Token.EOF && nextStart < regionEnd) {
			token = flexer.advance();
			nextStart += flexer.getTokenLength();
		}
		return new TextRegion(prevStart, nextStart - prevStart);
	} catch(IOException e) {
		throw new RuntimeException(e); // cannot happen since StringReader doesn't throw IOException
	}
}
 

@Override
public Token nextToken() {
	try {
		int type = flexer.advance();
		if (type == Token.EOF) {
			return Token.EOF_TOKEN;
		}
		int length = flexer.getTokenLength();
		final String tokenText = flexer.getTokenText();
		CommonToken result = new CommonTokenWithText(tokenText, type, Token.DEFAULT_CHANNEL, offset);
		offset += length;
		return result;
	} catch (IOException e) {
		throw new RuntimeException(e);
	}
}
 

/**
 * <p>
 * Promotes EOL which may lead to an automatically inserted semicolon. This is probably the most important method
 * for automatic semicolon insertion, as it is only possible to insert a semicolon in case of line breaks (even if
 * they are hidden in a multi-line comment!).
 * </p>
 */
public static void promoteEOL(Callback callback) {
	RecognizerSharedState state = callback.getState();
	TokenStream input = callback.getInput();
	// Don't promote EOL if there was a syntax error at EOF
	if (state.lastErrorIndex == input.size()) {
		return;
	}
	// Get current token and its type (the possibly offending token).
	Token prev = input.LT(-1);
	Token next = input.LT(1);
	int la = next.getType();
	if (la == InternalN4JSParser.Semicolon) {
		return;
	}

	// Promoting an EOL means switching it from off channel to on channel.
	// A ML_COMMENT gets promoted when it contains an EOL.
	for (int idx = prev == null ? 0 : prev.getTokenIndex() + 1, max = la == Token.EOF ? input.size()
			: next.getTokenIndex(); idx < max; idx++) {
		Token lt = input.get(idx);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning (previously promoted)
			break;
		} else if (isSemicolonEquivalent(lt)) {
			// We found our EOL: promote the token to on channel, position the input on it and reset the rule
			// start.
			lt.setChannel(Token.DEFAULT_CHANNEL);
			input.seek(idx);
			break;
		}
	}
}
 

@Override
protected void doSplitToken(Token token, ITokenAcceptor result) {
	int nextLevel = (token.getType() == Token.EOF) ? 0 : countTabs(token.getText());
	while (this.level > nextLevel) {
		this.level--;
		result.accept(new CommonToken(InternalIndentationAwareTestLanguageLexer.RULE_END, ""));
	}
	result.accept(token);
	while (this.level < nextLevel) {
		this.level++;
		result.accept(new CommonToken(InternalIndentationAwareTestLanguageLexer.RULE_BEGIN, ""));
	}
}
 

protected void doTestLookahead(int lookAhead, int tokenCount) {
	this.tokenCount = tokenCount;
	XtextTokenStream stream = new XtextTokenStream(this, Token.DEFAULT_CHANNEL);
	while(stream.LA(1) != Token.EOF) {
		for(int i = 0; i < lookAhead; i++) {
			for(int j = i + 1; j < lookAhead; j++) {
				stream.LA(j - i);					
			}
			stream.consume();
		}
	}
}
 

@Override
public void recover(IntStream stream, RecognitionException ex) {
	if (recoveryListener != null)
		recoveryListener.beginErrorRecovery();
	removeUnexpectedElements();
	if (ex instanceof FailedPredicateException && ex.token.getType() == Token.EOF) {
		failedPredicateAtEOF = true;
	}
	super.recover(stream, ex);
	if (recoveryListener != null)
		recoveryListener.endErrorRecovery();
}
 

@Test public void testConsumeUntilEOF_large() {
	tokenCount = 25000;
	XtextTokenStream stream = new XtextTokenStream(this, Token.DEFAULT_CHANNEL);
	while(stream.LA(1) != Token.EOF) {
		stream.consume();
		stream.LA(1);
	}
}
 

@Override
public int LA(int i) {
	Token lookaheadToken = LT(i);
	int result = lookaheadToken.getType();
	if (result == Token.EOF && getListener() != null)
		getListener().announceEof(i);
	return result;
}
 

@Override
public Token nextToken() {
	try {
		int type = flexer.advance();
		if (type == Token.EOF) {
			return Token.EOF_TOKEN;
		}
		int length = flexer.getTokenLength();
		CommonToken result = new CommonTokenWithoutText(type, Token.DEFAULT_CHANNEL, offset, length);
		offset += length;
		return result;
	} catch (IOException e) {
		throw new RuntimeException(e);
	}
}
 

protected boolean equalTokenSequence(InternalFlexer first, InternalFlexer second) throws IOException {
	int token;
	while((token = first.advance()) != Token.EOF) {
		int otherToken = second.advance();
		if (token != otherToken) {
			return false;
		}
		if (first.getTokenLength() != second.getTokenLength()) {
			return false;
		}
	}
	return true;
}
 

/** Add an EOF transition to any rule end ATNState that points to nothing
    *  (i.e., for all those rules not invoked by another rule).  These
    *  are start symbols then.
 *
 *  Return the number of grammar entry points; i.e., how many rules are
 *  not invoked by another rule (they can only be invoked from outside).
 *  These are the start rules.
    */
public int addEOFTransitionToStartRules() {
	int n = 0;
	ATNState eofTarget = newState(null); // one unique EOF target for all rules
	for (Rule r : g.rules.values()) {
		ATNState stop = atn.ruleToStopState[r.index];
		if ( stop.getNumberOfTransitions()>0 ) continue;
		n++;
		Transition t = new AtomTransition(eofTarget, Token.EOF);
		stop.addTransition(t);
	}
	return n;
}
 

protected boolean isSameTokenSequence(TokenSource originalSource, TokenSource newSource, int expectedLength) {
	Token token = originalSource.nextToken();
	int newLength = 0;
	while(Token.EOF != token.getType()) {
		Token newToken = newSource.nextToken();
		if (token.getType() != newToken.getType()) {
			return false;
		}
		newLength += TokenTool.getLength(newToken);
		token = originalSource.nextToken();
	}
	return newLength == expectedLength;
}
 

public String getTokenErrorDisplay(Token t) {
    String s = t.getText();
    if (s == null) {
        if (t.getType() == Token.EOF) {
            s = "<EOF>";
        } else {
            s = "<" + t.getType() + ">";
        }
    }
    s = s.replaceAll("\n", "\\\\n");
    s = s.replaceAll("\r", "\\\\r");
    s = s.replaceAll("\t", "\\\\t");
    return "'" + s + "'";
}
 

@Override
protected boolean shouldSplitToken(Token token) {
	return token.getType() == Token.EOF || shouldSplitTokenImpl(token);
}
 

public void testPushPopFromEOF() throws Exception {
	// ^(101 ^(102 103) ^(104 105) ^(106 107) 108 109)
	// stream has 9 real + 8 nav nodes
	// Sequence of types: 101 DN 102 DN 103 UP 104 DN 105 UP 106 DN 107 UP 108 109 UP
	Tree r0 = new CommonTree(new CommonToken(101));
	Tree r1 = new CommonTree(new CommonToken(102));
	r1.addChild(new CommonTree(new CommonToken(103)));
	r0.addChild(r1);
	Tree r2 = new CommonTree(new CommonToken(104));
	r2.addChild(new CommonTree(new CommonToken(105)));
	r0.addChild(r2);
	Tree r3 = new CommonTree(new CommonToken(106));
	r3.addChild(new CommonTree(new CommonToken(107)));
	r0.addChild(r3);
	r0.addChild(new CommonTree(new CommonToken(108)));
	r0.addChild(new CommonTree(new CommonToken(109)));

	CommonTreeNodeStream stream = new CommonTreeNodeStream(r0);

	while ( stream.LA(1)!=Token.EOF ) {
		stream.consume();
	}
	int indexOf102 = 2;
	int indexOf104 = 6;
	assertEquals(Token.EOF, ((Tree)stream.LT(1)).getType());

	// CALL 102
	stream.push(indexOf102);
	assertEquals(102, ((Tree)stream.LT(1)).getType());
	stream.consume(); // consume 102
	assertEquals(Token.DOWN, ((Tree)stream.LT(1)).getType());
	stream.consume(); // consume DN
	assertEquals(103, ((Tree)stream.LT(1)).getType());
	stream.consume(); // consume 103
	assertEquals(Token.UP, ((Tree)stream.LT(1)).getType());
	// RETURN (to empty stack)
	stream.pop();
	assertEquals(Token.EOF, ((Tree)stream.LT(1)).getType());

	// CALL 104
	stream.push(indexOf104);
	assertEquals(104, ((Tree)stream.LT(1)).getType());
	stream.consume(); // consume 102
	assertEquals(Token.DOWN, ((Tree)stream.LT(1)).getType());
	stream.consume(); // consume DN
	assertEquals(105, ((Tree)stream.LT(1)).getType());
	stream.consume(); // consume 103
	assertEquals(Token.UP, ((Tree)stream.LT(1)).getType());
	// RETURN (to empty stack)
	stream.pop();
	assertEquals(Token.EOF, ((Tree)stream.LT(1)).getType());
}