Java Code Examples for org.antlr.runtime.Token#EOF_TOKEN

@Override
protected void announce(Token start, Token stop, AbstractElement element) {
	if (start != null && start != Token.EOF_TOKEN) {
		if (start == stop) {
			announce(start, element);
		} else {
			CommonToken castedStart = (CommonToken) start;
			if (stop == null) { // possible error condition
				if (start.getTokenIndex() == state.lastErrorIndex) {
					return;
				}
			}
			CommonToken castedEnd = (CommonToken) stop;
			Integer newType = rewriter.rewrite(castedStart, element);
			if (newType != null && castedEnd != null && castedEnd != Token.EOF_TOKEN) {
				LazyTokenStream castedInput = (LazyTokenStream) this.input;
				for (int i = castedStart.getTokenIndex() + 1; i < castedEnd.getTokenIndex(); i++) {
					Token token = castedInput.get(i);
					if (token.getChannel() != Token.HIDDEN_CHANNEL)
						token.setType(newType);
				}
				castedEnd.setType(newType);
			}
		}
	}
}
 

private ObservableXtextTokenStream toTokenStream(FollowElement element,
		ITokenDefProvider tokenDefProvider) {
	Iterator<LookAheadTerminal> iter = element.getLookAheadTerminals().iterator();
	return new ObservableXtextTokenStream(new TokenSource() {
		@Override
		public Token nextToken() {
			if (iter.hasNext()) {
				LookAheadTerminal lookAhead = iter.next();
				return lookAhead.getToken();
			}
			return Token.EOF_TOKEN;
		}

		@Override
		public String getSourceName() {
			return "LookAheadTerminalTokenSource";
		}
	}, tokenDefProvider);
}
 

@Override
protected TokenSource createTokenSource(String string) {
	List<Token> tokens = highlightingParser.getTokens(string);
	Iterator<Token> iter = tokens.iterator();
	return new TokenSource() {

		@Override
		public Token nextToken() {
			if (iter.hasNext()) {
				return iter.next();
			}
			return Token.EOF_TOKEN;
		}

		@Override
		public String getSourceName() {
			return "Text: " + string;
		}
	};
}
 

@Override
protected void createContextsForLastCompleteNode(EObject previousModel, boolean strict) {
	String currentNodePrefix = getPrefix(currentNode);
	if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) {
		lexer.setCharStream(new ANTLRStringStream(currentNodePrefix));
		Token token = lexer.nextToken();
		if (token == Token.EOF_TOKEN) {
			return;
		}
		while (token != Token.EOF_TOKEN) {
			if (isErrorToken(token)) {
				return;
			}
			token = lexer.nextToken();
		}
	}
	String prefix = "";
	Collection<FollowElement> followElements = parseFollowElements(completionOffset, strict);
	doCreateContexts(lastCompleteNode, currentNode, prefix, previousModel, followElements);
}
 

@Override
public int LA(int i) {
	Token lookaheadToken = LT(i);
	if (markerCount > 0) { // predicting with a marker
		if (Token.EOF_TOKEN == lookaheadToken) { // predicated past EOF
			if (indexOfLookAhead != size()) {
				indexOfLookAhead = size();
				myLookAhead++;
			}
		} else {
			int laTokenIndex = lookaheadToken.getTokenIndex();
			if (indexOfLookAhead < laTokenIndex) {
				indexOfLookAhead = laTokenIndex;
				myLookAhead++;
			}
		}
	} else {
		myLookAhead = Math.max(i, myLookAhead);
	}
	// return super.LA(i); // inlined
	return lookaheadToken.getType();
}
 

protected void createContextsForLastCompleteNode(EObject previousModel, boolean strict) throws BadLocationException {
	String currentNodePrefix = getPrefix(currentNode);
	if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) {
		lexer.setCharStream(new ANTLRStringStream(currentNodePrefix));
		Token token = lexer.nextToken();
		if (token == Token.EOF_TOKEN) { // error case - nothing could be parsed
			return;
		}
		while(token != Token.EOF_TOKEN) {
			if (isErrorToken(token))
				return;
			token = lexer.nextToken();
		}
	}
	String prefix = "";
	String completeInput = getInputToParse(completionOffset);
	Collection<FollowElement> followElements = parser.getFollowElements(completeInput, strict);
	doCreateContexts(lastCompleteNode, currentNode, prefix, previousModel, followElements);
}
 

@Override
public Token nextToken() {
	try {
		int type = flexer.advance();
		if (type == Token.EOF) {
			return Token.EOF_TOKEN;
		}
		int length = flexer.getTokenLength();
		final String tokenText = flexer.getTokenText();
		CommonToken result = new CommonTokenWithText(tokenText, type, Token.DEFAULT_CHANNEL, offset);
		offset += length;
		return result;
	} catch (IOException e) {
		throw new RuntimeException(e);
	}
}
 

public Token nextToken() {
  
  if ( state.token != null && state.token.getType() == SCRIPTCLOSE ){
    return Token.EOF_TOKEN;
  }
  
  while (true) {
    state.token = null;
    state.channel = Token.DEFAULT_CHANNEL;
    state.tokenStartCharIndex = input.index();
    state.tokenStartCharPositionInLine = input.getCharPositionInLine();
    state.tokenStartLine = input.getLine();
    state.text = null;
    if ( input.LA(1)==CharStream.EOF ) {
      return Token.EOF_TOKEN;
    }
    try {
      mTokens();
      if ( state.token==null ) {
        emit();
      }
      else if ( state.token==Token.SKIP_TOKEN ) {
        continue;
      }
      return state.token;
    }
    catch (RecognitionException re) {
      //reportError(re);
      return Token.EOF_TOKEN;
      //throw new RuntimeException("Bailing out!"); // or throw Error
    }
  }
}
 

@Override
public Token nextToken() {
	while (true) {
		state.token = null;
		state.channel = Token.DEFAULT_CHANNEL;
		state.tokenStartCharIndex = input.index();
		state.tokenStartCharPositionInLine = input.getCharPositionInLine();
		state.tokenStartLine = input.getLine();
		state.text = null;
		if ( input.LA(1)==CharStream.EOF ) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if ( state.token==null ) {
				emit();
			}
			else if ( state.token==Token.SKIP_TOKEN ) {
				continue;
			}
			return state.token;
		}
		/*
		 * Avoid infinite loop (editor freeze) on {@link FailedPredicateException}
		 */
		catch (NoViableAltException | FailedPredicateException e) {
			reportError(e);
			recover(e); // throw out current char and try again
		}
		catch (RecognitionException re) {
			reportError(re);
			// match() routine has already called recover()
		}
	}
}
 

@Override
public Token nextToken() {
	while (true) {
		this.state.token = null;
		this.state.channel = Token.DEFAULT_CHANNEL;
		this.state.tokenStartCharIndex = input.index();
		this.state.tokenStartCharPositionInLine = input.getCharPositionInLine();
		this.state.tokenStartLine = input.getLine();
		this.state.text = null;
		if (input.LA(1) == CharStream.EOF) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if (this.state.token == null) {
				emit();
			}
			else if (this.state.token == Token.SKIP_TOKEN) {
				continue;
			}
			return this.state.token;
		}
		catch (RecognitionException re) {
			reportError(re);
			if (re instanceof NoViableAltException ||
				re instanceof FailedPredicateException) {
				recover(re);
			}
			// create token that holds mismatched char
			Token t = new CommonToken(input, Token.INVALID_TOKEN_TYPE, Token.HIDDEN_CHANNEL,
					this.state.tokenStartCharIndex, getCharIndex() - 1);
			t.setLine(this.state.tokenStartLine);
			t.setCharPositionInLine(this.state.tokenStartCharPositionInLine);
			tokenErrorMap.put(t, getErrorMessage(re, this.getTokenNames()));
			emit(t);
			return this.state.token;
		}
	}
}
 

@Override
public Token nextToken() {
	if (iter.hasNext()) {
		ILookAheadTerminal lookAhead = iter.next();
		return lookAhead.getToken();
	}
	return Token.EOF_TOKEN;
}
 

@Override
public Token nextToken() {
	try {
		int type = flexer.advance();
		if (type == Token.EOF) {
			return Token.EOF_TOKEN;
		}
		int length = flexer.getTokenLength();
		CommonToken result = new CommonTokenWithoutText(type, Token.DEFAULT_CHANNEL, offset, length);
		offset += length;
		return result;
	} catch (IOException e) {
		throw new RuntimeException(e);
	}
}
 

@Override
public Token nextToken() {
	while (true) {
		this.state.token = null;
		this.state.channel = Token.DEFAULT_CHANNEL;
		this.state.tokenStartCharIndex = input.index();
		this.state.tokenStartCharPositionInLine = input
				.getCharPositionInLine();
		this.state.tokenStartLine = input.getLine();
		this.state.text = null;
		if (input.LA(1) == CharStream.EOF) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if (this.state.token == null) {
				emit();
			} else if (this.state.token == Token.SKIP_TOKEN) {
				continue;
			}
			return this.state.token;
		} catch (RecognitionException re) {
			reportError(re);
			if (re instanceof NoViableAltException
					|| re instanceof FailedPredicateException) {
				recover(re);
			}
			// create token that holds mismatched char
			Token t = new CommonToken(input, Token.INVALID_TOKEN_TYPE,
					Token.HIDDEN_CHANNEL, this.state.tokenStartCharIndex,
					getCharIndex() - 1);
			t.setLine(this.state.tokenStartLine);
			t.setCharPositionInLine(
					this.state.tokenStartCharPositionInLine);
			tokenErrorMap.put(t, getErrorMessage(re, this.getTokenNames()));
			emit(t);
			return this.state.token;
		}
	}
}
 

@Override
public Token nextToken() {
	while (true) {
		state.token = null;
		state.channel = Token.DEFAULT_CHANNEL;
		state.tokenStartCharIndex = input.index();
		state.tokenStartCharPositionInLine = input.getCharPositionInLine();
		state.tokenStartLine = input.getLine();
		state.text = null;
		if ( input.LA(1)==CharStream.EOF ) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if ( state.token==null ) {
				emit();
			}
			else if ( state.token==Token.SKIP_TOKEN ) {
				continue;
			}
			return state.token;
		}
		/*
		 * Avoid infinite loop (editor freeze) on {@link FailedPredicateException}
		 */
		catch (NoViableAltException | FailedPredicateException e) {
			reportError(e);
			recover(e); // throw out current char and try again
		}
		catch (RecognitionException re) {
			reportError(re);
			// match() routine has already called recover()
		}
	}
}
 

private void tokenize(String content) {
    CharStream cs = new ANTLRStringStream(content);
    CTFLexer lexer = new CTFLexer(cs);

    tokens.clear();
    for (;;) {
      Token token = lexer.nextToken();
      if (token == Token.EOF_TOKEN) {
        return;
      }
      tokens.add(token);
    }
}
 

protected void doProvideHighlightingFor(String body, org.eclipse.xtext.ide.editor.syntaxcoloring.IHighlightedPositionAcceptor acceptor) {
	lexer.setCharStream(new ANTLRStringStream(body));
	Token token = lexer.nextToken();
	while(token != Token.EOF_TOKEN) {
		String id = tokenIdMapper.getId(token.getType());
		int offset = TokenTool.getOffset(token);
		int length = TokenTool.getLength(token);
		acceptor.addPosition(offset, length, id);
		token = lexer.nextToken();
	}
}
 

@Override
protected void error(NoViableAltException nvae) {
	if (nvae.token == Token.EOF_TOKEN) {
		int lookAheadAddOn = getRecognizer().lookAheadAddOn;
		int lookAhead = ((XtextTokenStream)nvae.input).getCurrentLookAhead();
		if ((lookAhead >= lookAheadAddOn && lookAheadAddOn > 0) || (lookAhead == 0 && lookAheadAddOn > 0) || lookAhead == -1)
			getRecognizer().failedPredicateAtEOF = true;
	}
	super.error(nvae);
}
 

private void exhaustTokenSource() {
	LazyTokenStream casted = (LazyTokenStream) this.input;
	int marked = casted.mark();
	try {
		while (casted.LT(1) != Token.EOF_TOKEN) {
			casted.consume();
		}
	} finally {
		casted.rewind(marked);
	}
}
 

/**
 * Skips the given leaf as it's hidden. If it was the last token to be returned, a hidden token may be syntesized if
 * would affect the semicolon insertion.
 */
private Token processHiddenToken(ILeafNode leaf) {
	Token result = nextToken();
	if (result == Token.EOF_TOKEN && Strings.countLineBreaks(leaf.getText()) > 0) {
		next = result;
		CommonToken hidden = new CommonToken(tokenTypeMapper.getInternalTokenType(leaf), leaf.getText());
		hidden.setChannel(Token.HIDDEN_CHANNEL);
		return hidden;
	}
	return result;
}
 

/**
 * Implementation of the {@link TokenSource} interface. Return new tokens as long as there are some, afterwards
 * return {@link Token#EOF_TOKEN}.
 */
@Override
public Token nextToken() {
	if (next != null) {
		Token result = next;
		next = null;
		return result;
	}
	if (!leafNodes.hasNext()) {
		return Token.EOF_TOKEN;
	}
	ILeafNode leaf = leafNodes.next();
	if (leaf.getTotalOffset() >= endOffset) {
		leafNodes = Collections.emptyIterator();
		return Token.EOF_TOKEN;
	}
	if (leaf.getTotalEndOffset() <= startOffset) {
		return nextToken();
	}
	if (leaf.getTotalEndOffset() > endOffset) {
		return toPrefixToken(leaf);
	}
	SyntaxErrorMessage syntaxErrorMessage = leaf.getSyntaxErrorMessage();
	if (syntaxErrorMessage != null && SEMICOLON_INSERTED.equals(syntaxErrorMessage.getIssueCode())) {
		return toASIToken(leaf);
	}
	if (leaf.isHidden()) {
		return processHiddenToken(leaf);
	}
	int tokenType = tokenTypeMapper.getInternalTokenType(leaf);
	return new CommonToken(tokenType, leaf.getText());
}