Java Code Examples for org.antlr.runtime.Token#DEFAULT_CHANNEL

/**
 * Returns true if there was an unexpected EOL.
 */
public static boolean hasDisallowedEOL(Callback callback) {
	TokenStream input = callback.getInput();
	Token lt = input.LT(1);

	// Start on the position before the current token and scan backwards off channel tokens until the previous on
	// channel token.
	for (int ix = lt.getTokenIndex() - 1; ix > 0; ix--) {
		lt = input.get(ix);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning.
			break;
		} else if (isSemicolonEquivalent(lt)) {
			return true;
		}
	}
	return false;
}
 

@Override
public Token nextToken() {
	try {
		int type = flexer.advance();
		if (type == Token.EOF) {
			return Token.EOF_TOKEN;
		}
		int length = flexer.getTokenLength();
		final String tokenText = flexer.getTokenText();
		CommonToken result = new CommonTokenWithText(tokenText, type, Token.DEFAULT_CHANNEL, offset);
		offset += length;
		return result;
	} catch (IOException e) {
		throw new RuntimeException(e);
	}
}
 

/**
 * Prevent ASIs to be skipped at certain locations, e.g. after a return keyword.
 */
private boolean maySkipASI(CommonToken lastToken, ObservableXtextTokenStream tokens) {
	int countDownFrom = lastToken.getTokenIndex();
	for (int i = countDownFrom - 1; i >= 0; i--) {
		Token prevToken = tokens.get(i);
		if (prevToken.getChannel() == Token.DEFAULT_CHANNEL) {
			if (mandatoryASI.get(prevToken.getType())) {
				return false;
			}
			return true;
		}
	}
	return true;
}
 

/**
 * <p>
 * Promotes EOL which may lead to an automatically inserted semicolon. This is probably the most important method
 * for automatic semicolon insertion, as it is only possible to insert a semicolon in case of line breaks (even if
 * they are hidden in a multi-line comment!).
 * </p>
 */
public static void promoteEOL(Callback callback) {
	RecognizerSharedState state = callback.getState();
	TokenStream input = callback.getInput();
	// Don't promote EOL if there was a syntax error at EOF
	if (state.lastErrorIndex == input.size()) {
		return;
	}
	// Get current token and its type (the possibly offending token).
	Token prev = input.LT(-1);
	Token next = input.LT(1);
	int la = next.getType();
	if (la == InternalN4JSParser.Semicolon) {
		return;
	}

	// Promoting an EOL means switching it from off channel to on channel.
	// A ML_COMMENT gets promoted when it contains an EOL.
	for (int idx = prev == null ? 0 : prev.getTokenIndex() + 1, max = la == Token.EOF ? input.size()
			: next.getTokenIndex(); idx < max; idx++) {
		Token lt = input.get(idx);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning (previously promoted)
			break;
		} else if (isSemicolonEquivalent(lt)) {
			// We found our EOL: promote the token to on channel, position the input on it and reset the rule
			// start.
			lt.setChannel(Token.DEFAULT_CHANNEL);
			input.seek(idx);
			break;
		}
	}
}
 

/**
 * A "," cannot be followed by an automatically inserted semicolon. This is in particular true in case of variable
 * statements, in which the last declaration is ended with a comma (which might easily happen in case of copying the
 * initializer from a list or object literal (cf. IDEBUG-214).
 */
private static boolean findCommaBeforeEOL(TokenStream casted, int startIndex) {
	for (int ix = startIndex - 1; ix > 0; ix--) {
		Token lt = casted.get(ix);
		if (lt.getType() == InternalN4JSParser.Comma) {
			// System.out.println("Found Comma, EOL is not valid");
			return true;
		}
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) { // any other real char ends this search
			break;
		}
	}
	return false;
}
 

private void clearAndResetTokenState() {
	this.state.token = null;
	this.state.channel = Token.DEFAULT_CHANNEL;
	this.state.tokenStartCharIndex = input.index();
	this.state.tokenStartCharPositionInLine = input.getCharPositionInLine();
	this.state.tokenStartLine = input.getLine();
	this.state.text = null;
}
 

@Override
public Token nextToken() {
	while (true) {
		state.token = null;
		state.channel = Token.DEFAULT_CHANNEL;
		state.tokenStartCharIndex = input.index();
		state.tokenStartCharPositionInLine = input.getCharPositionInLine();
		state.tokenStartLine = input.getLine();
		state.text = null;
		if ( input.LA(1)==CharStream.EOF ) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if ( state.token==null ) {
				emit();
			}
			else if ( state.token==Token.SKIP_TOKEN ) {
				continue;
			}
			return state.token;
		}
		/*
		 * Avoid infinite loop (editor freeze) on {@link FailedPredicateException}
		 */
		catch (NoViableAltException | FailedPredicateException e) {
			reportError(e);
			recover(e); // throw out current char and try again
		}
		catch (RecognitionException re) {
			reportError(re);
			// match() routine has already called recover()
		}
	}
}
 

@Override
public Token nextToken() {
	while (true) {
		this.state.token = null;
		this.state.channel = Token.DEFAULT_CHANNEL;
		this.state.tokenStartCharIndex = input.index();
		this.state.tokenStartCharPositionInLine = input
				.getCharPositionInLine();
		this.state.tokenStartLine = input.getLine();
		this.state.text = null;
		if (input.LA(1) == CharStream.EOF) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if (this.state.token == null) {
				emit();
			} else if (this.state.token == Token.SKIP_TOKEN) {
				continue;
			}
			return this.state.token;
		} catch (RecognitionException re) {
			reportError(re);
			if (re instanceof NoViableAltException
					|| re instanceof FailedPredicateException) {
				recover(re);
			}
			// create token that holds mismatched char
			Token t = new CommonToken(input, Token.INVALID_TOKEN_TYPE,
					Token.HIDDEN_CHANNEL, this.state.tokenStartCharIndex,
					getCharIndex() - 1);
			t.setLine(this.state.tokenStartLine);
			t.setCharPositionInLine(
					this.state.tokenStartCharPositionInLine);
			tokenErrorMap.put(t, getErrorMessage(re, this.getTokenNames()));
			emit(t);
			return this.state.token;
		}
	}
}
 

@Override
public Token nextToken() {
	try {
		int type = flexer.advance();
		if (type == Token.EOF) {
			return Token.EOF_TOKEN;
		}
		int length = flexer.getTokenLength();
		CommonToken result = new CommonTokenWithoutText(type, Token.DEFAULT_CHANNEL, offset, length);
		offset += length;
		return result;
	} catch (IOException e) {
		throw new RuntimeException(e);
	}
}
 

@Override
public Token nextToken() {
	while (true) {
		this.state.token = null;
		this.state.channel = Token.DEFAULT_CHANNEL;
		this.state.tokenStartCharIndex = input.index();
		this.state.tokenStartCharPositionInLine = input.getCharPositionInLine();
		this.state.tokenStartLine = input.getLine();
		this.state.text = null;
		if (input.LA(1) == CharStream.EOF) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if (this.state.token == null) {
				emit();
			}
			else if (this.state.token == Token.SKIP_TOKEN) {
				continue;
			}
			return this.state.token;
		}
		catch (RecognitionException re) {
			reportError(re);
			if (re instanceof NoViableAltException ||
				re instanceof FailedPredicateException) {
				recover(re);
			}
			// create token that holds mismatched char
			Token t = new CommonToken(input, Token.INVALID_TOKEN_TYPE, Token.HIDDEN_CHANNEL,
					this.state.tokenStartCharIndex, getCharIndex() - 1);
			t.setLine(this.state.tokenStartLine);
			t.setCharPositionInLine(this.state.tokenStartCharPositionInLine);
			tokenErrorMap.put(t, getErrorMessage(re, this.getTokenNames()));
			emit(t);
			return this.state.token;
		}
	}
}
 

@Test public void testConsumeUntilEOF_large() {
	tokenCount = 25000;
	XtextTokenStream stream = new XtextTokenStream(this, Token.DEFAULT_CHANNEL);
	while(stream.LA(1) != Token.EOF) {
		stream.consume();
		stream.LA(1);
	}
}
 

@Test public void testConsumeUntilEOF_small() {
	tokenCount = 25;
	XtextTokenStream stream = new XtextTokenStream(this, Token.DEFAULT_CHANNEL);
	while(stream.LA(1) != Token.EOF) {
		stream.consume();
		stream.LA(1);
	}
}
 

protected void doTestLookahead(int lookAhead, int tokenCount) {
	this.tokenCount = tokenCount;
	XtextTokenStream stream = new XtextTokenStream(this, Token.DEFAULT_CHANNEL);
	while(stream.LA(1) != Token.EOF) {
		for(int i = 0; i < lookAhead; i++) {
			for(int j = i + 1; j < lookAhead; j++) {
				stream.LA(j - i);					
			}
			stream.consume();
		}
	}
}
 

@Override
public Token nextToken() {
	while (true) {
		state.token = null;
		state.channel = Token.DEFAULT_CHANNEL;
		state.tokenStartCharIndex = input.index();
		state.tokenStartCharPositionInLine = input.getCharPositionInLine();
		state.tokenStartLine = input.getLine();
		state.text = null;
		if ( input.LA(1)==CharStream.EOF ) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if ( state.token==null ) {
				emit();
			}
			else if ( state.token==Token.SKIP_TOKEN ) {
				continue;
			}
			return state.token;
		}
		/*
		 * Avoid infinite loop (editor freeze) on {@link FailedPredicateException}
		 */
		catch (NoViableAltException | FailedPredicateException e) {
			reportError(e);
			recover(e); // throw out current char and try again
		}
		catch (RecognitionException re) {
			reportError(re);
			// match() routine has already called recover()
		}
	}
}
 

public Token nextToken() {
  
  if ( state.token != null && state.token.getType() == SCRIPTCLOSE ){
    return Token.EOF_TOKEN;
  }
  
  while (true) {
    state.token = null;
    state.channel = Token.DEFAULT_CHANNEL;
    state.tokenStartCharIndex = input.index();
    state.tokenStartCharPositionInLine = input.getCharPositionInLine();
    state.tokenStartLine = input.getLine();
    state.text = null;
    if ( input.LA(1)==CharStream.EOF ) {
      return Token.EOF_TOKEN;
    }
    try {
      mTokens();
      if ( state.token==null ) {
        emit();
      }
      else if ( state.token==Token.SKIP_TOKEN ) {
        continue;
      }
      return state.token;
    }
    catch (RecognitionException re) {
      //reportError(re);
      return Token.EOF_TOKEN;
      //throw new RuntimeException("Bailing out!"); // or throw Error
    }
  }
}
 

/**
 * Recover from an error found on the input stream. This is for {@link NoViableAltException} and
 * {@link MismatchedTokenException}. If you enable single token insertion and deletion, this will usually not handle
 * mismatched symbol exceptions but there could be a mismatched token that the
 * {@link Parser#match(IntStream, int, BitSet) match} routine could not recover from.
 */
public static void recover(IntStream inputStream, RecognitionException re, Callback callback) {
	RecognizerSharedState state = callback.getState();
	if (re instanceof MismatchedTokenException) {
		// We expected a specific token
		// if that is not a semicolon, ASI is pointless, perform normal recovery
		int expecting = ((MismatchedTokenException) re).expecting;
		if (expecting != InternalN4JSParser.Semicolon) {

			callback.discardError(); // delete ASI message, a previously added ASI may fix too much! cf.
			// IDEBUG-215
			callback.recoverBase(inputStream, re);
			return;
		}
	}

	// System.out.println("Line: " + re.line + ":" + re.index);

	int unexpectedTokenType = re.token.getType();
	if (!followedBySemicolon(state, callback.getRecoverySets(), re.index)
			|| isOffendingToken(unexpectedTokenType)) {
		callback.recoverBase(inputStream, re);
	} else {
		int la = inputStream.LA(1);
		TokenStream casted = (TokenStream) inputStream;
		if (!isOffendingToken(la)) {
			// Start on the position before the current token and scan backwards off channel tokens until the
			// previous on channel token.
			for (int ix = re.token.getTokenIndex() - 1; ix > 0; ix--) {
				Token lt = casted.get(ix);
				if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
					// On channel token found: stop scanning.
					callback.recoverBase(inputStream, re);
					return;
				} else if (lt.getType() == InternalN4JSParser.RULE_EOL) {
					// We found our EOL: everything's good, no need to do additional recovering
					// rule start.
					if (!callback.allowASI(re)) {
						callback.recoverBase(inputStream, re);
						return;
					}
					if (!findCommaBeforeEOL(casted, ix)) {
						callback.addASIMessage();
						return;
					}
				} else if (lt.getType() == InternalN4JSParser.RULE_ML_COMMENT) {
					String tokenText = lt.getText();
					if (!findCommaBeforeEOL(casted, ix)
							&& (tokenText.indexOf('\n', 2) >= 2 || tokenText.indexOf('\r', 2) >= 2)) {
						callback.addASIMessage();
						return;
					}
				}
			}
			callback.recoverBase(inputStream, re);
		}
	}
}