Java Code Examples for org.antlr.v4.runtime.misc.IntervalSet#contains()

/**
 * Consume all tokens a whole line at a time until the next token is one expected by the current
 * rule. Each line (as delimited by supplied separator token) starting from the current line up to
 * the last line consumed is placed in an {@link ErrorNode} and inserted as a child of the current
 * rule.
 *
 * @param recognizer The {@link Parser} to whom to delegate creation of each {@link ErrorNode}
 */
private void consumeBlocksUntilWanted(Parser recognizer) {
  IntervalSet expecting = recognizer.getExpectedTokens();
  IntervalSet whatFollowsLoopIterationOrRule = expecting.or(getErrorRecoverySet(recognizer));

  int nextToken;
  do {
    // Eat tokens until we are at the end of the line
    consumeUntilEndOfLine(recognizer);

    // Get the line number and separator text from the separator token
    Token separatorToken = recognizer.getCurrentToken();

    // Insert the current line as an {@link ErrorNode} as a child of the current rule
    createErrorNode(recognizer, recognizer.getContext(), separatorToken);

    // Eat the separator token
    recognizer.consume();

    nextToken = recognizer.getInputStream().LA(1);
  } while (!whatFollowsLoopIterationOrRule.contains(nextToken) && nextToken != Lexer.EOF);
}
 

/**
 * Consumes token until lexer state is function-balanced and
 * token from follow is matched. Matched token is also consumed
 */
protected void consumeUntilGreedy(Parser recognizer, IntervalSet set, CSSLexerState.RecoveryMode mode) {
    CSSToken t;
    do {
        Token next = recognizer.getInputStream().LT(1);
        if (next instanceof CSSToken) {
            t = (CSSToken) recognizer.getInputStream().LT(1);
            if (t.getType() == Token.EOF) {
                logger.trace("token eof ");
                break;
            }
        } else
            break; /* not a CSSToken, probably EOF */
        logger.trace("Skipped greedy: {}", t.getText());
        // consume token even if it will match
        recognizer.consume();
    }
    while (!(t.getLexerState().isBalanced(mode, null, t) && set.contains(t.getType())));
}
 

/**
 * Consumes token until lexer state is function-balanced and
 * token from follow is matched.
 */
public void consumeUntil(Parser recognizer, IntervalSet follow, CSSLexerState.RecoveryMode mode, CSSLexerState ls) {
    CSSToken t;
    boolean finish;
    TokenStream input = recognizer.getInputStream();
    do {
        Token next = input.LT(1);
        if (next instanceof CSSToken) {
            t = (CSSToken) input.LT(1);
            if (t.getType() == Token.EOF) {
                logger.trace("token eof ");
                break;
            }
        } else
            break; /* not a CSSToken, probably EOF */
        // consume token if does not match
        finish = (t.getLexerState().isBalanced(mode, ls, t) && follow.contains(t.getType()));
        if (!finish) {
            logger.trace("Skipped: {}", t);
            input.consume();
        }
    } while (!finish);
}
 

public List<GrammarAST> getNodesWithType(IntervalSet types) {
	List<GrammarAST> nodes = new ArrayList<GrammarAST>();
	List<GrammarAST> work = new LinkedList<GrammarAST>();
	work.add(this);
	GrammarAST t;
	while ( !work.isEmpty() ) {
		t = work.remove(0);
		if ( types==null || types.contains(t.getType()) ) nodes.add(t);
		if ( t.children!=null ) {
			work.addAll(Arrays.asList(t.getChildrenAsArray()));
		}
	}
	return nodes;
}
 

public void getNodesWithTypePreorderDFS_(List<GrammarAST> nodes, IntervalSet types) {
	if ( types.contains(this.getType()) ) nodes.add(this);
	// walk all children of root.
	for (int i= 0; i < getChildCount(); i++) {
		GrammarAST child = (GrammarAST)getChild(i);
		child.getNodesWithTypePreorderDFS_(nodes, types);
	}
}
 

protected void processLexer() {
	// make sure all non-fragment lexer rules must match at least one symbol
	for (Rule rule : g.rules.values()) {
		if (rule.isFragment()) {
			continue;
		}

		LL1Analyzer analyzer = new LL1Analyzer(g.atn);
		IntervalSet look = analyzer.LOOK(g.atn.ruleToStartState[rule.index], null);
		if (look.contains(Token.EPSILON)) {
			g.tool.errMgr.grammarError(ErrorType.EPSILON_TOKEN, g.fileName, ((GrammarAST)rule.ast.getChild(0)).getToken(), rule.name);
		}
	}
}
 

private Set<Integer> process(ParsingState start, int precedence)
{
    Set<Integer> result = memo.get(start);
    if (result != null) {
        return result;
    }

    ImmutableSet.Builder<Integer> endTokens = ImmutableSet.builder();

    // Simulates the ATN by consuming input tokens and walking transitions.
    // The ATN can be in multiple states (similar to an NFA)
    Deque<ParsingState> activeStates = new ArrayDeque<>();
    activeStates.add(start);

    while (!activeStates.isEmpty()) {
        ParsingState current = activeStates.pop();

        ATNState state = current.state;
        int tokenIndex = current.tokenIndex;
        boolean suppressed = current.suppressed;

        while (stream.get(tokenIndex).getChannel() == Token.HIDDEN_CHANNEL) {
            // Ignore whitespace
            tokenIndex++;
        }
        int currentToken = stream.get(tokenIndex).getType();

        if (state.getStateType() == RULE_START) {
            int rule = state.ruleIndex;

            if (specialRules.containsKey(rule)) {
                if (!suppressed) {
                    record(tokenIndex, specialRules.get(rule));
                }
                suppressed = true;
            }
            else if (ignoredRules.contains(rule)) {
                // TODO expand ignored rules like we expand special rules
                continue;
            }
        }

        if (state instanceof RuleStopState) {
            endTokens.add(tokenIndex);
            continue;
        }

        for (int i = 0; i < state.getNumberOfTransitions(); i++) {
            Transition transition = state.transition(i);

            if (transition instanceof RuleTransition) {
                RuleTransition ruleTransition = (RuleTransition) transition;
                for (int endToken : process(new ParsingState(ruleTransition.target, tokenIndex, suppressed, parser), ruleTransition.precedence)) {
                    activeStates.push(new ParsingState(ruleTransition.followState, endToken, suppressed, parser));
                }
            }
            else if (transition instanceof PrecedencePredicateTransition) {
                if (precedence < ((PrecedencePredicateTransition) transition).precedence) {
                    activeStates.push(new ParsingState(transition.target, tokenIndex, suppressed, parser));
                }
            }
            else if (transition.isEpsilon()) {
                activeStates.push(new ParsingState(transition.target, tokenIndex, suppressed, parser));
            }
            else if (transition instanceof WildcardTransition) {
                throw new UnsupportedOperationException("not yet implemented: wildcard transition");
            }
            else {
                IntervalSet labels = transition.label();

                if (transition instanceof NotSetTransition) {
                    labels = labels.complement(IntervalSet.of(Token.MIN_USER_TOKEN_TYPE, atn.maxTokenType));
                }

                // Surprisingly, TokenStream (i.e. BufferedTokenStream) may not have loaded all the tokens from the
                // underlying stream. TokenStream.get() does not force tokens to be buffered -- it just returns what's
                // in the current buffer, or fail with an IndexOutOfBoundsError. Since Antlr decided the error occurred
                // within the current set of buffered tokens, stop when we reach the end of the buffer.
                if (labels.contains(currentToken) && tokenIndex < stream.size() - 1) {
                    activeStates.push(new ParsingState(transition.target, tokenIndex + 1, false, parser));
                }
                else {
                    if (!suppressed) {
                        record(tokenIndex, getTokenNames(labels));
                    }
                }
            }
        }
    }

    result = endTokens.build();
    memo.put(start, result);
    return result;
}
 

@Override
public void sync(Parser recognizer) throws RecognitionException {
  /*
   * BEGIN: Copied from super
   */
  ATNState s = recognizer.getInterpreter().atn.states.get(recognizer.getState());
  if (inErrorRecoveryMode(recognizer)) {
    return;
  }
  TokenStream tokens = recognizer.getInputStream();
  int la = tokens.LA(1);
  IntervalSet nextTokens = recognizer.getATN().nextTokens(s);
  /*
   * If next token is unmatchable (i.e. from a lexer error), we need to hide the whole line before
   * returning so we don't unnecessarily pop out of the star or plus loop (if in one) afterwards.
   */
  int atnStateType = s.getStateType();
  boolean atLoopExitDecision =
      (atnStateType == ATNState.STAR_LOOP_BACK
          || atnStateType == ATNState.PLUS_LOOP_BACK
          || atnStateType == ATNState.STAR_LOOP_ENTRY);
  boolean lexerErrorAtLoopExitDecision =
      la == BatfishLexer.UNMATCHABLE_TOKEN && atLoopExitDecision;
  boolean lexerErrorAtStartOfLineAtLoopExitDecision =
      lexerErrorAtLoopExitDecision
          && ((BatfishParser) recognizer).getLastConsumedToken() == _separatorToken;
  if (!lexerErrorAtStartOfLineAtLoopExitDecision
      && (nextTokens.contains(Token.EPSILON) || nextTokens.contains(la))) {
    return;
  }
  /*
   * END: Copied from super
   */

  boolean topLevel = recognizer.getContext().parent == null;

  switch (atnStateType) {
    case ATNState.BLOCK_START:
    case ATNState.STAR_BLOCK_START:
    case ATNState.PLUS_BLOCK_START:
    case ATNState.STAR_LOOP_ENTRY:
    case ATNState.PLUS_LOOP_BACK:
    case ATNState.STAR_LOOP_BACK:
      if (topLevel || lexerErrorAtStartOfLineAtLoopExitDecision) {
        /*
         * When at top level, we cannot pop up. So consume every "line" until we have one that
         * starts with a token acceptable at the top level.
         *
         * We also don't want to pop out of star or plus loops whose elements start at the
         * beginning of a line, or else we'd lose the whole loop.
         */
        reportUnwantedToken(recognizer);
        consumeBlocksUntilWanted(recognizer);
        return;
      } else {
        /*
         * If not at the top level, error out to pop up a level. This may repeat until the next
         * token is acceptable at the given level.
         *
         * Note that this branch is also taken for errors occurring in start or plus loops in the
         * middle in the middle of a line; in that case we want to throw the whole loop (and its
         * containing context) away.
         */
        beginErrorCondition(recognizer);
        throw new InputMismatchException(recognizer);
      }

    default:
      return;
  }
}
 

/**
 * Eats characters until on from follow is found and Lexer state
 * is balanced at the moment
 */
private void consumeUntilBalanced(IntervalSet follow) {

    if (log.isDebugEnabled()) {
        log.debug("Lexer entered consumeUntilBalanced with {} and follow {}",
            ls, follow);
    }

    int c;
    do {
        c = input.LA(1);
        // change apostrophe state
        if (c == '\'' && ls.quotOpen == false) {
            ls.aposOpen = !ls.aposOpen;
        }
        // change quot state
        else if (c == '"' && ls.aposOpen == false) {
            ls.quotOpen = !ls.quotOpen;
        } else if (c == '(') {
            ls.parenNest++;
        } else if (c == ')' && ls.parenNest > 0) {
            ls.parenNest--;
        } else if (c == '{') {
            ls.curlyNest++;
        } else if (c == '}' && ls.curlyNest > 0) {
            ls.curlyNest--;
        }
        // handle end of line in string
        else if (c == '\n') {
            if (ls.quotOpen) ls.quotOpen = false;
            else if (ls.aposOpen) ls.aposOpen = false;
        } else if (c == Token.EOF) {
            log.info("Unexpected EOF during consumeUntilBalanced, EOF not consumed");
            return;
        }

        input.consume();
        // log result
        if (log.isTraceEnabled())
            log.trace("Lexer consumes '{}'({}) until balanced ({}).",
                    new Object[]{
                            Character.toString((char) c),
                            Integer.toString(c),
                            ls});

    } while (!(ls.isBalanced() && follow.contains(c)));
}