Java Code Examples for org.antlr.v4.runtime.Token#getTokenIndex()

private Token findDocTokenBefore(Token token)
{
    if (currentTokenStream == null)
        return null;

    final int tokenIndex = token.getTokenIndex();
    final List<Token> docList = currentTokenStream.getHiddenTokensToLeft(tokenIndex, ZserioLexer.DOC);
    if (docList != null && !docList.isEmpty())
    {
        final Token docToken = docList.get(docList.size() - 1);
        currentUsedComments.add(docToken);
        return docToken;
    }

    return null;
}
 

/** Walk upwards from node until we find a child of p at t's char position.
 *  Don't see alignment with self, t, or element *after* us.
 *  return null if there is no such ancestor p.
 */
public static Pair<ParserRuleContext,Integer> earliestAncestorWithChildStartingAtCharPos(ParserRuleContext node, Token t, int charpos) {
	ParserRuleContext p = node;
	while ( p!=null ) {
		// check all children of p to see if one of them starts at charpos
		for (int i = 0; i<p.getChildCount(); i++) {
			ParseTree child = p.getChild(i);
			Token start;
			if ( child instanceof ParserRuleContext ) {
				start = ((ParserRuleContext) child).getStart();
			}
			else { // must be token
				start = ((TerminalNode)child).getSymbol();
			}
			// check that we don't see alignment with self or element *after* us
			if ( start.getTokenIndex()<t.getTokenIndex() && start.getCharPositionInLine()==charpos ) {
				return new Pair<>(p,i);
			}
		}
		p = p.getParent();
	}
	return null;
}
 

public static int getMatchingSymbolStartsLine(Corpus corpus,
	                                              InputDocument doc,
	                                              TerminalNode node)
	{
		TerminalNode matchingLeftNode = getMatchingLeftSymbol(corpus, doc, node);
		if ( matchingLeftNode != null ) {
			Token matchingLeftToken = matchingLeftNode.getSymbol();
			int i = matchingLeftToken.getTokenIndex();
			if ( i==0 ) return 1; // first token is considered first on line
			Token tokenBeforeMatchingToken = doc.tokens.getPreviousRealToken(i);
//			System.out.printf("doc=%s node=%s, pair=%s, before=%s\n",
//			                  new File(doc.fileName).getName(), node.getSymbol(), matchingLeftToken, tokenBeforeMatchingToken);
			if ( tokenBeforeMatchingToken!=null ) {
				return matchingLeftToken.getLine()>tokenBeforeMatchingToken.getLine() ? 1 : 0;
			}
			else { // matchingLeftToken must be first in file
				return 1;
			}
		}
		return NOT_PAIR;
	}
 

public static int getMatchingSymbolEndsLine(Corpus corpus,
	                                            InputDocument doc,
	                                            TerminalNode node)
	{
		TerminalNode matchingLeftNode = getMatchingLeftSymbol(corpus, doc, node);
		if ( matchingLeftNode != null ) {
			Token matchingLeftToken = matchingLeftNode.getSymbol();
			int i = matchingLeftToken.getTokenIndex();
			Token tokenAfterMatchingToken = doc.tokens.getNextRealToken(i);
//			System.out.printf("doc=%s node=%s, pair=%s, after=%s\n",
//			                  new File(doc.fileName).getName(), node.getSymbol(), matchingLeftToken, tokenAfterMatchingToken);
			if ( tokenAfterMatchingToken!=null ) {
				if ( tokenAfterMatchingToken.getType()==Token.EOF ) {
					return 1;
				}
				return tokenAfterMatchingToken.getLine()>matchingLeftToken.getLine() ? 1 : 0;
			}
		}
		return NOT_PAIR;
	}
 

public static void wipeCharPositionInfoAndWhitespaceTokens(CodeBuffTokenStream tokens) {
	tokens.fill();
	CommonToken dummy = new CommonToken(Token.INVALID_TYPE, "");
	dummy.setChannel(Token.HIDDEN_CHANNEL);
	Token firstRealToken = tokens.getNextRealToken(-1);
	for (int i = 0; i<tokens.size(); i++) {
		if ( i==firstRealToken.getTokenIndex() ) continue; // don't wack first token
		CommonToken t = (CommonToken)tokens.get(i);
		if ( t.getText().matches("\\s+") ) {
			tokens.getTokens().set(i, dummy); // wack whitespace token so we can't use it during prediction
		}
		else {
			t.setLine(0);
			t.setCharPositionInLine(-1);
		}
	}
}
 

private String findHiddenTextAfter(ParserRuleContext ctx)
{
    Token endCtx = ctx.getStop();
    int i = endCtx.getTokenIndex();
    List<Token> normalTextChannel =
                tokens.getHiddenTokensToRight(i, BNFLexer.HIDDEN);
    if (normalTextChannel != null) {
        // the quasi-comment (description) may be the end of a rule or start of the next. separation is on
        // a blank line
        int nextLine = endCtx.getLine() + 1;
        List<String> content = new ArrayList<>();
        for (Token lineToken : normalTextChannel) {
            if (lineToken.getLine() == nextLine) {
                content.add(lineToken.getText().replaceFirst("// ?", ""));
                nextLine++;
            } else {
                break;
            }
        }
        return content.stream().collect(Collectors.joining("\n"));
    }
    return "";
}
 

private FreeTextItem findHiddenText(ParserRuleContext ctx)
{
    // to suppress lexing, !! normal english text is a special comment //!! -> hidden
    // not sure i need to do that
    Token endAlt = ctx.getStop();
    int i = endAlt.getTokenIndex();
    List<Token> normalTextChannel = tokens.getHiddenTokensToRight(i, Gee4Lexer.HIDDEN);
    if (normalTextChannel != null) {
        // there should be only one line now
        String content = normalTextChannel.stream().map(tk -> tk.getText().replaceFirst("//!!\\s*", ""))
                .collect(Collectors.joining());
        return new FreeTextItem(content);
    }
    return null;
}
 

public int getTokenMode(Token t) {
  int tokenIndex = t.getTokenIndex();
  if (tokenIndex == -1) {
    // token probably added manually, not by parser
    return -1;
  }
  if (tokenIndex < _tokenModes.size()) {
    return _tokenModes.get(tokenIndex);
  } else {
    return _lexer._mode;
  }
}
 

private String findHiddenTextBefore(ParserRuleContext ctx, boolean forHeader)
{
    Token startCtx = ctx.getStart();
    int i = startCtx.getTokenIndex();
    List<Token> normalTextChannel =
                tokens.getHiddenTokensToLeft(i, BNFLexer.HIDDEN);
    if (normalTextChannel != null) {
        // find where the blank lines are
        // when called for a rule, is the quasi-comment part of the content of the previous rule or
        // the description of this one. Immaterial for grammar header
        
        List<Token> lineTokens = normalTextChannel.stream().collect(Collectors.toList());

        int precedingBlankLines = startCtx.getLine() - lineTokens.get(lineTokens.size()-1).getLine() - 1;
        if (precedingBlankLines > 0) {
            if (forHeader) {
                // this will preserve the linefeeds
                return lineTokens.stream().map(tk -> tk.getText().replaceFirst("// ?", ""))
                        .collect(Collectors.joining("\n"));
            }  // it wasn't a description (just a stray comment ?)
        } else {
            if (forHeader) {
                // no blank line, so this is a description to the first 
                return "";
            }
            // description - go back and find any gap showing a last blank line
            int lastGoodLine = startCtx.getLine() - 1;        
            int currentIndex = lineTokens.size() - 1;
            while (currentIndex >= 0 && lineTokens.get(currentIndex).getLine() == lastGoodLine) {
                currentIndex--;
                lastGoodLine--;
            }
            List<String> content = new ArrayList<>();
            for (int j = currentIndex + 1; j <lineTokens.size(); j++) {
                content.add(lineTokens.get(j).getText().replaceFirst("// ?", ""));
            }
            return content.stream().collect(Collectors.joining("\n"));
        }
    }
    return "";
}
 

@Override
	public boolean isInContext(@NotNull PsiFile file, PsiElement element, int offset) {
//		System.out.println("offset="+offset);
		CommonTokenStream tokens = ParsingUtils.tokenizeANTLRGrammar(file.getText());
		Token tokenUnderCursor = ParsingUtils.getTokenUnderCursor(tokens, offset);
		if ( tokenUnderCursor==null ) {
			return false; // sometimes happens at the eof
		}
//		System.out.println(tokenUnderCursor);
		int tokenIndex = tokenUnderCursor.getTokenIndex();
		Token nextRealToken = ParsingUtils.nextRealToken(tokens, tokenIndex);
		Token previousRealToken = ParsingUtils.previousRealToken(tokens, tokenIndex);

		if ( nextRealToken==null || previousRealToken==null ) {
			return false;
		}

		int previousRealTokenType = previousRealToken.getType();
		int nextRealTokenType = nextRealToken.getType();

		if ( previousRealTokenType== ANTLRv4Parser.BEGIN_ACTION ) {
			// make sure we're not in a rule; has to be @lexer::header {...} stuff
			Token prevPrevRealToken = ParsingUtils.previousRealToken(tokens, previousRealToken.getTokenIndex());
			if ( prevPrevRealToken==null ) {
				return false;
			}
//			System.out.println("prevPrevRealToken="+prevPrevRealToken);
			Token prevPrevPrevRealToken = ParsingUtils.previousRealToken(tokens, prevPrevRealToken.getTokenIndex());
			if ( prevPrevPrevRealToken==null ) {
				return false;
			}
//			System.out.println("prevPrevPrevRealToken="+prevPrevPrevRealToken);
			if ( prevPrevPrevRealToken.getType()!=ANTLRv4Parser.AT &&
				 prevPrevPrevRealToken.getType()!=ANTLRv4Parser.COLONCOLON )
			{
				return false;
			}
		}

//		System.out.println("next = "+(nextRealTokenType!=Token.EOF?ANTLRv4Parser.tokenNames[nextRealTokenType]:"<EOF>"));
//		System.out.println("prev = "+ANTLRv4Parser.tokenNames[previousRealTokenType]);
//		System.out.println(tokens.getTokens());

		boolean okBefore =
			previousRealTokenType == ANTLRv4Parser.RBRACE ||
				previousRealTokenType == ANTLRv4Parser.SEMI ||
				previousRealTokenType == ANTLRv4Parser.BEGIN_ACTION;
		boolean okAfter =
			nextRealTokenType == ANTLRv4Parser.TOKEN_REF ||
				nextRealTokenType == ANTLRv4Parser.RULE_REF ||
				nextRealTokenType == Token.EOF;
		if ( okBefore && okAfter) {
//			System.out.println("in context");
			return true;
		}
		return false;
	}