Java Code Examples for org.netbeans.api.lexer.TokenSequence#offset()

/**
 * For CDATA and XML comments, there is no start and end tokens differentiator.
 * XML lexer just gives us one token and we have to find the origin in there.
 */
private int[] findGenericOrigin(TokenSequence ts, String startTag, String endTag) {
    Token token = ts.token();
    String text = token.text().toString();
    int start = ts.offset();
    int end = start+startTag.length();
    
    //if context.getSearchOffset() is inside start tag such as "<!--"
    if(text.startsWith(startTag) &&
       start <= searchOffset && end > searchOffset)
        return new int[] {start, end};
    
    //if context.getSearchOffset() is inside end tag such as "-->"
    start = ts.offset() + token.length()-endTag.length();
    end = start+endTag.length();
    if(text.toString().endsWith(endTag) &&
       start <= searchOffset && end >= searchOffset)
        return new int[] {start, end};
    
    //if none works return null
    return null;
}
 

/** Search forwards in the token sequence until a token of type <code>down</code> is found. */
public static OffsetRange findFwd(BaseDocument doc, TokenSequence<GroovyTokenId> ts, TokenId up,
    TokenId down) {
    int balance = 0;

    while (ts.moveNext()) {
        Token<GroovyTokenId> token = ts.token();
        TokenId id = token.id();

        if (id == up) {
            balance++;
        } else if (id == down) {
            if (balance == 0) {
                return new OffsetRange(ts.offset(), ts.offset() + token.length());
            }

            balance--;
        }
    }

    return OffsetRange.NONE;
}
 

/**
 * Match paired tokens such as PI_START/PI_END.
 */
private int[] findMatchingPair(TokenSequence ts, XMLTokenId idToMatch, boolean isForward) {
    while(isForward?ts.moveNext():ts.movePrevious()) {
        Token t = ts.token();
        //we don't want to scan the entire document
        //if it hits a tag before match, return null
        if(t.id() == XMLTokenId.TAG)
            return null;
        if(t.id() == idToMatch) {
            int start = ts.offset();
            int end = start+t.length();
            return new int[] {start, end};
        }
    }
    return null;        
}
 

/**
     * Note - Because of {@link Comment.Style#WHITESPACE}, whitespaces are also
     * recorded
     */
    private CommentsCollection getCommentsCollection(TokenSequence<JavaTokenId> ts, int maxTension) {
        CommentsCollection result = new CommentsCollection();
        Token<JavaTokenId> t = ts.token();
        result.add(t);
        boolean isLC = t.id() == JavaTokenId.LINE_COMMENT;
        int lastCommentIndex = ts.index();
        int start = ts.offset();
        int end = ts.offset() + ts.token().length();
        while (ts.moveNext()) {
            if (ts.index() < tokenIndexAlreadyAdded) continue;
            t = ts.token();
            if (isComment(t.id())) {
                if (t.id() == JavaTokenId.JAVADOC_COMMENT &&
                    mixedJDocTokenIndexes.contains(ts.index())) {
                    // skip javadocs already added
                    continue;
                }
                result.add(t);
                start = Math.min(ts.offset(), start);
                end = Math.max(ts.offset() + t.length(), end);
                isLC = t.id() == JavaTokenId.LINE_COMMENT;
                lastCommentIndex = ts.index();                
            } else if (t.id() == JavaTokenId.WHITESPACE) {
                if ((numberOfNL(t) + (isLC ? 1 : 0)) > maxTension) {
                    break;
                }
            } else {
                break;                
            }
        }
        ts.moveIndex(lastCommentIndex);
        ts.moveNext();
        tokenIndexAlreadyAdded = ts.index();
        result.setBounds(new int[]{start, end});
//        System.out.println("tokenIndexAlreadyAdded = " + tokenIndexAlreadyAdded);
        return result;
    }
 

private static int findNextCorrectOffset(TokenSequence<? extends JsTokenId> ts, int offset) {
    ts.move(offset);
    if (ts.moveNext()) {
        LexUtilities.findNextIncluding(ts, Collections.singletonList(JsTokenId.BRACKET_LEFT_CURLY));
        LexUtilities.findNextIncluding(ts, Collections.singletonList(JsTokenId.EOL));
    }
    return ts.offset();
}
 

public static List<OffsetRange> findForwardMatching(
        TokenSequence<? extends LatteTopTokenId> topTs,
        LatteTokenText start,
        LatteTokenText end,
        List<LatteTokenText> middle) {
    List<OffsetRange> result = new ArrayList<>();
    topTs.moveNext();
    int originalOffset = topTs.offset();
    int balance = 1;
    while (topTs.moveNext()) {
        Token<? extends LatteTopTokenId> token = topTs.token();
        if (token != null && (token.id() == LatteTopTokenId.T_LATTE)) {
            TokenSequence<LatteMarkupTokenId> markupTs = topTs.embedded(LatteMarkupTokenId.language());
            if (markupTs != null) {
                while (markupTs.moveNext()) {
                    Token<? extends LatteMarkupTokenId> markupToken = markupTs.token();
                    if (start.matches(markupToken)) {
                        balance++;
                    } else if (end.matches(markupToken)) {
                        balance--;
                        if (balance == 0) {
                            result.add(new OffsetRange(markupTs.offset(), markupTs.offset() + markupToken.length()));
                            break;
                        }
                    } else if (matchesToken(middle, markupToken)) {
                        if (balance == 1) {
                            result.add(new OffsetRange(markupTs.offset(), markupTs.offset() + markupToken.length()));
                            break;
                        }
                    }
                }
                if (balance == 0) {
                    break;
                }
            }
        }
    }
    topTs.move(originalOffset);
    return result;
}
 

private static int getOffset(TokenSequence seq, StringDirection direction) {
    if (direction == StringDirection.BACKWARD) {
        return seq.offset();
    } else {
        return seq.offset() + seq.token().length();
    }
}
 

private static boolean posWithinAnyQuote(TypedTextInterceptor.MutableContext context, TokenSequence<JavaTokenId> javaTS) throws BadLocationException {
    if (javaTS.token().id() == JavaTokenId.STRING_LITERAL || javaTS.token().id() == JavaTokenId.CHAR_LITERAL) {
        char chr = context.getDocument().getText(context.getOffset(), 1).charAt(0);
        return (context.getOffset() - javaTS.offset() == 1 || (chr != '"' && chr != '\''));
    }
    return false;
}
 

private void handleScript(Snapshot snapshot, TokenSequence<HTMLTokenId> ts, JsAnalyzerState state, List<Embedding> embeddings) {
    String scriptType = (String) ts.token().getProperty(HTMLTokenId.SCRIPT_TYPE_TOKEN_PROPERTY);
    if (isValidScriptTypeAttributeValue(scriptType)) {
        state.in_javascript = true;
        // Emit the block verbatim
        int sourceStart = ts.offset();
        String text = ts.token().text().toString();
        List<EmbeddingPosition> jsEmbeddings = extractJsEmbeddings(text, sourceStart);
        for (EmbeddingPosition embedding : jsEmbeddings) {
            embeddings.addAll(createEmbedding(snapshot, embedding.getOffset(), embedding.getLength()));
        }
    }
}
 

private static Token[] getPreceedingTokens(TokenSequence tokenSequence) {
    int orgOffset = tokenSequence.offset();
    LinkedList<Token> tokens = new LinkedList<>();

    boolean success = true;

    // in case we are at the last token
    // include it in the result, see #154055
    if (tokenSequence.moveNext()) {
        success = tokenSequence.movePrevious()
                && tokenSequence.movePrevious();
    }

    if (success) {
        Token<PHPTokenId> token = tokenSequence.token();
        while (token != null && !CTX_DELIMITERS.contains(token.id())) {
            tokens.addFirst(token);
            if (!tokenSequence.movePrevious()) {
                break;
            } else {
                token = tokenSequence.token();
            }
        }
    }

    tokenSequence.move(orgOffset);
    tokenSequence.moveNext();
    return tokens.toArray(new Token[tokens.size()]);
}
 

static int moveBackToToken(TokenSequence<JavaTokenId> tokenSequence, 
        final int pos,
        JavaTokenId id)
{
    tokenSequence.move(pos);
    tokenSequence.moveNext(); // Assumes the pos is located within input bounds
    while (!id.equals(tokenSequence.token().id())) {
        if (!tokenSequence.movePrevious())
            return -1;
    }
    return tokenSequence.offset();
}
 

private void comment(BaseDocument baseDocument, TokenSequence<? extends LatteTopTokenId> topTs, AtomicBoolean processedByLatte) {
    if (moveToOpeningDelimiter(topTs)) {
        int start = topTs.offset() + topTs.token().length();
        if (moveToClosingDelimiter(topTs)) {
            int end = topTs.offset() + COMMENT_DELIMITER_PART_LENGTH;
            try {
                baseDocument.insertString(start, COMMENT_DELIMITER_PART, null);
                baseDocument.insertString(end, COMMENT_DELIMITER_PART, null);
            } catch (BadLocationException ex) {
                LOGGER.log(Level.WARNING, null, ex);
            }
            processedByLatte.set(true);
        }
    }
}
 

private static boolean posWithinAnyQuote(BaseDocument doc, int offset, TokenSequence<GroovyTokenId> ts) throws BadLocationException {
    if (ts.token().id() == GroovyTokenId.STRING_LITERAL || ts.token().id() == GroovyTokenId.STRING_CH) {
        char chr = doc.getText(offset, 1).charAt(0);
        return (offset - ts.offset() == 1 || (chr != '"' && chr != '\''));
    }
    return false;
}
 

/**
 * From Java.
 * 
 * Returns position of the first unpaired closing paren/brace/bracket from the caretOffset
 * till the end of caret row. If there is no such element, position after the last non-white
 * character on the caret row is returned.
 */
private int getRowOrBlockEnd(BaseDocument doc, int caretOffset, boolean[] insert) throws BadLocationException {
    int rowEnd = org.netbeans.editor.Utilities.getRowLastNonWhite(doc, caretOffset);
    if (rowEnd == -1 || caretOffset >= rowEnd) {
        return caretOffset;
    }
    rowEnd += 1;
    int parenBalance = 0;
    int braceBalance = 0;
    int bracketBalance = 0;
    TokenSequence<GroovyTokenId> ts = LexUtilities.getPositionedSequence(doc, caretOffset);
    if (ts == null) {
        return caretOffset;
    }
    while (ts.offset() < rowEnd) {
        GroovyTokenId id = ts.token().id();
        switch (id) {
            case SEMI:
                return ts.offset() + 1;
            case COMMA:
                return ts.offset();
            case LPAREN:
                parenBalance++;
                break;
            case RPAREN:
                if (parenBalance-- == 0) {
                    return ts.offset();
                }
                break;
            case LBRACE:
                braceBalance++;
                break;
            case RBRACE:
                if (braceBalance-- == 0) {
                    return ts.offset();
                }
                break;
            case LBRACKET:
                bracketBalance++;
                break;
            case RBRACKET:
                if (bracketBalance-- == 0) {
                    return ts.offset();
                }
                break;
        }
        if (!ts.moveNext()) {
            // this might happen in embedded case - line is not at the end
            // but there are no more tokens - for example <script>function foo() {</script>
            if ((caretOffset - ts.offset()) == 1
                    && (bracketBalance == 1 || parenBalance == 1 || braceBalance == 1)) {
                return caretOffset;
            }
            break;
        }
    }

    insert[0] = false;
    return rowEnd;
}
 

private static Token<JavaTokenId> createHighlightImpl(CompilationInfo info, Document doc, TreePath tree) {
    Tree leaf = tree.getLeaf();
    SourcePositions positions = info.getTrees().getSourcePositions();
    CompilationUnitTree cu = info.getCompilationUnit();
    
    //XXX: do not use instanceof:
    if (leaf instanceof MethodTree || leaf instanceof VariableTree || leaf instanceof ClassTree
            || leaf instanceof MemberSelectTree || leaf instanceof AnnotatedTypeTree || leaf instanceof MemberReferenceTree
            || "BINDING_PATTERN".equals(leaf.getKind().name())) {
        return findIdentifierSpan(info, doc, tree);
    }
    
    int start = (int) positions.getStartPosition(cu, leaf);
    int end = (int) positions.getEndPosition(cu, leaf);
    
    if (start == Diagnostic.NOPOS || end == Diagnostic.NOPOS) {
        return null;
    }
    
    TokenHierarchy<?> th = info.getTokenHierarchy();
    TokenSequence<JavaTokenId> ts = th.tokenSequence(JavaTokenId.language());
    
    if (ts.move(start) == Integer.MAX_VALUE) {
        return null;
    }
    
    if (ts.moveNext()) {
        Token<JavaTokenId> token = ts.token();
        if (ts.offset() == start && token != null) {
            final JavaTokenId id = token.id();
            if (id == JavaTokenId.IDENTIFIER) {
                return token;
            }
            if (id == JavaTokenId.THIS || id == JavaTokenId.SUPER) {
                return ts.offsetToken();
            }
        }
    }
    
    return null;
}
 

private int findDestinationOffset(CompilationInfo cInfo, BaseDocument doc, int offset, boolean insideBlock) {
    TreeUtilities tu = cInfo.getTreeUtilities();
    SourcePositions sp = cInfo.getTrees().getSourcePositions();
    while (true) {
        if (offset < 0 || offset > doc.getLength()) {
            return -1;
        }
        int destinationOffset = downward ? getLineEnd(doc, offset) : getLineStart(doc, offset);
        if (doc instanceof GuardedDocument && ((GuardedDocument)doc).isPosGuarded(destinationOffset)) {
            return -1;
        }
        if (destinationOffset < doc.getLength()) {
            TokenSequence<JavaTokenId> ts = SourceUtils.getJavaTokenSequence(cInfo.getTokenHierarchy(), destinationOffset);
            if (ts != null && (ts.moveNext() || ts.movePrevious())) {
                if (ts.offset() < destinationOffset && ts.token().id() != JavaTokenId.WHITESPACE) {
                    offset = downward ? ts.offset() + ts.token().length() : ts.offset();
                    continue;
                }
            }
        }
        TreePath destinationPath = tu.pathFor(destinationOffset);
        Tree leaf = destinationPath.getLeaf();
        if (insideBlock) {
            switch (leaf.getKind()) {
                case COMPILATION_UNIT:
                    return -1;
                case BLOCK:
                    return destinationOffset;
                case IF:
                case FOR_LOOP:
                case ENHANCED_FOR_LOOP:
                case WHILE_LOOP:
                case DO_WHILE_LOOP:
                case SWITCH:
                case CASE:
                case SYNCHRONIZED:
                case TRY:
                case CATCH:
                    offset = destinationOffset + (downward ? 1 : -1);
                    break;
                default:
                    offset = downward
                            ? (int) sp.getEndPosition(destinationPath.getCompilationUnit(), leaf)
                            : (int) sp.getStartPosition(destinationPath.getCompilationUnit(), leaf);
            }
        } else {
            switch (leaf.getKind()) {
                case COMPILATION_UNIT:
                    return -1;
                case CLASS:
                case INTERFACE:
                case ANNOTATION_TYPE:
                case ENUM:
                    return destinationOffset;
                default:
                    offset = downward
                            ? (int) sp.getEndPosition(destinationPath.getCompilationUnit(), leaf)
                            : (int) sp.getStartPosition(destinationPath.getCompilationUnit(), leaf);
            }
        }
    }
}
 

private void reindent(BaseDocument doc, int offset, TokenId id, Caret caret) throws BadLocationException {
    TokenSequence<? extends PHPTokenId> ts = LexUtilities.getPHPTokenSequence(doc, offset);
    if (ts != null) {
        ts.move(offset);
        if (!ts.moveNext() && !ts.movePrevious()) {
            return;
        }
        Token<? extends PHPTokenId> token = ts.token();
        if ((token.id() == id)) {
            final int rowFirstNonWhite = Utilities.getRowFirstNonWhite(doc, offset);
            if (id == PHPTokenId.PHP_CURLY_OPEN && ts.offset() == rowFirstNonWhite
                    && ts.movePrevious()) {
                // The curly is at the first nonwhite char at the line.
                // Do we need to indent the { according previous line?
                int previousExprestion = LexUtilities.findStartTokenOfExpression(ts);
                int previousIndent = Utilities.getRowIndent(doc, previousExprestion);
                int currentIndent = Utilities.getRowIndent(doc, offset);
                int newIndent = IndentUtils.countIndent(doc, offset, previousIndent);
                if (newIndent != currentIndent) {
                    GsfUtilities.setLineIndentation(doc, offset, Math.max(newIndent, 0));
                }
            } else if (id == PHPTokenId.WHITESPACE || (id == PHPTokenId.PHP_TOKEN && token.text().charAt(0) == ':')) { // ":" handles "default:"
                if (id == PHPTokenId.WHITESPACE) {
                    LexUtilities.findPreviousToken(ts, Arrays.asList(PHPTokenId.PHP_CASE));
                } else {
                    LexUtilities.findPreviousToken(ts, Arrays.asList(PHPTokenId.PHP_DEFAULT));
                }
                if (ts.offset() >= rowFirstNonWhite) { //previous "case" or "default" on one line with typed char
                    LexUtilities.findPreviousToken(ts, Arrays.asList(PHPTokenId.PHP_SWITCH));
                    Token<? extends PHPTokenId> firstCaseInSwitch = LexUtilities.findNextToken(ts, Arrays.asList(PHPTokenId.PHP_CASE));
                    if (firstCaseInSwitch != null && firstCaseInSwitch.id() == PHPTokenId.PHP_CASE) {
                        int indentOfFirstCase = GsfUtilities.getLineIndent(doc, ts.offset());
                        GsfUtilities.setLineIndentation(doc, offset, indentOfFirstCase);
                    }
                }
            } else if (id == PHPTokenId.PHP_CURLY_CLOSE) {
                OffsetRange begin = LexUtilities.findBwd(doc, ts, PHPTokenId.PHP_CURLY_OPEN, '{', PHPTokenId.PHP_CURLY_CLOSE, '}');
                if (begin != OffsetRange.NONE) {
                    int beginOffset = begin.getStart();
                    int indent = GsfUtilities.getLineIndent(doc, beginOffset);
                    previousAdjustmentIndent = GsfUtilities.getLineIndent(doc, offset);
                    GsfUtilities.setLineIndentation(doc, offset, indent);
                    previousAdjustmentOffset = caret.getDot();
                }
            }
        }
    }
}
 

private int[] findNameSpan(String name, Tree t, JavaTokenId... allowedTokens) {
    if (!SourceVersion.isIdentifier(name)) {
        //names like "<error>", etc.
        return null;
    }
    
    JCTree jcTree = (JCTree) t;
    int pos = jcTree.pos;
    
    if (pos < 0)
        return null;
    
    Set<JavaTokenId> allowedTokensSet = EnumSet.noneOf(JavaTokenId.class);
    
    allowedTokensSet.addAll(Arrays.asList(allowedTokens));
    
    TokenSequence<JavaTokenId> tokenSequence = info.getTokenHierarchy().tokenSequence(JavaTokenId.language());
    
    tokenSequence.move(pos);
    
    boolean wasNext;
    
    while ((wasNext = tokenSequence.moveNext()) && allowedTokensSet.contains(tokenSequence.token().id()))
        ;
    
    if (wasNext) {
        if (tokenSequence.token().id() == JavaTokenId.IDENTIFIER) {
            boolean nameMatches;
            
            if (!(nameMatches = name.contentEquals(tokenSequence.token().text()))) {
                ExpressionTree expr = info.getTreeUtilities().parseExpression(tokenSequence.token().text().toString(), new SourcePositions[1]);
                
                nameMatches = expr.getKind() == Kind.IDENTIFIER && name.contentEquals(((IdentifierTree) expr).getName());
            }

            if (nameMatches) {
                return new int[] {
                    tokenSequence.offset(),
                    tokenSequence.offset() + tokenSequence.token().length()
                };
            }
        }
    }
    
    tokenSequence.move(pos);
    
    if (tokenSequence.moveNext() && tokenSequence.token().id() == JavaTokenId.JAVADOC_COMMENT) {
        //TODO: this is not precise enough
        return new int[] {
            pos + 1,
            pos + name.length() + 1
        };
    }
    return null;
}
 

@SuppressWarnings("fallthrough")
private static int[] computeNameSpan(Tree tree, HintContext context) {
    switch (tree.getKind()) {
        case LABELED_STATEMENT:
            return context.getInfo().getTreeUtilities().findNameSpan((LabeledStatementTree) tree);
        case METHOD:
            return context.getInfo().getTreeUtilities().findNameSpan((MethodTree) tree);
        case ANNOTATION_TYPE:
        case CLASS:
        case ENUM:
        case INTERFACE:
            return context.getInfo().getTreeUtilities().findNameSpan((ClassTree) tree);
        case VARIABLE:
            return context.getInfo().getTreeUtilities().findNameSpan((VariableTree) tree);
        case MEMBER_SELECT:
            //XXX:
            MemberSelectTree mst = (MemberSelectTree) tree;
            int[] span = context.getInfo().getTreeUtilities().findNameSpan(mst);

            if (span == null) {
                int end = (int) context.getInfo().getTrees().getSourcePositions().getEndPosition(context.getInfo().getCompilationUnit(), tree);
                span = new int[] {end - mst.getIdentifier().length(), end};
            }
            return span;
        case METHOD_INVOCATION:
            return computeNameSpan(((MethodInvocationTree) tree).getMethodSelect(), context);
        case BLOCK:
            Collection<? extends TreePath> prefix = context.getMultiVariables().get("$$1$");
            
            if (prefix != null) {
                BlockTree bt = (BlockTree) tree;
                
                if (bt.getStatements().size() > prefix.size()) {
                    return computeNameSpan(bt.getStatements().get(prefix.size()), context);
                }
            }
        default:
            int start = (int) context.getInfo().getTrees().getSourcePositions().getStartPosition(context.getInfo().getCompilationUnit(), tree);
            if (    StatementTree.class.isAssignableFrom(tree.getKind().asInterface())
                && tree.getKind() != Kind.EXPRESSION_STATEMENT
                && tree.getKind() != Kind.BLOCK) {
                TokenSequence<?> ts = context.getInfo().getTokenHierarchy().tokenSequence();
                ts.move(start);
                if (ts.moveNext()) {
                    return new int[] {ts.offset(), ts.offset() + ts.token().length()};
                }
            }
            return new int[] {
                start,
                Math.min((int) context.getInfo().getTrees().getSourcePositions().getEndPosition(context.getInfo().getCompilationUnit(), tree),
                         findLineEnd(context.getInfo(), start)),
            };
    }
}
 

/**
 * From Java.
 * 
 * Returns position of the first unpaired closing paren/brace/bracket from the caretOffset
 * till the end of caret row. If there is no such element, position after the last non-white
 * character on the caret row is returned.
 */
private int getRowOrBlockEnd(BaseDocument doc, int caretOffset, boolean[] insert) throws BadLocationException {
    int rowEnd = org.netbeans.editor.Utilities.getRowLastNonWhite(doc, caretOffset);
    if (rowEnd == -1 || caretOffset >= rowEnd) {
        return caretOffset;
    }
    rowEnd += 1;
    int parenBalance = 0;
    int braceBalance = 0;
    int bracketBalance = 0;
    TokenSequence<? extends JsTokenId> ts = LexUtilities.getPositionedSequence(
            doc, caretOffset, language);
    if (ts == null) {
        return caretOffset;
    }
    while (ts.offset() < rowEnd) {
        JsTokenId id = ts.token().id();
        switch (id) {
            case OPERATOR_SEMICOLON:
                return ts.offset() + 1;
            case OPERATOR_COMMA:
                return ts.offset();
            case BRACKET_LEFT_PAREN:
                parenBalance++;
                break;
            case BRACKET_RIGHT_PAREN:
                if (parenBalance-- == 0) {
                    return ts.offset();
                }
                break;
            case BRACKET_LEFT_CURLY:
                braceBalance++;
                break;
            case BRACKET_RIGHT_CURLY:
                if (braceBalance-- == 0) {
                    return ts.offset();
                }
                break;
            case BRACKET_LEFT_BRACKET:
                bracketBalance++;
                break;
            case BRACKET_RIGHT_BRACKET:
                if (bracketBalance-- == 0) {
                    return ts.offset();
                }
                break;
        }
        if (!ts.moveNext()) {
            // this might happen in embedded case - line is not at the end
            // but there are no more tokens - for example <script>function foo() {</script>
            if ((caretOffset - ts.offset()) == 1
                    && (bracketBalance == 1 || parenBalance == 1 || braceBalance == 1)) {
                return caretOffset;
            }
            break;
        }
    }

    insert[0] = false;
    return rowEnd;
}