Java Code Examples for org.netbeans.api.lexer.TokenSequence#index()

private void lookForInline(TokenSequence<JavaTokenId> seq, Tree tree) {
    seq.move((int) positions.getEndPosition(unit, tree));
    CommentsCollection result = new CommentsCollection();
    while (seq.moveNext()) {
        if (seq.token().id() == JavaTokenId.WHITESPACE) {
            if (numberOfNL(seq.token()) > 0) {
                break;
            }
        } else if (isComment(seq.token().id())) {
            if (alreadySeenJavadoc(seq.token(), seq)) {
                continue;
            }
            if (seq.index() > tokenIndexAlreadyAdded)
                result.add(seq.token());
            tokenIndexAlreadyAdded = seq.index();
            if (seq.token().id() == JavaTokenId.LINE_COMMENT) {
                break;
            }
        } else {
            break;
        }
    }
    attachComments(tree, result, CommentSet.RelativePosition.INLINE);
}
 

private void lookWithinEmptyBlock(TokenSequence<JavaTokenId> seq, Tree tree) {
    // moving into opening brace.
    if (moveTo(seq, JavaTokenId.LBRACE, true)) {
        int idx = -1;
        if (seq.moveNext()) {
            JavaTokenId id = seq.token().id();
            idx = seq.index();
            if (id == JavaTokenId.WHITESPACE || isComment(id)) {
                CommentsCollection cc = getCommentsCollection(seq, Integer.MAX_VALUE);
                attachComments(tree, cc, CommentSet.RelativePosition.INNER);
            }
        }
        if (tokenIndexAlreadyAdded < idx) {
            tokenIndexAlreadyAdded = idx;
        }
    } else {
        int end = (int) positions.getEndPosition(unit, tree);
        seq.move(end); seq.moveNext();
    }
}
 

@SuppressWarnings("empty-statement")
private static int goAfterLastNewLine(final TokenSequence<JavaTokenId> seq) {
    int base = seq.offset();
    seq.movePrevious();
    while (seq.moveNext() && nonRelevant.contains(seq.token().id())) ;
    
    while (seq.movePrevious() && nonRelevant.contains(seq.token().id())) {
        switch (seq.token().id()) {
            case LINE_COMMENT:
                seq.moveNext();
                return seq.offset();
            case WHITESPACE:
                char[] c = seq.token().text().toString().toCharArray();
                for (int i = c.length; i > 0; ) {
                    if (c[--i] == '\n') {
                        return seq.offset() + i + 1;
                    }
                }
        }
    }
    if ((seq.index() == 0 || seq.moveNext()) && nonRelevant.contains(seq.token().id())) {
        return seq.offset();
    }
    return base;
}
 

/**
     * Note - Because of {@link Comment.Style#WHITESPACE}, whitespaces are also
     * recorded
     */
    private CommentsCollection getCommentsCollection(TokenSequence<JavaTokenId> ts, int maxTension) {
        CommentsCollection result = new CommentsCollection();
        Token<JavaTokenId> t = ts.token();
        result.add(t);
        boolean isLC = t.id() == JavaTokenId.LINE_COMMENT;
        int lastCommentIndex = ts.index();
        int start = ts.offset();
        int end = ts.offset() + ts.token().length();
        while (ts.moveNext()) {
            if (ts.index() < tokenIndexAlreadyAdded) continue;
            t = ts.token();
            if (isComment(t.id())) {
                if (t.id() == JavaTokenId.JAVADOC_COMMENT &&
                    mixedJDocTokenIndexes.contains(ts.index())) {
                    // skip javadocs already added
                    continue;
                }
                result.add(t);
                start = Math.min(ts.offset(), start);
                end = Math.max(ts.offset() + t.length(), end);
                isLC = t.id() == JavaTokenId.LINE_COMMENT;
                lastCommentIndex = ts.index();                
            } else if (t.id() == JavaTokenId.WHITESPACE) {
                if ((numberOfNL(t) + (isLC ? 1 : 0)) > maxTension) {
                    break;
                }
            } else {
                break;                
            }
        }
        ts.moveIndex(lastCommentIndex);
        ts.moveNext();
        tokenIndexAlreadyAdded = ts.index();
        result.setBounds(new int[]{start, end});
//        System.out.println("tokenIndexAlreadyAdded = " + tokenIndexAlreadyAdded);
        return result;
    }
 

/**
 * Called to add semicolon after bracket for some conditions
 *
 * @param context
 * @return relative caretOffset change
 * @throws BadLocationException
 */
static int moveOrSkipSemicolon(TypedTextInterceptor.MutableContext context) throws BadLocationException {
    TokenSequence<JavaTokenId> javaTS = javaTokenSequence(context, false);
    if (javaTS == null || isStringOrComment(javaTS.token().id())) {
        return -1;
    }
    if (javaTS.token().id() == JavaTokenId.SEMICOLON) {
        context.setText("", 0); // NOI18N
        return javaTS.offset() + 1;
    }
    int lastParenPos = context.getOffset();
    int index = javaTS.index();
    // Move beyond semicolon
    while (javaTS.moveNext()
            && !(javaTS.token().id() == JavaTokenId.WHITESPACE && javaTS.token().text().toString().contains("\n"))
            && javaTS.token().id() != JavaTokenId.RBRACE) {  // NOI18N
        switch (javaTS.token().id()) {
            case RPAREN:
                lastParenPos = javaTS.offset();
                break;
            case WHITESPACE:
                break;
            default:
                return -1;
        }
    }
    // Restore javaTS position
    javaTS.moveIndex(index);
    javaTS.moveNext();
    if (isForLoopTryWithResourcesOrLambdaSemicolon(javaTS) || posWithinAnyQuote(context, javaTS) || (lastParenPos == context.getOffset() && !javaTS.token().id().equals(JavaTokenId.RPAREN))) {
        return -1;
    }
    context.setText("", 0); // NOI18N
    context.getDocument().insertString(lastParenPos + 1, ";", null); // NOI18N
    return lastParenPos + 2;
}
 

/**
 * simple check whether selected token is type parameter {@code @param <T>}
 * @param seq token sequence with selected token
 * @return {@code true} when the token should not be interpreted.
 */
private static boolean isTypeParameterTag(TokenSequence<? extends TokenId> seq) {
    int index = seq.index();
    try {
        if (!seq.movePrevious() || seq.token().id() != JavadocTokenId.OTHER_TEXT) {
            return false;
        }

        return seq.movePrevious() && seq.token().id() == JavadocTokenId.TAG
                && "@param".contentEquals(seq.token().text()); // NOI18N
    } finally {
        seq.moveIndex(index);
        seq.moveNext();
    }
}
 

public static OffsetRange getMultilineRange(Document doc, TokenSequence<? extends JsTokenId> ts) {
    int index = ts.index();
    OffsetRange offsetRange = findMultilineRange(ts);
    ts.moveIndex(index);
    ts.moveNext();
    return offsetRange;
}
 

/**
 * Called to add semicolon after bracket for some conditions
 *
 * @param context
 * @return relative caretOffset change
 * @throws BadLocationException
 */
private static boolean moveSemicolon(BaseDocument doc, int dotPos, Caret caret) throws BadLocationException {
    TokenSequence<GroovyTokenId> ts = LexUtilities.getPositionedSequence(doc, dotPos);
    if (ts == null || isStringOrComment(ts.token().id())) {
        return false;
    }
    int lastParenPos = dotPos;
    int index = ts.index();
    // Move beyond semicolon
    while (ts.moveNext() && !(ts.token().id() == GroovyTokenId.NLS)) {
        switch (ts.token().id()) {
            case RPAREN:
                lastParenPos = ts.offset();
                break;
            case WHITESPACE:
                break;
            default:
                return false;
        }
    }
    // Restore javaTS position
    ts.moveIndex(index);
    ts.moveNext();
    if (isForLoopOrTryWithResourcesSemicolon(ts) || posWithinAnyQuote(doc, dotPos, ts) || (lastParenPos == dotPos && !ts.token().id().equals(GroovyTokenId.RPAREN))) {
        return false;
    }
    
    doc.remove(dotPos, 1);
    doc.insertString(lastParenPos, ";", null); // NOI18N
    caret.setDot(lastParenPos + 1);
    return true;
}
 

public static int getTokenSequenceStartOffset(TokenSequence<? extends TokenId> ts) {
    int currentIndex = ts.index();
    ts.moveStart();
    ts.moveNext();
    int offset = ts.offset();
    ts.moveIndex(currentIndex);
    return offset;
}
 

public static int getTokenSequenceEndOffset(TokenSequence<? extends TokenId> ts) {
    int currentIndex = ts.index();
    ts.moveEnd();
    ts.movePrevious();
    int offset = ts.offset() + ts.token().length();
    ts.moveIndex(currentIndex);
    return offset;
}
 

private CacheBlockContent(CharSequence code, TokenSequence<HTMLTokenId> tokenSequence, int firstTokenIndex) {
    this.firstTokenIndex = firstTokenIndex;
    //load the elements
    ElementsParser parser = ElementsParser.forTokenIndex(code, tokenSequence, firstTokenIndex);
    elements = new ArrayList<>(CACHE_BLOCK_SIZE);
    int limit = CACHE_BLOCK_SIZE;
    while (limit-- > 0 && parser.hasNext()) {
        elements.add(parser.next());
    }
    lastTokenIndex = tokenSequence.index();

}
 

public static ElementsParser forTokenIndex(CharSequence sourceCode, TokenSequence<HTMLTokenId> tokenSequence, int tokenIndex) {
    if (tokenIndex < 0) {
        throw new IllegalArgumentException(String.format("TokenSequence index (%s) must be positive", tokenIndex));
    }
    tokenSequence.moveEnd();
    int lastTokenIndex = tokenSequence.index();
    if(tokenIndex > lastTokenIndex) {
        throw new IllegalArgumentException(String.format("token index (%s) is bigger than last index in the sequence (%s)", tokenIndex, lastTokenIndex));
    }
    tokenSequence.moveIndex(tokenIndex);
    return new ElementsParser(sourceCode, tokenSequence);
}
 

public void test() {
    String commentText = "/* test comment  */";
    String text = "abc+ " + commentText + "def public publica publi static x";
    int commentTextStartOffset = 5;
    TokenHierarchy<?> hi = TokenHierarchy.create(text,TestTokenId.language());
    TokenSequence<?> ts = hi.tokenSequence();
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "abc", 0);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.PLUS, "+", 3);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.WHITESPACE, " ", 4);
    assertTrue(ts.moveNext());
    int offset = commentTextStartOffset;
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.BLOCK_COMMENT, commentText, offset);
    offset += commentText.length();
    int commentIndex = ts.index();

    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "def", offset);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.WHITESPACE, " ", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.PUBLIC, "public", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.WHITESPACE, " ", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "publica", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.WHITESPACE, " ", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "publi", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.WHITESPACE, " ", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.STATIC, "static", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.WHITESPACE, " ", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "x", -1);
    assertFalse(ts.moveNext());

    // Go back to block comment
    assertEquals(0, ts.moveIndex(commentIndex));
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.BLOCK_COMMENT, commentText, commentTextStartOffset);

    // Test embedded token sequence
    TokenSequence<?> embedded = ts.embedded();
    assertNotNull("Null embedded sequence", embedded);
    assertTrue(embedded.moveNext());
    offset = commentTextStartOffset + 2; // skip "/*"
    LexerTestUtilities.assertTokenEquals(embedded,TestPlainTokenId.WHITESPACE, " ", offset);
    offset += 1;
    assertTrue(embedded.moveNext());
    LexerTestUtilities.assertTokenEquals(embedded,TestPlainTokenId.WORD, "test", offset);
    offset += 4;
    assertTrue(embedded.moveNext());
    LexerTestUtilities.assertTokenEquals(embedded,TestPlainTokenId.WHITESPACE, " ", offset);
    offset += 1;
    assertTrue(embedded.moveNext());
    LexerTestUtilities.assertTokenEquals(embedded,TestPlainTokenId.WORD, "comment", offset);
    offset += 7;
    assertTrue(embedded.moveNext());
    LexerTestUtilities.assertTokenEquals(embedded,TestPlainTokenId.WHITESPACE, "  ", offset);
    assertFalse(embedded.moveNext());

}
 

@Override
public void insert(MutableContext context) throws BadLocationException {
    final BaseDocument doc = (BaseDocument) context.getDocument();
    final int offset = context.getCaretOffset();
    final int lineStart = LineDocumentUtils.getLineStart(doc, offset);
    final int lineEnd = LineDocumentUtils.getLineEnd(doc, offset);
    if (lineStart == lineEnd) {
        //Empty line
        return;
    }
    final TokenSequence<DockerfileTokenId> seq = TokenHierarchy.get(doc).tokenSequence(DockerfileTokenId.language());
    if (seq == null) {
        return;
    }
    seq.move(offset);
    if (!seq.moveNext() && !seq.movePrevious()) {
        return;
    }
    final int tokenEnd = seq.index();
    Token<DockerfileTokenId> token;
    while ((token = seq.token()) != null && token.id() == DockerfileTokenId.WHITESPACE) {
        if (!seq.movePrevious()) {
            break;
        }
    }
    final boolean lineContinuation = isLineContinuation(token, offset-seq.offset());
    final Pair<Integer,TokenSequence<DockerfileTokenId>> p = findImportantLine(seq, doc, lineStart, tokenEnd);
    int indent;
    if (lineContinuation) {
        indent = IndentUtils.lineIndent(doc, p.first());
        if (p.second().token().id().isKeyword()) {
            indent+=IndentUtils.indentLevelSize(doc);
        }
    } else {
        token = p.second().token();
        if (token.id() == DockerfileTokenId.WHITESPACE) {
            indent = IndentUtils.lineIndent(doc, lineStart);
        } else {
            if (!token.id().isKeyword() && token.id() != DockerfileTokenId.LINE_COMMENT) {
                int kwOffset = findPrevKeyword(p.second());
                if (kwOffset != -1) {
                    indent = IndentUtils.lineIndent(doc, kwOffset);
                } else {
                    indent = IndentUtils.lineIndent(doc, p.first()) - IndentUtils.indentLevelSize(doc);
                }
            } else {
                indent = IndentUtils.lineIndent(doc, p.first());
            }
        }
    }
    final StringBuilder sb = new StringBuilder("\n");   //NOI18N
    sb.append(IndentUtils.createIndentString(doc, indent));
    context.setText(sb.toString(), 0, sb.length());
}
 

private void navigateToUserBeanDef(Document doc, JspSyntaxSupport jspSup,
        JTextComponent target, String bean)
        throws BadLocationException {
    String text = doc.getText(0, doc.getLength());
    int index = text.indexOf(bean);
    TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);
    TokenSequence tokenSequence = tokenHierarchy.tokenSequence();

    while (index > 0) {
        tokenSequence.move(index);
        if (!tokenSequence.moveNext() && !tokenSequence.movePrevious()) {
            return; //no token found
        }
        Token token = tokenSequence.token();

        if (token.id() == JspTokenId.ATTR_VALUE) {

            while (!(token.id() == JspTokenId.ATTRIBUTE
                    && (token.text().toString().equals("class")
                    || token.text().toString().equals("type")))
                    && !(token.id() == JspTokenId.SYMBOL
                    && token.text().toString().equals("/>")) && tokenSequence.moveNext()) {
                token = tokenSequence.token();
            }

            if (tokenSequence.index() != -1 && token.id() == JspTokenId.SYMBOL) {
                while (!(token.id() == JspTokenId.ATTRIBUTE
                        && (token.text().toString().equals("class")
                        || token.text().toString().equals("type")))
                        && !(token.id() != JspTokenId.SYMBOL
                        && token.text().toString().equals("<")) && tokenSequence.movePrevious()) {
                    token = tokenSequence.token();
                }
            }

            if (tokenSequence.index() != -1 && token.id() == JspTokenId.ATTRIBUTE) {
                while (token.id() != JspTokenId.ATTR_VALUE && tokenSequence.moveNext()) {
                    token = tokenSequence.token();
                }
            }

            if (tokenSequence.index() != -1 && token.id() == JspTokenId.ATTR_VALUE) {
                target.setCaretPosition(token.offset(tokenHierarchy) + 1);
                break;
            }
        }
        index = text.indexOf(bean, index + bean.length());
    }
}
 

public void test2() {
    String commentText = "/* test comment  */";
    String text = "abc+ " + commentText + "def public publica publi static x";
    int commentTextStartOffset = 5;
    TokenHierarchy<?> hi = TokenHierarchy.create(text,GroovyTokenId.language());
    TokenSequence<?> ts = hi.tokenSequence();
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.IDENTIFIER, "abc", 0);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.PLUS, "+", 3);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.WHITESPACE, " ", 4);
    assertTrue(ts.moveNext());
    int offset = commentTextStartOffset;
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.BLOCK_COMMENT, commentText, offset);
    offset += commentText.length();
    int commentIndex = ts.index();

    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.LITERAL_def, "def", offset);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.WHITESPACE, " ", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.LITERAL_public, "public", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.WHITESPACE, " ", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.IDENTIFIER, "publica", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.WHITESPACE, " ", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.IDENTIFIER, "publi", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.WHITESPACE, " ", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.LITERAL_static, "static", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.WHITESPACE, " ", -1);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.IDENTIFIER, "x", -1);
    assertFalse(ts.moveNext());

    // Go back to block comment
    assertEquals(0, ts.moveIndex(commentIndex));
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,GroovyTokenId.BLOCK_COMMENT, commentText, commentTextStartOffset);

}
 

private static boolean isForLoopOrTryWithResourcesSemicolon(TokenSequence<GroovyTokenId> ts) {
    int parenDepth = 0; // parenthesis depth
    int braceDepth = 0; // brace depth
    boolean semicolonFound = false; // next semicolon
    int tsOrigIndex = ts.index();
    try {
        while (ts.movePrevious()) {
            switch (ts.token().id()) {
                case LPAREN:
                    if (parenDepth == 0) { // could be a 'for (' or 'try ('
                        while (ts.movePrevious()) {
                            switch (ts.token().id()) {
                                case WHITESPACE:
                                case BLOCK_COMMENT:
                                case LINE_COMMENT:
                                    break; // skip
                                case LITERAL_for:
                                case LITERAL_try:
                                    return true;
                                default:
                                    return false;
                            }
                        }
                        return false;
                    } else { // non-zero depth
                        parenDepth--;
                    }
                    break;

                case RPAREN:
                    parenDepth++;
                    break;

                case LBRACE:
                    if (braceDepth == 0) { // unclosed left brace
                        return false;
                    }
                    braceDepth--;
                    break;

                case RBRACE:
                    braceDepth++;
                    break;

                case SEMI:
                    if (semicolonFound) { // one semicolon already found
                        return false;
                    }
                    semicolonFound = true;
                    break;
            }
        }
    } finally {
        // Restore orig TS's location
        ts.moveIndex(tsOrigIndex);
        ts.moveNext();
    }
    return false;
}
 

private static boolean isForLoopTryWithResourcesOrLambdaSemicolon(TokenSequence<JavaTokenId> ts) {
    int parenDepth = 0; // parenthesis depth
    int braceDepth = 0; // brace depth
    boolean semicolonFound = false; // next semicolon
    int tsOrigIndex = ts.index();
    try {
        while (ts.movePrevious()) {
            switch (ts.token().id()) {
                case LPAREN:
                    if (parenDepth == 0) { // could be a 'for (' or 'try ('
                        while (ts.movePrevious()) {
                            switch (ts.token().id()) {
                                case WHITESPACE:
                                case BLOCK_COMMENT:
                                case JAVADOC_COMMENT:
                                case LINE_COMMENT:
                                    break; // skip
                                case FOR:
                                case TRY:
                                    return true;
                                default:
                                    return false;
                            }
                        }
                        return false;
                    } else { // non-zero depth
                        parenDepth--;
                    }
                    break;

                case RPAREN:
                    parenDepth++;
                    break;

                case LBRACE:
                    if (braceDepth == 0) { // unclosed left brace
                        if (!semicolonFound) {
                            while (ts.movePrevious()) {
                                switch (ts.token().id()) {
                                    case WHITESPACE:
                                    case BLOCK_COMMENT:
                                    case JAVADOC_COMMENT:
                                    case LINE_COMMENT:
                                        break; // skip
                                    case ARROW:
                                        return true;
                                    default:
                                        return false;
                                }
                            }
                        }
                        return false;
                    }
                    braceDepth--;
                    break;

                case RBRACE:
                    braceDepth++;
                    break;

                case SEMICOLON:
                    if (semicolonFound) { // one semicolon already found
                        return false;
                    }
                    semicolonFound = true;
                    break;
            }
        }
    } finally {
        // Restore orig TS's location
        ts.moveIndex(tsOrigIndex);
        ts.moveNext();
    }
    return false;
}