Java Code Examples for org.netbeans.api.lexer.TokenSequence#moveEnd()

private boolean hasPrevious() {
    boolean anotherSeq = false;
    
    if (index == -1) {
        index = list.size() - 1;
        anotherSeq = true;
    }
    
    for( ; index >= 0; index--) {
        TokenSequence<?> seq = list.get(index);
        if (anotherSeq) {
            seq.moveEnd();
        }
        
        if (seq.movePrevious()) {
            return true;
        }
        
        anotherSeq = true;
    }
    
    return false;
}
 

private boolean hasPrevious() {
    boolean anotherSeq = false;

    if (index == -1) {
        index = list.size() - 1;
        anotherSeq = true;
    }

    for( ; index >= 0; index--) {
        TokenSequence<?> seq = list.get(index);
        if (anotherSeq) {
            seq.moveEnd();
        }

        if (seq.movePrevious()) {
            return true;
        }

        anotherSeq = true;
    }

    return false;
}
 

private boolean hasPrevious() {
    boolean anotherSeq = false;
    if (index == -1) {
        index = list.size() - 1;
        anotherSeq = true;
    }
    for (; index >= 0; index--) {
        TokenSequence<?> seq = list.get(index);
        if (anotherSeq) {
            seq.moveEnd();
        }
        if (seq.movePrevious()) {
            return true;
        }
        anotherSeq = true;
    }
    return false;
}
 

private int embeddedEpilogLength(
    TokenSequence<?> embeddingSeq, 
    TokenSequence<?> embeddedSeq) 
{
    embeddedSeq.moveEnd();
    if (embeddedSeq.movePrevious()) {
        return (embeddingSeq.offset() + embeddingSeq.token().length()) - (embeddedSeq.offset() + embeddedSeq.token().length());
    } else {
        return -1;
    }
}
 

public void testTokenCountWasCalledInUpdater() throws Exception {
    Document doc = new ModificationTextDocument();
    String text = "+/* */";
    doc.insertString(0, text, null);
    
    doc.putProperty(Language.class, TestTokenId.language());
    TokenHierarchy<?> hi = TokenHierarchy.get(doc);
    TokenSequence<?> ts;
    ((AbstractDocument)doc).readLock();
    try {
        ts = hi.tokenSequence();
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.PLUS, "+", -1);
    } finally {
        ((AbstractDocument)doc).readUnlock();
    }

    doc.remove(1, 3); // Remove "/* "
    ((AbstractDocument)doc).readLock();
    try {
        ts = hi.tokenSequence();
        ts.moveEnd();
        // Extra ending '\n' of the document returned by DocumentUtilities.getText(doc) and lexed
        assertTrue(ts.movePrevious());
        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.WHITESPACE, "\n", -1);
        assertTrue(ts.movePrevious());
        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.DIV, "/", -1);
    } finally {
        ((AbstractDocument)doc).readUnlock();
    }
}
 

public void testSaveTokensWithinLookahead() throws Exception {
    Document doc = new ModificationTextDocument();
    String text = "aabc";
    doc.insertString(0, text, null);
    
    doc.putProperty(Language.class, TestSaveTokensInLATokenId.language());
    TokenHierarchy<?> hi = TokenHierarchy.get(doc);
    ((AbstractDocument)doc).readLock();
    try {
        TokenSequence<?> ts = hi.tokenSequence();
        ts.moveEnd(); // Force creation of all tokens
        LexerTestUtilities.initLastTokenHierarchyEventListening(doc);
    } finally {
        ((AbstractDocument)doc).readUnlock();
    }
        
    doc.remove(1, 1);

    ((AbstractDocument)doc).readLock();
    try {
        TokenHierarchyEvent evt = LexerTestUtilities.getLastTokenHierarchyEvent(doc);
        TokenChange<?> change = evt.tokenChange();
        assertEquals(1, change.addedTokenCount());
    } finally {
        ((AbstractDocument)doc).readUnlock();
    }
    
}
 

private OffsetRange getCssAreaRange(Document doc, int from, int to) {
    //limit the search just to one embedded css section
    TokenSequence<CssTokenId> ts = getCssTokenSequence(doc, from);
    if (ts == null) {
        return OffsetRange.NONE;
    }
    ts.moveStart();
    int limitedFrom = ts.moveNext() ? ts.offset() : from;
    ts.moveEnd();
    int limitedTo = ts.moveNext() ? ts.offset() + ts.token().length() : to;

    return new OffsetRange(limitedFrom, limitedTo);
}
 

public void testLanguagesEmbeddingMapMT() throws Exception {
    Document doc = new PlainDocument();
    doc.putProperty("mimeType", "text/x-simple-plain");
    // All words have to be longer than 3 characters
    doc.insertString(0, "Hello 1234 0xFF00", SimpleAttributeSet.EMPTY);
    
    TokenHierarchy th = TokenHierarchy.get(doc);
    assertNotNull("Can't find token hierarchy for a text/x-simple-plain document", th);
    
    TokenSequence seq = th.tokenSequence();
    Language lang = seq.language();
    assertNotNull("Can't find language for text/x-simple-plain", lang);
    assertEquals("Wrong language", "text/x-simple-plain", lang.mimeType());
    
    for(int i = 0; i < seq.tokenCount(); i++) {
        seq.moveIndex(i);
        assertTrue(seq.moveNext());
        Token token = seq.token();
        
        if (token.id() == SimplePlainTokenId.WORD) {
            TokenSequence embeddedSeq = seq.embedded();
            assertNotNull("Can't find embedded token sequence", embeddedSeq);

            Language embeddedLang = embeddedSeq.language();
            assertNotNull("Can't find language of the embedded sequence", embeddedLang);
            assertEquals("Wrong language of the embedded sequence", "text/x-simple-char", embeddedLang.mimeType());
            
            embeddedSeq.moveStart();
            assertTrue("Embedded sequence has no tokens (moveFirst)", embeddedSeq.moveNext());
            assertEquals("Wrong startSkipLength", 1, embeddedSeq.offset() - seq.offset());
            
            embeddedSeq.moveEnd();
            assertTrue("Embedded sequence has no tokens (moveLast)", embeddedSeq.movePrevious());
            assertEquals("Wrong endSkipLength", 2, 
                (seq.offset() + seq.token().length()) - (embeddedSeq.offset() + embeddedSeq.token().length()));
        }
    }
}
 

private TextBounds findTokenSequenceBounds(BaseDocument doc, TokenSequence tokenSequence) throws BadLocationException {
        tokenSequence.moveStart();
        tokenSequence.moveNext();
        int absoluteStart = tokenSequence.offset();
        tokenSequence.moveEnd();
        tokenSequence.movePrevious();
        int absoluteEnd = tokenSequence.offset() + tokenSequence.token().length();

//         trim whitespaces from both ends
        while (isWSToken(tokenSequence.token())) {
            if (!tokenSequence.movePrevious()) {
                return new TextBounds(absoluteStart, absoluteEnd); // a block of empty text
            }
        }

        int whiteSpaceSuffixLen = 0;

        while (Character.isWhitespace(tokenSequence.token().text().charAt(tokenSequence.token().length() - 1 - whiteSpaceSuffixLen))) {
            whiteSpaceSuffixLen++;
        }

        int languageBlockEnd = tokenSequence.offset() + tokenSequence.token().length() - whiteSpaceSuffixLen;

        tokenSequence.moveStart();

        do {
            tokenSequence.moveNext();
        } while (isWSToken(tokenSequence.token()));

        int whiteSpacePrefixLen = 0;

        while (Character.isWhitespace(tokenSequence.token().text().charAt(whiteSpacePrefixLen))) {
            whiteSpacePrefixLen++;
        }

        int languageBlockStart = tokenSequence.offset() + whiteSpacePrefixLen;
        int firstLineOfTheLanguageBlock = Utilities.getLineOffset(doc, languageBlockStart);
        int lastLineOfTheLanguageBlock = Utilities.getLineOffset(doc, languageBlockEnd);
        return new TextBounds(absoluteStart, absoluteEnd, languageBlockStart, languageBlockEnd, firstLineOfTheLanguageBlock, lastLineOfTheLanguageBlock);
    }
 

private static <T1 extends TokenId> List<JoinedTokenSequence.CodeBlock<T1>> calculateCodeBlock(List<TokenSequence<T1>> tss,
        VirtualSource virtualSource
    ) throws BadLocationException {

    List<JoinedTokenSequence.CodeBlock<T1>> blocks = new ArrayList<JoinedTokenSequence.CodeBlock<T1>>();

    for (int i=0; i<tss.size(); i++) {
        TokenSequence<T1> ts =tss.get(i);

        List<TokenSequenceWrapper<T1>> tss2 = new ArrayList<TokenSequenceWrapper<T1>>();
        tss2.add(new TokenSequenceWrapper<T1>(ts, false));

        // try to find additional token sequences which comprise this language block:
        for (int j=i+1; j<tss.size(); j++) {
            TokenSequence<T1> prev = tss.get(j-1);
            prev.moveEnd();
            prev.movePrevious();
            TokenSequence<T1> next = tss.get(j);
            next.moveStart();
            next.moveNext();
            // check whether current token sequence is continuation of previous one:
            TokenSequence<T1> tsVirtual = LexUtilities.getVirtualTokens(virtualSource, prev.offset()+prev.token().length(), next.offset(), ts.language());
            if (tsVirtual != null) {
                tss2.add(new TokenSequenceWrapper<T1>(tsVirtual, true));
                tss2.add(new TokenSequenceWrapper<T1>(next, false));
                i++;
            } else {
                break;
            }
        }


        blocks.add(new JoinedTokenSequence.CodeBlock<T1>(tss2));
    }

    return blocks;
}
 

public static int getTokenSequenceEndOffset(TokenSequence<? extends TokenId> ts) {
    int currentIndex = ts.index();
    ts.moveEnd();
    ts.movePrevious();
    int offset = ts.offset() + ts.token().length();
    ts.moveIndex(currentIndex);
    return offset;
}
 

public TokenSequenceWrapper(TokenSequence<T1> ts, boolean virtual) {
    this.ts = ts;
    this.virtual = virtual;
    ts.moveStart();
    ts.moveNext();
    start = ts.offset();
    ts.moveEnd();
    ts.movePrevious();
    end = ts.offset() + ts.token().length();
}
 

public static ElementsParser forTokenIndex(CharSequence sourceCode, TokenSequence<HTMLTokenId> tokenSequence, int tokenIndex) {
    if (tokenIndex < 0) {
        throw new IllegalArgumentException(String.format("TokenSequence index (%s) must be positive", tokenIndex));
    }
    tokenSequence.moveEnd();
    int lastTokenIndex = tokenSequence.index();
    if(tokenIndex > lastTokenIndex) {
        throw new IllegalArgumentException(String.format("token index (%s) is bigger than last index in the sequence (%s)", tokenIndex, lastTokenIndex));
    }
    tokenSequence.moveIndex(tokenIndex);
    return new ElementsParser(sourceCode, tokenSequence);
}
 

/**
 * Checks whether tree is part of compound variable declaration.
 *
 * @param tree tree to examine.
 * @return true if {@code tree} is part of compound var declaration.
 * @since 2.34.0
 */
public boolean isPartOfCompoundVariableDeclaration(@NonNull Tree tree) {
    TokenSequence<JavaTokenId> tokenSequence = tokensFor(tree);

    if (tree.getKind() != Tree.Kind.VARIABLE) {
        return false;
    }

    // If tree ends with comma then tree is part of compound variable declaration.
    tokenSequence.moveEnd();
    if (tokenSequence.movePrevious() && tokenSequence.token().id() == JavaTokenId.COMMA) {
        return true;
    }

    int startPos = (int) info.getTrees().getSourcePositions().getStartPosition(info.getCompilationUnit(), tree);
    tokenSequence.moveStart();

    int tokensLength = 0;

    // To find out the first subtree from compound varaible declaration statement(if any).
    while (tokenSequence.moveNext()) {
        tokensLength += tokenSequence.token().length();
        if (tokenSequence.token().id() == JavaTokenId.IDENTIFIER) {

            Tree path = pathFor(startPos + tokensLength).getLeaf();
            TokenSequence<JavaTokenId> TokenSeq = tokensFor(path);
            TokenSeq.moveEnd();

            if (TokenSeq.movePrevious() && TokenSeq.token().id() == JavaTokenId.COMMA) {
                return true;
            }
            break;
        }
    }

    return false;
}
 

/**Check the tree is the end of compound declaration. {@link Tree}.
 *
 * @param tree the tree {@link Tree}
 * @return the true if tree is end of compound declaration else return false
 * @since 2.33.0
 */
public boolean isEndOfCompoundVariableDeclaration(@NonNull Tree tree) {
    TokenSequence<JavaTokenId> tokenSequence = tokensFor(tree);
    tokenSequence.moveEnd();
    if (tokenSequence.movePrevious() && tokenSequence.token().id() != JavaTokenId.COMMA) {
        return true;
    }
    return false;
}
 

private void printBlock(JCTree tree, List<? extends JCTree> stats, BracePlacement bracePlacement, boolean spaceBeforeLeftBrace, boolean members, boolean printComments) {
       if (printComments) printPrecedingComments(tree, true);
int old = indent();
int bcol = old;
       switch(bracePlacement) {
       case NEW_LINE:
           newline();
           toColExactly(old);
           break;
       case NEW_LINE_HALF_INDENTED:
           newline();
    bcol += (indentSize >> 1);
           toColExactly(bcol);
           break;
       case NEW_LINE_INDENTED:
           newline();
    bcol = out.leftMargin;
           toColExactly(bcol);
           break;
       }
       String trailing = null;
       if (conditionStartHack != (-1)) {
           TokenSequence<JavaTokenId> ts = TokenHierarchy.create(toString().substring(conditionStartHack), JavaTokenId.language()).tokenSequence(JavaTokenId.language());
           boolean found;
           ts.moveEnd();
           while ((found = ts.movePrevious()) && PositionEstimator.nonRelevant.contains(ts.token().id()))
               ;
           if (found) {
               String content = toString();
               trailing = content.substring(conditionStartHack + ts.offset() + ts.token().text().length());
               out.used -= trailing.length();
               out.col -= trailing.length();
           }
       }
       if (spaceBeforeLeftBrace)
           needSpace();
print('{');
       if (trailing != null) print(trailing);
       boolean emptyBlock = true;
       for (List<? extends JCTree> l = stats; l.nonEmpty(); l = l.tail) {
           if (!isSynthetic(l.head)) {
               emptyBlock = false;
               break;
           }
       }
if (emptyBlock) {
           printEmptyBlockComments(tree, members);
       } else {
           if (innerCommentsHandled.add(tree)) {
               java.util.List<Comment> comments = commentHandler.getComments(tree).getComments(CommentSet.RelativePosition.INNER);
               for (Comment c : comments) {
                   printComment(c, false, members);
               }
           }
           if (members)
               blankLines(cs.getBlankLinesAfterAnonymousClassHeader());
           else
               newline();
    printStats(stats, members);
       }
       toColExactly(bcol);
undent(old);
print('}');
       if (printComments) printTrailingComments(tree, true);
   }
 

public static List<OffsetRange> findBackwardMatching(
        TokenSequence<? extends TwigTopTokenId> topTs,
        TwigTokenText start,
        TwigTokenText end,
        List<TwigTokenText> middle) {
    List<OffsetRange> result = new ArrayList<>();
    topTs.movePrevious();
    int originalOffset = topTs.offset();
    int balance = 1;
    while (topTs.movePrevious()) {
        Token<? extends TwigTopTokenId> token = topTs.token();
        if (token != null && (token.id() == TwigTopTokenId.T_TWIG_BLOCK || token.id() == TwigTopTokenId.T_TWIG_VAR)) {
            TokenSequence<TwigBlockTokenId> markupTs = topTs.embedded(TwigBlockTokenId.language());
            if (markupTs != null) {
                markupTs.moveEnd();
                while (markupTs.movePrevious()) {
                    Token<? extends TwigBlockTokenId> markupToken = markupTs.token();
                    if (start.matches(markupToken)) {
                        balance++;
                    } else if (end.matches(markupToken)) {
                        balance--;
                        if (balance == 0) {
                            result.add(new OffsetRange(markupTs.offset(), markupTs.offset() + markupToken.length()));
                            break;
                        }
                    } else if (matchesToken(middle, markupToken)) {
                        if (balance == 1) {
                            result.add(new OffsetRange(markupTs.offset(), markupTs.offset() + markupToken.length()));
                            break;
                        }
                    }
                }
                if (balance == 0) {
                    break;
                }
            }
        }
    }
    topTs.move(originalOffset);
    return result;
}
 

public static List<OffsetRange> findBackwardMatching(
        TokenSequence<? extends LatteTopTokenId> topTs,
        LatteTokenText start,
        LatteTokenText end,
        List<LatteTokenText> middle) {
    List<OffsetRange> result = new ArrayList<>();
    topTs.movePrevious();
    int originalOffset = topTs.offset();
    int balance = 1;
    while (topTs.movePrevious()) {
        Token<? extends LatteTopTokenId> token = topTs.token();
        if (token != null && (token.id() == LatteTopTokenId.T_LATTE)) {
            TokenSequence<LatteMarkupTokenId> markupTs = topTs.embedded(LatteMarkupTokenId.language());
            if (markupTs != null) {
                markupTs.moveEnd();
                while (markupTs.movePrevious()) {
                    Token<? extends LatteMarkupTokenId> markupToken = markupTs.token();
                    if (start.matches(markupToken)) {
                        balance++;
                    } else if (end.matches(markupToken)) {
                        balance--;
                        if (balance == 0) {
                            result.add(new OffsetRange(markupTs.offset(), markupTs.offset() + markupToken.length()));
                            break;
                        }
                    } else if (matchesToken(middle, markupToken)) {
                        if (balance == 1) {
                            result.add(new OffsetRange(markupTs.offset(), markupTs.offset() + markupToken.length()));
                            break;
                        }
                    }
                }
                if (balance == 0) {
                    break;
                }
            }
        }
    }
    topTs.move(originalOffset);
    return result;
}