Java Code Examples for org.netbeans.api.lexer.TokenSequence#token()

private void performLatteAction(BaseDocument baseDocument, int caretOffset, AtomicBoolean processedByLatte) {
    TokenSequence<? extends LatteTopTokenId> topTs = LatteLexerUtils.getLatteTopTokenSequence(baseDocument, caretOffset);
    if (topTs != null) {
        topTs.move(caretOffset);
        if (topTs.moveNext() || topTs.movePrevious()) {
            Token<? extends LatteTopTokenId> token = topTs.token();
            if (token != null) {
                LatteTopTokenId tokenId = token.id();
                if (tokenId == LatteTopTokenId.T_LATTE_COMMENT || tokenId == LatteTopTokenId.T_LATTE_COMMENT_DELIMITER) {
                    uncomment(baseDocument, topTs);
                    processedByLatte.set(true);
                } else if (tokenId == LatteTopTokenId.T_LATTE || tokenId == LatteTopTokenId.T_LATTE_OPEN_DELIMITER
                        || tokenId == LatteTopTokenId.T_LATTE_CLOSE_DELIMITER || tokenId == LatteTopTokenId.T_LATTE_ERROR) {
                    comment(baseDocument, topTs, processedByLatte);
                }
            }
        }
    } else {
        processedByLatte.set(false);
    }
}
 

public static OffsetRange findEnd(BaseDocument doc, TokenSequence<GroovyTokenId> ts) {
    int balance = 0;

    while (ts.moveNext()) {
        Token<GroovyTokenId> token = ts.token();
        TokenId id = token.id();

        if (isBeginToken(id, doc, ts)) {
            balance--;
        } else if (id == GroovyTokenId.RBRACE) {
            if (balance == 0) {
                return new OffsetRange(ts.offset(), ts.offset() + token.length());
            }

            balance++;
        }
    }

    return OffsetRange.NONE;
}
 

@Override
public void run() {
    TokenHierarchy th = TokenHierarchy.get(document);
    TokenSequence<XMLTokenId> xml = th.tokenSequence(XMLTokenId.language());
    xml.move(offset);
    xml.moveNext();
    Token<XMLTokenId> token = xml.token();

    // when it's not a value -> do nothing.
    if (token == null) {
        return;
    }

    if (token.id() == XMLTokenId.TEXT) {
        hyperLinkInfo.calculateInfo(token, xml);
    }
}
 

@Override
public OffsetRange getReferenceSpan(Document doc, int caretOffset) {
    final TokenSequence<HTMLTokenId> ts = Utils.getJoinedHtmlSequence(doc, caretOffset);
    if (ts == null) {
        return super.getReferenceSpan(doc, caretOffset);
    }

    ts.move(caretOffset);
    if (ts.moveNext()) {
        Token<HTMLTokenId> token = ts.token();
        int offset = ts.offset();
        if (getDataMainValue(ts, caretOffset) != null) {
            return new OffsetRange(offset + 1, offset + token.length() - 1);
        }

    }
    return super.getReferenceSpan(doc, caretOffset);
}
 

/**
 * If the current token is WS, then this method scans tokens bacwards until it finds
 * a non white token. 
 * 
 * @since 1.57
 * @return the non-white token id or null if there isn't any.
 */
public CssTokenId getNonWhiteTokenIdBackward() {
    TokenSequence<CssTokenId> ts = getTokenSequence();
    restoreTokenSequence();
    try {
        for(;;) {
            Token<CssTokenId> t = ts.token();
            if(t == null) {
                //empty file
                return null;
            }
            if(!CssTokenIdCategory.WHITESPACES.name().toLowerCase().equals(t.id().primaryCategory())) {
                return t.id();
            } else {
                if(!ts.movePrevious()) {
                    break;
                }
            }
        }
        return null;
    } finally {
        //reposition the token sequence back
        restoreTokenSequence();
    }
}
 

@Override
public int[] findOrigin() throws InterruptedException, BadLocationException {
    document.readLock();
    try {
        TokenHierarchy<BaseDocument> tokenHierarchy = TokenHierarchy.get(document);
        TokenSequence<GspTokenId> tokenSequence = tokenHierarchy.tokenSequence(GspLexerLanguage.getLanguage());

        if (tokenSequence == null) {
            return null;
        }

        tokenSequence.move(context.getSearchOffset());
        tokenSequence.moveNext();
        Token<GspTokenId> gspToken = tokenSequence.token();

        if (gspToken == null || isIndependentTag(tokenSequence)) {
            return null;
        }
        return findTagBoundaries(tokenSequence, gspToken);

    } finally {
        document.readUnlock();
    }
}
 

public static String getPrefix(OJETContext ojContext, Document document, int offset) {
    TokenHierarchy th = TokenHierarchy.get(document);
    String empty = "";
    switch (ojContext) {
        case DATA_BINDING:
            TokenSequence<KODataBindTokenId> ts = LexerUtils.getTokenSequence(th, offset, KODataBindTokenId.language(), false);
            if (ts != null) {
                int diff = ts.move(offset);
                if (diff == 0 && ts.movePrevious() || ts.moveNext()) {
                    //we are on a token of ko-data-bind token sequence
                    Token<KODataBindTokenId> etoken = ts.token();
                    if (etoken.id() == KODataBindTokenId.KEY) {
                        //ke|
                        CharSequence prefix = diff == 0 ? etoken.text() : etoken.text().subSequence(0, diff);
                        return prefix.toString();
                    }
                }
                break;
            } 
    }
    return empty;
}
 

public void testGCedE() {
    TokenHierarchy th = TokenHierarchy.create("abc", TestPlainTokenId.language());
    TokenSequence tokens = th.tokenSequence();
    tokens.moveStart();
    assertEquals(true, tokens.moveNext());
    
    TokenSequence embedded = tokens.embedded();
    assertNotNull("There should be an embedded language", embedded);
    
    WeakReference<Language> refLang = new WeakReference<Language>(embedded.language());
    embedded = null;

    WeakReference<Token> refToken = new WeakReference<Token>(tokens.token());
    tokens = null;
    th = null;
    
    // This no longer works after the language is statically held in the xxTokenId by the new convention
    //assertGC("The embedded language has not been GCed", refLang);
    assertGC("The token with embedded language has not been GCed", refToken);
}
 

/**
 * Checks to see if this document declares any DOCTYPE or not?
 * If exists, it must appear before the first xml tag.
 * @return true if found, else false.
 */
public static boolean isDTDBasedDocument(Document document) {
    ((AbstractDocument)document).readLock();
    try {
        TokenHierarchy th = TokenHierarchy.get(document);
        TokenSequence ts = th.tokenSequence();
        while(ts.moveNext()) {
            Token token = ts.token();
            //if an xml tag is found, we have come too far.
            if(token.id() == XMLTokenId.TAG)
                return false;
            if(token.id() == XMLTokenId.DECLARATION)
                return true;
        }
    } finally {
        ((AbstractDocument)document).readUnlock();
    }
    return false;
}
 

@Override
protected void renderProcess() throws BadLocationException {
    processIncludes(false, null);

    TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);
    TokenSequence tokenSequence = tokenHierarchy.tokenSequence(); //get top level token sequence
    if (!tokenSequence.moveNext()) {
        return; //no tokens in token sequence
    }

    /**
     * process java code blocks one by one
     * note: We count on the fact the scripting language in JSP is Java
     */
    do {
        Token token = tokenSequence.token();

        if (token.id() == JspTokenId.SCRIPTLET) {
            JavaCodeType blockType = (JavaCodeType) token.getProperty(JspTokenId.SCRIPTLET_TOKEN_TYPE_PROPERTY);
            StringBuilder buff = blockType == JavaCodeType.DECLARATION ? declarations : scriptlets;

            if (blockType != JavaCodeType.EXPRESSION) {
                buff.append(token.text() + "\n"); //NOI18N
            }
        }
    } while (tokenSequence.moveNext());


    importsDeclarations.append(createImplicitImportStatements(Collections.<String>emptyList()));
    // no need to do it, the JSP parser will return the beans for the including page
    //declarations.append(createBeanVarDeclarations());
}
 

/**
 * Returns of set of the array declaration, where is the exexpression.
 *
 * @param startExpression
 * @param ts
 * @return
 */
private static int offsetArrayDeclaration(int startExpression, TokenSequence ts) {
    int result = -1;
    int origOffset = ts.offset();
    Token token;
    int balance = 0;
    int squaredBalance = 0;
    do {
        token = ts.token();
        if (token.id() == PHPTokenId.PHP_TOKEN) {
            switch (token.text().charAt(0)) {
                case ')':
                    balance--;
                    break;
                case '(':
                    balance++;
                    break;
                case ']':
                    squaredBalance--;
                    break;
                case '[':
                    squaredBalance++;
                    break;
                default:
                    //no-op
            }
        }
    } while (ts.offset() > startExpression
            && !(token.id() == PHPTokenId.PHP_ARRAY && balance == 1)
            && !(token.id() == PHPTokenId.PHP_TOKEN && squaredBalance == 1)
            && ts.movePrevious());

    if ((token.id() == PHPTokenId.PHP_ARRAY && balance == 1)
            || (token.id() == PHPTokenId.PHP_TOKEN && squaredBalance == 1)) {
        result = ts.offset();
    }
    ts.move(origOffset);
    ts.moveNext();
    return result;
}
 

/**
 * The same as braceBalance but generalized to any pair of matching
 * tokens.
 * @param open the token that increses the count
 * @param close the token that decreses the count
 */
public static int getTokenBalance(BaseDocument doc, char open, char close, int offset) throws BadLocationException {
    TokenSequence<?extends PHPTokenId> ts = LexUtilities.getPHPTokenSequence(doc, 0);
    if (ts == null) {
        return 0;
    }

    // XXX Why 0? Why not offset?
    ts.moveIndex(0);

    if (!ts.moveNext()) {
        return 0;
    }

    int balance = 0;

    do {
        Token t = ts.token();

        if (textEquals(t.text(), open)) {
            balance++;
        } else if (textEquals(t.text(), close)) {
            balance--;
        }
    } while (ts.moveNext());

    return balance;
}
 

public void testPerf() {
    StringBuilder sb = new StringBuilder();
    for (int i = 0; i < 7000; i++) {
        sb.append("public static x + y /* test comment */ abc * def\n");
    }
    String text = sb.toString();

    long tm;
    Language<TestTokenId> language = TestTokenId.language();
    tm = System.currentTimeMillis();
    TokenHierarchy<?> hi = TokenHierarchy.create(text, language);
    tm = System.currentTimeMillis() - tm;
    assertTrue("Timeout tm = " + tm + "msec", tm < 100); // Should be fast
    
    tm = System.currentTimeMillis();
    TokenSequence<?> ts = hi.tokenSequence();
    tm = System.currentTimeMillis() - tm;
    assertTrue("Timeout tm = " + tm + "msec", tm < 100); // Should be fast
    
    // Fetch 2 initial tokens - should be lexed lazily
    tm = System.currentTimeMillis();
    ts.moveNext();
    ts.token();
    ts.moveNext();
    ts.token();
    tm = System.currentTimeMillis() - tm;
    assertTrue("Timeout tm = " + tm + "msec", tm < 100); // Should be fast
    
    tm = System.currentTimeMillis();
    ts.moveIndex(0);
    int cntr = 1; // On the first token
    while (ts.moveNext()) {
        Token t = ts.token();
        cntr++;
    }
    tm = System.currentTimeMillis() - tm;
    assertTrue("Timeout tm = " + tm + "msec", tm < 1000); // Should be fast
    System.out.println("SimpleLexerBatchTest.testPerf(): Lexed input " + text.length()
            + " chars long and created " + cntr + " tokens in " + tm + " ms.");
}
 

private boolean afterCommentTag(TokenHierarchy<Document> th, TokenSequence<TplTopTokenId> ts, int[] delimsLength, int searchOffset) {
    if (ts.movePrevious()) {
        Token<TplTopTokenId> prevToken = ts.token();
        if (prevToken != null) {
            ts.moveNext();
            return searchOffset - prevToken.offset(th) <= delimsLength[1] + 1;
        }
    }
    return false;
}
 

private void reindent(BaseDocument doc, int offset, TokenId id, Caret caret)
    throws BadLocationException {
    TokenSequence<GroovyTokenId> ts = LexUtilities.getGroovyTokenSequence(doc, offset);

    if (ts != null) {
        ts.move(offset);

        if (!ts.moveNext() && !ts.movePrevious()) {
            return;
        }

        Token<GroovyTokenId> token = ts.token();

        if ((token.id() == id)) {
            final int rowFirstNonWhite = Utilities.getRowFirstNonWhite(doc, offset);
            // Ensure that this token is at the beginning of the line
            if (ts.offset() > rowFirstNonWhite) {
                return;
            }

            OffsetRange begin = OffsetRange.NONE;

            if (id == GroovyTokenId.RBRACE) {
                begin = LexUtilities.findBwd(doc, ts, GroovyTokenId.LBRACE, GroovyTokenId.RBRACE);
            } else if (id == GroovyTokenId.RBRACKET) {
                begin = LexUtilities.findBwd(doc, ts, GroovyTokenId.LBRACKET, GroovyTokenId.RBRACKET);
            }

            if (begin != OffsetRange.NONE) {
                int beginOffset = begin.getStart();
                int indent = GsfUtilities.getLineIndent(doc, beginOffset);
                previousAdjustmentIndent = GsfUtilities.getLineIndent(doc, offset);
                GsfUtilities.setLineIndentation(doc, offset, indent);
                previousAdjustmentOffset = caret.getDot();
            }
        }
    }
}
 

private void checkHeredocNowdocIndentationAndNewline(TokenSequence<PHPTokenId> ts) {
    Token<? extends PHPTokenId> endTag;
    List<PHPTokenId> lookforEndTokens = Arrays.asList(PHPTokenId.PHP_HEREDOC_TAG_END, PHPTokenId.PHP_NOWDOC_TAG_END);
    while (ts.movePrevious()
            && (endTag = LexUtilities.findPreviousToken(ts, lookforEndTokens)) != null) {
        if (endTag.id() != PHPTokenId.PHP_HEREDOC_TAG_END
                && endTag.id() != PHPTokenId.PHP_NOWDOC_TAG_END) {
            // NETBEANS-1285 the last token may be returned
            continue;
        }
        String endId = endTag.text().toString();
        // indentation of closing marker
        int offset = ts.offset();
        if (endId.contains(" ") || endId.contains("\t")) { // NOI18N
            createError(offset, offset + endId.length());
        }
        // new line of closing marker
        if (ts.moveNext()) {
            Token<PHPTokenId> newLine = ts.token();
            if (newLine != null) {
                if (TokenUtilities.startsWith(newLine.text(), "\r") // NOI18N
                        || (TokenUtilities.startsWith(newLine.text(), "\n") // NOI18N
                        || TokenUtilities.textEquals(newLine.text(), ";"))) { // NOI18N
                    // noop
                } else {
                    createError(ts.offset(), ts.offset() + newLine.length());
                }
            }
        }
        ts.move(offset);
    }
}
 

private Token<HTMLTokenId> getHtmlToken(int offset) {
    TokenSequence<HTMLTokenId> htmlts = ts.embeddedJoined(HTMLTokenId.language());
    assertNotNull(htmlts);
    htmlts.move(offset);
    assertTrue(htmlts.moveNext());
    return htmlts.token();
}
 

@CheckForNull
private static Map<OffsetRange, ColoringAttributes> calculateOccurences(
        @NonNull final ParserResult result,
        final int caretPosition,
        boolean includeQuotes,
        @NonNull final AtomicBoolean cancelled) {
    if (cancelled.getAndSet(false)) {
        return null;
    }
    TokenHierarchy<?> th = result.getSnapshot().getTokenHierarchy();
    if (th == null) {
        return null;
    }
    TokenSequence<JsTokenId> ts = th.tokenSequence(JsTokenId.jsonLanguage());
    if (ts == null) {
        return null;
    }
    int offset = result.getSnapshot().getEmbeddedOffset(caretPosition);
    int delta = ts.move(offset);
    if (!ts.moveNext() && !ts.movePrevious()){
        return null;
    }
    final Model model = Model.getModel(result, false);
    if (model == null) {
        return null;
    }
    Token<? extends JsTokenId> token = ts.token();
    JsTokenId tokenId = token.id();
    if (tokenId != JsTokenId.STRING && delta == 0 && ts.movePrevious()) {
        token = ts.token();
        tokenId = token.id();
    }
    ts.movePrevious();
    final Token<? extends JsTokenId> prevToken = LexUtilities.findPreviousNonWsNonComment(ts);
    final JsTokenId prevTokenId = prevToken.id();
    Set<OffsetRange> ranges = new HashSet<>();
    if (tokenId == JsTokenId.STRING && (prevTokenId == JsTokenId.BRACKET_LEFT_CURLY || prevTokenId == JsTokenId.OPERATOR_COMMA)) {
        CharSequence text = token.text();
        findRanges(model.getGlobalObject(), text.subSequence(1, text.length() - 1).toString(), includeQuotes, ranges);
    }
    final Map<OffsetRange, ColoringAttributes> res = new HashMap<>();
    if (cancelled.getAndSet(false)) {
        return null;
    }
    for (OffsetRange offsetRange : ranges) {
        res.put(ModelUtils.documentOffsetRange(result, offsetRange.getStart(), offsetRange.getEnd()), ColoringAttributes.MARK_OCCURRENCES);
    }
    return res;
}
 

private BraceContext getBraceContextForIfStatement(TokenSequence<? extends PHPTokenId> ts) throws BadLocationException {
    // find "if"
    int elseStart = ts.offset();
    if (elseStart < 0 || matchingOffset < elseStart) {
        return null;
    }
    int balance = 0;
    int ifStart = -1;
    int ifEnd = -1;
    String lastBrace = null;
    boolean found = false;
    boolean isAlternative = ":".equals(matchingText); // NOI18N
    while(ts.movePrevious()) {
        Token<? extends PHPTokenId> token = ts.token();
        PHPTokenId id = token.id();
        switch (id) {
            case PHP_ENDIF:
                if (isAlternative) {
                    balance++;
                }
                break;
            case PHP_ELSEIF: // fall-through
            case PHP_IF:
                if (matchingText.equals(lastBrace)) {
                    if (balance == 0) {
                        ifStart = ts.offset();
                        found = true;
                    }
                    if (isAlternative && id == PHPTokenId.PHP_IF) {
                        balance--;
                    }
                }
                break;
            case PHP_CURLY_CLOSE:
                if (!isAlternative) {
                    balance++;
                }
                break;
            case PHP_CURLY_OPEN:
                if (!isAlternative) {
                    balance--;
                    ifEnd = ts.offset();
                }
                lastBrace = token.text().toString();
                break;
            case PHP_TOKEN:
                if (isColon(token)) {
                    if (isAlternative) {
                        ifEnd = ts.offset();
                    }
                    lastBrace = token.text().toString();
                }
                if (isComplexSyntaxOpen(token)) {
                    if (!isAlternative) {
                        balance--;
                    }
                    lastBrace = token.text().toString();
                }
                break;
            default:
                break;
        }
        if (found) {
            break;
        }
    }
    if (!found || ifStart == -1 || ifEnd == -1) {
        // broken code
        return getBraceContext(elseStart);
    }
    BraceContext braceContext = BraceContext.create(
            context.getDocument().createPosition(ifStart),
            context.getDocument().createPosition(ifEnd + 1) // + "{" or ":"
    );
    return braceContext.createRelated(
            context.getDocument().createPosition(elseStart),
            context.getDocument().createPosition(matchingOffset + 1) // + "{" or ":"
    );
}
 

public static Set<String> findComponents(TokenSequence<? extends JsTokenId> ts, int offset) {
    if (ts == null || offset == -1) {
        return Collections.emptySet();
    }
    ts.move(offset);
    if (!ts.moveNext()) {
        return Collections.emptySet();
    }
    Token<? extends JsTokenId> token = ts.token();
    if (token.id() != JsTokenId.BRACKET_LEFT_BRACKET) {
        token = LexUtilities.findNextToken(ts, Arrays.asList(JsTokenId.BRACKET_LEFT_BRACKET));
    }
    Set<String> result = new HashSet<>();
    while (token != null && token.id() != JsTokenId.BRACKET_RIGHT_BRACKET) {
        token = LexUtilities.findNextToken(ts, Arrays.asList(JsTokenId.BRACKET_LEFT_CURLY, JsTokenId.BRACKET_RIGHT_BRACKET));
        if (token.id() == JsTokenId.BRACKET_LEFT_CURLY) {
            // we are in the anonymous object defining the component
            while (token != null && token.id() != JsTokenId.BRACKET_RIGHT_CURLY) {
                // find the property identifier or end of an object
                token = LexUtilities.findNextToken(ts, Arrays.asList(JsTokenId.IDENTIFIER, JsTokenId.BRACKET_RIGHT_CURLY));
                if (token.text().toString().startsWith(COMPONENT_PROP)) {
                    // we have found "component" or "components" property identifier, now find the string value or an anonymous object
                    token = LexUtilities.findNextToken(ts, Arrays.asList(JsTokenId.STRING, JsTokenId.BRACKET_LEFT_CURLY));
                    if (token != null) {
                        if (token.id() == JsTokenId.STRING) {
                            // simple case "component: 'mycomponent'"
                            result.add(token.text().toString());
                        } else if (token.id() == JsTokenId.BRACKET_LEFT_CURLY) {
                            // case "components: { prop1: 'comp1', prop2 'comp2' }:
                            while (token != null && token.id() != JsTokenId.BRACKET_RIGHT_CURLY) {
                                // try to find string values/names of components inside this object (value of components property)
                                token = LexUtilities.findNextToken(ts, Arrays.asList(JsTokenId.STRING, JsTokenId.BRACKET_RIGHT_CURLY));
                                if (token != null && token.id() == JsTokenId.STRING) {
                                    // component name has been found
                                    result.add(token.text().toString());
                                }
                                ts.moveNext();
                            }
                        }
                    }
                } else {
                    ts.moveNext();
                }
            }
        }
    }
    return result;
}