org.netbeans.api.lexer.TokenSequence Java Examples

private static boolean hasCommentEnd(TokenSequence ts) {
    while (ts.moveNext()) {
        Token<JsDocumentationTokenId> token = ts.token();
        if (token.id() == JsDocumentationTokenId.COMMENT_END) {
            return true;
        } else if (CharSequenceUtilities.endsWith(token.text(), "/")) { //NOI18N
            if (ts.moveNext()) {
                Token<JsDocumentationTokenId> nextToken = ts.token();
                if (CharSequenceUtilities.textEquals(nextToken.text(), "/")) { //NOI18N
                    ts.movePrevious();
                    continue;
                } else if (nextToken.id() == JsDocumentationTokenId.ASTERISK) {
                    return false;
                }
            }
        }
    }
    return false;
}
 

private int[] findContentPositions(PropertySetter p) {
    int start = env.getTreeUtilities().positions(p).getStart();
    int len = 1;
    TokenSequence<XMLTokenId>  seq = (TokenSequence<XMLTokenId>)env.getHierarchy().tokenSequence();
    seq.move(start);
    if (seq.moveNext()) {
        Token<XMLTokenId>   t = seq.token();
        if (t.id() == XMLTokenId.TEXT) {
            String tokenText = t.text().toString();
            String trimmed = tokenText.trim();
            int indexOfTrimmed = tokenText.indexOf(trimmed);
            int indexOfNl = trimmed.indexOf('\n');

            start = seq.offset() + indexOfTrimmed;
            if (indexOfNl > -1) {
                len = indexOfNl;
            } else {
                len = trimmed.length();
            }
        } else {
            start = seq.offset();
            len = t.length();
        }
    }
    return new int[] { start, len };
}
 

private static boolean isPartOfHereOrNowDoc(TokenSequence<? extends PHPTokenId> ts) {
    boolean result = false;
    int originalOffset = ts.offset();
    Token<? extends PHPTokenId> token = ts.token();
    if (token != null && TypingHooksUtils.isStringToken(token)) {
        while (ts.movePrevious()) {
            token = ts.token();
            if (token != null) {
                if (!TypingHooksUtils.isStringToken(token)) {
                    PHPTokenId tokenId = token.id();
                    if (tokenId == PHPTokenId.PHP_HEREDOC_TAG_START || tokenId == PHPTokenId.PHP_NOWDOC_TAG_START) {
                        result = true;
                        break;
                    } else if (tokenId == PHPTokenId.PHP_HEREDOC_TAG_END || tokenId == PHPTokenId.PHP_NOWDOC_TAG_END) {
                        break;
                    }
                }
            }
        }
    }
    ts.move(originalOffset);
    ts.moveNext();
    return result;
}
 

/** Compute the initial balance of brackets at the given offset. */
private int getFormatStableStart(BaseDocument doc, int offset) {
    TokenSequence<GroovyTokenId> ts = LexUtilities.getGroovyTokenSequence(doc, offset);
    if (ts == null) {
        return 0;
    }

    ts.move(offset);

    if (!ts.movePrevious()) {
        return 0;
    }

    // Look backwards to find a suitable context - a class, module or method definition
    // which we will assume is properly indented and balanced
    do {
        Token<GroovyTokenId> token = ts.token();
        TokenId id = token.id();

        if (id == GroovyTokenId.LITERAL_class) {
            return ts.offset();
        }
    } while (ts.movePrevious());

    return ts.offset();
}
 

private static List<Token> tokenize(CharSequence input) {
    List<Token> stack = new LinkedList<>();
    TokenHierarchy<CharSequence> th = TokenHierarchy.create(input, CssTokenId.language());
    TokenSequence<CssTokenId> ts = th.tokenSequence(CssTokenId.language());
    ts.moveStart();
    while(ts.moveNext()) {
        org.netbeans.api.lexer.Token<CssTokenId> t = ts.token();
        switch(t.id()) {
            case WS:
            case NL:
                continue; //ignore WS
        }
        stack.add(new Token(t.id(), ts.offset(), t.length(), input));
    }
    return stack;
}
 

public void testGstringSimple3(){
    
    TokenSequence<?> ts = seqForText("def s = \"hallo: ${a}\"");
    
    next(ts, GroovyTokenId.LITERAL_def, "def");
    next(ts, GroovyTokenId.WHITESPACE, " ");
    next(ts, GroovyTokenId.IDENTIFIER, "s");
    next(ts, GroovyTokenId.WHITESPACE, " ");
    next(ts, GroovyTokenId.ASSIGN, "=");
    next(ts, GroovyTokenId.WHITESPACE, " ");
    next(ts, GroovyTokenId.STRING_LITERAL, "\"hallo: $");
    next(ts, GroovyTokenId.LBRACE, "{");
    next(ts, GroovyTokenId.IDENTIFIER, "a");
    next(ts, GroovyTokenId.RBRACE, "}");
    next(ts, GroovyTokenId.STRING_LITERAL, "\"");
    
    dumpTokenStream(ts);  
    
}
 

/**
 * @since 0.21
 */
public static TokenSequence<JavaTokenId> getJavaTokenSequence(final TokenHierarchy hierarchy, final int offset) {
    if (hierarchy != null) {
        TokenSequence<?> ts = hierarchy.tokenSequence();
        while(ts != null && (offset == 0 || ts.moveNext())) {
            ts.move(offset);
            if (ts.language() == JavaTokenId.language()) {
                return (TokenSequence<JavaTokenId>)ts;
            }
            if (!ts.moveNext() && !ts.movePrevious()) {
                return null;
            }
            ts = ts.embedded();
        }
    }
    return null;
}
 

/**
 * @return all preceding tokens for current line
 */
private static List<? extends Token<PHPTokenId>> getPreceedingLineTokens(Token<PHPTokenId> token, int tokenOffset, TokenSequence<PHPTokenId> tokenSequence) {
    int orgOffset = tokenSequence.offset();
    LinkedList<Token<PHPTokenId>> tokens = new LinkedList<>();
    if (token.id() != PHPTokenId.WHITESPACE
            || TokenUtilities.indexOf(token.text().subSequence(0, Math.min(token.text().length(), tokenOffset)), '\n') == -1) { // NOI18N
        while (true) {
            if (!tokenSequence.movePrevious()) {
                break;
            }
            Token<PHPTokenId> cToken = tokenSequence.token();
            if (cToken.id() == PHPTokenId.WHITESPACE
                    && TokenUtilities.indexOf(cToken.text(), '\n') != -1) { // NOI18N
                break;
            }
            tokens.addLast(cToken);
        }
    }

    tokenSequence.move(orgOffset);
    tokenSequence.moveNext();

    return tokens;
}
 

private static void checkHideCompletion(final BaseDocument doc, final int caretOffset) {
    //test whether we are just in text and eventually close the opened completion
    //this is handy after end tag autocompletion when user doesn't complete the
    //end tag and just types a text
    //test whether the user typed an ending quotation in the attribute value
    doc.render(new Runnable() {

        @Override
        public void run() {
            TokenHierarchy tokenHierarchy = TokenHierarchy.get(doc);
            TokenSequence tokenSequence = tokenHierarchy.tokenSequence();

            tokenSequence.move(caretOffset == 0 ? 0 : caretOffset - 1);
            if (!tokenSequence.moveNext()) {
                return;
            }

            Token tokenItem = tokenSequence.token();
            if (tokenItem.id() == HTMLTokenId.TEXT && !tokenItem.text().toString().startsWith("<") && !tokenItem.text().toString().startsWith("&")) {
                hideCompletion();
            }
        }
    });
}
 

private TokenSequence<JavaTokenId> findFirstOtherToken(int startOffset, int endOffset, EnumSet<JavaTokenId> ids) {
    if (startOffset == endOffset) {
        return null;
    }
    ts.move(startOffset);
    boolean backward = startOffset > endOffset;
    while (backward ? ts.movePrevious() : ts.moveNext()) {
        if (backward && ts.offset() < endOffset || !backward && ts.offset() > endOffset) {
            return null;
        }
        if (!ids.contains(ts.token().id())) {
            return ts;
        }
    }
    return null;
}
 

private static boolean hasReturnClause(JsParserResult jsParserResult, JsObject jsObject) {
    OffsetRange offsetRange = jsObject.getOffsetRange();
    TokenHierarchy<?> tokenHierarchy = jsParserResult.getSnapshot().getTokenHierarchy();
    TokenSequence<? extends JsTokenId> ts = tokenHierarchy.tokenSequence(JsTokenId.javascriptLanguage());
    if (ts == null) {
        return false;
    }
    ts.move(offsetRange.getStart());
    if (!ts.moveNext() || !ts.movePrevious()) {
        return false;
    }

    while (ts.moveNext() && ts.offset() <= offsetRange.getEnd()) {
        if (ts.token().id() == JsTokenId.KEYWORD_RETURN) {
            return true;
        }
    }
    return false;
}
 

public CompletionSurrounding(
    Token<GroovyTokenId> beforeLiteral,
    Token<GroovyTokenId> before2,
    Token<GroovyTokenId> before1,
    Token<GroovyTokenId> active,
    Token<GroovyTokenId> after1,
    Token<GroovyTokenId> after2,
    Token<GroovyTokenId> afterLiteral,
    TokenSequence<GroovyTokenId> ts) {

    this.beforeLiteral = beforeLiteral;
    this.before2 = before2;
    this.before1 = before1;
    this.active = active;
    this.after1 = after1;
    this.after2 = after2;
    this.afterLiteral = afterLiteral;
    this.ts = ts;
}
 

private boolean completeFieldTypes(TokenSequence<PHPTokenId> tokenSequence, int caretOffset, TokenHierarchy<?> th, FileObject fileObject) {
    if (!isPhp74OrNewer(fileObject)) {
        return false;
    }
    boolean completeTypes = false;
    tokenSequence.move(caretOffset);
    if (!(!tokenSequence.moveNext() && !tokenSequence.movePrevious())) {
        Token<PHPTokenId> token = tokenSequence.token();
        int tokenIdOffset = tokenSequence.token().offset(th);
        completeTypes = !CompletionContextFinder.lineContainsAny(token, caretOffset - tokenIdOffset, tokenSequence, Arrays.asList(new PHPTokenId[]{
            PHPTokenId.PHP_TYPE_BOOL,
            PHPTokenId.PHP_TYPE_INT,
            PHPTokenId.PHP_TYPE_FLOAT,
            PHPTokenId.PHP_TYPE_STRING,
            PHPTokenId.PHP_ARRAY,
            PHPTokenId.PHP_TYPE_OBJECT,
            PHPTokenId.PHP_ITERABLE,
            PHPTokenId.PHP_SELF,
            PHPTokenId.PHP_PARENT,
            PHPTokenId.PHP_STRING,
            PHPTokenId.PHP_CONST
        }));
    }
    return completeTypes;
}
 

public static OffsetRange findFwd(BaseDocument doc, TokenSequence<?extends PHPTokenId> ts, PHPTokenId tokenUpId, char up, PHPTokenId tokenDownId, char down) {
    int balance = 0;

    while (ts.moveNext()) {
        Token<?extends PHPTokenId> token = ts.token();

        if ((token.id() == tokenUpId && textEquals(token.text(), up))
                || (tokenUpId == PHPTokenId.PHP_CURLY_OPEN && token.id() == PHPTokenId.PHP_TOKEN && token.text().charAt(token.text().length() - 1) == '{')) {
            balance++;
        } else if (token.id() == tokenDownId && textEquals(token.text(), down)) {
            if (balance == 0) {
                return new OffsetRange(ts.offset(), ts.offset() + token.length());
            }

            balance--;
        }
    }

    return OffsetRange.NONE;
}
 

public void testVarIdentWithStringVersion() {
    String text = "var var = 0;";
    InputAttributes attr = new InputAttributes();
    attr.setValue(JavaTokenId.language(), "version", (Supplier<String>) () -> {return "1.8";}, true);
    TokenHierarchy<?> hi = TokenHierarchy.create(text, false, JavaTokenId.language(), EnumSet.noneOf(JavaTokenId.class), attr);
    TokenSequence<?> ts = hi.tokenSequence();

    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "var");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "var");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.EQ, "=");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "0");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.SEMICOLON, ";");
}
 

private static boolean isSingleLineEmbedding(FormatToken token, Document doc, Snapshot snapshot) {
    final FormatToken prevNonVirtual = FormatTokenStream.getPreviousNonVirtual(token);
    if (prevNonVirtual != null) {
        final int originalOffset = snapshot.getOriginalOffset(prevNonVirtual.getOffset());
        final List<TokenSequence<?>> tokenSeqs = TokenHierarchy.get(doc)
                .embeddedTokenSequences(originalOffset, false);
        final String snapshotMimePath = snapshot.getMimePath().getPath();
        for (TokenSequence<?> ts : tokenSeqs) {
            if (ts.languagePath().mimePath()
                    .concat("/") //NOI18N
                    .concat(snapshot.getMimeType()).equals(snapshotMimePath)) {
                ts.move(originalOffset);
                ts.moveNext();
                if (ts.token() != null) {
                    final String tokenText = ts.token().text().toString();
                    // if text of the token already contains newline char
                    // it is not single-line embedding
                    return !tokenText.contains("\n"); //NOI18N
                }
                return false;
            }
        }
    }
    return false;
}
 

public void testPairGTagWithExpression() {
    String text = "<g:if test=\"${}\">"
                + "</g:if>";
    TokenSequence<?> sequence = createTokenSequence(text);

    checkNext(sequence, GspTokenId.GTAG_OPENING_START, "<g:");
    checkNext(sequence, GspTokenId.GTAG_OPENING_NAME, "if");
    checkNext(sequence, GspTokenId.GTAG_ATTRIBUTE_NAME, " test=");
    checkNext(sequence, GspTokenId.GTAG_ATTRIBUTE_VALUE, "\"");
    checkNext(sequence, GspTokenId.GSTRING_START, "${");
    checkNext(sequence, GspTokenId.GSTRING_END, "}");
    checkNext(sequence, GspTokenId.GTAG_ATTRIBUTE_VALUE, "\"");
    checkNext(sequence, GspTokenId.GTAG_OPENING_END, ">");
    checkNext(sequence, GspTokenId.GTAG_CLOSING_START, "</g:");
    checkNext(sequence, GspTokenId.GTAG_CLOSING_NAME, "if");
    checkNext(sequence, GspTokenId.GTAG_CLOSING_END, ">");
}
 

public void testCommonBlockComment03() {
    String text = "/* \n\n */";
    TokenHierarchy hi = TokenHierarchy.create(text, JsDocumentationTokenId.language());
    TokenSequence<?extends JsDocumentationTokenId> ts = hi.tokenSequence();
    LexerTestUtilities.assertNextTokenEquals(ts, JsDocumentationTokenId.COMMENT_BLOCK_START, "/*");
    LexerTestUtilities.assertNextTokenEquals(ts, JsDocumentationTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JsDocumentationTokenId.EOL, "\n");
    LexerTestUtilities.assertNextTokenEquals(ts, JsDocumentationTokenId.EOL, "\n");
    LexerTestUtilities.assertNextTokenEquals(ts, JsDocumentationTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JsDocumentationTokenId.COMMENT_END, "*/");
}
 

private void comment(BaseDocument baseDocument, TokenSequence<? extends LatteTopTokenId> topTs, AtomicBoolean processedByLatte) {
    if (moveToOpeningDelimiter(topTs)) {
        int start = topTs.offset() + topTs.token().length();
        if (moveToClosingDelimiter(topTs)) {
            int end = topTs.offset() + COMMENT_DELIMITER_PART_LENGTH;
            try {
                baseDocument.insertString(start, COMMENT_DELIMITER_PART, null);
                baseDocument.insertString(end, COMMENT_DELIMITER_PART, null);
            } catch (BadLocationException ex) {
                LOGGER.log(Level.WARNING, null, ex);
            }
            processedByLatte.set(true);
        }
    }
}
 

private static LatteCompletionContext findContext(TokenSequence<? extends LatteMarkupTokenId> ts) {
    LatteCompletionContext result = LatteCompletionContext.ALL;
    do {
        Token<? extends LatteMarkupTokenId> token = ts.token();
        if (token == null) {
            break;
        }
        LatteMarkupTokenId tokenId = token.id();
        if (acceptTokenChains(ts, FILTER_TOKEN_CHAINS, false)) {
            result = LatteCompletionContext.HELPER;
            break;
        } else if (acceptTokenChains(ts, ITERATOR_ITEMS_TOKEN_CHAINS, false)) {
            result = LatteCompletionContext.ITERATOR_ITEM;
            break;
        } else if (acceptTokenChains(ts, VARIABLE_TOKEN_CHAINS, false)) {
            result = LatteCompletionContext.VARIABLE;
            break;
        } else if (acceptTokenChains(ts, END_MACRO_TOKEN_CHAINS, false)) {
            result = LatteCompletionContext.END_MACRO;
            break;
        } else if (acceptTokenChains(ts, CONTROL_MACRO_TOKEN_CHAINS, false)) {
            result = LatteCompletionContext.CONTROL_MACRO;
            break;
        } else if (LatteMarkupTokenId.T_SYMBOL.equals(tokenId) || LatteMarkupTokenId.T_MACRO_START.equals(tokenId)) {
            result = LatteCompletionContext.MACRO;
            break;
        }
    } while (ts.movePrevious());
    return result;
}
 

@Override
public List<Embedding> getEmbeddings(Snapshot snapshot) {
    TokenHierarchy<?> th = snapshot.getTokenHierarchy();
    TokenSequence<JspTokenId> sequence = th.tokenSequence(JspTokenId.language());
    List<Embedding> embeddings = new ArrayList<Embedding>();
    sequence.moveStart();
    boolean inAttributeValueWithEL = false;
    while (sequence.moveNext()) {
        Token t = sequence.token();
        if (t.id() == JspTokenId.ATTR_VALUE && t.length() == 1 && 
                (t.text().charAt(0) == '"' || t.text().charAt(0) == '\'')) {
            //a quote before/after attribute value with EL inside
            inAttributeValueWithEL = !inAttributeValueWithEL;
        }
        if (t.id() == JspTokenId.EL) {
            embeddings.add(snapshot.create(sequence.offset(), t.length(), "text/x-el")); //NOI18N
            //XXX hack - there's a need to distinguish between ELs inside or outside of attribute values
            if(inAttributeValueWithEL) {
                embeddings.add(snapshot.create(ATTRIBUTE_EL_MARKER, "text/x-el")); //NOI18N
            }
            
            // just to separate expressions for easier handling in EL parser
            embeddings.add(snapshot.create(Constants.LANGUAGE_SNIPPET_SEPARATOR, "text/x-el")); //NOI18N
         
        }
    }
    if (embeddings.isEmpty()) {
        return Collections.emptyList();
    } else {
        return Collections.singletonList(Embedding.create(embeddings));
    }
}
 

public void test233097b() {
    String text = "{@code null}";
    
    TokenHierarchy<?> hi = TokenHierarchy.create(text, JavadocTokenId.language());
    TokenSequence<?> ts = hi.tokenSequence();
    
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.OTHER_TEXT, "{");
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.TAG, "@code");
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.OTHER_TEXT, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.IDENT, "null");
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.OTHER_TEXT, "}");
}
 

private int[] findMatchesInTopSequence(TokenSequence<? extends TwigTopTokenId> topTs) {
    assert topTs != null;
    int[] result = null;
    topTs.move(context.getSearchOffset());
    topTs.moveNext();
    TokenSequence<TwigBlockTokenId> ts = topTs.embeddedJoined(TwigBlockTokenId.language());
    if (ts != null) {
        result = findMatchesInEmbeddedSequence(topTs, ts);
    }
    return result;
}
 

private static TokenSequence<SQLTokenId> getTokenSequence(String sql) throws BadLocationException {
    ModificationTextDocument doc = new ModificationTextDocument();
    doc.insertString(0, sql, null);
    doc.putProperty(Language.class, SQLTokenId.language());
    TokenHierarchy<?> hi = TokenHierarchy.get(doc);
    doc.readLock();
    TokenSequence<SQLTokenId> seq = hi.tokenSequence(SQLTokenId.language());
    doc.readUnlock();
    seq.moveStart();
    return seq;
}
 

/**
 * Finds the first previous token that is not in the ignore list
 * @param ts
 * @param ignores
 * @return 
 */
public static Token<? extends JsTokenId> findPrevious(TokenSequence<? extends JsTokenId> ts, List<JsTokenId> ignores) {
    if (ignores.contains(ts.token().id())) {
        while (ts.movePrevious() && ignores.contains(ts.token().id())) {}
    }
    return ts.token();
}
 

public void testIdentifiers01() throws Exception {
    String text = "session";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.assertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "session");

    text = "myBean.property";
    ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.assertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "myBean");
    LexerTestUtilities.assertNextTokenEquals(ts, ELTokenId.DOT, ".");
    LexerTestUtilities.assertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "property");
}
 

public void testEmptyKeyword01() throws Exception {
    String text = "empty bean.list";
    TokenSequence ts = TokenHierarchy.create(text, ELTokenId.language()).tokenSequence();
    LexerTestUtilities.assertNextTokenEquals(ts, ELTokenId.EMPTY_KEYWORD, "empty");
    LexerTestUtilities.assertNextTokenEquals(ts, ELTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "bean");
    LexerTestUtilities.assertNextTokenEquals(ts, ELTokenId.DOT, ".");
    LexerTestUtilities.assertNextTokenEquals(ts, ELTokenId.IDENTIFIER, "list");
}
 

private static Token[] getPreceedingTokens(TokenSequence tokenSequence) {
    int orgOffset = tokenSequence.offset();
    LinkedList<Token> tokens = new LinkedList<>();

    boolean success = true;

    // in case we are at the last token
    // include it in the result, see #154055
    if (tokenSequence.moveNext()) {
        success = tokenSequence.movePrevious()
                && tokenSequence.movePrevious();
    }

    if (success) {
        Token<PHPTokenId> token = tokenSequence.token();
        while (token != null && !CTX_DELIMITERS.contains(token.id())) {
            tokens.addFirst(token);
            if (!tokenSequence.movePrevious()) {
                break;
            } else {
                token = tokenSequence.token();
            }
        }
    }

    tokenSequence.move(orgOffset);
    tokenSequence.moveNext();
    return tokens.toArray(new Token[tokens.size()]);
}
 

/**
 * Returns offset of the package statement or {@literal -1} if no package
 * statement was found within this in {@link BaseDocument}.
 *
 * @param doc document
 * @return offset of the package statement or {@literal -1} if no package
 *         statement was found within this {@link BaseDocument}.
 */
private static int getLastPackageStatementOffset(BaseDocument doc) {
    TokenSequence<GroovyTokenId> ts = LexUtilities.getGroovyTokenSequence(doc, 1);

    int packageOffset = -1;

    while (ts.moveNext()) {
        if (ts.token().id() == GroovyTokenId.LITERAL_package) {
            packageOffset = ts.offset();
        }
    }
    return packageOffset;
}
 

public void testPHP05() throws Exception {
    TokenSequence<?> ts = PHPLexerUtils.seqForText("<html>\n" +
            "   <? \n" +
            "   echo '\\\'?>\\\'';\n" +
            "   ?>\n" +
            "</html>", PHPTopTokenId.language());
    PHPLexerUtils.next(ts, PHPTopTokenId.T_HTML, "<html>\n   ");
    PHPLexerUtils.next(ts, PHPTopTokenId.T_PHP_OPEN_DELIMITER, "<?");
    PHPLexerUtils.next(ts, PHPTopTokenId.T_PHP, " \n   echo '\\\'?>\\\'';\n   ");
    PHPLexerUtils.next(ts, PHPTopTokenId.T_PHP_CLOSE_DELIMITER, "?>");
    PHPLexerUtils.next(ts, PHPTopTokenId.T_HTML, "\n</html>");
}