jdk.nashorn.internal.parser.Token Java Examples

/**
 * Dump a token stream to stdout
 *
 * TODO: most other bugging goes to stderr, change?
 *
 * @param source the source
 * @param lexer  the lexer
 * @param stream the stream to dump
 */
public static void dumpTokens(final Source source, final Lexer lexer, final TokenStream stream) {
    TokenType type;
    int k = 0;
    do {
        while (k > stream.last()) {
            // Get more tokens.
            lexer.lexify();
        }

        final long token = stream.get(k);
        type = Token.descType(token);
        System.out.println("" + k + ": " + Token.toString(source, token, true));
        k++;
    } while(type != EOF);
}
 

@Override
public Node leaveExpressionStatement(final ExpressionStatement expressionStatement) {
    final Expression expr = expressionStatement.getExpression();
    ExpressionStatement node = expressionStatement;

    final FunctionNode currentFunction = lc.getCurrentFunction();

    if (currentFunction.isProgram()) {
        if (!isInternalExpression(expr) && !isEvalResultAssignment(expr)) {
            node = expressionStatement.setExpression(
                new BinaryNode(
                    Token.recast(
                        expressionStatement.getToken(),
                        TokenType.ASSIGN),
                    compilerConstant(RETURN),
                expr));
        }
    }

    return addStatement(node);
}
 

@Override
public Node leaveExpressionStatement(final ExpressionStatement expressionStatement) {
    final Expression expr = expressionStatement.getExpression();
    ExpressionStatement node = expressionStatement;

    final FunctionNode currentFunction = lc.getCurrentFunction();

    if (currentFunction.isProgram()) {
        if (!isInternalExpression(expr) && !isEvalResultAssignment(expr)) {
            node = expressionStatement.setExpression(
                new BinaryNode(
                    Token.recast(
                        expressionStatement.getToken(),
                        TokenType.ASSIGN),
                    compilerConstant(RETURN),
                expr));
        }
    }

    return addStatement(node);
}
 

@Override
public Node leaveExpressionStatement(final ExpressionStatement expressionStatement) {
    final Expression expr = expressionStatement.getExpression();
    ExpressionStatement node = expressionStatement;

    final FunctionNode currentFunction = lc.getCurrentFunction();

    if (currentFunction.isProgram()) {
        if (!isInternalExpression(expr) && !isEvalResultAssignment(expr)) {
            node = expressionStatement.setExpression(
                new BinaryNode(
                    Token.recast(
                        expressionStatement.getToken(),
                        TokenType.ASSIGN),
                    compilerConstant(RETURN),
                expr));
        }
    }

    return addStatement(node);
}
 

/**
 * Dump a token stream to stdout
 *
 * TODO: most other bugging goes to stderr, change?
 *
 * @param source the source
 * @param lexer  the lexer
 * @param stream the stream to dump
 */
public static void dumpTokens(final Source source, final Lexer lexer, final TokenStream stream) {
    TokenType type;
    int k = 0;
    do {
        while (k > stream.last()) {
            // Get more tokens.
            lexer.lexify();
        }

        final long token = stream.get(k);
        type = Token.descType(token);
        System.out.println("" + k + ": " + Token.toString(source, token, true));
        k++;
    } while(type != EOF);
}
 

@Override
public Node leaveBlock(final Block block) {
    if (!artificialBlock) {
        if (lc.isFunctionBody()) {
            // Prepend declaration-only var statements to the top of the statement list.
            lc.prependStatements(getCurrentFunctionState().varStatements);
        } else if (lc.isSplitBody()) {
            appendSplitReturn(FALLTHROUGH_STATE, NO_LINE_NUMBER);
            if (getCurrentFunctionState().fn.isProgram()) {
                // If we're splitting the program, make sure every shard ends with "return :return" and
                // begins with ":return = :return-in;".
                lc.prependStatement(new ExpressionStatement(NO_LINE_NUMBER, NO_TOKEN, NO_FINISH,
                        new BinaryNode(Token.toDesc(TokenType.ASSIGN, 0, 0), createReturnIdent(), createReturnParamIdent())));
            }
        }
    }
    return block;
}
 

@Override
public Node leaveBlock(final Block block) {
    if (!artificialBlock) {
        if (lc.isFunctionBody()) {
            // Prepend declaration-only var statements to the top of the statement list.
            lc.prependStatements(getCurrentFunctionState().varStatements);
        } else if (lc.isSplitBody()) {
            appendSplitReturn(FALLTHROUGH_STATE, NO_LINE_NUMBER);
            if (getCurrentFunctionState().fn.isProgram()) {
                // If we're splitting the program, make sure every shard ends with "return :return" and
                // begins with ":return = :return-in;".
                lc.prependStatement(new ExpressionStatement(NO_LINE_NUMBER, NO_TOKEN, NO_FINISH,
                        new BinaryNode(Token.toDesc(TokenType.ASSIGN, 0, 0), createReturnIdent(), createReturnParamIdent())));
            }
        }
    }
    return block;
}
 

@Override
public Node leaveExpressionStatement(final ExpressionStatement expressionStatement) {
    final Expression expr = expressionStatement.getExpression();
    ExpressionStatement node = expressionStatement;

    final FunctionNode currentFunction = lc.getCurrentFunction();

    if (currentFunction.isProgram()) {
        if (!isInternalExpression(expr) && !isEvalResultAssignment(expr)) {
            node = expressionStatement.setExpression(
                new BinaryNode(
                    Token.recast(
                        expressionStatement.getToken(),
                        TokenType.ASSIGN),
                    compilerConstant(RETURN),
                expr));
        }
    }

    return addStatement(node);
}
 

@Override
public Node leaveBlock(final Block block) {
    if (!artificialBlock) {
        if (lc.isFunctionBody()) {
            // Prepend declaration-only var statements to the top of the statement list.
            lc.prependStatements(getCurrentFunctionState().varStatements);
        } else if (lc.isSplitBody()) {
            appendSplitReturn(FALLTHROUGH_STATE, NO_LINE_NUMBER);
            if (getCurrentFunctionState().fn.isProgram()) {
                // If we're splitting the program, make sure every shard ends with "return :return" and
                // begins with ":return = :return-in;".
                lc.prependStatement(new ExpressionStatement(NO_LINE_NUMBER, NO_TOKEN, NO_FINISH,
                        new BinaryNode(Token.toDesc(TokenType.ASSIGN, 0, 0), createReturnIdent(), createReturnParamIdent())));
            }
        }
    }
    return block;
}
 

/**
 * Dump a token stream to stdout
 *
 * TODO: most other bugging goes to stderr, change?
 *
 * @param source the source
 * @param lexer  the lexer
 * @param stream the stream to dump
 */
public static void dumpTokens(final Source source, final Lexer lexer, final TokenStream stream) {
    TokenType type;
    int k = 0;
    do {
        while (k > stream.last()) {
            // Get more tokens.
            lexer.lexify();
        }

        final long token = stream.get(k);
        type = Token.descType(token);
        System.out.println("" + k + ": " + Token.toString(source, token, true));
        k++;
    } while(type != EOF);
}
 

@Override
public Node leaveBlock(final Block block) {
    if (!artificialBlock) {
        if (lc.isFunctionBody()) {
            // Prepend declaration-only var statements to the top of the statement list.
            lc.prependStatements(getCurrentFunctionState().varStatements);
        } else if (lc.isSplitBody()) {
            appendSplitReturn(FALLTHROUGH_STATE, NO_LINE_NUMBER);
            if (getCurrentFunctionState().fn.isProgram()) {
                // If we're splitting the program, make sure every shard ends with "return :return" and
                // begins with ":return = :return-in;".
                lc.prependStatement(new ExpressionStatement(NO_LINE_NUMBER, NO_TOKEN, NO_FINISH,
                        new BinaryNode(Token.toDesc(TokenType.ASSIGN, 0, 0), createReturnIdent(), createReturnParamIdent())));
            }
        }
    }
    return block;
}
 

private static IfNode makeIfStateEquals(final int lineNumber, final long token, final int finish,
        final int value, final Block pass, final Statement fail) {
    return new IfNode(lineNumber, token, finish,
            new BinaryNode(Token.recast(token, TokenType.EQ_STRICT),
                    GetSplitState.INSTANCE, intLiteral(value)),
            pass,
            fail == null ? null : new Block(NO_TOKEN, NO_FINISH, fail));
}
 

/**
 * Constructor
 *
 * @param token   token
 * @param finish  finish
 * @param value   array literal value, a Node array
 */
protected ArrayLiteralNode(final long token, final int finish, final Expression[] value) {
    super(Token.recast(token, TokenType.ARRAY), finish, value);
    this.elementType = Type.UNKNOWN;
    this.presets     = null;
    this.postsets    = null;
    this.splitRanges = null;
}
 

private static long tokenFor(final FunctionNode fn) {
    final int  position  = Token.descPosition(fn.getFirstToken());
    final long lastToken = Token.withDelimiter(fn.getLastToken());
    // EOL uses length field to store the line number
    final int  length    = Token.descPosition(lastToken) - position + (Token.descType(lastToken) == TokenType.EOL ? 0 : Token.descLength(lastToken));

    return Token.toDesc(TokenType.FUNCTION, position, length);
}
 

@Override
String toSource() {
    if (source != null && token != 0) {
        return source.getString(Token.descPosition(token), Token.descLength(token));
    }

    return "function " + (name == null ? "" : name) + "() { [native code] }";
}
 

@Override
String toSource() {
    if (source != null && token != 0) {
        return source.getString(Token.descPosition(token), Token.descLength(token));
    }

    return "function " + (name == null ? "" : name) + "() { [native code] }";
}
 

@Override
String toSource() {
    if (source != null && token != 0) {
        return source.getString(Token.descPosition(token), Token.descLength(token));
    }

    return "function " + (name == null ? "" : name) + "() { [native code] }";
}
 

/**
 * Format an error message to include source and line information.
 * @param message Error message string.
 * @param source  Source file information.
 * @param line    Source line number.
 * @param column  Source column number.
 * @param token   Offending token descriptor.
 * @return formatted string
 */
public static String format(final String message, final Source source, final int line, final int column, final long token) {
    final String        eoln     = System.lineSeparator();
    final int           position = Token.descPosition(token);
    final StringBuilder sb       = new StringBuilder();

    // Source description and message.
    sb.append(source.getName()).
        append(':').
        append(line).
        append(':').
        append(column).
        append(' ').
        append(message).
        append(eoln);

    // Source content.
    final String sourceLine = source.getSourceLine(position);
    sb.append(sourceLine).append(eoln);

    // Pointer to column.
    for (int i = 0; i < column; i++) {
        if (i < sourceLine.length() && sourceLine.charAt(i) == '\t') {
            sb.append('\t');
        } else {
            sb.append(' ');
        }
    }

    sb.append('^');
    // Use will append eoln.
    // buffer.append(eoln);

    return sb.toString();
}
 

/**
 * Format an error message to include source and line information.
 * @param message Error message string.
 * @param source  Source file information.
 * @param line    Source line number.
 * @param column  Source column number.
 * @param token   Offending token descriptor.
 * @return formatted string
 */
public static String format(final String message, final Source source, final int line, final int column, final long token) {
    final String        eoln     = System.lineSeparator();
    final int           position = Token.descPosition(token);
    final StringBuilder sb       = new StringBuilder();

    // Source description and message.
    sb.append(source.getName()).
        append(':').
        append(line).
        append(':').
        append(column).
        append(' ').
        append(message).
        append(eoln);

    // Source content.
    final String sourceLine = source.getSourceLine(position);
    sb.append(sourceLine).append(eoln);

    // Pointer to column.
    for (int i = 0; i < column; i++) {
        if (sourceLine.charAt(i) == '\t') {
            sb.append('\t');
        } else {
            sb.append(' ');
        }
    }

    sb.append('^');
    // Use will append eoln.
    // buffer.append(eoln);

    return sb.toString();
}
 

private static IfNode makeIfStateEquals(final int lineNumber, final long token, final int finish,
        final int value, final Block pass, final Statement fail) {
    return new IfNode(lineNumber, token, finish,
            new BinaryNode(Token.recast(token, TokenType.EQ_STRICT),
                    GetSplitState.INSTANCE, intLiteral(value)),
            pass,
            fail == null ? null : new Block(NO_TOKEN, NO_FINISH, fail));
}
 

@Override
String toSource() {
    if (source != null && token != 0) {
        return source.getString(Token.descPosition(token), Token.descLength(token));
    }

    return "function " + (name == null ? "" : name) + "() { [native code] }";
}
 

private static long tokenFor(final FunctionNode fn) {
    final int  position  = Token.descPosition(fn.getFirstToken());
    final long lastToken = Token.withDelimiter(fn.getLastToken());
    // EOL uses length field to store the line number
    final int  length    = Token.descPosition(lastToken) - position + (Token.descType(lastToken) == TokenType.EOL ? 0 : Token.descLength(lastToken));

    return Token.toDesc(TokenType.FUNCTION, position, length);
}
 

@Override
String toSource() {
    if (source != null && token != 0) {
        return source.getString(Token.descPosition(token), Token.descLength(token));
    }

    return "function " + (name == null ? "" : name) + "() { [native code] }";
}
 

private static long tokenFor(final FunctionNode fn) {
    final int  position  = Token.descPosition(fn.getFirstToken());
    final long lastToken = Token.withDelimiter(fn.getLastToken());
    // EOL uses length field to store the line number
    final int  length    = Token.descPosition(lastToken) - position + (Token.descType(lastToken) == TokenType.EOL ? 0 : Token.descLength(lastToken));

    return Token.toDesc(TokenType.FUNCTION, position, length);
}
 

/**
 * Format an error message to include source and line information.
 * @param message Error message string.
 * @param source  Source file information.
 * @param line    Source line number.
 * @param column  Source column number.
 * @param token   Offending token descriptor.
 * @return formatted string
 */
public static String format(final String message, final Source source, final int line, final int column, final long token) {
    final String        eoln     = System.lineSeparator();
    final int           position = Token.descPosition(token);
    final StringBuilder sb       = new StringBuilder();

    // Source description and message.
    sb.append(source.getName()).
        append(':').
        append(line).
        append(':').
        append(column).
        append(' ').
        append(message).
        append(eoln);

    // Source content.
    final String sourceLine = source.getSourceLine(position);
    sb.append(sourceLine).append(eoln);

    // Pointer to column.
    for (int i = 0; i < column; i++) {
        if (i < sourceLine.length() && sourceLine.charAt(i) == '\t') {
            sb.append('\t');
        } else {
            sb.append(' ');
        }
    }

    sb.append('^');
    // Use will append eoln.
    // buffer.append(eoln);

    return sb.toString();
}
 

private LexerTokenLiteralNode(final long token, final int finish, final LexerToken value) {
    super(Token.recast(token, TokenType.STRING), finish, value); //TODO is string the correct token type here?
}
 

private static long tokenFor(final FunctionNode fn) {
    final int  position   = Token.descPosition(fn.getFirstToken());
    final int  length     = Token.descPosition(fn.getLastToken()) - position + Token.descLength(fn.getLastToken());

    return Token.toDesc(TokenType.FUNCTION, position, length);
}
 

/**
 * Fetch a portion of source content associated with a token.
 * @param token Token descriptor.
 * @return Source content portion.
 */
public String getString(final long token) {
    final int start = Token.descPosition(token);
    final int len = Token.descLength(token);
    return new String(data(), start, len);
}
 

private StringLiteralNode(final long token, final int finish, final String value) {
    super(Token.recast(token, TokenType.STRING), finish, value);
}
 

private NullLiteralNode(final long token, final int finish) {
    super(Token.recast(token, TokenType.OBJECT), finish, null);
}