Java Code Examples for org.antlr.runtime.Token#HIDDEN_CHANNEL

@Override
protected void announce(Token start, Token stop, AbstractElement element) {
	if (start != null && start != Token.EOF_TOKEN) {
		if (start == stop) {
			announce(start, element);
		} else {
			CommonToken castedStart = (CommonToken) start;
			if (stop == null) { // possible error condition
				if (start.getTokenIndex() == state.lastErrorIndex) {
					return;
				}
			}
			CommonToken castedEnd = (CommonToken) stop;
			Integer newType = rewriter.rewrite(castedStart, element);
			if (newType != null && castedEnd != null && castedEnd != Token.EOF_TOKEN) {
				LazyTokenStream castedInput = (LazyTokenStream) this.input;
				for (int i = castedStart.getTokenIndex() + 1; i < castedEnd.getTokenIndex(); i++) {
					Token token = castedInput.get(i);
					if (token.getChannel() != Token.HIDDEN_CHANNEL)
						token.setType(newType);
				}
				castedEnd.setType(newType);
			}
		}
	}
}
 

@Test
public void TestLexer() throws IOException {
    CharStream input = new QueryParserFileStream( "test/org/apache/pig/parser/TestLexer.pig" );
    QueryLexer lexer = new QueryLexer( input );
    int tokenCount = 0;
    Token token;
    while( ( token = lexer.nextToken() ).getType() != Token.EOF ) {
        if( token.getChannel() == Token.HIDDEN_CHANNEL )
            continue;
        tokenCount++;
        if( token.getText().equals( ";" ) ) {
            System.out.println( token.getText() );
        } else {
            System.out.print( token.getText() + "(" + token.getType() + ") " );
        }
    }

    // While we can check more conditions, such as type of each token, for now I think the following
    // is enough. If the token type is wrong, it will be most likely caught by the parser.
    Assert.assertEquals( 455, tokenCount );
    Assert.assertEquals( 0, lexer.getNumberOfSyntaxErrors() );
}
 

@Test
public void test2() throws IOException {
    String query = "A = load 'input' using PigStorage(';');" +
                   "B = foreach ^ A generate string.concatsep( ';', $1, $2 );";
    CharStream input = new QueryParserStringStream( query, null );
    QueryLexer lexer = new QueryLexer( input );
    Token token;
    try {
        while( ( token = lexer.nextToken() ).getType() != Token.EOF ) {
            if( token.getChannel() == Token.HIDDEN_CHANNEL )
                continue;
            if( token.getText().equals( ";" ) ) {
                System.out.println( token.getText() );
            } else {
                System.out.print( token.getText() + "(" + token.getType() + ") " );
            }
        }
    } catch(Exception ex) {
        Assert.assertTrue( ex.getMessage().contains( "Unexpected character" ) );
        return;
    }
    Assert.fail( "Query should fail." );
}
 

private String getDoc(CommonTree ast, String prefix) {
    CommonTokenStream ts = ((TarsRoot) ast.getAncestor(TarsParser.TARS_ROOT)).getTokenStream();
    Token t = ts.get(ast.getTokenStartIndex() - 1);
    if (t != null && t.getChannel() == Token.HIDDEN_CHANNEL) {
        return prefix + t.getText().replaceAll("\n\\s*\\*", "\n" + prefix + " *");
    } else {
        return "";
    }
}
 

/**
 * Adds token and adjust channel.
 */
@Override
public boolean add(Token tok) {
	super.add(tok);
	int type = tok.getType();
	if (type == InternalN4JSParser.EqualsSignGreaterThanSign) {
		// The arrow expression may not follow a semicolon thus we promote those here
		// to the default channel if they precede the arrow => operator
		for (int i = size() - 2; i >= 0; i--) {
			Token prev = get(i);
			if (prev.getChannel() == Token.HIDDEN_CHANNEL) {
				if (SemicolonInjectionHelper.isSemicolonEquivalent(prev)) {
					prev.setChannel(Token.DEFAULT_CHANNEL);
					break;
				}
			} else {
				break;
			}
		}
	} else if (type == InternalN4JSParser.RULE_EOL
			|| type == InternalN4JSParser.RULE_ML_COMMENT
			|| type == InternalN4JSParser.RULE_WS
			|| type == InternalN4JSParser.RULE_SL_COMMENT) {
		tok.setChannel(Token.HIDDEN_CHANNEL);
	} else {
		tok.setChannel(Token.DEFAULT_CHANNEL);
	}
	return true;
}
 

@Override
public Token nextToken() {
	while (true) {
		this.state.token = null;
		this.state.channel = Token.DEFAULT_CHANNEL;
		this.state.tokenStartCharIndex = input.index();
		this.state.tokenStartCharPositionInLine = input
				.getCharPositionInLine();
		this.state.tokenStartLine = input.getLine();
		this.state.text = null;
		if (input.LA(1) == CharStream.EOF) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if (this.state.token == null) {
				emit();
			} else if (this.state.token == Token.SKIP_TOKEN) {
				continue;
			}
			return this.state.token;
		} catch (RecognitionException re) {
			reportError(re);
			if (re instanceof NoViableAltException
					|| re instanceof FailedPredicateException) {
				recover(re);
			}
			// create token that holds mismatched char
			Token t = new CommonToken(input, Token.INVALID_TOKEN_TYPE,
					Token.HIDDEN_CHANNEL, this.state.tokenStartCharIndex,
					getCharIndex() - 1);
			t.setLine(this.state.tokenStartLine);
			t.setCharPositionInLine(
					this.state.tokenStartCharPositionInLine);
			tokenErrorMap.put(t, getErrorMessage(re, this.getTokenNames()));
			emit(t);
			return this.state.token;
		}
	}
}
 

protected void skipHiddenTokens() {
	if (hiddenTokens == null || hiddenTokens.isEmpty())
		return;
	Token token = LT(1);
	while(token.getChannel() == Token.HIDDEN_CHANNEL) {
		p++;
		token = LT(1);
	}
}
 

@Override
public Token nextToken() {
	while (true) {
		this.state.token = null;
		this.state.channel = Token.DEFAULT_CHANNEL;
		this.state.tokenStartCharIndex = input.index();
		this.state.tokenStartCharPositionInLine = input.getCharPositionInLine();
		this.state.tokenStartLine = input.getLine();
		this.state.text = null;
		if (input.LA(1) == CharStream.EOF) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if (this.state.token == null) {
				emit();
			}
			else if (this.state.token == Token.SKIP_TOKEN) {
				continue;
			}
			return this.state.token;
		}
		catch (RecognitionException re) {
			reportError(re);
			if (re instanceof NoViableAltException ||
				re instanceof FailedPredicateException) {
				recover(re);
			}
			// create token that holds mismatched char
			Token t = new CommonToken(input, Token.INVALID_TOKEN_TYPE, Token.HIDDEN_CHANNEL,
					this.state.tokenStartCharIndex, getCharIndex() - 1);
			t.setLine(this.state.tokenStartLine);
			t.setCharPositionInLine(this.state.tokenStartCharPositionInLine);
			tokenErrorMap.put(t, getErrorMessage(re, this.getTokenNames()));
			emit(t);
			return this.state.token;
		}
	}
}