Java Code Examples for org.antlr.v4.runtime.Lexer#nextToken()

The following examples show how to use org.antlr.v4.runtime.Lexer#nextToken() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AddMissingImportPrefixInDTTransformer.java    From jdmn with Apache License 2.0 5 votes vote down vote up 
String addMissingPrefix(String text, Set<String> names) {
    if (StringUtils.isEmpty(text)) {
        return text;
    }

    // Extract tokens
    Lexer lexer = makeLexer(text);
    List<Token> tokens = new ArrayList<>();
    Token token;
    do {
        token = lexer.nextToken();
        tokens.add(token);
    } while (token.getType() != FEELLexer.EOF);

    // Add prefix
    StringBuilder newText = new StringBuilder();
    int card = tokens.size() - 1;
    for (int i = 0; i < card; i++) {
        token = tokens.get(i);
        String tokenText = token.getText();
        if (isInputName(i, tokens)) {
            // Replace name
            if (names.contains(tokenText)) {
                // Prefix same with DRG name
                newText.append(tokenText).append(".").append(tokenText);
            } else {
                newText.append(tokenText);
            }
        } else {
            newText.append(tokenText);
        }
        newText.append(" ");
    }
    return newText.toString().trim();
}
Example 2
Source File: AbstractLexerTest.java    From jdmn with Apache License 2.0 5 votes vote down vote up 
protected Token checkToken(String inputTape, int expectedTokenCode, String expectedLexeme) {
    Lexer tokenStream = makeLexer(inputTape);
    Token token = tokenStream.nextToken();
    assertEquals(expectedLexeme, token.getText());
    assertEquals(expectedTokenCode, token.getType());
    return token;
}
Example 3
Source File: AbstractLexerTest.java    From jdmn with Apache License 2.0 5 votes vote down vote up 
public void checkTokenList(boolean allowSpaceInNames, String text, Integer[] expectedTokenCodes, String[] expectedLexemes) {
    Lexer lexer = makeLexer(text);
    for(int i=0; i<expectedTokenCodes.length; i++) {
        Token token = lexer.nextToken();
        assertEquals(expectedLexemes[i], token.getText());
        assertEquals((long)expectedTokenCodes[i], token.getType());
    }
}
Example 4
Source File: ZserioLexerTest.java    From zserio with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
private void checkToken(Lexer lexer, int type, int channel, String text)
{
    final Token token = lexer.nextToken();
    assertNotNull(token);
    assertEquals("expecting token '" + type + ":" + lexer.getVocabulary().getSymbolicName(type) + "' - ",
            type, token.getType());
    assertEquals("expecting channel '" + channel + ":" + lexer.getChannelNames()[channel] + "' - ",
            channel, token.getChannel());
    if (text != null)
        assertEquals("expecting text \"" + text + "\" - ", text, token.getText());
}
Example 5
Source File: NormalizeDateTimeLiteralsTransformer.java    From jdmn with Apache License 2.0 4 votes vote down vote up 
String normalize(String text) {
    if (StringUtils.isEmpty(text)) {
        return text;
    }

    // Extract tokens
    Lexer lexer = makeLexer(text);
    List<Token> tokens = new ArrayList<>();
    Token token;
    do {
        token = lexer.nextToken();
        tokens.add(token);
    } while (token.getType() != FEELLexer.EOF);

    // Do not process unless neecessary
    if (!containsDateTimeLiterals(tokens)) {
        return text;
    }

    // Normalize
    StringBuilder newText = new StringBuilder();
    int size = tokens.size();
    int i=0;
    while (i < size -1) {
        token = tokens.get(i);
        String tokenText = token.getText();
        if (isDateTimeLiteralStart(i, tokens)) {
            // Replace literal
            int textIndex = i + 2;
            if (textIndex < size && tokens.get(textIndex).getType() == FEELLexer.STRING) {
                newText.append(tokens.get(i).getText());
                newText.append("(");
                String newTokenText = normalizeLiteral(tokenText, tokens.get(textIndex).getText());
                newText.append(newTokenText);
                newText.append(")");
                i += 4;
            } else {
                newText.append(tokenText);
                i++;
            }
        } else {
            newText.append(tokenText);
            i++;
        }
        newText.append(" ");
    }
    return newText.toString().trim();
}
Example 6
Source File: ZserioLexerTest.java    From zserio with BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up 
private void checkEOF(Lexer lexer)
{
    final Token token = lexer.nextToken();
    assertEquals(Lexer.EOF, token.getType());
}