org.antlr.runtime.Lexer Java Examples

private String lexerResult(Lexer lexer, Map<Integer, String> tokenNames, CharSequence text) {
	lexer.setCharStream(new ANTLRStringStream(text.toString()));
	List<String> result = new ArrayList<>();
	while (true) {
		Token token = lexer.nextToken();
		if (token == Token.EOF_TOKEN) {
			return Joiner.on(System.lineSeparator()).join(result);
		}
		Object nameOrType = null;
		String tokenName = tokenNames.get(token.getType());
		if (tokenName != null) {
			nameOrType = tokenName;
		} else {
			nameOrType = Integer.valueOf(token.getType());
		}
		result.add(nameOrType + " \'" + token.getText() + "\'");
	}
}
 

public void testSimpleSupportsStatementTokens() throws Exception {
    String source = "@supports not (text-align: center) {} ";
    
    Lexer lexer = createLexer(source);
    
    assertANTLRToken("@supports", Css3Lexer.SUPPORTS_SYM, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("not" ,Css3Lexer.NOT, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("(" ,Css3Lexer.LPAREN, lexer.nextToken());
    assertANTLRToken("text-align" ,Css3Lexer.IDENT, lexer.nextToken());
    assertANTLRToken(":", Css3Lexer.COLON, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("center" ,Css3Lexer.IDENT, lexer.nextToken());
    assertANTLRToken(")" ,Css3Lexer.RPAREN, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("{" ,Css3Lexer.LBRACE, lexer.nextToken());
    assertANTLRToken("}" ,Css3Lexer.RBRACE, lexer.nextToken());
    
}
 

public void testLexingOfPercentageWithoutNumberPrefix() throws Exception {
    String source = "font: %/20 ";
    Lexer lexer = createLexer(source);
    assertANTLRToken("font" ,Css3Lexer.IDENT, lexer.nextToken());
    assertANTLRToken(":" ,Css3Lexer.COLON, lexer.nextToken());
    assertANTLRToken(" " ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("%" , Css3Lexer.PERCENTAGE_SYMBOL, lexer.nextToken());
    assertANTLRToken("/" ,Css3Lexer.SOLIDUS, lexer.nextToken());
    assertANTLRToken("20" , Css3Lexer.NUMBER, lexer.nextToken());
    assertANTLRToken(" " , Css3Lexer.WS, lexer.nextToken());
}
 

public void testLineComment() throws Exception {
    String source = "//comment\na";
    Lexer lexer = createLexer(source);
    assertANTLRToken("//comment" ,Css3Lexer.LINE_COMMENT, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.NL, lexer.nextToken());
    assertANTLRToken("a",Css3Lexer.IDENT, lexer.nextToken());
}
 

public void testSASS_Else() throws Exception {
    String source = "@else cau";
    Lexer lexer = createLexer(source);
    assertANTLRToken(null ,Css3Lexer.SASS_ELSE, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.IDENT, lexer.nextToken());
}
 

public void testExtendOnlySelector2() throws Exception {
    String source = "#context a%extreme {";
    Lexer lexer = createLexer(source);
    assertANTLRToken(null ,Css3Lexer.HASH, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.IDENT, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.SASS_EXTEND_ONLY_SELECTOR, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.LBRACE, lexer.nextToken());
}
 

public void testCaseInsensivityOfSomeAtTokens() throws Exception {
    String source = "@FONT-face @charset @CHARSET @charSeT ";

    Lexer lexer = createLexer(source);

    assertANTLRToken("@FONT-face" ,Css3Lexer.FONT_FACE_SYM, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("@charset" ,Css3Lexer.CHARSET_SYM, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("@CHARSET" ,Css3Lexer.CHARSET_SYM, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());
    assertANTLRToken("@charSeT" ,Css3Lexer.CHARSET_SYM, lexer.nextToken());

}
 

public Block(Architecture arch) {
	this.architecture = arch;
	Lexer lex = null;
	Assembler assembler = null;
	
	switch (architecture) {
		// no parsing necessary if we're writing bytes to memory
		case raw:
			return;
		// arm architectures trickle down
		case armv7:
		case armv9:
			// lex = new armLexer(new ANTLRStringStream(program));
			// assembler = new armAssembler(new CommonTokenStream(lex));
			break;
		case x64:
			lex = new x64Lexer(new ANTLRStringStream(program));
			assembler = new x64Assembler(new CommonTokenStream(lex));
			break;
		case x86:
			break;
		default:
			break;
	}
	
	try {
		if (assembler != null) {
			assembler.start();
			this.instructions = assembler.getMachineCode();
		} else {
			throw new JssemblyException("Assembler not found for architecture: " + this.architecture.name());
		}
	} catch (RecognitionException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
}
 

public String getLexerErrorMessage(Token invalidToken) {
	if (tokenSource instanceof org.eclipse.xtext.parser.antlr.Lexer) {
		return ((org.eclipse.xtext.parser.antlr.Lexer) tokenSource).getErrorMessage(invalidToken);
	}
	return (invalidToken.getType() == Token.INVALID_TOKEN_TYPE) ? "Invalid token " + invalidToken.getText() : null;
}
 

public static <T extends Lexer> LexerProvider<T> create(Class<T> clazz) {
	return new LexerProvider<T>(clazz);
}
 

protected String lex(Lexer lexer, InputStream tokensStream, CharSequence text) throws Exception {
	InputStreamReader _inputStreamReader = new InputStreamReader(tokensStream);
	LinkedHashMap<Integer, String> tokenNames = tokenNames(_inputStreamReader);
	return lexerResult(lexer, tokenNames, text);
}
 

public void testIssue236649() throws Exception {
    String source = "url(http://fonts.googleapis.com/css?family=Josefin+Sans|Sigmar+One|Maven+Pro)";
    Lexer lexer = createLexer(source);
    assertANTLRToken("url(http://fonts.googleapis.com/css?family=Josefin+Sans|Sigmar+One|Maven+Pro)", Css3Lexer.URI, lexer.nextToken());
}
 

public void testURLWithAtSign() throws Exception {
    String source = "url([email protected])";
    Lexer lexer = createLexer(source);
    assertANTLRToken("url([email protected])" ,Css3Lexer.URI, lexer.nextToken());
}
 

public void testLESS_JS_STRING() throws Exception {
    String source = "`\"hello\".toUpperCase() + '!'`;";
    Lexer lexer = createLexer(source);
    assertANTLRToken("`\"hello\".toUpperCase() + '!'`" ,Css3Lexer.LESS_JS_STRING, lexer.nextToken());
    assertANTLRToken(";" ,Css3Lexer.SEMI, lexer.nextToken());
}
 

public Lexer[] getDelegates() {
	return new Lexer[] {};
}
 

public Lexer[] getDelegates() {
  return new Lexer[] {};
}
 

public Lexer[] getDelegates() {
  return new Lexer[] {};
}
 

public void testLineCommentAtTheFileEnd() throws Exception {
    String source = "//comment";
    Lexer lexer = createLexer(source);
    assertANTLRToken("//comment" ,Css3Lexer.LINE_COMMENT, lexer.nextToken());
}
 

public void testExtendOnlySelector() throws Exception {
    String source = "body%my";
    Lexer lexer = createLexer(source);
    assertANTLRToken(null ,Css3Lexer.IDENT, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.SASS_EXTEND_ONLY_SELECTOR, lexer.nextToken());
}
 

public void testCPLineComment() throws Exception {
    String source = "//line comment\n";
    Lexer lexer = createLexer(source);
    assertANTLRToken(null ,Css3Lexer.LINE_COMMENT, lexer.nextToken());

}
 

public void testSassVar() throws Exception {
    String source = "$var ";
    Lexer lexer = createLexer(source);
    assertANTLRToken(null ,Css3Lexer.SASS_VAR, lexer.nextToken());
}
 

public void testLexingURLToken() throws Exception {
    String source = "url(http://fonts.googleapis.com/css?family=Syncopate) ";
    Lexer lexer = createLexer(source);
    assertANTLRToken(null ,Css3Lexer.URI, lexer.nextToken());
}
 

@Override
public Lexer get() {
	return new InternalN4JSLexer(null);
}
 

@Override
public Lexer get() {
	return new InternalN4JSLexer(null);
}
 

public Lexer[] getDelegates() {
	return new Lexer[] {};
}
 

public Lexer[] getDelegates() {
	return new Lexer[] {};
}
 

public void testRemUnit() throws Exception {
    String source = "10rad 20rem ";

    Lexer lexer = createLexer(source);

    assertANTLRToken(null ,Css3Lexer.ANGLE, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());

    assertANTLRToken(null ,Css3Lexer.REM, lexer.nextToken());
    assertANTLRToken(null ,Css3Lexer.WS, lexer.nextToken());

}
 

public void testSupportsToken() throws Exception {
    String source = "@supports ";
    
    Lexer lexer = createLexer(source);
    
    assertANTLRToken("@supports", Css3Lexer.SUPPORTS_SYM, lexer.nextToken());
    
}
 

protected abstract Lexer lexer();