org.antlr.runtime.TokenSource Java Examples

protected IParseResult doParse(String ruleName, CharStream in, NodeModelBuilder nodeModelBuilder, int initialLookAhead) {
	TokenSource tokenSource = createLexer(in);
	XtextTokenStream tokenStream = createTokenStream(tokenSource);
	tokenStream.initCurrentLookAhead(initialLookAhead);
	setInitialHiddenTokens(tokenStream);
	AbstractInternalAntlrParser parser = createParser(tokenStream);
	parser.setTokenTypeMap(getTokenDefProvider().getTokenDefMap());
	parser.setSyntaxErrorProvider(getSyntaxErrorProvider());
	parser.setNodeModelBuilder(nodeModelBuilder);
	parser.setSemanticModelBuilder(getElementFactory());
	IUnorderedGroupHelper helper = getUnorderedGroupHelper().get();
	parser.setUnorderedGroupHelper(helper);
	helper.initializeWith(parser);
	try {
		if(ruleName != null)
			return parser.parse(ruleName);
		return parser.parse();
	} catch (Exception re) {
		throw new ParseException(re.getMessage(),re);
	}
}
 

@Override
protected TokenSource createTokenSource(String string) {
	List<Token> tokens = highlightingParser.getTokens(string);
	Iterator<Token> iter = tokens.iterator();
	return new TokenSource() {

		@Override
		public Token nextToken() {
			if (iter.hasNext()) {
				return iter.next();
			}
			return Token.EOF_TOKEN;
		}

		@Override
		public String getSourceName() {
			return "Text: " + string;
		}
	};
}
 

/**
 * First pass. Use the tokens that have been computed from the production parser's result and collect the follow
 * elements from those.
 */
private CustomInternalN4JSParser collectFollowElements(
		TokenSource tokens,
		AbstractElement grammarElement,
		String ruleName,
		boolean strict,
		Set<FollowElement> result) {
	CustomInternalN4JSParser parser = createParser();
	parser.setStrict(strict);
	try {
		if (grammarElement != null)
			parser.getGrammarElements().add(grammarElement);
		ObservableXtextTokenStream tokenStream = new ObservableXtextTokenStream(tokens, parser);
		result.addAll(doGetFollowElements(parser, tokenStream, ruleName));
	} catch (InfiniteRecursion infinite) {
		// this guards against erroneous infinite recovery loops in Antlr.
		// Grammar dependent and not expected something that is expected for N4JS
		// is used in the base class thus also used here.
		result.addAll(parser.getFollowElements());
	}
	return parser;
}
 

private ObservableXtextTokenStream toTokenStream(FollowElement element,
		ITokenDefProvider tokenDefProvider) {
	Iterator<LookAheadTerminal> iter = element.getLookAheadTerminals().iterator();
	return new ObservableXtextTokenStream(new TokenSource() {
		@Override
		public Token nextToken() {
			if (iter.hasNext()) {
				LookAheadTerminal lookAhead = iter.next();
				return lookAhead.getToken();
			}
			return Token.EOF_TOKEN;
		}

		@Override
		public String getSourceName() {
			return "LookAheadTerminalTokenSource";
		}
	}, tokenDefProvider);
}
 

@Override
protected TokenSource createTokenSource(String string) {
	final FlexTokenSource delegate = flexerFactory.createTokenSource(new StringReader(string));
	return new TokenSource() {

		@Override
		public Token nextToken() {
			Token token = delegate.nextToken();
			if(token.getType() == multilineTokenType) {
				String text = token.getText();
				if(text.startsWith("/**") && !text.startsWith("/***")){
					token.setType(JAVA_DOC_TOKEN_TYPE);
				}
			}
			return token;
		}

		@Override
		public String getSourceName() {
			return delegate.getSourceName();
		}
	};
}
 

@Override
protected void assertTokens(String value, TokenSource tokenSource,
		String escapedString) {
	if (tokenSource == null)
		return;
	Token token = tokenSource.nextToken();

	// customization start
	if ("ID".equals(getRuleName()) && "TEXT".equals(getRuleName(token))) {
		return;
	}
	// customization end

	if (!escapedString.equals(token.getText())) {
		throw createTokenContentMismatchException(value, escapedString,
				token);
	}
	if (!getRuleName().toUpperCase().equals(getRuleName(token))) {
		throw createTokenTypeMismatchException(value, escapedString, token);
	}
	String reparsedValue = toValue(token.getText(), null);
	if (value != reparsedValue && !value.equals(reparsedValue)) {
		throw createTokenContentMismatchException(value, escapedString,
				token);
	}
}
 

public Collection<FE> getFollowElements(String input, boolean strict) {
	TokenSource tokenSource = createTokenSource(input);
	InternalParser parser = createParser();
	parser.setStrict(strict);
	ObservableXtextTokenStream tokens = new ObservableXtextTokenStream(tokenSource, parser);
	tokens.setInitialHiddenTokens(getInitialHiddenTokens());
	parser.setTokenStream(tokens);
	IUnorderedGroupHelper helper = createUnorderedGroupHelper();
	parser.setUnorderedGroupHelper(helper);
	helper.initializeWith(parser);
	tokens.setListener(parser);
	try {
		return Lists.newArrayList(getFollowElements(parser));
	} catch (InfiniteRecursion infinite) {
		return Lists.newArrayList(parser.getFollowElements());
	}
}
 

public XtextTokenStream(TokenSource tokenSource, ITokenDefProvider tokenDefProvider) {
	super(tokenSource);
	tokens = new TokenList(500);
	rulenameToTokenType = new HashMap<String, Integer>(tokenDefProvider.getTokenDefMap().size());
	for(Map.Entry<Integer, String> entry: tokenDefProvider.getTokenDefMap().entrySet()) {
		rulenameToTokenType.put(entry.getValue(), entry.getKey());
	}
}
 

@Override
protected void initTokenSource(String text, TokenSource tokenSource, ContentAssistContext context) {
	EObject currentModel = context.getLastCompleteNode().getSemanticElement();
	Variable variable = currentModel != null ? EcoreUtil2.getContainerOfType(currentModel, Variable.class) : null;
	TemplateBody body = currentModel != null ? EcoreUtil2.getContainerOfType(currentModel, TemplateBody.class) : null;
	Lexer lexer = (Lexer) tokenSource;
	CharStream stream = new ANTLRStringStream(text);
	lexer.setCharStream(stream);
	initLexer(lexer, body != null, variable != null);
}
 

protected boolean isSameTokenSequence(TokenSource originalSource, TokenSource newSource, int expectedLength) {
	Token token = originalSource.nextToken();
	int newLength = 0;
	while(Token.EOF != token.getType()) {
		Token newToken = newSource.nextToken();
		if (token.getType() != newToken.getType()) {
			return false;
		}
		newLength += TokenTool.getLength(newToken);
		token = originalSource.nextToken();
	}
	return newLength == expectedLength;
}
 

@Override
public Collection<FollowElement> getFollowElements(IParseResult parseResult, int offset, boolean strict) {
	try (Measurement m = dataCollectors.dcParseContexts().getMeasurement()) {
		Set<FollowElement> result = Sets.newLinkedHashSet();
		ICompositeNode entryPoint = entryPointFinder.findEntryPoint(parseResult, offset);
		String ruleName;
		if (entryPoint == null) {
			entryPoint = parseResult.getRootNode();
			ruleName = getRuleNames().getAntlrRuleName(getEntryRule());
		} else {
			ruleName = getRuleName(entryPoint);
		}
		// Yeah Xpect fallback, Lord this is annoying.
		if (ruleName == null) {
			entryPoint = parseResult.getRootNode();
			ruleName = "ruleScript";
		}
		TokenSource tokenSource = tokenSourceFactory.toTokenSource(entryPoint, entryPoint.getOffset(), offset,
				true);
		CustomInternalN4JSParser parser = collectFollowElements(tokenSource, getEntryGrammarElement(entryPoint),
				ruleName, strict, result);
		adjustASIAndCollectFollowElements(parser, ruleName, strict, result);

		/*
		 * Lists are easier to debug
		 */
		return Lists.newArrayList(result);
	}
}
 

public LexerMultiplexer(TokenSource... modes) {
	this.modes = modes;

	this.stack = new LinkedList<>();
	this.stack.push(0);

	this.channels = new HashSet<>();
	this.channels.add(Token.DEFAULT_CHANNEL);
}
 

@Override
public Token nextToken() {
	Integer mode = stack.peekFirst();
	TokenSource src = modes[mode];
	Token t;
	do {
		t = src.nextToken();
		//System.out.println("Token(" + mode + "," + src.getSourceName() + "): " + t);
	}
	while (!channels.contains(t.getChannel()));
	return t;
}
 

protected boolean equalTokenSequence(TokenSource first, TokenSource second) {
	Token token = null;
	while (!(token = first.nextToken()).equals(Token.EOF_TOKEN)) {
		Token otherToken = second.nextToken();
		if (otherToken.equals(Token.EOF_TOKEN)) {
			return false;
		}
		if (!token.getText().equals(otherToken.getText())) {
			return false;
		}
	}
	return true;
}
 

public RepairEntryData(int offset, int index, CommonToken newToken, TokenSource lexer) {
	super();
	this.offset = offset;
	this.index = index;
	this.newToken = newToken;
	this.tokenSource = lexer;
}
 

/**
 * @since 2.4
 */
protected RepairEntryData getRepairEntryData(DocumentEvent e) throws Exception {
	int tokenStartsAt = 0;
	int tokenInfoIdx = 0;
	for(tokenInfoIdx = 0; tokenInfoIdx< getInternalModifyableTokenInfos().size(); ++tokenInfoIdx) {
		TokenInfo oldToken = getInternalModifyableTokenInfos().get(tokenInfoIdx);
		if(tokenStartsAt <= e.getOffset() && tokenStartsAt + oldToken.getLength() >= e.getOffset())
			break;
		tokenStartsAt += oldToken.getLength();
	}
	final TokenSource delegate = createTokenSource(e.fDocument.get(tokenStartsAt, e.fDocument.getLength() - tokenStartsAt));
	final int offset = tokenStartsAt;
	TokenSource source = new TokenSource() {
		@Override
		public Token nextToken() {
			CommonToken commonToken = (CommonToken) delegate.nextToken();
			commonToken.setText(commonToken.getText());
			commonToken.setStartIndex(commonToken.getStartIndex()+offset);
			commonToken.setStopIndex(commonToken.getStopIndex()+offset);
			return commonToken;
		}

		@Override
		public String getSourceName() {
			return delegate.getSourceName();
		}
	};
	final CommonToken token = (CommonToken) source.nextToken();
	return new RepairEntryData(offset, tokenInfoIdx, token, source);
}
 

protected TokenSource getTokenSource(String escapedValue) {
	Lexer result = getLexer();
	if (result == null)
		return null;
	result.setCharStream(new ANTLRStringStream(escapedValue));
	return result;
}
 

protected void assertTokens(T value, TokenSource tokenSource, String escapedString) {
	if (tokenSource == null)
		return;
	Token token = tokenSource.nextToken();
	if (!escapedString.equals(token.getText())) {
		throw createTokenContentMismatchException(value, escapedString, token);
	}
	if (!getRuleName().toUpperCase().equals(getRuleName(token))) {
		throw createTokenTypeMismatchException(value, escapedString, token);
	}
	T reparsedValue = toValue(token.getText(), null);
	if (value != reparsedValue && !value.equals(reparsedValue)) {
		throw createTokenContentMismatchException(value, escapedString, token);
	}
}
 

@Override
protected TokenSource createLexer(CharStream stream) {
	if (stream instanceof ReaderCharStream) {
		Reader reader = ((ReaderCharStream) stream).getReader();
		return flexerFactory.createTokenSource(reader);
	}
	throw new IllegalArgumentException(stream.getClass().getName());
}
 

@Override
public TokenSource createTokenSource(CharStream stream) {
	if (parser instanceof AbstractAntlrParser) {
		return ((AbstractAntlrParser) parser).createLexer(stream);
	}
	String msg = parser.getClass().getName() + " should be a subclass of " + AbstractAntlrParser.class.getName();
	throw new IllegalStateException(msg);
}
 

protected boolean equalTokenSequence(TokenSource first, TokenSource second) {
	Token token = null;
	while(!(token = first.nextToken()).equals(Token.EOF_TOKEN)) {
		Token otherToken = second.nextToken();
		if (otherToken.equals(Token.EOF_TOKEN)) {
			return false;
		}
		if (!token.getText().equals(otherToken.getText())) {
			return false;
		}
	}
	return true;
}
 

@Override
public TokenSource createTokenSource(CharSequence text) {
	return createTokenSource(getReader(text));
}
 

/**
 * @since 2.7
 */
protected void assertTokens(T value, String result) {
	TokenSource tokenSource = getTokenSource(result);
	assertTokens(value, tokenSource, result);
}
 

public TokenSource getDelegate() {
	return delegate;
}
 

public TokenSource getCombinedLexer() {
	return combinedLexer;
}
 

public void setDelegate(TokenSource delegate) {
	this.delegate = delegate;
}
 

@Override
public TokenSource createTokenSource(Reader reader) {
	return createTokenSource(getCharStream(reader));
}
 

public ObservableXtextTokenStream(TokenSource tokenSource, ITokenDefProvider tokenDefProvider) {
	super(tokenSource, tokenDefProvider);
}
 

@Override
protected TokenSource createLexer(CharStream stream) {
	Lexer lexer = lexerProvider.get();
	lexer.setCharStream(stream);
	return lexer;
}
 

protected AbstractIndentationTokenSource(TokenSource delegate) {
	setDelegate(delegate);
}