Java Code Examples for org.antlr.runtime.CommonToken#setStartIndex()

public void test_getTrimmedNodeRange() {
    String source = " hello! ";
    //               012345678
    CommonToken token = new CommonToken(Css3Lexer.IDENT);
    token.setText(source);
    token.setStartIndex(0);
    token.setStopIndex(7); //len - 1 -> points to last char not the end!
    
    Node node = new TokenNode(source, token);
    
    assertEquals(" hello! ", node.image().toString());
    int[] result = NodeUtil.getTrimmedNodeRange(node);

    assertEquals(1, result[0]);
    assertEquals(7, result[1]);
    
}
 

/**
 * Create a new token from the given prototype. Any argument besides the prototype is optional and
 * will be ignored if its value is <code>null</code>.
 */
protected CommonToken createToken(CommonToken prototype, String text, 
		Integer charPosInLine, Integer channel, Integer start, Integer stop, Integer type) {
	if (prototype == null)
		throw new IllegalArgumentException("Prototype may not be null.");
	CommonToken result = new CommonToken(prototype);
	if (text != null)
		result.setText(text);
	if (charPosInLine != null)
		result.setCharPositionInLine(charPosInLine.intValue());
	if (channel != null)
		result.setChannel(channel.intValue());
	if (start != null)
		result.setStartIndex(start.intValue());
	if (stop != null)
		result.setStopIndex(stop.intValue());
	if (type != null)
		result.setType(type.intValue());
	return result;
}
 

/**
 * @since 2.4
 */
protected RepairEntryData getRepairEntryData(DocumentEvent e) throws Exception {
	int tokenStartsAt = 0;
	int tokenInfoIdx = 0;
	for(tokenInfoIdx = 0; tokenInfoIdx< getInternalModifyableTokenInfos().size(); ++tokenInfoIdx) {
		TokenInfo oldToken = getInternalModifyableTokenInfos().get(tokenInfoIdx);
		if(tokenStartsAt <= e.getOffset() && tokenStartsAt + oldToken.getLength() >= e.getOffset())
			break;
		tokenStartsAt += oldToken.getLength();
	}
	final TokenSource delegate = createTokenSource(e.fDocument.get(tokenStartsAt, e.fDocument.getLength() - tokenStartsAt));
	final int offset = tokenStartsAt;
	TokenSource source = new TokenSource() {
		@Override
		public Token nextToken() {
			CommonToken commonToken = (CommonToken) delegate.nextToken();
			commonToken.setText(commonToken.getText());
			commonToken.setStartIndex(commonToken.getStartIndex()+offset);
			commonToken.setStopIndex(commonToken.getStopIndex()+offset);
			return commonToken;
		}

		@Override
		public String getSourceName() {
			return delegate.getSourceName();
		}
	};
	final CommonToken token = (CommonToken) source.nextToken();
	return new RepairEntryData(offset, tokenInfoIdx, token, source);
}
 

protected Token createEndToken(int offset) {
	CommonToken result = new CommonToken(getEndTokenType());
	result.setText("");
	result.setChannel(Token.DEFAULT_CHANNEL);
	result.setStartIndex(offset);
	result.setStopIndex(offset-1);
	return result;
}
 

protected Token createBeginToken(int offset) {
	CommonToken result = new CommonToken(getBeginTokenType());
	result.setText("");
	result.setChannel(Token.DEFAULT_CHANNEL);
	result.setStartIndex(offset);
	result.setStopIndex(offset-1);
	return result;
}
 

@Test public void testMergeFirstToken() {
	CommonToken commonToken = new CommonToken(TOKEN_TYPE, "text");
	commonToken.setStartIndex(4);
	AntlrDatatypeRuleToken token = new AntlrDatatypeRuleToken();
	token.merge(commonToken);
	assertEquals("text", token.getText());
	assertEquals(8, token.getExpectedOffset());
}
 

@Test public void testMergeSecondToken() {
	CommonToken tokenToMerge = new CommonToken(TOKEN_TYPE, "ext");
	tokenToMerge.setStartIndex(6);
	AntlrDatatypeRuleToken token = new AntlrDatatypeRuleToken();
	token.setStartOffset(tokenToMerge.getStartIndex() - 1);
	token.setExpectedOffset(tokenToMerge.getStartIndex());
	token.setText("t");
	token.merge(tokenToMerge);
	assertEquals("text", token.getText());
	assertEquals(9, token.getExpectedOffset());
}
 

@Test public void testMergeSecondTokenUnmatched() {
	CommonToken tokenToMerge = new CommonToken(TOKEN_TYPE, "ext");
	tokenToMerge.setStartIndex(6);
	AntlrDatatypeRuleToken token = new AntlrDatatypeRuleToken();
	token.setStartOffset(tokenToMerge.getStartIndex() - 4);
	token.setExpectedOffset(tokenToMerge.getStartIndex() - 3);
	token.setText("t");
	token.merge(tokenToMerge);
	assertEquals("t ext", token.getText());
	assertEquals(9, token.getExpectedOffset());
}