Java Code Examples for org.eclipse.jface.text.rules.IToken#isUndefined()

public IToken nextToken() {
IToken token;
tokenOffset = offset;
column = UNDEFINED;
Iterator<?> ruleIterator = rules.iterator();
while(ruleIterator.hasNext()) {
    IRule rule = (IRule)ruleIterator.next();
    token = rule.evaluate(this);
    if(!token.isUndefined()) {
        return token;
    }
    if(ruleIterator.hasNext()) {
    	offset = tokenOffset;
    }
}
if(rules.size() <= 0) {
	read();
}
if (read() == EOF) {
	return Token.EOF;
} else {
	unread();
	return defaultToken;
}
  }
 

@Override
public IToken nextToken()
{
	IToken token;
	if (!hasTokens)
	{
		hasTokens = true;
		token = findBrokenToken();
		if (!token.isUndefined())
		{
			return token;
		}
	}
	token = super.nextToken();
	return token;
}
 

protected void runRuleTest(String text, Integer[] tokenIndexes, Integer[] tokenLen) {
	StringCharacterScanner scanner = new StringCharacterScanner(text);
	 
	FullPatternRule fpRule = new FullPatternRule(new Token(null), sequences, new SampleJavaWordDetector());
	 
	while (scanner.peekNext() != ICharacterScanner.EOF) {
		int beginOffset = scanner.textOffset;
		IToken token = fpRule.evaluate(scanner);
		if(token.isUndefined()) {
			assertTrue(scanner.textOffset == beginOffset);
			assertTrue(ArrayUtil.contains(tokenIndexes, beginOffset) == false);
			scanner.read(); // advance
		} else {
			int indexOf = ArrayUtil.indexOf(tokenIndexes, beginOffset);
			assertTrue(indexOf != -1);
			assertTrue(scanner.textOffset == beginOffset + tokenLen[indexOf]);
		}
	}
}
 

protected IToken doNextToken() {
	fTokenOffset= fOffset;
	fColumn= UNDEFINED;

	for(int i = 0; i < fRules.length; i++) {
		fOffset = fTokenOffset; // Revert position changes that rules might have changed
		
		IToken token = fRules[i].evaluate(this);
		if(!token.isUndefined()) {
			return token; 
		}
	}
	
	if (read() == EOF)
		return Token.EOF;
	return fDefaultReturnToken;
}
 

protected IToken doEvaluate(ICharacterScanner scanner, boolean resume)
{
	int column = scanner.getColumn();
	IToken token = super.doEvaluate(scanner, resume);
	if (token.isUndefined())
		return token;

	// Make sure whole thing matches pattern
	int read = scanner.getColumn() - column;
	for (int i = 0; i < read; i++)
	{
		scanner.unread();
	}
	StringBuilder builder = new StringBuilder();
	for (int i = 0; i < read; i++)
	{
		builder.append((char) scanner.read());
	}
	String word = builder.toString();
	if (word.length() > 2 && ENTITY_PATTERN.matcher(word).find())
	{
		return token;
	}
	for (int i = 0; i < read; i++)
	{
		scanner.unread();
	}
	return Token.UNDEFINED;
}
 

@Override
public IToken evaluate(ICharacterScanner scanner, CombinedWordRule.CharacterBuffer word) {
	IToken token= super.evaluate(scanner, word);

	if (fIsVersionMatch || token.isUndefined())
		return token;

	return fDefaultToken;
}
 

public IToken evaluate(ICharacterScanner scanner) {
	int c= scanner.read();
	if (fDetector.isWordStart((char) c)) {
		if (fColumn == UNDEFINED || (fColumn == scanner.getColumn() - 1)) {

			fBuffer.clear();
			do {
				fBuffer.append((char) c);
				c= scanner.read();
			} while (c != ICharacterScanner.EOF && fDetector.isWordPart((char) c));
			scanner.unread();

			for (int i= 0, n= fMatchers.size(); i < n; i++) {
				IToken token= fMatchers.get(i).evaluate(scanner, fBuffer);
				if (!token.isUndefined())
					return token;
			}

			if (fDefaultToken.isUndefined())
				unreadBuffer(scanner);

			return fDefaultToken;
		}
	}

	scanner.unread();
	return Token.UNDEFINED;
}
 

@Override
public IToken nextToken()
{
	IToken token;
	if (!hasTokens)
	{
		hasTokens = true;
		token = findBrokenToken();
		if (!token.isUndefined())
		{
			return token;
		}
	}
	token = super.nextToken();
	if (rescanNestedLanguages && (doubleQuotedStringToken == token || singleQuotedStringToken == token))
	{
		IToken attributeToken = getAttributeToken();
		ITokenScanner tokenScanner = null;
		if (attributeScriptToken == attributeToken)
		{
			tokenScanner = jsTokenScanner;
		}
		else if (attributeStyleToken == attributeToken)
		{
			tokenScanner = cssTokenScanner;
		}
		tokenHistory.clear();
		int offset = getTokenOffset();
		int length = getTokenLength() - 2;
		if (tokenScanner != null && length > 0)
		{
			queueToken(token, offset, 1);
			queueDelegate(tokenScanner, offset + 1, length);
			queueToken(token, offset + length + 1, 1);
			return super.nextToken();
		}
	}
	if (!token.isWhitespace())
	{
		tokenHistory.push(token);
	}
	return token;
}
 

private IToken doResumeContentType() {
	if (fContentType == null) {
		return null;
	}
	// inside a partition
	fColumn = UNDEFINED;
	boolean resume = (fPartitionOffset > -1 && fPartitionOffset < fOffset);
	fTokenOffset = resume ? fPartitionOffset : fOffset;
	if (hasResume) {
		resume = true;
		hasResume = false;
	}

	IToken token;
	boolean doResetRules;
	do {
		doResetRules = false;
		for (IPredicateRule rule : currentPartitionScanner.getRules()) {
			token = rule.getSuccessToken();
			if (fContentType.equals(token.getData())) {
				token = rule.evaluate(currentPartitionScanner.getCharacterScanner(), resume);
				if (!token.isUndefined() && fOffset != fTokenOffset) {
					fContentType = null;
					currentPartitionScanner.setLastToken(token);
					currentPartitionScanner.doResetRules();
					return returnToken(token);
				}
				if (doResetRules = currentPartitionScanner.doResetRules()) {
					break;
				}
				if (hasSwitchingSequence()) {
					fContentType = null;
					return getDefaultToken();
				}
			}
		}
	} while (doResetRules);

	// haven't found any rule for this type of partition
	fContentType = null;
	if (resume && fPartitionOffset >= 0) {
		fOffset = fPartitionOffset;
		fPartitionOffset = -1;
	}
	return null;
}
 

public IToken evaluate( ICharacterScanner scanner, boolean resume )
{
	int column = scanner.getColumn( );
	int iCh = ' ';//Default it to space. This will be checked if the column
	// is zero
	//First check whether we are at the first column
	if ( column > 0 )
	{
		//if not unread and read the character
		scanner.unread( );
		iCh = scanner.read( );
	}
	IToken tokenToReturn = Token.UNDEFINED;
	buf.setLength( 0 );
	//We should only apply this rule if we have a valid preceding character
	if ( isValidPrecedingCharacter( iCh ) )
	{
		do
		{
			//Read the character
			iCh = scanner.read( );
			//append it to the buffer
			buf.append( Character.toLowerCase( (char) iCh ) );
		} while ( isKeywordStart( buf.toString( ) )
				&& iCh != ICharacterScanner.EOF );
	}

	//Check whether there is anything in the buffer
	if ( buf.length( ) > 0 )
	{
		//System.out.println("buffer contains " + buf.toString());
		//Check whether the last character read was the EOF character
		//or a space character
		if ( isValidTerminatingCharacter( iCh ) )
		{
			//If the length of the buffer is greater than 1
			if ( buf.length( ) > 1 )
			{
				//Strip out the last character
				String sToCompare = buf.substring( 0, buf.length( ) - 1 );
				//System.out.println("String is " + sToCompare);

				//Now check whether it is a keyword
				if ( isKeyword( sToCompare ) )
				{
					scanner.unread( );
					tokenToReturn = token;
				}
			}
		}

		if ( tokenToReturn.isUndefined( ) )
		{
			//if the token is undefined
			//then just unread the buffer
			unreadBuffer( scanner );
		}
	}

	return tokenToReturn;
}