org.eclipse.jface.text.rules.WhitespaceRule Java Examples

/**
 * Creates a new JSScanner object.
 * 
 */
public JSScanner( )
{
	List rules = new ArrayList( );

	// Add generic whitespace rule.
	rules.add( new WhitespaceRule( new IWhitespaceDetector( ) {

		public boolean isWhitespace( char c )
		{
			return Character.isWhitespace( c );
		}
	} ) );

	IRule[] result = new IRule[rules.size( )];
	rules.toArray( result );
	setRules( result );
}
 

/**
 *  
 */
public SQLKeywordScanner( )
{
	super( );
	IToken sqlKeywordsToken = new Token( new TextAttribute( ColorManager.getColor(127, 0, 85), null, SWT.BOLD ) );
	ArrayList rules = new ArrayList( );
	rules.add( new SQLKeywordRule( sqlKeywordsToken, reservedwords ) );
	rules.add( new SQLKeywordRule( sqlKeywordsToken, types ) );
	rules.add( new SQLKeywordRule( sqlKeywordsToken, constants ) );
	rules.add( new SQLKeywordRule( sqlKeywordsToken, functions ) );
	rules.add( new SQLKeywordRule( sqlKeywordsToken, predicates ) );
	
	// Add generic whitespace rule.
	rules.add( new WhitespaceRule( new IWhitespaceDetector( ) {

		public boolean isWhitespace( char c )
		{
			return Character.isWhitespace( c );
		}
	} ) );

	setRules( (IRule[]) rules.toArray( new IRule[rules.size( )] ) );
	this.setDefaultReturnToken( new Token( new TextAttribute( Display.getDefault( ).getSystemColor( SWT.COLOR_LIST_FOREGROUND ))));
}
 

@Override
protected List<IRule> createRules() {
	setDefaultReturnToken(getToken(PreferenceConstants.PROPERTIES_FILE_COLORING_VALUE));
	List<IRule> rules= new ArrayList<IRule>();

	// Add rule for arguments.
	IToken token= getToken(PreferenceConstants.PROPERTIES_FILE_COLORING_ARGUMENT);
	rules.add(new ArgumentRule(token));

	// Add word rule for assignment operator.
	token= getToken(PreferenceConstants.PROPERTIES_FILE_COLORING_ASSIGNMENT);
	WordRule wordRule= new WordRule(new AssignmentDetector(), token);
	rules.add(wordRule);

	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new JavaWhitespaceDetector()));

	return rules;
}
 

/**
 * A default constructor.
 * @param manager
 */
public TexArgScanner(ColorManager manager) {
    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    //Commands are colored in argument color with command styles 
    IToken commandToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.CURLY_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.COMMAND_STYLE)));

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new WordRule(new TexWord(), commandToken));

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}
 

/**
 * A default constructor.
 * @param manager
 */
public TexOptArgScanner(ColorManager manager) {
    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    //Commands are colored in argument color with command styles 
    IToken commandToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.SQUARE_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.COMMAND_STYLE)));

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new WordRule(new TexWord(), commandToken));

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}
 

/**
 * XMLTokenScanner
 */
public XMLTokenScanner()
{
	List<IRule> rules = new ArrayList<IRule>();

	rules.add(new WhitespaceRule(new WhitespaceDetector()));
	rules.add(new MultiLineRule("<!--", "-->", createToken(XMLTokenType.COMMENT))); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new DocTypeRule(createToken(XMLTokenType.DOCTYPE), false));
	rules.add(new MultiLineRule("<![CDATA[", "]]>", createToken(XMLTokenType.CDATA))); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new TagRule("?xml", createToken(XMLTokenType.DECLARATION))); //$NON-NLS-1$
	rules.add(new TagRule("/", createToken(XMLTokenType.END_TAG))); //$NON-NLS-1$
	rules.add(new TagRule(createToken(XMLTokenType.START_TAG)));

	// text
	IToken token = createToken(XMLTokenType.TEXT);
	rules.add(new WordRule(new WordDetector(), token));

	setRules(rules.toArray(new IRule[rules.size()]));
	setDefaultReturnToken(token);
}
 

public XMLTagScanner( ColorManager manager) {
   IToken string = new Token(new TextAttribute(manager.getColor(IXMLColorConstants.STRING)));

   IRule[] rules = new IRule[3];

   // Add rule for double quotes
   rules[0] = new SingleLineRule("\"", "\"", string, '\\');
   // Add a rule for single quotes
   rules[1] = new SingleLineRule("'", "'", string, '\\');
   // Add generic whitespace rule.
   rules[2] = new WhitespaceRule(new XMLWhitespaceDetector());

   setRules(rules);
}
 

@Override
	protected List<IRule> createRules() {
		setDefaultReturnToken(getToken(PreferenceConstants.EDITOR_CONFIG_COLORING_SECTION));
		List<IRule> rules = new ArrayList<IRule>();

		// Add rule for arguments.
//		IToken token = getToken(PreferenceConstants.PROPERTIES_FILE_COLORING_ARGUMENT);
//		rules.add(new ArgumentRule(token));

		// Add generic whitespace rule.
		rules.add(new WhitespaceRule(new EditorConfigWhitespaceDetector()));

		return rules;
	}
 

@Override
protected List<IRule> createRules() {
	setDefaultReturnToken(getToken(PreferenceConstants.EDITOR_CONFIG_COLORING_PROPERTY_VALUE));
	List<IRule> rules = new ArrayList<IRule>();

	// Add word rule for assignment operator.
	IToken token = getToken(PreferenceConstants.EDITOR_CONFIG_COLORING_ASSIGNMENT);
	WordRule wordRule = new WordRule(new AssignmentDetector(), token);
	rules.add(wordRule);

	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new EditorConfigWhitespaceDetector()));

	return rules;
}
 

@Override
protected List<IRule> createRules() {

	List<IRule> list= new ArrayList<IRule>();

	// Add rule for tags
	Token token= getToken(IJavaColorConstants.JAVADOC_TAG);
	list.add(new TagRule(token));


	// Add rule for HTML comments
	WordRule wordRule= new WordRule(new HTMLCommentDetector(), token);
	wordRule.addWord("<!--", token); //$NON-NLS-1$
	wordRule.addWord("--!>", token); //$NON-NLS-1$
	list.add(wordRule);


	// Add rules for links
	token= getToken(IJavaColorConstants.JAVADOC_LINK);
	list.add(new MultiLineRule("{@link", "}", token)); //$NON-NLS-2$ //$NON-NLS-1$
	list.add(new MultiLineRule("{@value", "}", token)); //$NON-NLS-2$ //$NON-NLS-1$
	list.add(new MultiLineRule("{@inheritDoc", "}", token)); //$NON-NLS-2$ //$NON-NLS-1$

	// Add rules for @code and @literals
	token= getToken(IJavaColorConstants.JAVADOC_DEFAULT);
	list.add(new MultiLineRule("{@code", "}", token)); //$NON-NLS-2$ //$NON-NLS-1$
	list.add(new MultiLineRule("{@literal", "}", token)); //$NON-NLS-2$ //$NON-NLS-1$

	// Add generic whitespace rule
	token= getToken(IJavaColorConstants.JAVADOC_DEFAULT);
	list.add(new WhitespaceRule(new JavaWhitespaceDetector(), token));


	list.addAll(super.createRules());
	return list;
}
 

public HTMLTokenScanner()
{
	List<IRule> rules = new ArrayList<IRule>();
	// generic whitespace rule
	rules.add(new WhitespaceRule(new WhitespaceDetector(), createToken(getTokenName(HTMLTokens.TEXT))));
	// comments
	rules.add(new PartitionerSwitchingIgnoreRule(new MultiLineRule(
			"<!--", "-->", createToken(getTokenName(HTMLTokens.COMMENT)), (char) 0, true))); //$NON-NLS-1$ //$NON-NLS-2$
	// DOCTYPE
	IToken token = createToken(getTokenName(HTMLTokens.DOCTYPE));
	rules.add(new MultiLineRule("<!DOCTYPE ", ">", token)); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new MultiLineRule("<!doctype ", ">", token)); //$NON-NLS-1$ //$NON-NLS-2$
	// CDATA
	rules.add(new MultiLineRule("<![CDATA[", "]]>", createToken(getTokenName(HTMLTokens.CDATA)))); //$NON-NLS-1$ //$NON-NLS-2$
	// script
	rules.add(new TagRule("script", createToken(getTokenName(HTMLTokens.SCRIPT)), true)); //$NON-NLS-1$
	rules.add(new TagRule("/script", createToken(getTokenName(HTMLTokens.SCRIPT_END)), true)); //$NON-NLS-1$
	// style
	rules.add(new TagRule("style", createToken(getTokenName(HTMLTokens.STYLE)), true)); //$NON-NLS-1$
	rules.add(new TagRule("/style", createToken(getTokenName(HTMLTokens.STYLE_END)), true)); //$NON-NLS-1$
	// xml declaration
	rules.add(new TagRule("?xml", createToken(getTokenName(HTMLTokens.XML_DECL)))); //$NON-NLS-1$
	// tags
	rules.add(new TagRule("/", createToken(getTokenName(HTMLTokens.END_TAG)))); //$NON-NLS-1$
	rules.add(generalTagRule = new HTMLParserTagRule(createToken(getTokenName(HTMLTokens.START_TAG))));

	// text
	token = createToken(getTokenName(HTMLTokens.TEXT));
	rules.add(new WordRule(new WordDetector(), token));

	setRules(rules.toArray(new IRule[rules.size()]));
	setDefaultReturnToken(token);
}
 

public HTMLDoctypeScanner()
{
	List<IRule> rules = new ArrayList<IRule>();

	// Add rule for double quotes
	rules.add(new MultiLineRule(
			"\"", "\"", createToken("string.quoted.double.doctype.identifiers-and-DTDs.html"), '\\')); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

	// Add a rule for single quotes
	rules.add(new MultiLineRule(
			"'", "'", createToken("string.quoted.single.doctype.identifiers-and-DTDs.html"), '\\')); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new WhitespaceDetector()));

	// Tags
	WordRule wordRule = new WordRule(new WordDetector(), createToken(StringUtil.EMPTY), true);
	wordRule.addWord("DOCTYPE", createToken("entity.name.tag.doctype.html")); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(wordRule);

	CharacterMapRule rule = new CharacterMapRule();
	rule.add('>', createToken(HTMLTokenType.TAG_END));
	rule.add('=', createToken(HTMLTokenType.EQUAL));
	rules.add(rule);
	// Tag start <(/)?
	rules.add(new TagStartRule(new TagStartWordDetector(), createToken(HTMLTokenType.TAG_START), false));

	setRules(rules.toArray(new IRule[rules.size()]));
	setDefaultReturnToken(createToken(StringUtil.EMPTY));
}
 

public HTMLScanner()
{
	IRule[] rules = new IRule[4];
	rules[0] = new WhitespaceRule(new WhitespaceDetector());
	rules[1] = new EntityRule(createToken("constant.character.entity.html")); //$NON-NLS-1$
	// non-entity ampersands should be marked as invalid
	rules[2] = new SingleCharacterRule('&', createToken("invalid.illegal.bad-ampersand.html")); //$NON-NLS-1$
	rules[3] = new WordRule(new WordDetector(), createToken("text")); //$NON-NLS-1$
	setRules(rules);
	setDefaultReturnToken(createToken("text")); //$NON-NLS-1$
}
 

/**
 * XMLAttributeScanner
 */
XMLAttributeScanner()
{
	List<IRule> rules = new ArrayList<IRule>();

	rules.add(new WhitespaceRule(new WhitespaceDetector()));

	// attribute values
	rules.add(new MultiLineRule("\"", "\"", createToken(XMLTokenType.VALUE), '\\')); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new MultiLineRule("'", "'", createToken(XMLTokenType.VALUE), '\\')); //$NON-NLS-1$ //$NON-NLS-2$

	// attribute names
	rules.add(new XMLAttributeRule(createToken(XMLTokenType.ATTRIBUTE)));

	// tag name
	rules.add(new WordRule(new DTDNameDetector(), createToken(XMLTokenType.OTHER), true));

	// special characters
	CharacterMapRule characterRule = new CharacterMapRule();
	characterRule.add('<', createToken(XMLTokenType.OTHER));
	characterRule.add('>', createToken(XMLTokenType.OTHER));
	characterRule.add('=', createToken(XMLTokenType.OTHER));
	rules.add(characterRule);

	setRules(rules.toArray(new IRule[rules.size()]));

	setDefaultReturnToken(createToken(XMLTokenType.OTHER));
}
 

public XMLTagScanner()
{
	List<IRule> rules = new ArrayList<IRule>();

	// Add rule for double quotes
	rules.add(new MultiLineRule("\"", "\"", doubleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new BreakingMultiLineRule("\"", "\"", QUOTED_STRING_BREAKS, doubleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$

	// Add a rule for single quotes
	rules.add(new MultiLineRule("'", "'", singleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new BreakingMultiLineRule("'", "'", QUOTED_STRING_BREAKS, singleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$

	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new WhitespaceDetector()));

	// Tags
	WordRule tagWordRule = new TagWordRule(new TagNameWordDetector(), createToken(XMLTokenType.TAG_NAME), true);
	rules.add(tagWordRule);

	// Attributes
	WordRule attributeWordRule = new WordRule(new AttributeNameWordDetector(), createToken(XMLTokenType.ATTRIBUTE),
			true);
	rules.add(attributeWordRule);

	rules.add(new MultiCharacterRule("</", createToken(XMLTokenType.START_TAG))); //$NON-NLS-1$
	rules.add(new MultiCharacterRule("/>", createToken(XMLTokenType.TAG_SELF_CLOSE))); //$NON-NLS-1$

	CharacterMapRule charsRule = new CharacterMapRule();
	charsRule.add('<', createToken(XMLTokenType.START_TAG));
	charsRule.add('>', createToken(XMLTokenType.END_TAG));
	charsRule.add('=', equalToken);
	rules.add(charsRule);

	setRules(rules.toArray(new IRule[rules.size()]));
	setDefaultReturnToken(createToken(XMLTokenType.TEXT));
}
 

public XMLScanner()
{
	IRule[] rules = new IRule[4];
	rules[0] = new WhitespaceRule(new WhitespaceDetector());
	rules[1] = new EntityRule(createToken(CONSTANT_CHARACTER_ENTITY_XML));
	// non-entity ampersands should be marked as invalid
	rules[2] = new SingleCharacterRule('&', createToken(INVALID_ILLEGAL_BAD_AMPERSAND_XML));
	rules[3] = new WordRule(new WordDetector(), createToken(TEXT));
	setRules(rules);
	setDefaultReturnToken(createToken(TEXT));
}
 

/**
 * A default constructor.
 * @param manager
 */
public TexScanner(ColorManager manager) {
    // A token that defines how to color numbers
    IToken numberToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.TEX_NUMBER),
            null,
            manager.getStyle(ColorManager.TEX_NUMBER_STYLE)));

    // A token that defines how to color command words (\command_word)
    IToken commandToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMAND),
            null,
            manager.getStyle(ColorManager.COMMAND_STYLE)));

    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    // A token that defines how to color special characters (\_, \&, \~ ...)
    IToken specialCharToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.TEX_SPECIAL),
            null,
            manager.getStyle(ColorManager.TEX_SPECIAL_STYLE)));
    
    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new TexSpecialCharRule(specialCharToken));
    rules.add(new WordRule(new TexWord(), commandToken));
    rules.add(new NumberRule(numberToken));
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    
    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}
 

/**
 * A default constructor. 
 * @param manager
 */
public TexCommentScanner(ColorManager manager) {					
    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new WhitespaceRule(new WhitespaceDetector()));		
    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);		
}
 

/**
 * Creates a BibTeX document scanner
 * 
 * @param provider The color provider for syntax highlighting
 */
public BibCodeScanner(BibColorProvider provider) {

    IToken keyword = new Token(new TextAttribute(provider
            .getColor(BibColorProvider.KEYWORD)));
    IToken comment = new Token(new TextAttribute(provider
            .getColor(BibColorProvider.SINGLE_LINE_COMMENT)));
    IToken other = new Token(new TextAttribute(provider
            .getColor(BibColorProvider.DEFAULT)));

    List rules = new ArrayList();

    // Add rule for single line comments.
    rules.add(new EndOfLineRule("%", comment));

    rules.add(new BibCommandRule(keyword));

    // Add generic whitespace rule.
    rules.add(new WhitespaceRule(new WhitespaceDetector()));

    // Add word rule for keywords, types, and constants.
    WordRule wordRule = new WordRule(new BibWordDetector(), other);
    rules.add(wordRule);

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}
 

public XMLScanner( ColorManager manager) {
   IToken procInstr = new Token(new TextAttribute(manager.getColor(IXMLColorConstants.PROC_INSTR)));
   IToken docType = new Token(new TextAttribute(manager.getColor(IXMLColorConstants.DOCTYPE)));

   IRule[] rules = new IRule[3];
   // Add rule for processing instructions and doctype
   rules[0] = new MultiLineRule("<?", "?>", procInstr);
   rules[1] = new MultiLineRule("<!DOCTYPE", ">", docType);
   // Add generic whitespace rule.
   rules[2] = new WhitespaceRule(new XMLWhitespaceDetector());

   setRules(rules);
}
 

/**
 * 
 */
public DTDTagScanner()
{
	List<IRule> rules = new ArrayList<IRule>();

	rules.add(new WhitespaceRule(new WhitespaceDetector()));

	// NOTE: There is no String, but we're using this to generalize pubid,
	// att value, entity value
	rules.add(new MultiLineRule("\"", "\"", createToken(DTDTokenType.STRING), '\0', true)); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new MultiLineRule("'", "'", createToken(DTDTokenType.STRING), '\0', true)); //$NON-NLS-1$ //$NON-NLS-2$

	WordRule operatorRule = new WordRule(new DTDOperatorWordDetector(), Token.UNDEFINED);
	operatorRule.addWord("<![", createToken(DTDTokenType.SECTION_START)); //$NON-NLS-1$
	operatorRule.addWord("]]>", createToken(DTDTokenType.SECTION_END)); //$NON-NLS-1$
	rules.add(operatorRule);

	WordRule wordRule = new WordRule(new DTDWordDetector(), Token.UNDEFINED);
	wordRule.addWord("<!ATTLIST", createToken(DTDTokenType.ATTLIST)); //$NON-NLS-1$
	wordRule.addWord("<!ELEMENT", createToken(DTDTokenType.ELEMENT)); //$NON-NLS-1$
	wordRule.addWord("<!ENTITY", createToken(DTDTokenType.ENTITY)); //$NON-NLS-1$
	wordRule.addWord("<!NOTATION", createToken(DTDTokenType.NOTATION)); //$NON-NLS-1$
	wordRule.addWord("#FIXED", createToken(DTDTokenType.FIXED)); //$NON-NLS-1$
	wordRule.addWord("#IMPLIED", createToken(DTDTokenType.IMPLIED)); //$NON-NLS-1$
	wordRule.addWord("#PCDATA", createToken(DTDTokenType.PCDATA)); //$NON-NLS-1$
	wordRule.addWord("#REQUIRED", createToken(DTDTokenType.REQUIRED)); //$NON-NLS-1$
	wordRule.addWord("ANY", createToken(DTDTokenType.ANY)); //$NON-NLS-1$
	wordRule.addWord("CDATA", createToken(DTDTokenType.CDATA_TYPE)); //$NON-NLS-1$
	wordRule.addWord("EMPTY", createToken(DTDTokenType.EMPTY)); //$NON-NLS-1$
	wordRule.addWord("ENTITY", createToken(DTDTokenType.ENTITY_TYPE)); //$NON-NLS-1$
	wordRule.addWord("ENTITIES", createToken(DTDTokenType.ENTITIES_TYPE)); //$NON-NLS-1$
	wordRule.addWord("ID", createToken(DTDTokenType.ID_TYPE)); //$NON-NLS-1$
	wordRule.addWord("IDREF", createToken(DTDTokenType.IDREF_TYPE)); //$NON-NLS-1$
	wordRule.addWord("IDREFS", createToken(DTDTokenType.IDREFS_TYPE)); //$NON-NLS-1$
	wordRule.addWord("IGNORE", createToken(DTDTokenType.IGNORE)); //$NON-NLS-1$
	wordRule.addWord("INCLUDE", createToken(DTDTokenType.INCLUDE)); //$NON-NLS-1$
	wordRule.addWord("NDATA", createToken(DTDTokenType.NDATA)); //$NON-NLS-1$
	wordRule.addWord("NMTOKEN", createToken(DTDTokenType.NMTOKEN_TYPE)); //$NON-NLS-1$
	wordRule.addWord("NMTOKENS", createToken(DTDTokenType.NMTOKENS_TYPE)); //$NON-NLS-1$
	wordRule.addWord("NOTATION", createToken(DTDTokenType.NOTATION_TYPE)); //$NON-NLS-1$
	wordRule.addWord("PUBLIC", createToken(DTDTokenType.PUBLIC)); //$NON-NLS-1$
	wordRule.addWord("SYSTEM", createToken(DTDTokenType.SYSTEM)); //$NON-NLS-1$
	rules.add(wordRule);

	rules.add(new DTDEntityRule('%', createToken(DTDTokenType.PE_REF)));
	rules.add(new DTDEntityRule('&', createToken(DTDTokenType.PE_REF)));

	CharacterMapRule charsRule = new CharacterMapRule();
	charsRule.add('>', createToken(DTDTokenType.GREATER_THAN));
	charsRule.add('(', createToken(DTDTokenType.LPAREN));
	charsRule.add('|', createToken(DTDTokenType.PIPE));
	charsRule.add(')', createToken(DTDTokenType.RPAREN));
	charsRule.add('?', createToken(DTDTokenType.QUESTION));
	charsRule.add('*', createToken(DTDTokenType.STAR));
	charsRule.add('+', createToken(DTDTokenType.PLUS));
	charsRule.add(',', createToken(DTDTokenType.COMMA));
	charsRule.add('%', createToken(DTDTokenType.PERCENT));
	charsRule.add('[', createToken(DTDTokenType.LBRACKET));
	rules.add(charsRule);

	rules.add(new WordRule(new DTDNameDetector(), createToken(DTDTokenType.NAME)));
	rules.add(new WordRule(new DTDNmtokenWordDetector(), createToken(DTDTokenType.NMTOKEN)));

	setRules(rules.toArray(new IRule[rules.size()]));
}
 

/**
 * createRules
 * 
 * @return
 */
protected List<IRule> createRules()
{
	List<IRule> rules = new ArrayList<IRule>();

	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new WhitespaceDetector()));

	// multi-line comments
	rules.add(new MultiLineRule("/*", "*/", createToken(CSSTokenType.COMMENT), (char) 0, true));

	// strings
	rules.addAll(createStringRules());

	// at-keywords
	rules.add(createAtWordsRule());

	// units
	rules.addAll(createUnitRules());

	// numbers
	rules.add(new CSSNumberRule(createToken(CSSTokenType.NUMBER)));

	// hex colors
	// TODO: we need separate scanners for inside and outside of rules. This will erroneously pick up some ids as
	// well
	rules.add(new CSSHexColorRule(createToken(CSSTokenType.RGB)));

	// classes;
	rules.add(new WordRule(new IdentifierWithPrefixDetector('.'), createToken(CSSTokenType.CLASS)));

	// ids
	rules.add(new WordRule(new IdentifierWithPrefixDetector('#'), createToken(CSSTokenType.ID)));

	// !important
	rules.add(new CSSImportantRule(createToken(CSSTokenType.IMPORTANT)));

	// url
	rules.add(new CSSURLRule(createToken(CSSTokenType.URL)));

	// TODO: functions

	// TODO: Unicode

	// identifiers
	rules.add(new CSSIdentifierRule(createToken(CSSTokenType.IDENTIFIER)));

	// single character punctuators
	rules.add(createPunctuatorsRule());

	// multi-character punctuators
	WordRule punctuatorRule2 = new WordRule(new EqualOperatorWordDetector(), Token.UNDEFINED);
	punctuatorRule2.addWord("~=", createToken(CSSTokenType.INCLUDES));
	punctuatorRule2.addWord("|=", createToken(CSSTokenType.DASHMATCH));
	punctuatorRule2.addWord("^=", createToken(CSSTokenType.BEGINS_WITH));
	punctuatorRule2.addWord("$=", createToken(CSSTokenType.ENDS_WITH));
	rules.add(punctuatorRule2);

	return rules;
}
 

public JsniScanner(IColorManager colorManager) {
  // TODO: get these from GWT preference store instead of Java's

  // TODO: need to refresh colorManager when prefs change
  //
  IToken jsniJavaRef = new Token(new TextAttribute(
      colorManager.getColor(JsniColorConstants.JSNI_JAVA_REF)));

  IToken jsniComment = new Token(new TextAttribute(
      colorManager.getColor(JsniColorConstants.JSNI_COMMENT)));

  IToken jsniKeyword = new Token(new TextAttribute(
      colorManager.getColor(JsniColorConstants.JSNI_KEYWORD)));

  IToken jsniDefault = new Token(new TextAttribute(
      colorManager.getColor(JsniColorConstants.JSNI_DEFAULT)));

  IToken jsniString = new Token(new TextAttribute(
      colorManager.getColor(JsniColorConstants.JSNI_STRING)));

  ArrayList<IRule> rules = new ArrayList<IRule>();

  // Java references
  rules.add(new JavaRefRule(jsniJavaRef));

  // single line comments
  rules.add(new EndOfLineRule("//", jsniComment));

  // JS keywords
  WordRule keywordRules = new WordRule(new JsWordDetector(), jsniDefault);
  for (String keyword : JS_KEYWORDS) {
    keywordRules.addWord(keyword, jsniKeyword);
  }
  rules.add(keywordRules);

  // Strings
  rules.add(new SingleLineRule("\"", "\"", jsniString, '\\', true));
  rules.add(new SingleLineRule("'", "'", jsniString, '\\', true));

  // Add generic whitespace rule.
  rules.add(new WhitespaceRule(new WhitespaceDetector()));

  // convert to array and set them for RuleBasedScanner to use
  setRules(rules.toArray(new IRule[rules.size()]));
}
 

protected List createRules() {

		List rules= new ArrayList();

		// Add rule for character constants.
		Token token= getToken(IJavaScriptColorConstants.JAVA_STRING);
		rules.add(new SingleLineRule("'", "'", token, '\\')); //$NON-NLS-2$ //$NON-NLS-1$


		// Add generic whitespace rule.
		rules.add(new WhitespaceRule(new JavaWhitespaceDetector()));

		String version= getPreferenceStore().getString(SOURCE_VERSION);

		// Add JLS3 rule for /@\s*interface/ and /@\s*\w+/
		token = getToken(ITypeScriptColorConstants.DECORATOR);
		DecoratorRule decoratorRule = new DecoratorRule(token,
				"", version);
		rules.add(decoratorRule);
		
		// Add word rule for new keywords, 4077
		JavaWordDetector wordDetector= new JavaWordDetector();
		token= getToken(IJavaScriptColorConstants.JAVA_DEFAULT);
		CombinedWordRule combinedWordRule= new CombinedWordRule(wordDetector, token);
		
		// Add rule for operators
		token= getToken(IJavaScriptColorConstants.JAVA_OPERATOR);
		rules.add(new OperatorRule(token));

		// Add rule for brackets
		token= getToken(IJavaScriptColorConstants.JAVA_BRACKET);
		rules.add(new BracketRule(token));

		// Add word rule for keyword 'return'.
		CombinedWordRule.WordMatcher returnWordRule= new CombinedWordRule.WordMatcher();
		token= getToken(IJavaScriptColorConstants.JAVA_KEYWORD_RETURN);
		returnWordRule.addWord(RETURN, token);  
		combinedWordRule.addWordMatcher(returnWordRule);

		// Add word rule for keywords, types, and constants.
		CombinedWordRule.WordMatcher wordRule= new CombinedWordRule.WordMatcher();
		token= getToken(IJavaScriptColorConstants.JAVA_KEYWORD);
		for (int i=0; i<fgKeywords.length; i++)
			wordRule.addWord(fgKeywords[i], token);
		for (int i=0; i<fgConstants.length; i++)
			wordRule.addWord(fgConstants[i], token);
		// TypeScript key words
		for (int i=0; i<fgTypeScriptKeywords.length; i++)
			wordRule.addWord(fgTypeScriptKeywords[i], token);

		combinedWordRule.addWordMatcher(wordRule);

		rules.add(combinedWordRule);

		setDefaultReturnToken(getToken(IJavaScriptColorConstants.JAVA_DEFAULT));
		return rules;
	}
 

/**
 * A default constructor. 
 * @param manager
 */
public TexMathScanner(ColorManager manager) {
    IToken defaultToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.EQUATION),
                    null,
                    manager.getStyle(ColorManager.EQUATION_STYLE)));

    IToken commentToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.COMMENT),
                    null,
                    manager.getStyle(ColorManager.COMMENT_STYLE)));

    //Commands are colored in math color with command styles 
    IToken commandToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.EQUATION),
                    null,
                    manager.getStyle(ColorManager.COMMAND_STYLE)));
    // A token that defines how to color special characters (\_, \&, \~ ...)
    IToken specialCharToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.TEX_SPECIAL),
            null,
            manager.getStyle(ColorManager.TEX_SPECIAL_STYLE)));
    
    List<IRule> rules = new ArrayList<IRule>();
    
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new TexSpecialCharRule(specialCharToken));
    //rules.add(new SingleLineRule("\\%", " ", specialCharToken));
    rules.add(new EndOfLineRule("%", commentToken));
    /*rules.add(new TexEnvironmentRule("comment", commentToken));
    rules.add(new SingleLineRule("\\[", " ", defaultToken));
    rules.add(new SingleLineRule("\\]", " ", defaultToken));
    rules.add(new SingleLineRule("\\(", " ", defaultToken));
    rules.add(new SingleLineRule("\\)", " ", defaultToken));*/
    rules.add(new WordRule(new TexWord(), commandToken));
    
    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);		
}
 

/**
 * A default constructor.
 * @param manager
 */
public TexTikzScanner(ColorManager manager) {
    IToken commandToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.COMMAND),
                    null,
                    manager.getStyle(ColorManager.COMMAND_STYLE)));

    IToken specialCharToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.TEX_SPECIAL),
            null,
            manager.getStyle(ColorManager.TEX_SPECIAL_STYLE)));

    IToken numberToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.TEX_NUMBER),
            null,
            manager.getStyle(ColorManager.TEX_NUMBER_STYLE)));

    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    IToken argToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.CURLY_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.CURLY_BRACKETS_STYLE)));

    IToken optArgToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.SQUARE_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.SQUARE_BRACKETS_STYLE)));

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new TexSpecialCharRule(specialCharToken));
    rules.add(new WordRule(new TexWord(), commandToken));
    rules.add(new NumberRule(numberToken));
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new MultiLineRule("{", "}", argToken, '\\'));
    rules.add(new MultiLineRule("[", "]", optArgToken, '\\'));

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}
 

@Override
protected List<IRule> createRules() {

	List<IRule> rules= new ArrayList<IRule>();

	// Add rule for character constants.
	Token token= getToken(IJavaColorConstants.JAVA_STRING);
	rules.add(new SingleLineRule("'", "'", token, '\\')); //$NON-NLS-2$ //$NON-NLS-1$


	Token defaultToken= getToken(IJavaColorConstants.JAVA_DEFAULT);
	
	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new JavaWhitespaceDetector(), defaultToken));

	String version= getPreferenceStore().getString(SOURCE_VERSION);

	// Add JLS3 rule for /@\s*interface/ and /@\s*\w+/
	token= getToken(ANNOTATION_COLOR_KEY);
	AnnotationRule atInterfaceRule= new AnnotationRule(getToken(IJavaColorConstants.JAVA_KEYWORD), token, JavaCore.VERSION_1_5, version);
	rules.add(atInterfaceRule);
	fVersionDependentRules.add(atInterfaceRule);

	// Add word rule for new keywords, see bug 4077
	JavaWordDetector wordDetector= new JavaWordDetector();
	CombinedWordRule combinedWordRule= new CombinedWordRule(wordDetector, defaultToken);

	VersionedWordMatcher j14Matcher= new VersionedWordMatcher(defaultToken, JavaCore.VERSION_1_4, version);

	token= getToken(IJavaColorConstants.JAVA_KEYWORD);
	for (int i=0; i<fgJava14Keywords.length; i++)
		j14Matcher.addWord(fgJava14Keywords[i], token);

	combinedWordRule.addWordMatcher(j14Matcher);
	fVersionDependentRules.add(j14Matcher);

	VersionedWordMatcher j15Matcher= new VersionedWordMatcher(defaultToken, JavaCore.VERSION_1_5, version);
	
	token= getToken(IJavaColorConstants.JAVA_KEYWORD);
	for (int i=0; i<fgJava15Keywords.length; i++)
		j15Matcher.addWord(fgJava15Keywords[i], token);

	combinedWordRule.addWordMatcher(j15Matcher);
	fVersionDependentRules.add(j15Matcher);

	// Add rule for operators
	token= getToken(IJavaColorConstants.JAVA_OPERATOR);
	rules.add(new OperatorRule(token));

	// Add rule for brackets
	token= getToken(IJavaColorConstants.JAVA_BRACKET);
	rules.add(new BracketRule(token));

	// Add word rule for keyword 'return'.
	CombinedWordRule.WordMatcher returnWordRule= new CombinedWordRule.WordMatcher();
	token= getToken(IJavaColorConstants.JAVA_KEYWORD_RETURN);
	returnWordRule.addWord(RETURN, token);
	combinedWordRule.addWordMatcher(returnWordRule);

	// Add word rule for keywords, types, and constants.
	CombinedWordRule.WordMatcher wordRule= new CombinedWordRule.WordMatcher();
	token= getToken(IJavaColorConstants.JAVA_KEYWORD);
	for (int i=0; i<fgKeywords.length; i++)
		wordRule.addWord(fgKeywords[i], token);
	for (int i=0; i<fgTypes.length; i++)
		wordRule.addWord(fgTypes[i], token);
	for (int i=0; i<fgConstants.length; i++)
		wordRule.addWord(fgConstants[i], token);

	combinedWordRule.addWordMatcher(wordRule);

	rules.add(combinedWordRule);

	setDefaultReturnToken(defaultToken);
	return rules;
}
 

/**
     * Creates a BibTeX entry scanner
     */
    public BibEntryScanner(BibColorProvider provider) {

        IToken keyword = new Token(new TextAttribute(provider
                .getColor(BibColorProvider.KEYWORD)));
        IToken type = new Token(new TextAttribute(provider
                .getColor(BibColorProvider.TYPE)));
        IToken string = new Token(new TextAttribute(provider
                .getColor(BibColorProvider.STRING)));
        IToken comment = new Token(new TextAttribute(provider
                .getColor(BibColorProvider.SINGLE_LINE_COMMENT)));
        IToken other = new Token(new TextAttribute(provider
                .getColor(BibColorProvider.DEFAULT)));

        List rules = new ArrayList();

        // Add rule for single line comments.
        // Not supported inside entries.
//        rules.add(new EndOfLineRule("%", comment));

        rules.add(new BibSeparatorRule(keyword));

        // Add rule for strings and character constants.
        // Note that escaping is not possible in BibTeX.
//        rules.add(new MultiLineRule("\"", "\"", string));
//        rules.add(new MultiLineRule("\"", "\"", string, (char) 0, true));
        rules.add(new BibStringRule(string));

        // this must be preceded by # or =
//        rules.add(new BibBraceRule(false, string));

        // Add generic whitespace rule.
        rules.add(new WhitespaceRule(new WhitespaceDetector()));

        // Add word rule for keywords, types, and constants.
        WordRule wordRule = new WordRule(new BibWordDetector(), type);
//        wordRule.addWord("author", keyword);
        rules.add(wordRule);

        rules.add(new NumberRule(string));
        
        IRule[] result= new IRule[rules.size()];
        rules.toArray(result);
        setRules(result);
    }
 

@Override
protected void initRules(ArrayList2<IRule> rules) {
	
	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new LangWhitespaceDetector()));
	
	
	final IToken tkDefault = getToken(GoUIPreferenceConstants.DEFAULT);
	
	WordRule wordRule = new WordRule(new JavaWordDetector(), tkDefault);
	
	final IToken keyword         = getToken(GoUIPreferenceConstants.KEYWORD);
	final IToken primitive       = getToken(GoUIPreferenceConstants.KW_NATIVE_TYPES);
	final IToken builtinFunction = getToken(GoUIPreferenceConstants.BUILTIN_FUNCTION);
	final IToken textToken       = getToken(GoUIPreferenceConstants.DEFAULT);
	setDefaultReturnToken(textToken);
	
	// add tokens for each reserved word
	wordRule.addWord("break",       keyword);
	wordRule.addWord("default",     keyword);
	wordRule.addWord("func",        keyword);
	wordRule.addWord("interface",   keyword);
	wordRule.addWord("select",      keyword);
	wordRule.addWord("case",        keyword);
	wordRule.addWord("defer",       keyword);
	wordRule.addWord("go",          keyword);
	wordRule.addWord("map",         keyword);
	wordRule.addWord("struct",      keyword);
	wordRule.addWord("chan",        keyword);
	wordRule.addWord("else",        keyword);
	wordRule.addWord("goto",        keyword);
	wordRule.addWord("package",     keyword);
	wordRule.addWord("switch",      keyword);
	wordRule.addWord("const",       keyword);
	wordRule.addWord("fallthrough", keyword);
	wordRule.addWord("if",          keyword);
	wordRule.addWord("range",       keyword);
	wordRule.addWord("type",        keyword);
	wordRule.addWord("continue",    keyword);
	wordRule.addWord("for",         keyword);
	wordRule.addWord("import",      keyword);
	wordRule.addWord("return",      keyword);
	wordRule.addWord("var",         keyword);
	
	wordRule.addWord("iota",        keyword);
	
	wordRule.addWord("append",  builtinFunction);
	wordRule.addWord("cap",     builtinFunction);
	wordRule.addWord("close",   builtinFunction);
	wordRule.addWord("complex", builtinFunction);
	wordRule.addWord("copy",    builtinFunction);
	wordRule.addWord("delete",  builtinFunction);
	wordRule.addWord("imag",    builtinFunction);
	wordRule.addWord("len",     builtinFunction);
	wordRule.addWord("make",    builtinFunction);
	wordRule.addWord("new",     builtinFunction);
	wordRule.addWord("panic",   builtinFunction);
	wordRule.addWord("print",   builtinFunction);
	wordRule.addWord("println", builtinFunction);
	wordRule.addWord("real",    builtinFunction);
	wordRule.addWord("recover", builtinFunction);
	
	final IToken value = getToken(GoUIPreferenceConstants.KW_LITERAL);
	wordRule.addWord("nil",   value);
	wordRule.addWord("true",  value);
	wordRule.addWord("false", value);
	
	wordRule.addWord("uint8", primitive);
	wordRule.addWord("uint16", primitive);
	wordRule.addWord("uint32", primitive);
	wordRule.addWord("uint64", primitive);
	wordRule.addWord("int8", primitive);
	wordRule.addWord("int16", primitive);
	wordRule.addWord("int32", primitive);
	wordRule.addWord("int64", primitive);
	wordRule.addWord("float32", primitive);
	wordRule.addWord("float64", primitive);
	wordRule.addWord("complex64",  primitive);
	wordRule.addWord("complex128", primitive);
	wordRule.addWord("rune",       primitive);
	wordRule.addWord("byte",       primitive);
	wordRule.addWord("uint",       primitive);
	wordRule.addWord("int",        primitive);
	wordRule.addWord("uintptr",    primitive);
	
	wordRule.addWord("string", primitive);
	wordRule.addWord("bool",   primitive);
	wordRule.addWord("error",  primitive);
	
	rules.add(wordRule);
	
	rules.add(new LexingRule_RuleAdapter(new GoSubLexer(getToken(GoUIPreferenceConstants.NUMBER))));
	
	rules.add(new GoOperatorRule(getToken(GoUIPreferenceConstants.OPERATOR)));
	rules.add(new GoControlCharactersRule(getToken(GoUIPreferenceConstants.STRUCTURAL_SYMBOLS)));
}