com.intellij.lexer.Lexer#start

Source File: SelectWordUtil.java From consulo with Apache License 2.0

6 votes

public static void addWordHonoringEscapeSequences(CharSequence editorText,
                                                  TextRange literalTextRange,
                                                  int cursorOffset,
                                                  Lexer lexer,
                                                  List<TextRange> result) {
  lexer.start(editorText, literalTextRange.getStartOffset(), literalTextRange.getEndOffset());

  while (lexer.getTokenType() != null) {
    if (lexer.getTokenStart() <= cursorOffset && cursorOffset < lexer.getTokenEnd()) {
      if (StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(lexer.getTokenType())) {
        result.add(new TextRange(lexer.getTokenStart(), lexer.getTokenEnd()));
      }
      else {
        TextRange word = getWordSelectionRange(editorText, cursorOffset, JAVA_IDENTIFIER_PART_CONDITION);
        if (word != null) {
          result.add(new TextRange(Math.max(word.getStartOffset(), lexer.getTokenStart()),
                                   Math.min(word.getEndOffset(), lexer.getTokenEnd())));
        }
      }
      break;
    }
    lexer.advance();
  }
}

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

6 votes

@Test
@SuppressWarnings("unchecked")
public void testProperty() throws Exception {
	Lexer lexer = new LatteLexer();
	lexer.start("{$object->foo}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_IDENTIFIER, "foo"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{$object->foo->bar}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_IDENTIFIER, "foo"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_IDENTIFIER, "bar"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

6 votes

@Test
@SuppressWarnings("unchecked")
public void testBasic() throws Exception {
	Lexer lexer = new LatteLexer();
	lexer.start("{if $var}B{/if}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$var"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
		Pair.create(T_TEXT, "B"),
		Pair.create(T_MACRO_CLOSE_TAG_OPEN, "{/"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

6 votes

@Test
@SuppressWarnings("unchecked")
public void testVarDefinition() throws Exception {
	Lexer lexer = new LatteLexer();
	lexer.start("{var $foo = 123}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "var"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$foo"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_DEFINITION_OPERATOR, "="),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_NUMBER, "123"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Source File: LexerTestCase.java From consulo with Apache License 2.0

6 votes

public static String printTokens(CharSequence text, int start, Lexer lexer) {
  lexer.start(text, start, text.length());
  String result = "";
  while (true) {
    IElementType tokenType = lexer.getTokenType();
    if (tokenType == null) {
      break;
    }
    String tokenText = getTokenText(lexer);
    String tokenTypeName = tokenType.toString();
    String line = tokenTypeName + " ('" + tokenText + "')\n";
    result += line;
    lexer.advance();
  }
  return result;
}

Source File: AndroidUtils.java From flutter-intellij with BSD 3-Clause "New" or "Revised" License

5 votes

@Nullable
public static String isReservedKeyword(@NotNull String string) {
  Lexer lexer = JAVA_LEXER;
  lexer.start(string);
  if (lexer.getTokenType() != JavaTokenType.IDENTIFIER) {
    if (lexer.getTokenType() instanceof IKeywordElementType) {
      return "Package names cannot contain Java keywords like '" + string + "'";
    }
    if (string.isEmpty()) {
      return "Package segments must be of non-zero length";
    }
    return string + " is not a valid identifier";
  }
  return null;
}

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

5 votes

@Test
@SuppressWarnings("unchecked")
public void testArrayItemsWithDoubleArrow() throws Exception {
	Lexer lexer = new LatteHighlightingLexer(new LatteLexer());
	lexer.start("{block test, te => $item, test . 1 => 123}{/block}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "block"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "test"),
		Pair.create(T_MACRO_ARGS, ","),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "te"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS, "=>"), //todo: weird behavior, it must be T_PHP_DOUBLE_ARROW (it is because is workaround for "=>" in LatteParser.bnf)
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$item"),
		Pair.create(T_MACRO_ARGS, ","),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "test"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_CONCATENATION, "."),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_NUMBER, "1"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_DOUBLE_ARROW, "=>"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_NUMBER, "123"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
		Pair.create(T_MACRO_CLOSE_TAG_OPEN, "{/"),
		Pair.create(T_MACRO_NAME, "block"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Source File: LatteTopLexerAdapterTest.java From intellij-latte with MIT License

5 votes

@Test
@SuppressWarnings("unchecked")
public void testHtml() {
	Lexer lexer = new LatteTopLexerAdapter();

	lexer.start("");
	assertTokens(lexer, new Pair[] {});

	lexer.start("abc<strong>def</strong>ghi");
	assertTokens(lexer, new Pair[] {
			Pair.create(T_TEXT, "abc"),
			Pair.create(T_HTML_OPEN_TAG_OPEN, "<"),
			Pair.create(T_TEXT, "strong"),
			Pair.create(T_HTML_TAG_CLOSE, ">"),
			Pair.create(T_TEXT, "def"),
			Pair.create(T_HTML_CLOSE_TAG_OPEN, "</"),
			Pair.create(T_TEXT, "strong"),
			Pair.create(T_HTML_TAG_CLOSE, ">"),
			Pair.create(T_TEXT, "ghi"),
	});

	lexer.start("abc<img src=\"\" />ghi");
	assertTokens(lexer, new Pair[] {
			Pair.create(T_TEXT, "abc"),
			Pair.create(T_HTML_OPEN_TAG_OPEN, "<"),
			Pair.create(T_TEXT, "img src=\"\" "),
			Pair.create(T_HTML_OPEN_TAG_CLOSE, "/>"),
			Pair.create(T_TEXT, "ghi"),
	});

	lexer.start("abc<style></style>");
	assertTokens(lexer, new Pair[] {
			Pair.create(T_TEXT, "abc"),
			Pair.create(T_HTML_OPEN_TAG_OPEN, "<"),
			Pair.create(T_TEXT, "style></style"),
			Pair.create(T_HTML_TAG_CLOSE, ">"),
	});
}

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

5 votes

@Test
@SuppressWarnings("unchecked")
public void testStaticProperty() throws Exception {
	Lexer lexer = new LatteLexer();
	lexer.start("{$object::$foo}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_MACRO_ARGS_VAR, "$foo"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{\\Foo\\Bar::$foo::$bar}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_MACRO_ARGS_VAR, "$foo"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_MACRO_ARGS_VAR, "$bar"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{Foo\\Bar::$foo}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_IDENTIFIER, "Bar"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_MACRO_ARGS_VAR, "$foo"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

5 votes

@Test
@SuppressWarnings("unchecked")
public void testMethod() throws Exception {
	Lexer lexer = new LatteHighlightingLexer(new LatteLexer());
	lexer.start("{$object->getFoo()}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_METHOD, "getFoo"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{$object->getFoo()->getBar($var)}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_METHOD, "getFoo"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_METHOD, "getBar"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_MACRO_ARGS_VAR, "$var"),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Source File: LatteLexerTest.java From intellij-latte with MIT License

5 votes

@Test
@SuppressWarnings("unchecked")
public void testBasic() throws Exception {
	Lexer lexer = new LatteLexer();

	lexer.start("<e>{if XYZ}B{/if}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_HTML_OPEN_TAG_OPEN, "<"),
		Pair.create(T_TEXT, "e"),
		Pair.create(T_HTML_TAG_CLOSE, ">"),
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "XYZ"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
		Pair.create(T_TEXT, "B"),
		Pair.create(T_MACRO_CLOSE_TAG_OPEN, "{/"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{if XYZ}B{/if}<e>");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "XYZ"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
		Pair.create(T_TEXT, "B"),
		Pair.create(T_MACRO_CLOSE_TAG_OPEN, "{/"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
		Pair.create(T_HTML_OPEN_TAG_OPEN, "<"),
		Pair.create(T_TEXT, "e"),
		Pair.create(T_HTML_TAG_CLOSE, ">"),
	});
}

Source File: LanguageUtil.java From consulo with Apache License 2.0

5 votes

@Nonnull
@RequiredReadAction
public static ParserDefinition.SpaceRequirements canStickTokensTogetherByLexer(ASTNode left, ASTNode right, Lexer lexer) {
  String textStr = left.getText() + right.getText();

  lexer.start(textStr, 0, textStr.length());
  if(lexer.getTokenType() != left.getElementType()) return ParserDefinition.SpaceRequirements.MUST;
  if(lexer.getTokenEnd() != left.getTextLength()) return ParserDefinition.SpaceRequirements.MUST;
  lexer.advance();
  if(lexer.getTokenEnd() != textStr.length()) return ParserDefinition.SpaceRequirements.MUST;
  if(lexer.getTokenType() != right.getElementType()) return ParserDefinition.SpaceRequirements.MUST;
  return ParserDefinition.SpaceRequirements.MAY;
}

Source File: MacroParser.java From consulo with Apache License 2.0

5 votes

@Nonnull
public static Expression parse(@Nullable String expression) {
  if (StringUtil.isEmpty(expression)) {
    return new ConstantNode("");
  }
  Lexer lexer = new MacroLexer();
  lexer.start(expression);
  skipWhitespaces(lexer);
  return parseMacro(lexer, expression);
}

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

4 votes

@Test
@SuppressWarnings("unchecked")
public void testVarType() throws Exception {
	Lexer lexer = new LatteLexer();
	lexer.start("{varType \\Foo\\Bar|null $_var1}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "varType"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_NULL, "null"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$_var1"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{varType Permissions\\Permission[]|string|null $_var1}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "varType"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Permissions"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_IDENTIFIER, "Permission"),
		Pair.create(T_PHP_LEFT_BRACKET, "["),
		Pair.create(T_PHP_RIGHT_BRACKET, "]"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_TYPE, "string"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_NULL, "null"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$_var1"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{varType Foo\\Bar\\_TestClass1|Foo\\Bar|string|int $_var1}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "varType"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_IDENTIFIER, "_TestClass1"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_IDENTIFIER, "Bar"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_TYPE, "string"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_TYPE, "int"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$_var1"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{varType Foo\\Bar\\_TestClass1|\\Foo\\Bar|string|int $_var1}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "varType"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_IDENTIFIER, "_TestClass1"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_TYPE, "string"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_TYPE, "int"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$_var1"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{varType string|$int $_var1}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "varType"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_TYPE, "string"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_MACRO_ARGS_VAR, "$int"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$_var1"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Source File: HaxeProjectStructureDetector.java From intellij-haxe with Apache License 2.0

4 votes

public String fun(final CharSequence charSequence) {
  Lexer lexer = LanguageParserDefinitions.INSTANCE.forLanguage(HaxeLanguage.INSTANCE).createLexer(null);
  lexer.start(charSequence);
  return readPackageName(charSequence, lexer);
}

Source File: EnterHandler.java From consulo with Apache License 2.0

4 votes

public static boolean isCommentComplete(PsiComment comment, CodeDocumentationAwareCommenter commenter, Editor editor) {
  for (CommentCompleteHandler handler : CommentCompleteHandler.EP_NAME.getExtensionList()) {
    if (handler.isApplicable(comment, commenter)) {
      return handler.isCommentComplete(comment, commenter, editor);
    }
  }

  String commentText = comment.getText();
  final boolean docComment = isDocComment(comment, commenter);
  final String expectedCommentEnd = docComment ? commenter.getDocumentationCommentSuffix() : commenter.getBlockCommentSuffix();
  if (!commentText.endsWith(expectedCommentEnd)) return false;

  final PsiFile containingFile = comment.getContainingFile();
  final Language language = containingFile.getLanguage();
  ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(language);
  if (parserDefinition == null) {
    return true;
  }
  Lexer lexer = parserDefinition.createLexer(containingFile.getLanguageVersion());
  final String commentPrefix = docComment ? commenter.getDocumentationCommentPrefix() : commenter.getBlockCommentPrefix();
  lexer.start(commentText, commentPrefix == null ? 0 : commentPrefix.length(), commentText.length());
  QuoteHandler fileTypeHandler = TypedHandler.getQuoteHandler(containingFile, editor);
  JavaLikeQuoteHandler javaLikeQuoteHandler = fileTypeHandler instanceof JavaLikeQuoteHandler ? (JavaLikeQuoteHandler)fileTypeHandler : null;

  while (true) {
    IElementType tokenType = lexer.getTokenType();
    if (tokenType == null) {
      return false;
    }

    if (javaLikeQuoteHandler != null && javaLikeQuoteHandler.getStringTokenTypes() != null && javaLikeQuoteHandler.getStringTokenTypes().contains(tokenType)) {
      String text = commentText.substring(lexer.getTokenStart(), lexer.getTokenEnd());
      int endOffset = comment.getTextRange().getEndOffset();

      if (text.endsWith(expectedCommentEnd) && endOffset < containingFile.getTextLength() && containingFile.getText().charAt(endOffset) == '\n') {
        return true;
      }
    }
    if (tokenType == commenter.getDocumentationCommentTokenType() || tokenType == commenter.getBlockCommentTokenType()) {
      return false;
    }
    if (tokenType == commenter.getLineCommentTokenType() && lexer.getTokenText().contains(commentPrefix)) {
      return false;
    }
    if (lexer.getTokenEnd() == commentText.length()) {
      if (tokenType == commenter.getLineCommentTokenType()) {
        String prefix = commenter.getLineCommentPrefix();
        lexer.start(commentText, lexer.getTokenStart() + (prefix == null ? 0 : prefix.length()), commentText.length());
        lexer.advance();
        continue;
      }
      else if (isInvalidPsi(comment)) {
        return false;
      }
      return true;
    }
    lexer.advance();
  }
}

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

4 votes

@Test
@SuppressWarnings("unchecked")
public void testStaticMethod() throws Exception {
	Lexer lexer = new LatteLexer();
	lexer.start("{$object::getFoo()}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_PHP_IDENTIFIER, "getFoo"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{\\Foo\\Bar::getFoo($var, 'test', 123)::getBar()}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_PHP_IDENTIFIER, "getFoo"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_MACRO_ARGS_VAR, "$var"),
		Pair.create(T_MACRO_ARGS, ","),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_SINGLE_QUOTE_LEFT, "'"),
		Pair.create(T_MACRO_ARGS_STRING, "test"),
		Pair.create(T_PHP_SINGLE_QUOTE_RIGHT, "'"),
		Pair.create(T_MACRO_ARGS, ","),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_NUMBER, "123"),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_PHP_IDENTIFIER, "getBar"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{Foo\\Bar::getFoo()::$bar}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_IDENTIFIER, "Bar"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_PHP_IDENTIFIER, "getFoo"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_MACRO_ARGS_VAR, "$bar"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Source File: LatteHighlightingLexerTest.java From intellij-latte with MIT License

4 votes

@Test
@SuppressWarnings("unchecked")
public void testMacroLexer() {
	Lexer lexer = new LatteHighlightingLexer(new LatteLookAheadLexer(new LatteLexer()));

	lexer.start("{include #blockName}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "include"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_BLOCK_NAME, "#"),
		Pair.create(T_BLOCK_NAME, "blockName"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{= dump ()}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_SHORTNAME, "="),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_METHOD, "dump"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{= \\Foo\\Bar}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_SHORTNAME, "="),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{= \\Bar}");
	assertTokens(lexer, new Pair[] {
			Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
			Pair.create(T_MACRO_SHORTNAME, "="),
			Pair.create(T_WHITESPACE, " "),
			Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
			Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
			Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Source File: UnescapingPsiBuilder.java From BashSupport with Apache License 2.0

4 votes

private void initTokenListAndCharSequence(Lexer lexer) {
    lexer.start(processedText);

    myShrunkSequence = new ArrayList<MyShiftedToken>(512); //assume a larger token size by default
    StringBuilder charSequenceBuilder = new StringBuilder();

    int realPos = 0;
    int shrunkPos = 0;
    while (lexer.getTokenType() != null) {
        final IElementType tokenType = lexer.getTokenType();
        final String tokenText = lexer.getTokenText();

        int tokenStart = lexer.getTokenStart();
        int tokenEnd = lexer.getTokenEnd();
        int realLength = tokenEnd - tokenStart;

        int delta = textProcessor.getContentRange().getStartOffset();
        int originalStart = textProcessor.getOffsetInHost(tokenStart - delta);
        int originalEnd = textProcessor.getOffsetInHost(tokenEnd - delta);

        if (textProcessor.containsRange(tokenStart, tokenEnd) && originalStart != -1 && originalEnd != -1) {
            realLength = originalEnd - originalStart;
            int masqueLength = tokenEnd - tokenStart;

            myShrunkSequence.add(new MyShiftedToken(tokenType,
                    realPos, realPos + realLength,
                    shrunkPos, shrunkPos + masqueLength, tokenText));
            charSequenceBuilder.append(tokenText);

            shrunkPos += masqueLength;
        } else {
            myShrunkSequence.add(new MyShiftedToken(tokenType,
                    realPos, realPos + realLength,
                    shrunkPos, shrunkPos + realLength, tokenText));
            charSequenceBuilder.append(tokenText);

            shrunkPos += realLength;
        }

        realPos += realLength;

        lexer.advance();
    }

    myShrunkCharSequence = charSequenceBuilder.toString();
    myShrunkSequenceSize = myShrunkSequence.size();
}

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

4 votes

@Test
@SuppressWarnings("unchecked")
public void testLinkDestination() throws Exception {
	Lexer lexer = new LatteLookAheadLexer(new LatteLexer());
	lexer.start("{link default}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "link"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_LINK_DESTINATION, "default"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{link Presenter:default}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "link"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_LINK_DESTINATION, "Presenter"),
		Pair.create(T_LINK_DESTINATION, ":"),
		Pair.create(T_LINK_DESTINATION, "default"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{link :Presenter:default}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "link"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_LINK_DESTINATION, ":"),
		Pair.create(T_LINK_DESTINATION, "Presenter"),
		Pair.create(T_LINK_DESTINATION, ":"),
		Pair.create(T_LINK_DESTINATION, "default"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("<a n:href=\"default\"></a>");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_HTML_OPEN_TAG_OPEN, "<"),
		Pair.create(T_TEXT, "a "),
		Pair.create(T_HTML_TAG_NATTR_NAME, "n:href"),
		Pair.create(T_HTML_TAG_ATTR_EQUAL_SIGN, "="),
		Pair.create(T_HTML_TAG_ATTR_DQ, "\""),
		Pair.create(T_LINK_DESTINATION, "default"),
		Pair.create(T_HTML_TAG_ATTR_DQ, "\""),
		Pair.create(T_HTML_TAG_CLOSE, ">"),
		Pair.create(T_HTML_CLOSE_TAG_OPEN, "</"),
		Pair.create(T_TEXT, "a"),
		Pair.create(T_HTML_TAG_CLOSE, ">"),
	});
}

Java Code Examples for com.intellij.lexer.Lexer#start()