Java Code Examples for com.intellij.lang.PsiBuilder#eof()

private void parseHeader(@Nonnull final PsiBuilder builder) {
  boolean doLoop = true;
  while (doLoop && !builder.eof()) {
    final PsiBuilder.Marker marker = builder.mark();

    if (builder.getTokenType() == null) {
      marker.drop();
    } else {
      final IElementType token = builder.getTokenType();

      if (token == MMTokens.HEADER_DELIMITER) {
        marker.done(token);
        doLoop = false;
      } else if (token == MMTokens.HEADER_LINE
          || token == MMTokens.UNKNOWN
          || token == MMTokens.WHITE_SPACE
          || token == MMTokens.ATTRIBUTES) {
        marker.done(token);
      } else {
        throw Assertions.fail("Unexpected header token : " + token);
      }
    }
    builder.advanceLexer();
  }
}
 

private void parseTopics(@Nonnull final PsiBuilder builder) {
  while (!builder.eof()) {
    final PsiBuilder.Marker marker = builder.mark();
    final IElementType token = builder.getTokenType();

    if (token == null) {
      marker.drop();
    } else {
      if (token == MMTokens.TOPIC_LEVEL) {
        final PsiBuilder.Marker levelMarker = builder.mark();
        levelMarker.done(token);
        final int topicLevel = ModelUtils.calcCharsOnStart('#', builder.getTokenText());
        if (topicLevel != 1) {
          marker.done(MMTokens.UNKNOWN);
        } else {
          builder.advanceLexer();
          recursiveParseTopic(builder, topicLevel);
          marker.done(MMTokens.TOPIC);
        }
      } else {
        marker.done(MMTokens.UNKNOWN);
      }
    }
    builder.advanceLexer();
  }
}
 

@Nonnull
private static IElementType createElementType(String id, BiConsumer<PsiBuilder, LanguageVersion> consumer)
{
	return new IFileElementType(id, CSharpLanguage.INSTANCE)
	{
		@Override
		protected ASTNode doParseContents(@Nonnull ASTNode chameleon, @Nonnull PsiElement psi)
		{
			final Project project = psi.getProject();
			final Language languageForParser = getLanguageForParser(psi);
			final LanguageVersion tempLanguageVersion = chameleon.getUserData(LanguageVersion.KEY);
			final LanguageVersion languageVersion = tempLanguageVersion == null ? psi.getLanguageVersion() : tempLanguageVersion;
			final PsiBuilder builder = PsiBuilderFactory.getInstance().createBuilder(project, chameleon, null, languageForParser, languageVersion, chameleon.getChars());
			consumer.accept(builder, languageVersion);
			while(!builder.eof())
			{
				builder.advanceLexer();
			}
			return builder.getTreeBuilt();
		}
	};
}
 

@Override
public void parseLight(IElementType root, PsiBuilder builder) {
    if (crashing) {
        builder.setDebugMode(true);
    }
    final PsiBuilder.Marker rootMarker = builder.mark();
    if (!builder.eof()) { //Empty file is not an error
        final GLSLParsing theRealParser = new GLSLParsing(builder);

        theRealParser.parseTranslationUnit();
        while (!builder.eof()) // exhaust the file if unable to parse everything
            builder.advanceLexer();
    }

    rootMarker.done(root);
}
 

private static void readIdentifier(PsiBuilder psiBuilder, IElementType identifierEnd, IElementType identifierType) {
  PsiBuilder.Marker identifier = psiBuilder.mark();
  psiBuilder.advanceLexer();
  IElementType currentType;

  while(!psiBuilder.eof() && (currentType = psiBuilder.getTokenType()) != identifierEnd) {
    PsiBuilder.Marker marker = shouldProduceComposite(currentType) ? psiBuilder.mark() : null;
    psiBuilder.advanceLexer();
    if (marker != null) marker.done(currentType);
  }
  identifier.done(identifierType);
}
 

@NotNull
public ASTNode parse(IElementType iElementType, PsiBuilder psiBuilder) {
  final PsiBuilder.Marker marker = psiBuilder.mark();
  PsiBuilder.Marker statement = psiBuilder.mark();
  IElementType statementType = null;

  while(!psiBuilder.eof()) {
    final IElementType tokenType = psiBuilder.getTokenType();

    if (tokenType == MakefileTokenTypes.SEMANTIC_WHITESPACE) {
      psiBuilder.advanceLexer();
      statement.done(statementType != null ? statementType:MakefileTokenTypes.STATEMENT);
      statementType = null;
      statement = psiBuilder.mark();
      continue;
    } else if (shouldProduceComposite(tokenType)) {
      PsiBuilder.Marker identifier = psiBuilder.mark();
      psiBuilder.advanceLexer();
      identifier.done(tokenType);
      continue;
    } else if (tokenType == MakefileTokenTypes.VAR_DEFINITION) {
      statementType = tokenType;
    } else if (tokenType == MakefileTokenTypes.IDENTIFIER_START) {
      readIdentifier(psiBuilder, MakefileTokenTypes.IDENTIFIER_END, MakefileTokenTypes.IDENTIFIER);
    }
    else if (tokenType == MakefileTokenTypes.TARGET_IDENTIFIER_START) {
      readIdentifier(psiBuilder, MakefileTokenTypes.TARGET_IDENTIFIER_END, MakefileTokenTypes.TARGET_IDENTIFIER);
      if (!psiBuilder.eof()) psiBuilder.advanceLexer();

      statementType = MakefileTokenTypes.TARGET_DECLARATION;
      continue;
    }
    psiBuilder.advanceLexer();
  }

  statement.done(MakefileTokenTypes.STATEMENT);
  marker.done(iElementType);
  return psiBuilder.getTreeBuilt();
}
 

@Nonnull
@Override
public ASTNode parse(@Nonnull IElementType elementType, @Nonnull PsiBuilder builder, @Nonnull LanguageVersion languageVersion)
{
	PsiBuilder.Marker mark = builder.mark();
	SharedParsingHelpers.parseType(new CSharpBuilderWrapper(builder, languageVersion), SharedParsingHelpers.VAR_SUPPORT);
	while(!builder.eof())
	{
		builder.error("Unexpected token");
		builder.advanceLexer();
	}
	mark.done(elementType);
	return builder.getTreeBuilt();
}
 

@Nonnull
@Override
public ASTNode parse(@Nonnull IElementType elementType, @Nonnull PsiBuilder builder, @Nonnull LanguageVersion languageVersion)
{
	PsiBuilder.Marker mark = builder.mark();
	ExpressionParsing.parse(new CSharpBuilderWrapper(builder, languageVersion), ModifierSet.EMPTY);
	while(!builder.eof())
	{
		builder.error("Unexpected token");
		builder.advanceLexer();
	}
	mark.done(elementType);
	return builder.getTreeBuilt();
}
 

@Nonnull
@Override
public ASTNode parse(@Nonnull IElementType elementType, @Nonnull PsiBuilder builder, @Nonnull LanguageVersion languageVersion)
{
	PsiBuilder.Marker mark = builder.mark();

	while(!builder.eof())
	{
		PreprocessorParsing.parse(builder);
	}

	mark.done(elementType);
	return builder.getTreeBuilt();
}
 

public static boolean indented(@NotNull PsiBuilder builder, int level, Parser geq) {
    if (!(builder instanceof Builder)) return false;
    PsiParser wrapper = ((Builder) builder).parser;
    if (!(wrapper instanceof HaskellParserWrapper)) return false;
    if (builder.eof()) return false;

    // IElementType currtok = builder.getTokenType();
    Pair<Integer, IElementType> prevtok = previousElem(builder);
    if (prevtok == null) return true;

    int offs = builder.getCurrentOffset();
    int line = StringUtil.offsetToLineNumber(builder.getOriginalText(), offs);
    int prevline = StringUtil.offsetToLineNumber(builder.getOriginalText(), offs + prevtok.getFirst());
    if (prevline == line) return true;

    int thisLineStart = StringUtil.lineColToOffset(builder.getOriginalText(), line, 0);
    int prevLineStart = StringUtil.lineColToOffset(builder.getOriginalText(), prevline, 0);
    // CharSequence lineStuff = builder.getOriginalText().subSequence(thisLineStart, offs);
    CharSequence prevLineStuff = builder.getOriginalText().subSequence(prevLineStart, thisLineStart - 1);
    int indentation = indentationLevel(prevLineStuff);
    int myindentation = offs - thisLineStart;
    // String tokName = builder.getTokenText();

    boolean geqVal = geq.parse(builder, level);
    if (geqVal && myindentation >= indentation ||
            !geqVal && myindentation > indentation) return true;
    return false;
}
 

/**
 * External rule used to determine if the current lexer position is
 * in an indent. Semantically, this returns true unless the previous
 * character was a newline and the current one is not whitespace.
 * Via recoverWhile, this rule will be used to consume input until
 * it reaches an unindented token, presumably a top-level element
 * of some sort, and resume at that time. It's quite convenient to just
 * resume parsing once we encounter a new top-level element.
 */
public static boolean inIndentRecover(@NotNull PsiBuilder builder, int level) {
    if (builder.eof()) return false;
    final int offset = builder.getCurrentOffset();
    if (offset == 0) return false;
    final CharSequence text = builder.getOriginalText();
    return !(
        text.charAt(offset - 1) == '\n'
            && !Character.isWhitespace(text.charAt(offset)));
}
 

public static boolean toplevel_recover_debug(PsiBuilder builder, int level) {
    IElementType typ = builder.getTokenType();
    final boolean res = !(TOPLEVEL_RECOVER_TYPES.contains(typ) || builder.eof());
    System.out.println(
        "toplevel_recover_debug: res: " + res
          + "; offset: " + builder.getCurrentOffset()
          + "; type: " + typ
          + "; text: '" + builder.getTokenText() + "'");
    return res;
}
 

@NotNull
@Override
public ASTNode parse(IElementType root, PsiBuilder builder) {
    PsiBuilder.Marker marker = builder.mark();
    while (!builder.eof()) {
        builder.advanceLexer();
    }
    marker.done(root);
    return builder.getTreeBuilt();
}
 

/**
 * @param exact if true then positions should be equal;
 *              else delegate should be behind, not including exactly all foreign (skipped) or whitespace tokens
 */
private void synchronizePositions(boolean exact) {
    final PsiBuilder delegate = getDelegate();

    if (myLexPosition >= myShrunkSequenceSize || delegate.eof()) {
        myLexPosition = myShrunkSequenceSize;
        while (!delegate.eof()) {
            delegate.advanceLexer();
        }
        return;
    }

    if (delegate.getCurrentOffset() > myShrunkSequence.get(myLexPosition).realStart) {
        LOG.debug("delegate is ahead of my builder!");
        return;
    }

    final int keepUpPosition = getKeepUpPosition(exact);

    while (!delegate.eof()) {
        final int delegatePosition = delegate.getCurrentOffset();

        if (delegatePosition < keepUpPosition) {
            delegate.advanceLexer();
        } else {
            break;
        }
    }
}
 

private boolean parseExtraBlock(@Nonnull final PsiBuilder builder) {
  // read type
  final PsiBuilder.Marker type = builder.mark();
  if (builder.getTokenType() != MMTokens.EXTRA_TYPE) {
    throw Assertions.fail("Unexpected token " + builder.getTokenType());
  }
  builder.advanceLexer();
  type.done(MMTokens.EXTRA_TYPE);

  boolean dataFound = false;

  // read body
  while (!builder.eof()) {
    final PsiBuilder.Marker marker = builder.mark();
    if (builder.eof() || builder.getTokenType() == null) {
      marker.drop();
      break;
    } else {
      final IElementType token = builder.getTokenType();
      if (token == MMTokens.TOPIC_LEVEL || token == MMTokens.EXTRA_TYPE) {
        marker.rollbackTo();
        return true;
      } else if (token == MMTokens.EXTRA_BODY || token == MMTokens.WHITE_SPACE) {
        if (dataFound && token == MMTokens.EXTRA_BODY) {
          builder.advanceLexer();
          marker.done(MMTokens.UNKNOWN);
          break;
        } else {
          builder.advanceLexer();
          marker.done(token);
          dataFound = dataFound || token == MMTokens.EXTRA_BODY;
        }
      } else {
        marker.done(MMTokens.UNKNOWN);
        break;
      }
    }
  }
  return false;
}
 

public static boolean anyNonSemi(@NotNull PsiBuilder builder, int level) {
    return
        !builder.eof() && (
            builder.getTokenType() == HaskellTypes.SEMICOLON
              || builder.getTokenType() == HaskellTypes.WHITESPACESEMITOK);
}
 

public static void parseElementsInBraces(@Nonnull PsiBuilder builder, IElementType open, IElementType close, IElementType valid)
{
	if(builder.getTokenType() == open)
	{
		builder.advanceLexer();

		while(!builder.eof())
		{
			IElementType tokenType = builder.getTokenType();
			if(tokenType == close)
			{
				break;
			}

			if(valid != null)
			{
				if(tokenType == valid)
				{
					builder.advanceLexer();
				}
				else
				{
					doneError(builder, "Unexpected token");
				}
			}
			else
			{
				builder.advanceLexer();
			}

			if(builder.getTokenType() == ShaderLabTokens.COMMA)
			{
				builder.advanceLexer();
			}
			else if(builder.getTokenType() != close)
			{
				doneError(builder, "Unexpected token");
			}
		}
		expectWithError(builder, close, "Unexpected token");
	}
}
 

private int recursiveParseTopic(@Nonnull final PsiBuilder builder, final int level) {

    while (!builder.eof()) {
      final PsiBuilder.Marker marker = builder.mark();

      final IElementType token = builder.getTokenType();

      if (token == null) {
        marker.drop();
      } else {
        if (token == MMTokens.TOPIC_LEVEL) {
          final PsiBuilder.Marker levelMarker = builder.mark();
          levelMarker.done(token);

          final int theTopicLevel = ModelUtils.calcCharsOnStart('#', builder.getTokenText());
          if (theTopicLevel <= 1) {
            marker.done(MMTokens.UNKNOWN);
          } else {
            if (theTopicLevel <= level) {
              marker.rollbackTo();
              return theTopicLevel;
            } else {
              builder.advanceLexer();
              final int parsedTopicLevel = recursiveParseTopic(builder, theTopicLevel);
              marker.done(MMTokens.TOPIC);
              if (parsedTopicLevel < theTopicLevel) {
                return parsedTopicLevel;
              }
              if (parsedTopicLevel == theTopicLevel) {
                continue;
              }
            }
          }
        } else if (token == MMTokens.TOPIC_TITLE || token == MMTokens.CODE_SNIPPET_BODY || token == MMTokens.CODE_SNIPPET_END || token == MMTokens.CODE_SNIPPET_START || token == MMTokens.ATTRIBUTES) {
          marker.done(token);
        } else if (token == MMTokens.EXTRA_TYPE) {
          try {
            if (parseExtraBlock(builder)) {
              continue;
            }
          } finally {
            marker.done(MMTokens.EXTRA_DATA);
          }
        } else if (token == MMTokens.WHITE_SPACE) {
          marker.done(token);
        } else {
          marker.done(MMTokens.UNKNOWN);
        }
      }
      builder.advanceLexer();
    }


    return level;
  }
 

@NotNull
@Override
public ASTNode parse(IElementType root, PsiBuilder builder) {
	ProgressIndicatorProvider.checkCanceled();

	TokenSource source = new PSITokenSource(builder);
	TokenStream tokens = new CommonTokenStream(source);
	parser.setTokenStream(tokens);
	parser.setErrorHandler(new ErrorStrategyAdaptor()); // tweaks missing tokens
	parser.removeErrorListeners();
	parser.addErrorListener(new SyntaxErrorListener()); // trap errors
	ParseTree parseTree = null;
	PsiBuilder.Marker rollbackMarker = builder.mark();
	try {
		parseTree = parse(parser, root);
	}
	finally {
		rollbackMarker.rollbackTo();
	}

	// Now convert ANTLR parser tree to PSI tree by mimicking subtree
	// enter/exit with mark/done calls. I *think* this creates their parse
	// tree (AST as they call it) when you call {@link PsiBuilder#getTreeBuilt}
	ANTLRParseTreeToPSIConverter listener = createListener(parser, root, builder);
	PsiBuilder.Marker rootMarker = builder.mark();
	ParseTreeWalker.DEFAULT.walk(listener, parseTree);
	while (!builder.eof()) {
		ProgressIndicatorProvider.checkCanceled();
		builder.advanceLexer();
	}
	// NOTE: parse tree returned from parse will be the
	// usual ANTLR tree ANTLRParseTreeToPSIConverter will
	// convert that to the analogous jetbrains AST nodes
	// When parsing an entire file, the root IElementType
	// will be a IFileElementType.
	//
	// When trying to rename IDs and so on, you get a
	// dummy root and a type arg identifier IElementType.
	// This results in a weird tree that has for example
	// (ID (expr (primary ID))) with the ID IElementType
	// as a subtree root as well as the appropriate leaf
	// all the way at the bottom.  The dummy ID root is a
	// CompositeElement and created by
	// ParserDefinition.createElement() despite having
	// being TokenIElementType.
	rootMarker.done(root);
	return builder.getTreeBuilt(); // calls the ASTFactory.createComposite() etc...
}