org.eclipse.jdt.core.compiler.IScanner Java Examples

private static boolean isJustWhitespaceOrComment(int start, int end, IBuffer buffer) {
	if (start == end) {
		return true;
	}
	Assert.isTrue(start <= end);
	String trimmedText = buffer.getText(start, end - start).trim();
	if (0 == trimmedText.length()) {
		return true;
	} else {
		IScanner scanner = ToolFactory.createScanner(false, false, false, null);
		scanner.setSource(trimmedText.toCharArray());
		try {
			return scanner.getNextToken() == ITerminalSymbols.TokenNameEOF;
		} catch (InvalidInputException e) {
			return false;
		}
	}
}
 

private int getSurroundingComment(IScanner scanner) {
	try {
		int start= fLocation.getOffset();
		int end= start + fLocation.getLength();

		int token= scanner.getNextToken();
		while (token != ITerminalSymbols.TokenNameEOF) {
			if (TokenScanner.isComment(token)) {
				int currStart= scanner.getCurrentTokenStartPosition();
				int currEnd= scanner.getCurrentTokenEndPosition() + 1;
				if (currStart <= start && end <= currEnd) {
					return token;
				}
			}
			token= scanner.getNextToken();
		}

	} catch (InvalidInputException e) {
		// ignore
	}
	return ITerminalSymbols.TokenNameEOF;
}
 

private FoldingStructureComputationContext createContext(boolean allowCollapse) {
	if (!isInstalled())
		return null;
	ProjectionAnnotationModel model= getModel();
	if (model == null)
		return null;
	IDocument doc= getDocument();
	if (doc == null)
		return null;

	IScanner scanner= null;
	if (fUpdatingCount == 1)
		scanner= fSharedScanner; // reuse scanner

	return new FoldingStructureComputationContext(doc, model, allowCollapse, scanner);
}
 

private static boolean isJustWhitespaceOrComment(int start, int end, IBuffer buffer) {
	if (start == end)
		return true;
	Assert.isTrue(start <= end);
	String trimmedText= buffer.getText(start, end - start).trim();
	if (0 == trimmedText.length()) {
		return true;
	} else {
		IScanner scanner= ToolFactory.createScanner(false, false, false, null);
		scanner.setSource(trimmedText.toCharArray());
		try {
			return scanner.getNextToken() == ITerminalSymbols.TokenNameEOF;
		} catch (InvalidInputException e) {
			return false;
		}
	}
}
 

/**
 * Removes comments and whitespace
 * @param reference the type reference
 * @return the reference only consisting of dots and java identifier characters
 */
public static String normalizeReference(String reference) {
	IScanner scanner= ToolFactory.createScanner(false, false, false, false);
	scanner.setSource(reference.toCharArray());
	StringBuffer sb= new StringBuffer();
	try {
		int tokenType= scanner.getNextToken();
		while (tokenType != ITerminalSymbols.TokenNameEOF) {
			sb.append(scanner.getRawTokenSource());
			tokenType= scanner.getNextToken();
		}
	} catch (InvalidInputException e) {
		Assert.isTrue(false, reference);
	}
	reference= sb.toString();
	return reference;
}
 

private static void parseTags(NLSLine line, IScanner scanner) {
	String s= new String(scanner.getCurrentTokenSource());
	int pos= s.indexOf(NLSElement.TAG_PREFIX);
	while (pos != -1) {
		int start= pos + NLSElement.TAG_PREFIX_LENGTH;
		int end= s.indexOf(NLSElement.TAG_POSTFIX, start);
		if (end < 0)
			return; //no error recovery

		String index= s.substring(start, end);
		int i= 0;
		try {
			i= Integer.parseInt(index) - 1; 	// Tags are one based not zero based.
		} catch (NumberFormatException e) {
			return; //ignore the exception - no error recovery
		}
		if (line.exists(i)) {
			NLSElement element= line.get(i);
			element.setTagPosition(scanner.getCurrentTokenStartPosition() + pos, end - pos + 1);
		} else {
			return; //no error recovery
		}
		pos= s.indexOf(NLSElement.TAG_PREFIX, start);
	}
}
 

/**
 * @param source
 *            must be not null
 * @param range
 *            can be null
 * @return may return null, otherwise an initialized scanner which may
 *         answer which source offset index belongs to which source line
 * @throws JavaModelException
 */
private static IScanner initScanner(IType source, ISourceRange range) throws JavaModelException {
    if (range == null) {
        return null;
    }
    char[] charContent = getContent(source);
    if (charContent == null) {
        return null;
    }
    IScanner scanner = ToolFactory.createScanner(false, false, false, true);
    scanner.setSource(charContent);
    int offset = range.getOffset();
    try {
        while (scanner.getNextToken() != ITerminalSymbols.TokenNameEOF) {
            // do nothing, just wait for the end of stream
            if (offset <= scanner.getCurrentTokenEndPosition()) {
                break;
            }
        }
    } catch (InvalidInputException e) {
        FindbugsPlugin.getDefault().logException(e, "Could not init scanner for type: " + source);
    }
    return scanner;
}
 

private boolean isValidComment(String template) {
	IScanner scanner= ToolFactory.createScanner(true, false, false, false);
	scanner.setSource(template.toCharArray());
	try {
		int next= scanner.getNextToken();
		while (TokenScanner.isComment(next)) {
			next= scanner.getNextToken();
		}
		return next == ITerminalSymbols.TokenNameEOF;
	} catch (InvalidInputException e) {
	}
	return false;
}
 

protected IScanner getScanner(ICompilationUnit unit) {
	IJavaProject project= unit.getJavaProject();
	if (project.equals(fProjectCache)) {
		return fScannerCache;
	}

	fProjectCache= project;
	String sourceLevel= project.getOption(JavaCore.COMPILER_SOURCE, true);
	String complianceLevel= project.getOption(JavaCore.COMPILER_COMPLIANCE, true);
	fScannerCache= ToolFactory.createScanner(false, false, false, sourceLevel, complianceLevel);
	return fScannerCache;
}
 

private static void parseTags(NLSLine line, IScanner scanner) {
	String s= new String(scanner.getCurrentTokenSource());
	int pos= s.indexOf(NLSElement.TAG_PREFIX);
	while (pos != -1) {
		int start= pos + NLSElement.TAG_PREFIX_LENGTH;
		int end= s.indexOf(NLSElement.TAG_POSTFIX, start);
		if (end < 0)
		 {
			return; //no error recovery
		}

		String index= s.substring(start, end);
		int i= 0;
		try {
			i= Integer.parseInt(index) - 1; 	// Tags are one based not zero based.
		} catch (NumberFormatException e) {
			return; //ignore the exception - no error recovery
		}
		if (line.exists(i)) {
			NLSElement element= line.get(i);
			element.setTagPosition(scanner.getCurrentTokenStartPosition() + pos, end - pos + 1);
		} else {
			return; //no error recovery
		}
		pos= s.indexOf(NLSElement.TAG_PREFIX, start);
	}
}
 

private void check(RefactoringStatus result, Selection selection, IScanner scanner, int start, int end) {
	char[] characters= scanner.getSource();
	selection= adjustSelection(characters, selection, end);
	scanner.resetTo(start, end);

	int token= 0;
	try {
		loop: while (token != ITerminalSymbols.TokenNameEOF) {
			token= scanner.getNextToken();
			switch(token) {
				case ITerminalSymbols.TokenNameCOMMENT_LINE:
				case ITerminalSymbols.TokenNameCOMMENT_BLOCK:
				case ITerminalSymbols.TokenNameCOMMENT_JAVADOC:
					if (checkStart(scanner, selection.getOffset())) {
						result.addFatalError(RefactoringCoreMessages.CommentAnalyzer_starts_inside_comment);
						break loop;
					}
					if (checkEnd(scanner, selection.getInclusiveEnd())) {
						result.addFatalError(RefactoringCoreMessages.CommentAnalyzer_ends_inside_comment);
						break loop;
					}
					break;
			}
		}
	} catch (InvalidInputException e) {
		result.addFatalError(RefactoringCoreMessages.CommentAnalyzer_internal_error);
	}
}
 

public static RefactoringStatus perform(Selection selection, IScanner scanner, int start, int length) {
	RefactoringStatus result= new RefactoringStatus();
	if (length <= 0)
		return result;
	new CommentAnalyzer().check(result, selection, scanner, start, start + length - 1);
	return result;
}
 

public static NLSLine[] scan(ICompilationUnit cu) throws JavaModelException, BadLocationException, InvalidInputException {
	IJavaProject javaProject= cu.getJavaProject();
	IScanner scanner= null;
	if (javaProject != null) {
		String complianceLevel= javaProject.getOption(JavaCore.COMPILER_COMPLIANCE, true);
		String sourceLevel= javaProject.getOption(JavaCore.COMPILER_SOURCE, true);
		scanner= ToolFactory.createScanner(true, true, true, sourceLevel, complianceLevel);
	} else {
		scanner= ToolFactory.createScanner(true, true, false, true);
	}
	return scan(scanner, cu.getBuffer().getCharacters());
}
 

private boolean isValidComment(String template) {
	IScanner scanner= ToolFactory.createScanner(true, false, false, false);
	scanner.setSource(template.toCharArray());
	try {
		int next= scanner.getNextToken();
		while (TokenScanner.isComment(next)) {
			next= scanner.getNextToken();
		}
		return next == ITerminalSymbols.TokenNameEOF;
	} catch (InvalidInputException e) {
	}
	return false;
}
 

public static NLSLine[] scan(ICompilationUnit cu) throws JavaModelException, BadLocationException, InvalidInputException {
	IJavaProject javaProject= cu.getJavaProject();
	IScanner scanner= null;
	if (javaProject != null) {
		String complianceLevel= javaProject.getOption(JavaCore.COMPILER_COMPLIANCE, true);
		String sourceLevel= javaProject.getOption(JavaCore.COMPILER_SOURCE, true);
		scanner= ToolFactory.createScanner(true, true, true, sourceLevel, complianceLevel);
	} else {
		scanner= ToolFactory.createScanner(true, true, false, true);
	}
	return scan(scanner, cu.getBuffer().getCharacters());
}
 

private static int firstOpeningBraceOffset(IInitializer iInitializer) throws JavaModelException {
	try {
		IScanner scanner= ToolFactory.createScanner(false, false, false, false);
		scanner.setSource(iInitializer.getSource().toCharArray());
		int token= scanner.getNextToken();
		while (token != ITerminalSymbols.TokenNameEOF && token != ITerminalSymbols.TokenNameLBRACE)
			token= scanner.getNextToken();
		if (token == ITerminalSymbols.TokenNameLBRACE)
			return iInitializer.getSourceRange().getOffset() + scanner.getCurrentTokenStartPosition() + scanner.getRawTokenSource().length;
		return iInitializer.getSourceRange().getOffset();
	} catch (InvalidInputException e) {
		return iInitializer.getSourceRange().getOffset();
	}
}
 

/**
 * A visitor that maps a selection to a given ASTNode. The result node is
 * determined as follows:
 * <ul>
 *   <li>first the visitor tries to find a node that is covered by <code>start</code> and
 *       <code>length</code> where either <code>start</code> and <code>length</code> exactly
 *       matches the node or where the text covered before and after the node only consists
 *       of white spaces or comments.</li>
 * 	 <li>if no such node exists than the node that encloses the range defined by
 *       start and end is returned.</li>
 *   <li>if the length is zero than also nodes are considered where the node's
 *       start or end position matches <code>start</code>.</li>
 *   <li>otherwise <code>null</code> is returned.</li>
 * </ul>
 *
 * @param root the root node from which the search starts
 * @param start the start offset
 * @param length the length
 * @param source the source of the compilation unit
 *
 * @return the result node
 * @throws JavaModelException if an error occurs in the Java model
 *
 * @since		3.0
 */
public static ASTNode perform(ASTNode root, int start, int length, ITypeRoot source) throws JavaModelException {
	NodeFinder finder= new NodeFinder(start, length);
	root.accept(finder);
	ASTNode result= finder.getCoveredNode();
	if (result == null)
		return null;
	Selection selection= Selection.createFromStartLength(start, length);
	if (selection.covers(result)) {
		IBuffer buffer= source.getBuffer();
		if (buffer != null) {
			IScanner scanner= ToolFactory.createScanner(false, false, false, false);
			scanner.setSource(buffer.getText(start, length).toCharArray());
			try {
				int token= scanner.getNextToken();
				if (token != ITerminalSymbols.TokenNameEOF) {
					int tStart= scanner.getCurrentTokenStartPosition();
					if (tStart == result.getStartPosition() - start) {
						scanner.resetTo(tStart + result.getLength(), length - 1);
						token= scanner.getNextToken();
						if (token == ITerminalSymbols.TokenNameEOF)
							return result;
					}
				}
			} catch (InvalidInputException e) {
			}
		}
	}
	return finder.getCoveringNode();
}
 

protected IScanner getScanner(ICompilationUnit unit) {
	IJavaProject project= unit.getJavaProject();
	if (project.equals(fProjectCache))
		return fScannerCache;

	fProjectCache= project;
	String sourceLevel= project.getOption(JavaCore.COMPILER_SOURCE, true);
	String complianceLevel= project.getOption(JavaCore.COMPILER_COMPLIANCE, true);
	fScannerCache= ToolFactory.createScanner(false, false, false, sourceLevel, complianceLevel);
	return fScannerCache;
}
 

private boolean isValidComment(String template) {
  IScanner scanner = ToolFactory.createScanner(true, false, false, false);
  scanner.setSource(template.toCharArray());
  try {
    int next = scanner.getNextToken();
    while (TokenScanner.isComment(next)) {
      next = scanner.getNextToken();
    }
    return next == ITerminalSymbols.TokenNameEOF;
  } catch (InvalidInputException e) {
    // If there are lexical errors, the comment is invalid
  }
  return false;
}
 

/**
 * Creates a token comparator for the given string.
 *
 * @param text the text to be tokenized
 * @param textTokenComparatorFactory a factory to create text token comparators
 */
public SJavaTokenComparator(String text) {


  fText= text;

  int length= fText.length();
  fStarts= new int[length];
  fLengths= new int[length];
  fTokens= new String[length];
  fCount= 0;

  IScanner scanner= ToolFactory.createScanner(true, false, false, false); // returns comments & whitespace
  scanner.setSource(fText.toCharArray());
  int endPos= 0;
  try {
    int tokenType;
    while ((tokenType= scanner.getNextToken()) != ITerminalSymbols.TokenNameEOF) {
      int start= scanner.getCurrentTokenStartPosition();
      int end= scanner.getCurrentTokenEndPosition()+1;
      // Comments are treated as a single token (see bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=78063)
      if (!TokenScanner.isComment(tokenType)) {
        recordTokenRange(start, end - start);
      }
      endPos= end;
    }
  } catch (InvalidInputException ex) {
    // We couldn't parse part of the input. Fall through and make the rest a single token
  }
  // Workaround for https://bugs.eclipse.org/bugs/show_bug.cgi?id=13907
  if (endPos < length) {
    recordTokenRange(endPos, length - endPos);
  }
}
 

/**
 * Creates a token comparator for the given string.
 *
 * @param text the text to be tokenized
 */
public JavaTokenComparator(String text) {
	Assert.isLegal(text != null);

	fText= text;

	int length= fText.length();
	fStarts= new int[length];
	fLengths= new int[length];
	fCount= 0;

	IScanner scanner= ToolFactory.createScanner(true, true, false, false); // returns comments & whitespace
	scanner.setSource(fText.toCharArray());
	int endPos= 0;
	try {
		int tokenType;
		while ((tokenType= scanner.getNextToken()) != ITerminalSymbols.TokenNameEOF) {
			int start= scanner.getCurrentTokenStartPosition();
			int end= scanner.getCurrentTokenEndPosition()+1;
			// Comments and strings should not be treated as a single token, see https://bugs.eclipse.org/78063
			if (TokenScanner.isComment(tokenType) || tokenType == ITerminalSymbols.TokenNameStringLiteral) {
				// Line comments are often commented code, so lets treat them as code. See https://bugs.eclipse.org/216707
				boolean parseAsJava= tokenType == ITerminalSymbols.TokenNameCOMMENT_LINE;
				int dl= parseAsJava ? getCommentStartTokenLength(tokenType) : 0;
				if (dl > 0)
					recordTokenRange(start, dl);
				parseSubrange(start + dl, text.substring(start + dl, end), parseAsJava);
			} else {
				recordTokenRange(start, end - start);
			}
			endPos= end;
		}
	} catch (InvalidInputException ex) {
		// We couldn't parse part of the input. Fall through and make the rest a single token
	}
	// Workaround for https://bugs.eclipse.org/bugs/show_bug.cgi?id=13907
	if (endPos < length) {
		recordTokenRange(endPos, length - endPos);
	}
}
 

/**
 * @return start line of given type, or {@link #DONT_KNOW_LINE} if line could not be found
 */
private static int getLineStart(IType source) throws JavaModelException {
    ISourceRange range = source.getNameRange();
    if (range == null) {
        range = source.getSourceRange();
    }
    IScanner scanner = initScanner(source, range);
    if (scanner != null && range != null) {
        return scanner.getLineNumber(range.getOffset());
    }
    return DONT_KNOW_LINE;
}
 

private void computeTypeRanges(List<FoldingRange> foldingRanges, IType unit, IScanner scanner) throws CoreException {
	ISourceRange typeRange = unit.getSourceRange();
	foldingRanges.add(new FoldingRange(scanner.getLineNumber(unit.getNameRange().getOffset()) - 1, scanner.getLineNumber(typeRange.getOffset() + typeRange.getLength()) - 1));
	IJavaElement[] children = unit.getChildren();
	for (IJavaElement c : children) {
		if (c instanceof IMethod) {
			computeMethodRanges(foldingRanges, (IMethod) c, scanner);
		} else if (c instanceof IType) {
			computeTypeRanges(foldingRanges, (IType) c, scanner);
		}
	}
}
 

private void computeTypeRootRanges(List<FoldingRange> foldingRanges, ITypeRoot unit, IScanner scanner) throws CoreException {
	if (unit.hasChildren()) {
		for (IJavaElement child : unit.getChildren()) {
			if (child instanceof IImportContainer) {
				ISourceRange importRange = ((IImportContainer) child).getSourceRange();
				FoldingRange importFoldingRange = new FoldingRange(scanner.getLineNumber(importRange.getOffset()) - 1, scanner.getLineNumber(importRange.getOffset() + importRange.getLength()) - 1);
				importFoldingRange.setKind(FoldingRangeKind.Imports);
				foldingRanges.add(importFoldingRange);
			} else if (child instanceof IType) {
				computeTypeRanges(foldingRanges, (IType) child, scanner);
			}
		}
	}
}
 

private int getNextToken(IScanner scanner) {
	int token = 0;
	while (token == 0) {
		try {
			token = scanner.getNextToken();
		} catch (InvalidInputException e) {
			// ignore
			// JavaLanguageServerPlugin.logException("Problem with folding range", e);
		}
	}
	return token;
}
 

private FoldingStructureComputationContext(IDocument document, ProjectionAnnotationModel model, boolean allowCollapsing, IScanner scanner) {
	Assert.isNotNull(document);
	Assert.isNotNull(model);
	fDocument= document;
	fModel= model;
	fAllowCollapsing= allowCollapsing;
	fScanner= scanner;
}
 

private boolean isValidComment(String template) {
	IScanner scanner= ToolFactory.createScanner(true, false, false, false);
	scanner.setSource(template.toCharArray());
	try {
		int next= scanner.getNextToken();
		while (TokenScanner.isComment(next)) {
			next= scanner.getNextToken();
		}
		return next == ITerminalSymbols.TokenNameEOF;
	} catch (InvalidInputException e) {
	}
	return false;
}
 

/**
 * Creates a TokenScanner
 * @param scanner The scanner to be wrapped
 * @param document The document used for line information if specified
 */
public TokenScanner(IScanner scanner, IDocument document) {
	fScanner= scanner;
	fEndPosition= fScanner.getSource().length - 1;
	fDocument= document;
}
 

private IScanner getScanner() {
	if (fScanner == null)
		fScanner= ToolFactory.createScanner(true, false, false, false);
	return fScanner;
}
 

private boolean checkEnd(IScanner scanner, int position) {
	return scanner.getCurrentTokenStartPosition() <= position && position < scanner.getCurrentTokenEndPosition();
}