Java Code Examples for org.eclipse.jdt.core.ToolFactory#createScanner()

private static boolean isJustWhitespaceOrComment(int start, int end, IBuffer buffer) {
	if (start == end) {
		return true;
	}
	Assert.isTrue(start <= end);
	String trimmedText = buffer.getText(start, end - start).trim();
	if (0 == trimmedText.length()) {
		return true;
	} else {
		IScanner scanner = ToolFactory.createScanner(false, false, false, null);
		scanner.setSource(trimmedText.toCharArray());
		try {
			return scanner.getNextToken() == ITerminalSymbols.TokenNameEOF;
		} catch (InvalidInputException e) {
			return false;
		}
	}
}
 

private static boolean isJustWhitespaceOrComment(int start, int end, IBuffer buffer) {
	if (start == end)
		return true;
	Assert.isTrue(start <= end);
	String trimmedText= buffer.getText(start, end - start).trim();
	if (0 == trimmedText.length()) {
		return true;
	} else {
		IScanner scanner= ToolFactory.createScanner(false, false, false, null);
		scanner.setSource(trimmedText.toCharArray());
		try {
			return scanner.getNextToken() == ITerminalSymbols.TokenNameEOF;
		} catch (InvalidInputException e) {
			return false;
		}
	}
}
 

/**
 * Removes comments and whitespace
 * @param reference the type reference
 * @return the reference only consisting of dots and java identifier characters
 */
public static String normalizeReference(String reference) {
	IScanner scanner= ToolFactory.createScanner(false, false, false, false);
	scanner.setSource(reference.toCharArray());
	StringBuffer sb= new StringBuffer();
	try {
		int tokenType= scanner.getNextToken();
		while (tokenType != ITerminalSymbols.TokenNameEOF) {
			sb.append(scanner.getRawTokenSource());
			tokenType= scanner.getNextToken();
		}
	} catch (InvalidInputException e) {
		Assert.isTrue(false, reference);
	}
	reference= sb.toString();
	return reference;
}
 

public void scan(ICompilationUnit cu)	throws JavaModelException {
		char[] chars= cu.getBuffer().getCharacters();
		fMatches= new HashSet<>();
		fScanner= ToolFactory.createScanner(true, true, false, true);
		fScanner.setSource(chars);

//		IImportContainer importContainer= cu.getImportContainer();
//		if (importContainer.exists())
//			fNoFlyZone= importContainer.getSourceRange();
//		else
//			fNoFlyZone= null;

		doScan();
		fScanner= null;
	}
 

/**
 * Creates a TokenScanner
 * @param document The textbuffer to create the scanner on
 * @param project the current Java project
 */
public TokenScanner(IDocument document, IJavaProject project) {
	String sourceLevel= project.getOption(JavaCore.COMPILER_SOURCE, true);
	String complianceLevel= project.getOption(JavaCore.COMPILER_COMPLIANCE, true);
	fScanner= ToolFactory.createScanner(true, false, false, sourceLevel, complianceLevel); // no line info required
	fScanner.setSource(document.get().toCharArray());
	fDocument= document;
	fEndPosition= fScanner.getSource().length - 1;
}
 

protected IScanner getScanner(ICompilationUnit unit) {
	IJavaProject project= unit.getJavaProject();
	if (project.equals(fProjectCache)) {
		return fScannerCache;
	}

	fProjectCache= project;
	String sourceLevel= project.getOption(JavaCore.COMPILER_SOURCE, true);
	String complianceLevel= project.getOption(JavaCore.COMPILER_COMPLIANCE, true);
	fScannerCache= ToolFactory.createScanner(false, false, false, sourceLevel, complianceLevel);
	return fScannerCache;
}
 

public static NLSLine[] scan(ICompilationUnit cu) throws JavaModelException, BadLocationException, InvalidInputException {
	IJavaProject javaProject= cu.getJavaProject();
	IScanner scanner= null;
	if (javaProject != null) {
		String complianceLevel= javaProject.getOption(JavaCore.COMPILER_COMPLIANCE, true);
		String sourceLevel= javaProject.getOption(JavaCore.COMPILER_SOURCE, true);
		scanner= ToolFactory.createScanner(true, true, true, sourceLevel, complianceLevel);
	} else {
		scanner= ToolFactory.createScanner(true, true, false, true);
	}
	return scan(scanner, cu.getBuffer().getCharacters());
}
 

/**
 * Creates a token comparator for the given string.
 *
 * @param text the text to be tokenized
 */
public JavaTokenComparator(String text) {
	Assert.isLegal(text != null);

	fText= text;

	int length= fText.length();
	fStarts= new int[length];
	fLengths= new int[length];
	fCount= 0;

	IScanner scanner= ToolFactory.createScanner(true, true, false, false); // returns comments & whitespace
	scanner.setSource(fText.toCharArray());
	int endPos= 0;
	try {
		int tokenType;
		while ((tokenType= scanner.getNextToken()) != ITerminalSymbols.TokenNameEOF) {
			int start= scanner.getCurrentTokenStartPosition();
			int end= scanner.getCurrentTokenEndPosition()+1;
			// Comments and strings should not be treated as a single token, see https://bugs.eclipse.org/78063
			if (TokenScanner.isComment(tokenType) || tokenType == ITerminalSymbols.TokenNameStringLiteral) {
				// Line comments are often commented code, so lets treat them as code. See https://bugs.eclipse.org/216707
				boolean parseAsJava= tokenType == ITerminalSymbols.TokenNameCOMMENT_LINE;
				int dl= parseAsJava ? getCommentStartTokenLength(tokenType) : 0;
				if (dl > 0)
					recordTokenRange(start, dl);
				parseSubrange(start + dl, text.substring(start + dl, end), parseAsJava);
			} else {
				recordTokenRange(start, end - start);
			}
			endPos= end;
		}
	} catch (InvalidInputException ex) {
		// We couldn't parse part of the input. Fall through and make the rest a single token
	}
	// Workaround for https://bugs.eclipse.org/bugs/show_bug.cgi?id=13907
	if (endPos < length) {
		recordTokenRange(endPos, length - endPos);
	}
}
 

/**
 * Scan the given text.
 * <p>
 * <strong>NOTE:</strong> Use only for testing.
 * </p>
 * 
 * @param text the text
 */
public void scan(String text) {
	char[] chars= text.toCharArray();
	fMatches= new HashSet<TextMatch>();
	fScanner= ToolFactory.createScanner(true, true, false, true);
	fScanner.setSource(chars);
	doScan();
	fScanner= null;
}
 

/**
 * Creates a token comparator for the given string.
 *
 * @param text the text to be tokenized
 * @param textTokenComparatorFactory a factory to create text token comparators
 */
public SJavaTokenComparator(String text) {


  fText= text;

  int length= fText.length();
  fStarts= new int[length];
  fLengths= new int[length];
  fTokens= new String[length];
  fCount= 0;

  IScanner scanner= ToolFactory.createScanner(true, false, false, false); // returns comments & whitespace
  scanner.setSource(fText.toCharArray());
  int endPos= 0;
  try {
    int tokenType;
    while ((tokenType= scanner.getNextToken()) != ITerminalSymbols.TokenNameEOF) {
      int start= scanner.getCurrentTokenStartPosition();
      int end= scanner.getCurrentTokenEndPosition()+1;
      // Comments are treated as a single token (see bug https://bugs.eclipse.org/bugs/show_bug.cgi?id=78063)
      if (!TokenScanner.isComment(tokenType)) {
        recordTokenRange(start, end - start);
      }
      endPos= end;
    }
  } catch (InvalidInputException ex) {
    // We couldn't parse part of the input. Fall through and make the rest a single token
  }
  // Workaround for https://bugs.eclipse.org/bugs/show_bug.cgi?id=13907
  if (endPos < length) {
    recordTokenRange(endPos, length - endPos);
  }
}
 

private boolean isValidComment(String template) {
	IScanner scanner= ToolFactory.createScanner(true, false, false, false);
	scanner.setSource(template.toCharArray());
	try {
		int next= scanner.getNextToken();
		while (TokenScanner.isComment(next)) {
			next= scanner.getNextToken();
		}
		return next == ITerminalSymbols.TokenNameEOF;
	} catch (InvalidInputException e) {
	}
	return false;
}
 

protected IScanner getScanner(ICompilationUnit unit) {
	IJavaProject project= unit.getJavaProject();
	if (project.equals(fProjectCache))
		return fScannerCache;

	fProjectCache= project;
	String sourceLevel= project.getOption(JavaCore.COMPILER_SOURCE, true);
	String complianceLevel= project.getOption(JavaCore.COMPILER_COMPLIANCE, true);
	fScannerCache= ToolFactory.createScanner(false, false, false, sourceLevel, complianceLevel);
	return fScannerCache;
}
 

public static NLSLine[] scan(String s) throws InvalidInputException, BadLocationException {
	IScanner scanner= ToolFactory.createScanner(true, true, false, true);
	return scan(scanner, s.toCharArray());
}
 

public List<List<String>> tokenizeLines(String text) {
	IScanner scanner = ToolFactory.createScanner(false, false, true, "1.8");
	scanner.setSource(text.toCharArray());
	List<List<String>> lineTokens = new ArrayList<>();
	List<String> tokens = new ArrayList<>();
	lineTokens.add(new ArrayList<>());
	int nextToken = 0;
	int line = 1;
	while (true) {
		try {
			nextToken = scanner.getNextToken();
			int ln = scanner.getLineNumber(scanner.getCurrentTokenStartPosition());
			if (ln > line) {
				for (int i = line + 1; i <= ln; i++) lineTokens.add(new ArrayList<>());
				line = ln;
			}
			if (nextToken == ITerminalSymbols.TokenNameEOF) break;
		} catch (InvalidInputException e) {
			continue;
		}
		String val = new String(scanner.getCurrentTokenSource());
		if (val.startsWith("\"") && val.endsWith("\"") && val.length() > 2) {
			if (val.length() >= 15) {
				val = "\"\"";
			}
			else {
				String body = val.substring(1, val.length() - 1);
				body = body.replaceAll("\\\\", "\\\\\\\\");
				body = body.replaceAll("\"", "\\\\\"");
				body = body.replaceAll("\n", "\\n");
				body = body.replaceAll("\r", "\\r");
				body = body.replaceAll("\t", "\\t");
				val = "\"" + body + "\"";
			}
		}
		else if (val.startsWith("\'") && val.endsWith("\'")) {
			val = val.replaceAll("\n", "\\n");
			val = val.replaceAll("\r", "\\r");
			val = val.replaceAll("\t", "\\t");
		}
		// For Java, we have to add heuristic check regarding breaking up >>
		else if (val.matches(">>+")) {
			boolean split = false;
			for (int i = tokens.size() - 1; i >= 0; i--) {
				String token = tokens.get(i);
				if (token.matches("[,\\.\\?\\[\\]]") || Character.isUpperCase(token.charAt(0))
						|| token.equals("extends") || token.equals("super")
						|| token.matches("(byte|short|int|long|float|double)")) {
					continue;
				}
				else if (token.matches("(<|>)+")) {
					split = true;
					break;
				}
				else {
					break;
				}
			}
			if (split) {
				for (int i = 0; i < val.length(); i++) {
					tokens.add(">");
					lineTokens.get(lineTokens.size() - 1).add(">");
				}
				continue;
			}
		}
		tokens.add(val);
		lineTokens.get(lineTokens.size() - 1).add(val);
	}
	return lineTokens;
}
 

public Collector(IPackageFragment source, ReferencesInBinaryContext binaryRefs) {
	super(binaryRefs);
	fSource= source;
	fScanner= ToolFactory.createScanner(false, false, false, false);
}
 

/**
 * Finds the key defined by the given match. The assumption is that the key is the only argument
 * and it is a string literal i.e. quoted ("...") or a string constant i.e. 'static final
 * String' defined in the same class.
 * 
 * @param keyPositionResult reference parameter: will be filled with the position of the found
 *            key
 * @param enclosingElement enclosing java element
 * @return a string denoting the key, {@link #NO_KEY} if no key can be found and
 *         <code>null</code> otherwise
 * @throws CoreException if a problem occurs while accessing the <code>enclosingElement</code>
 */
private String findKey(Position keyPositionResult, IJavaElement enclosingElement) throws CoreException {
	ICompilationUnit unit= (ICompilationUnit)enclosingElement.getAncestor(IJavaElement.COMPILATION_UNIT);
	if (unit == null)
		return null;

	String source= unit.getSource();
	if (source == null)
		return null;

	IJavaProject javaProject= unit.getJavaProject();
	IScanner scanner= null;
	if (javaProject != null) {
		String complianceLevel= javaProject.getOption(JavaCore.COMPILER_COMPLIANCE, true);
		String sourceLevel= javaProject.getOption(JavaCore.COMPILER_SOURCE, true);
		scanner= ToolFactory.createScanner(false, false, false, sourceLevel, complianceLevel);
	} else {
		scanner= ToolFactory.createScanner(false, false, false, false);
	}
	scanner.setSource(source.toCharArray());
	scanner.resetTo(keyPositionResult.getOffset() + keyPositionResult.getLength(), source.length());

	try {
		if (scanner.getNextToken() != ITerminalSymbols.TokenNameDOT)
			return null;

		if (scanner.getNextToken() != ITerminalSymbols.TokenNameIdentifier)
			return null;

		String src= new String(scanner.getCurrentTokenSource());
		int tokenStart= scanner.getCurrentTokenStartPosition();
		int tokenEnd= scanner.getCurrentTokenEndPosition();

		if (scanner.getNextToken() == ITerminalSymbols.TokenNameLPAREN) {
			// Old school
			// next must be key string. Ignore methods which do not take a single String parameter (Bug 295040).
			int nextToken= scanner.getNextToken();
			if (nextToken != ITerminalSymbols.TokenNameStringLiteral && nextToken != ITerminalSymbols.TokenNameIdentifier)
				return null;

			tokenStart= scanner.getCurrentTokenStartPosition();
			tokenEnd= scanner.getCurrentTokenEndPosition();
			int token;
			while ((token= scanner.getNextToken()) == ITerminalSymbols.TokenNameDOT) {
				if ((nextToken= scanner.getNextToken()) != ITerminalSymbols.TokenNameIdentifier) {
						return null;
				}
				tokenStart= scanner.getCurrentTokenStartPosition();
				tokenEnd= scanner.getCurrentTokenEndPosition();
			}
			if (token != ITerminalSymbols.TokenNameRPAREN)
				return null;
			
			if (nextToken == ITerminalSymbols.TokenNameStringLiteral) {
				keyPositionResult.setOffset(tokenStart + 1);
				keyPositionResult.setLength(tokenEnd - tokenStart - 1);
				return source.substring(tokenStart + 1, tokenEnd);
			} else if (nextToken == ITerminalSymbols.TokenNameIdentifier) {
				keyPositionResult.setOffset(tokenStart);
				keyPositionResult.setLength(tokenEnd - tokenStart + 1);
				IType parentClass= (IType)enclosingElement.getAncestor(IJavaElement.TYPE);
				IField[] fields= parentClass.getFields();
				String identifier= source.substring(tokenStart, tokenEnd + 1);
				for (int i= 0; i < fields.length; i++) {
					if (fields[i].getElementName().equals(identifier)) {
						if (!Signature.getSignatureSimpleName(fields[i].getTypeSignature()).equals("String")) //$NON-NLS-1$
							return null;
						Object obj= fields[i].getConstant();
						return obj instanceof String ? ((String)obj).substring(1, ((String)obj).length() - 1) : NO_KEY;
					}
				}
			}
			return NO_KEY;
		} else {
			IJavaElement[] keys= unit.codeSelect(tokenStart, tokenEnd - tokenStart + 1);

			// an interface can't be a key
			if (keys.length == 1 && keys[0].getElementType() == IJavaElement.TYPE && ((IType) keys[0]).isInterface())
				return null;

			keyPositionResult.setOffset(tokenStart);
			keyPositionResult.setLength(tokenEnd - tokenStart + 1);
			return src;
		}
	} catch (InvalidInputException e) {
		throw new CoreException(JavaUIStatus.createError(IStatus.ERROR, e));
	}
}
 

private IScanner getScanner() {
	if (fScanner == null)
		fScanner= ToolFactory.createScanner(true, false, false, false);
	return fScanner;
}
 

private void initializeRanges() throws CoreException {
	fRanges = new HashMap<>();
	if (fSelectedNodes.length == 0) {
		return;
	}

	fRanges.put(fSelectedNodes[0], super.computeSourceRange(fSelectedNodes[0]));
	int last = fSelectedNodes.length - 1;
	fRanges.put(fSelectedNodes[last], super.computeSourceRange(fSelectedNodes[last]));

	IJavaProject javaProject = ((CompilationUnit) fSelectedNodes[0].getRoot()).getTypeRoot().getJavaProject();
	String sourceLevel = javaProject.getOption(JavaCore.COMPILER_SOURCE, true);
	String complianceLevel = javaProject.getOption(JavaCore.COMPILER_COMPLIANCE, true);
	IScanner scanner = ToolFactory.createScanner(true, false, false, sourceLevel, complianceLevel);
	char[] source = fDocumentPortionToScan.toCharArray();
	scanner.setSource(source);
	fDocumentPortionToScan = null; // initializeRanges() is only called once

	TokenScanner tokenizer = new TokenScanner(scanner);
	int pos = tokenizer.getNextStartOffset(0, false);

	ASTNode currentNode = fSelectedNodes[0];
	int newStart = Math.min(fSelectionStart + pos, currentNode.getStartPosition());
	SourceRange range = fRanges.get(currentNode);
	fRanges.put(currentNode, new SourceRange(newStart, range.getLength() + range.getStartPosition() - newStart));

	currentNode = fSelectedNodes[last];
	int scannerStart = currentNode.getStartPosition() + currentNode.getLength() - fSelectionStart;
	tokenizer.setOffset(scannerStart);
	pos = scannerStart;
	int token = -1;
	try {
		while (true) {
			token = tokenizer.readNext(false);
			pos = tokenizer.getCurrentEndOffset();
		}
	} catch (CoreException e) {
	}
	if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
		int index = pos - 1;
		while (index >= 0 && IndentManipulation.isLineDelimiterChar(source[index])) {
			pos--;
			index--;
		}
	}

	int newEnd = Math.max(fSelectionStart + pos, currentNode.getStartPosition() + currentNode.getLength());
	range = fRanges.get(currentNode);
	fRanges.put(currentNode, new SourceRange(range.getStartPosition(), newEnd - range.getStartPosition()));

	// The extended source range of the last child node can end after the selection.
	// We have to ensure that the source range of such child nodes is also capped by the selection range.
	// Example: (/*]*/TRANSFORMER::transform/*[*/)
	// Selection is between /*]*/ and /*[*/, but the extended range of the "transform" node actually includes /*[*/ as well.
	while (true) {
		List<ASTNode> children = ASTNodes.getChildren(currentNode);
		if (!children.isEmpty()) {
			ASTNode lastChild = children.get(children.size() - 1);
			SourceRange extRange = super.computeSourceRange(lastChild);
			if (extRange.getStartPosition() + extRange.getLength() > newEnd) {
				fRanges.put(lastChild, new SourceRange(extRange.getStartPosition(), newEnd - extRange.getStartPosition()));
				currentNode = lastChild;
				continue;
			}
		}
		break;
	}
}
 

public static NLSLine[] scan(String s) throws InvalidInputException, BadLocationException {
	IScanner scanner= ToolFactory.createScanner(true, true, false, true);
	return scan(scanner, s.toCharArray());
}
 

public Collector(IPackageFragment source, ReferencesInBinaryContext binaryRefs) {
	super(binaryRefs);
	fSource = source;
	fScanner = ToolFactory.createScanner(false, false, false, false);
}