org.eclipse.jdt.internal.corext.dom.TokenScanner Java Examples

private int getSurroundingComment(IScanner scanner) {
	try {
		int start= fLocation.getOffset();
		int end= start + fLocation.getLength();

		int token= scanner.getNextToken();
		while (token != ITerminalSymbols.TokenNameEOF) {
			if (TokenScanner.isComment(token)) {
				int currStart= scanner.getCurrentTokenStartPosition();
				int currEnd= scanner.getCurrentTokenEndPosition() + 1;
				if (currStart <= start && end <= currEnd) {
					return token;
				}
			}
			token= scanner.getNextToken();
		}

	} catch (InvalidInputException e) {
		// ignore
	}
	return ITerminalSymbols.TokenNameEOF;
}
 

/**
 * Creates a token comparator for the given string.
 *
 * @param text the text to be tokenized
 */
public JavaTokenComparator(String text) {
	Assert.isLegal(text != null);

	fText= text;

	int length= fText.length();
	fStarts= new int[length];
	fLengths= new int[length];
	fCount= 0;

	IScanner scanner= ToolFactory.createScanner(true, true, false, false); // returns comments & whitespace
	scanner.setSource(fText.toCharArray());
	int endPos= 0;
	try {
		int tokenType;
		while ((tokenType= scanner.getNextToken()) != ITerminalSymbols.TokenNameEOF) {
			int start= scanner.getCurrentTokenStartPosition();
			int end= scanner.getCurrentTokenEndPosition()+1;
			// Comments and strings should not be treated as a single token, see https://bugs.eclipse.org/78063
			if (TokenScanner.isComment(tokenType) || tokenType == ITerminalSymbols.TokenNameStringLiteral) {
				// Line comments are often commented code, so lets treat them as code. See https://bugs.eclipse.org/216707
				boolean parseAsJava= tokenType == ITerminalSymbols.TokenNameCOMMENT_LINE;
				int dl= parseAsJava ? getCommentStartTokenLength(tokenType) : 0;
				if (dl > 0)
					recordTokenRange(start, dl);
				parseSubrange(start + dl, text.substring(start + dl, end), parseAsJava);
			} else {
				recordTokenRange(start, end - start);
			}
			endPos= end;
		}
	} catch (InvalidInputException ex) {
		// We couldn't parse part of the input. Fall through and make the rest a single token
	}
	// Workaround for https://bugs.eclipse.org/bugs/show_bug.cgi?id=13907
	if (endPos < length) {
		recordTokenRange(endPos, length - endPos);
	}
}
 

private boolean needsSemicolon(Expression expression) {
	if ((expression.getParent().getFlags() & ASTNode.RECOVERED) != 0) {
		try {
			TokenScanner scanner= new TokenScanner(getCompilationUnit());
			return scanner.readNext(expression.getStartPosition() + expression.getLength(), true) != ITerminalSymbols.TokenNameSEMICOLON;
		} catch (CoreException e) {
			// ignore
		}
	}
	return false;
}
 

private boolean isValidComment(String template) {
  IScanner scanner = ToolFactory.createScanner(true, false, false, false);
  scanner.setSource(template.toCharArray());
  try {
    int next = scanner.getNextToken();
    while (TokenScanner.isComment(next)) {
      next = scanner.getNextToken();
    }
    return next == ITerminalSymbols.TokenNameEOF;
  } catch (InvalidInputException e) {
    // If there are lexical errors, the comment is invalid
  }
  return false;
}
 

private boolean isValidComment(String template) {
	IScanner scanner= ToolFactory.createScanner(true, false, false, false);
	scanner.setSource(template.toCharArray());
	try {
		int next= scanner.getNextToken();
		while (TokenScanner.isComment(next)) {
			next= scanner.getNextToken();
		}
		return next == ITerminalSymbols.TokenNameEOF;
	} catch (InvalidInputException e) {
	}
	return false;
}
 

@Override
public boolean visit(LambdaExpression node) {
	Selection selection= getSelection();
	int selectionStart= selection.getOffset();
	int selectionExclusiveEnd= selection.getExclusiveEnd();
	int lambdaStart= node.getStartPosition();
	int lambdaExclusiveEnd= lambdaStart + node.getLength();
	ASTNode body= node.getBody();
	int bodyStart= body.getStartPosition();
	int bodyExclusiveEnd= bodyStart + body.getLength();

	boolean isValidSelection= false;
	if ((body instanceof Block) && (bodyStart < selectionStart && selectionExclusiveEnd <= bodyExclusiveEnd)) {
		// if selection is inside lambda body's block
		isValidSelection= true;
	} else if (body instanceof Expression) {
		try {
			TokenScanner scanner= new TokenScanner(fCUnit);
			int arrowExclusiveEnd= scanner.getTokenEndOffset(ITerminalSymbols.TokenNameARROW, lambdaStart);
			if (selectionStart >= arrowExclusiveEnd) {
				isValidSelection= true;
			}
		} catch (CoreException e) {
			// ignore
		}
	}
	if (selectionStart <= lambdaStart && selectionExclusiveEnd >= lambdaExclusiveEnd) {
		// if selection covers the lambda node
		isValidSelection= true;
	}

	if (!isValidSelection) {
		return false;
	}
	return super.visit(node);
}
 

private boolean needsSemicolon(Expression expression) {
	if ((expression.getParent().getFlags() & ASTNode.RECOVERED) != 0) {
		try {
			TokenScanner scanner= new TokenScanner(getCompilationUnit());
			return scanner.readNext(expression.getStartPosition() + expression.getLength(), true) != ITerminalSymbols.TokenNameSEMICOLON;
		} catch (CoreException e) {
			// ignore
		}
	}
	return false;
}
 

public StatementAnalyzer(ICompilationUnit cunit, Selection selection, boolean traverseSelectedNode) throws CoreException {
	super(selection, traverseSelectedNode);
	Assert.isNotNull(cunit);
	fCUnit= cunit;
	fStatus= new RefactoringStatus();
	fScanner= new TokenScanner(fCUnit);
}
 

@Override
public boolean visit(LambdaExpression node) {
	Selection selection = getSelection();
	int selectionStart = selection.getOffset();
	int selectionExclusiveEnd = selection.getExclusiveEnd();
	int lambdaStart = node.getStartPosition();
	int lambdaExclusiveEnd = lambdaStart + node.getLength();
	ASTNode body = node.getBody();
	int bodyStart = body.getStartPosition();
	int bodyExclusiveEnd = bodyStart + body.getLength();

	boolean isValidSelection = false;
	if ((body instanceof Block) && (bodyStart < selectionStart && selectionExclusiveEnd <= bodyExclusiveEnd)) {
		// if selection is inside lambda body's block
		isValidSelection = true;
	} else if (body instanceof Expression) {
		try {
			TokenScanner scanner = new TokenScanner(fCUnit);
			int arrowExclusiveEnd = scanner.getTokenEndOffset(ITerminalSymbols.TokenNameARROW, lambdaStart);
			if (selectionStart >= arrowExclusiveEnd) {
				isValidSelection = true;
			}
		} catch (CoreException e) {
			// ignore
		}
	}
	if (selectionStart <= lambdaStart && selectionExclusiveEnd >= lambdaExclusiveEnd) {
		// if selection covers the lambda node
		isValidSelection = true;
	}

	if (!isValidSelection) {
		return false;
	}
	return super.visit(node);
}
 

private boolean isValidComment(String template) {
	IScanner scanner= ToolFactory.createScanner(true, false, false, false);
	scanner.setSource(template.toCharArray());
	try {
		int next= scanner.getNextToken();
		while (TokenScanner.isComment(next)) {
			next= scanner.getNextToken();
		}
		return next == ITerminalSymbols.TokenNameEOF;
	} catch (InvalidInputException e) {
	}
	return false;
}
 

private int getMemberStartOffset(IMember curr, IDocument document) throws JavaModelException {
	int offset= curr.getSourceRange().getOffset();
	TokenScanner scanner= new TokenScanner(document, curr.getJavaProject());
	try {
		return scanner.getNextStartOffset(offset, true); // read to the first real non comment token
	} catch (CoreException e) {
		// ignore
	}
	return offset;
}
 

private boolean isValidComment(String template) {
	IScanner scanner= ToolFactory.createScanner(true, false, false, false);
	scanner.setSource(template.toCharArray());
	try {
		int next= scanner.getNextToken();
		while (TokenScanner.isComment(next)) {
			next= scanner.getNextToken();
		}
		return next == ITerminalSymbols.TokenNameEOF;
	} catch (InvalidInputException e) {
	}
	return false;
}
 

private OccurrenceLocation getLocationForFirstToken(ASTNode node) {
	try {
		int nextEndOffset= new TokenScanner(fASTRoot.getTypeRoot()).getNextEndOffset(node.getStartPosition(), true);
		return new OccurrenceLocation(node.getStartPosition(), nextEndOffset - node.getStartPosition(), 0, fDescription);
	} catch (CoreException e) {
		// ignore
	}
	return new OccurrenceLocation(node.getStartPosition(), node.getLength(), 0, fDescription);
}
 

private int getNextElseOffset(Statement then, ITypeRoot editorInput) {
	int thenEnd= ASTNodes.getExclusiveEnd(then);
	try {
		TokenScanner scanner= new TokenScanner(editorInput);
		return scanner.getNextStartOffset(thenEnd, true);
	} catch (CoreException e) {
		// ignore
	}
	return -1;
}
 

protected TokenScanner getTokenScanner() {
	return fScanner;
}
 

private void initializeRanges() throws CoreException {
	fRanges= new HashMap<ASTNode, SourceRange>();
	if (fSelectedNodes.length == 0)
		return;

	fRanges.put(fSelectedNodes[0], super.computeSourceRange(fSelectedNodes[0]));
	int last= fSelectedNodes.length - 1;
	fRanges.put(fSelectedNodes[last], super.computeSourceRange(fSelectedNodes[last]));

	IScanner scanner= ToolFactory.createScanner(true, false, false, false);
	char[] source= fDocumentPortionToScan.toCharArray();
	scanner.setSource(source);
	fDocumentPortionToScan= null; // initializeRanges() is only called once

	TokenScanner tokenizer= new TokenScanner(scanner);
	int pos= tokenizer.getNextStartOffset(0, false);

	ASTNode currentNode= fSelectedNodes[0];
	int newStart= Math.min(fSelectionStart + pos, currentNode.getStartPosition());
	SourceRange range= fRanges.get(currentNode);
	fRanges.put(currentNode, new SourceRange(newStart, range.getLength() + range.getStartPosition() - newStart));

	currentNode= fSelectedNodes[last];
	int scannerStart= currentNode.getStartPosition() + currentNode.getLength() - fSelectionStart;
	tokenizer.setOffset(scannerStart);
	pos= scannerStart;
	int token= -1;
	try {
		while (true) {
			token= tokenizer.readNext(false);
			pos= tokenizer.getCurrentEndOffset();
		}
	} catch (CoreException e) {
	}
	if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
		int index= pos - 1;
		while (index >= 0 && IndentManipulation.isLineDelimiterChar(source[index])) {
			pos--;
			index--;
		}
	}

	int newEnd= Math.max(fSelectionStart + pos, currentNode.getStartPosition() + currentNode.getLength());
	range= fRanges.get(currentNode);
	fRanges.put(currentNode, new SourceRange(range.getStartPosition(), newEnd - range.getStartPosition()));
}
 

private Location computeBreakContinue(ITypeRoot typeRoot, int line, int column) throws CoreException {
	int offset = JsonRpcHelpers.toOffset(typeRoot.getBuffer(), line, column);
	if (offset >= 0) {
		CompilationUnit unit = SharedASTProviderCore.getAST(typeRoot, SharedASTProviderCore.WAIT_YES, null);
		if (unit == null) {
			return null;
		}
		ASTNode selectedNode = NodeFinder.perform(unit, offset, 0);
		ASTNode node = null;
		SimpleName label = null;
		if (selectedNode instanceof BreakStatement) {
			node = selectedNode;
			label = ((BreakStatement) node).getLabel();
		} else if (selectedNode instanceof ContinueStatement) {
			node = selectedNode;
			label = ((ContinueStatement) node).getLabel();
		} else if (selectedNode instanceof SimpleName && selectedNode.getParent() instanceof BreakStatement) {
			node = selectedNode.getParent();
			label = ((BreakStatement) node).getLabel();
		} else if (selectedNode instanceof SimpleName && selectedNode.getParent() instanceof ContinueStatement) {
			node = selectedNode.getParent();
			label = ((ContinueStatement) node).getLabel();
		}
		if (node != null) {
			ASTNode parent = node.getParent();
			ASTNode target = null;
			while (parent != null) {
				if (parent instanceof MethodDeclaration || parent instanceof Initializer) {
					break;
				}
				if (label == null) {
					if (parent instanceof ForStatement || parent instanceof EnhancedForStatement || parent instanceof WhileStatement || parent instanceof DoStatement) {
						target = parent;
						break;
					}
					if (node instanceof BreakStatement) {
						if (parent instanceof SwitchStatement || parent instanceof SwitchExpression) {
							target = parent;
							break;
						}
					}
					if (node instanceof LabeledStatement) {
						target = parent;
						break;
					}
				} else if (LabeledStatement.class.isInstance(parent)) {
					LabeledStatement ls = (LabeledStatement) parent;
					if (ls.getLabel().getIdentifier().equals(label.getIdentifier())) {
						target = ls;
						break;
					}
				}
				parent = parent.getParent();
			}
			if (target != null) {
				int start = target.getStartPosition();
				int end = new TokenScanner(unit.getTypeRoot()).getNextEndOffset(node.getStartPosition(), true) - start;
				if (start >= 0 && end >= 0) {
					return JDTUtils.toLocation((ICompilationUnit) typeRoot, start, end);
				}
			}
		}
	}
	return null;
}
 

private void initializeRanges() throws CoreException {
	fRanges = new HashMap<>();
	if (fSelectedNodes.length == 0) {
		return;
	}

	fRanges.put(fSelectedNodes[0], super.computeSourceRange(fSelectedNodes[0]));
	int last = fSelectedNodes.length - 1;
	fRanges.put(fSelectedNodes[last], super.computeSourceRange(fSelectedNodes[last]));

	IJavaProject javaProject = ((CompilationUnit) fSelectedNodes[0].getRoot()).getTypeRoot().getJavaProject();
	String sourceLevel = javaProject.getOption(JavaCore.COMPILER_SOURCE, true);
	String complianceLevel = javaProject.getOption(JavaCore.COMPILER_COMPLIANCE, true);
	IScanner scanner = ToolFactory.createScanner(true, false, false, sourceLevel, complianceLevel);
	char[] source = fDocumentPortionToScan.toCharArray();
	scanner.setSource(source);
	fDocumentPortionToScan = null; // initializeRanges() is only called once

	TokenScanner tokenizer = new TokenScanner(scanner);
	int pos = tokenizer.getNextStartOffset(0, false);

	ASTNode currentNode = fSelectedNodes[0];
	int newStart = Math.min(fSelectionStart + pos, currentNode.getStartPosition());
	SourceRange range = fRanges.get(currentNode);
	fRanges.put(currentNode, new SourceRange(newStart, range.getLength() + range.getStartPosition() - newStart));

	currentNode = fSelectedNodes[last];
	int scannerStart = currentNode.getStartPosition() + currentNode.getLength() - fSelectionStart;
	tokenizer.setOffset(scannerStart);
	pos = scannerStart;
	int token = -1;
	try {
		while (true) {
			token = tokenizer.readNext(false);
			pos = tokenizer.getCurrentEndOffset();
		}
	} catch (CoreException e) {
	}
	if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
		int index = pos - 1;
		while (index >= 0 && IndentManipulation.isLineDelimiterChar(source[index])) {
			pos--;
			index--;
		}
	}

	int newEnd = Math.max(fSelectionStart + pos, currentNode.getStartPosition() + currentNode.getLength());
	range = fRanges.get(currentNode);
	fRanges.put(currentNode, new SourceRange(range.getStartPosition(), newEnd - range.getStartPosition()));

	// The extended source range of the last child node can end after the selection.
	// We have to ensure that the source range of such child nodes is also capped by the selection range.
	// Example: (/*]*/TRANSFORMER::transform/*[*/)
	// Selection is between /*]*/ and /*[*/, but the extended range of the "transform" node actually includes /*[*/ as well.
	while (true) {
		List<ASTNode> children = ASTNodes.getChildren(currentNode);
		if (!children.isEmpty()) {
			ASTNode lastChild = children.get(children.size() - 1);
			SourceRange extRange = super.computeSourceRange(lastChild);
			if (extRange.getStartPosition() + extRange.getLength() > newEnd) {
				fRanges.put(lastChild, new SourceRange(extRange.getStartPosition(), newEnd - extRange.getStartPosition()));
				currentNode = lastChild;
				continue;
			}
		}
		break;
	}
}