org.eclipse.jdt.core.compiler.ITerminalSymbols Java Examples

public static boolean isModifier(int token) {
	switch (token) {
		case ITerminalSymbols.TokenNamepublic:
		case ITerminalSymbols.TokenNameprotected:
		case ITerminalSymbols.TokenNameprivate:
		case ITerminalSymbols.TokenNamestatic:
		case ITerminalSymbols.TokenNamefinal:
		case ITerminalSymbols.TokenNameabstract:
		case ITerminalSymbols.TokenNamenative:
		case ITerminalSymbols.TokenNamevolatile:
		case ITerminalSymbols.TokenNamestrictfp:
		case ITerminalSymbols.TokenNametransient:
		case ITerminalSymbols.TokenNamesynchronized:
			return true;
		default:
			return false;
	}
}
 

@Override
public boolean visit(DoStatement node) {
	boolean result = super.visit(node);

	try {
		int actionStart = getTokenScanner().getTokenEndOffset(ITerminalSymbols.TokenNamedo, node.getStartPosition());
		if (getSelection().getOffset() == actionStart) {
			invalidSelection(RefactoringCoreMessages.ExtractMethodAnalyzer_after_do_keyword, JavaStatusContext.create(fCUnit, getSelection()));
			return false;
		}
	} catch (CoreException e) {
		// ignore
	}

	return result;
}
 

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			for (final String cToken : getConvertedToken(scanner, token)) {
				tokens.add(new FullToken(cToken, ""));
			}
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}
 

private int getLastSimpleNameStart(String reference) {
	fScanner.setSource(reference.toCharArray());
	int lastIdentifierStart = -1;
	try {
		int tokenType = fScanner.getNextToken();
		while (tokenType != ITerminalSymbols.TokenNameEOF) {
			if (tokenType == ITerminalSymbols.TokenNameIdentifier) {
				lastIdentifierStart = fScanner.getCurrentTokenStartPosition();
			}
			tokenType = fScanner.getNextToken();
		}
	} catch (InvalidInputException e) {
		JavaLanguageServerPlugin.logException(e.getMessage(), e);
	}
	return lastIdentifierStart;
}
 

/**
 * @param source
 *            must be not null
 * @param range
 *            can be null
 * @return may return null, otherwise an initialized scanner which may
 *         answer which source offset index belongs to which source line
 * @throws JavaModelException
 */
private static IScanner initScanner(IType source, ISourceRange range) throws JavaModelException {
    if (range == null) {
        return null;
    }
    char[] charContent = getContent(source);
    if (charContent == null) {
        return null;
    }
    IScanner scanner = ToolFactory.createScanner(false, false, false, true);
    scanner.setSource(charContent);
    int offset = range.getOffset();
    try {
        while (scanner.getNextToken() != ITerminalSymbols.TokenNameEOF) {
            // do nothing, just wait for the end of stream
            if (offset <= scanner.getCurrentTokenEndPosition()) {
                break;
            }
        }
    } catch (InvalidInputException e) {
        FindbugsPlugin.getDefault().logException(e, "Could not init scanner for type: " + source);
    }
    return scanner;
}
 

private void doScan() {
	try{
		int token = fScanner.getNextToken();
		while (token != ITerminalSymbols.TokenNameEOF) {
			switch (token) {
				case ITerminalSymbols.TokenNameStringLiteral :
				case ITerminalSymbols.TokenNameCOMMENT_JAVADOC :
				case ITerminalSymbols.TokenNameCOMMENT_LINE :
				case ITerminalSymbols.TokenNameCOMMENT_BLOCK :
					parseCurrentToken();
			}
			token = fScanner.getNextToken();
		}
	} catch (InvalidInputException e){
		//ignore
	}
}
 

private static boolean isJustWhitespaceOrComment(int start, int end, IBuffer buffer) {
	if (start == end) {
		return true;
	}
	Assert.isTrue(start <= end);
	String trimmedText = buffer.getText(start, end - start).trim();
	if (0 == trimmedText.length()) {
		return true;
	} else {
		IScanner scanner = ToolFactory.createScanner(false, false, false, null);
		scanner.setSource(trimmedText.toCharArray());
		try {
			return scanner.getNextToken() == ITerminalSymbols.TokenNameEOF;
		} catch (InvalidInputException e) {
			return false;
		}
	}
}
 

private static boolean isJustWhitespaceOrComment(int start, int end, IBuffer buffer) {
	if (start == end)
		return true;
	Assert.isTrue(start <= end);
	String trimmedText= buffer.getText(start, end - start).trim();
	if (0 == trimmedText.length()) {
		return true;
	} else {
		IScanner scanner= ToolFactory.createScanner(false, false, false, null);
		scanner.setSource(trimmedText.toCharArray());
		try {
			return scanner.getNextToken() == ITerminalSymbols.TokenNameEOF;
		} catch (InvalidInputException e) {
			return false;
		}
	}
}
 

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position, stripTokenIfNeeded(nxtToken));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			for (final String cToken : getConvertedToken(scanner, token)) {
				tokens.add(new FullToken(cToken, ""));
			}
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}
 

/**
 * Removes comments and whitespace
 * @param reference the type reference
 * @return the reference only consisting of dots and java identifier characters
 */
public static String normalizeReference(String reference) {
	IScanner scanner= ToolFactory.createScanner(false, false, false, false);
	scanner.setSource(reference.toCharArray());
	StringBuffer sb= new StringBuffer();
	try {
		int tokenType= scanner.getNextToken();
		while (tokenType != ITerminalSymbols.TokenNameEOF) {
			sb.append(scanner.getRawTokenSource());
			tokenType= scanner.getNextToken();
		}
	} catch (InvalidInputException e) {
		Assert.isTrue(false, reference);
	}
	reference= sb.toString();
	return reference;
}
 

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			tokens.addAll(getConvertedToken(scanner, token));
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}
 

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position, stripTokenIfNeeded(nxtToken));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			tokens.addAll(getConvertedToken(scanner, token));
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}
 

@Override
public boolean visit(DoStatement node) {
	boolean result= super.visit(node);

	try {
		int actionStart= getTokenScanner().getTokenEndOffset(ITerminalSymbols.TokenNamedo, node.getStartPosition());
		if (getSelection().getOffset() == actionStart) {
			invalidSelection(RefactoringCoreMessages.ExtractMethodAnalyzer_after_do_keyword, JavaStatusContext.create(fCUnit, getSelection()));
			return false;
		}
	} catch (CoreException e) {
		// ignore
	}

	return result;
}
 

private void doScan() {
	try{
		int token = fScanner.getNextToken();
		while (token != ITerminalSymbols.TokenNameEOF) {
			switch (token) {
				case ITerminalSymbols.TokenNameStringLiteral :
				case ITerminalSymbols.TokenNameCOMMENT_JAVADOC :
				case ITerminalSymbols.TokenNameCOMMENT_LINE :
				case ITerminalSymbols.TokenNameCOMMENT_BLOCK :
					parseCurrentToken();
			}
			token = fScanner.getNextToken();
		}
	} catch (InvalidInputException e){
		//ignore
	}
}
 

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position, stripTokenIfNeeded(nxtToken));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 

@Override
public List<String> tokenListFromCode(final char[] code) {
	final List<String> tokens = Lists.newArrayList();
	tokens.add(SENTENCE_START);
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			tokens.addAll(getConvertedToken(scanner, token));
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(SENTENCE_END);
	return tokens;
}
 

@Override
public List<FullToken> getTokenListFromCode(final char[] code) {
	final List<FullToken> tokens = Lists.newArrayList();
	tokens.add(new FullToken(SENTENCE_START, SENTENCE_START));
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			for (final String cToken : getConvertedToken(scanner, token)) {
				tokens.add(new FullToken(cToken, ""));
			}
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new FullToken(SENTENCE_END, SENTENCE_END));
	return tokens;
}
 

private boolean isValidComment(String template) {
	IScanner scanner= ToolFactory.createScanner(true, false, false, false);
	scanner.setSource(template.toCharArray());
	try {
		int next= scanner.getNextToken();
		while (TokenScanner.isComment(next)) {
			next= scanner.getNextToken();
		}
		return next == ITerminalSymbols.TokenNameEOF;
	} catch (InvalidInputException e) {
	}
	return false;
}
 

/**
 * Evaluates the start offset of comments directly ahead of a token specified by its start offset
 *
 * @param lastPos An offset to before the node start offset. Can be 0 but better is the end location of the previous node.
 * @param nodeStart Start offset of the node to find the comments for.
 * @return Returns the start offset of comments directly ahead of a token.
 * @exception CoreException Thrown when a lexical error was detected while scanning (code LEXICAL_ERROR)
 */
public int getTokenCommentStart(int lastPos, int nodeStart) throws CoreException {
	setOffset(lastPos);

	int prevEndPos= lastPos;
	int prevEndLine= prevEndPos > 0 ? getLineOfOffset(prevEndPos - 1) : 0;
	int nodeLine= getLineOfOffset(nodeStart);

	int res= -1;

	int curr= readNextWithEOF(false);
	int currStartPos= getCurrentStartOffset();
	int currStartLine= getLineOfOffset(currStartPos);
	while (curr != ITerminalSymbols.TokenNameEOF && nodeStart > currStartPos) {
		if (TokenScanner.isComment(curr)) {
			int linesDifference= currStartLine - prevEndLine;
			if ((linesDifference > 1) || (res == -1 && (linesDifference != 0 || nodeLine == currStartLine))) {
				res= currStartPos; // begin new
			}
		} else {
			res= -1;
		}

		if (curr == ITerminalSymbols.TokenNameCOMMENT_LINE) {
			prevEndLine= currStartLine;
		} else {
			prevEndLine= getLineOfOffset(getCurrentEndOffset() - 1);
		}
		curr= readNextWithEOF(false);
		currStartPos= getCurrentStartOffset();
		currStartLine= getLineOfOffset(currStartPos);
	}
	if (res == -1 || curr == ITerminalSymbols.TokenNameEOF) {
		return nodeStart;
	}
	if (currStartLine - prevEndLine > 1) {
		return nodeStart;
	}
	return res;
}
 

/**
 * Reads the next token.
 * @param ignoreComments If set, comments will be overread
 * @return Return the token id.
 * @exception CoreException Thrown when the end of the file has been reached (code END_OF_FILE)
 * or a lexical error was detected while scanning (code LEXICAL_ERROR)
 */
public int readNext(boolean ignoreComments) throws CoreException {
	int curr= 0;
	do {
		try {
			curr= fScanner.getNextToken();
			if (curr == ITerminalSymbols.TokenNameEOF) {
				throw new CoreException(createError(END_OF_FILE, "End Of File", null)); //$NON-NLS-1$
			}
		} catch (InvalidInputException e) {
			throw new CoreException(createError(LEXICAL_ERROR, e.getMessage(), e));
		}
	} while (ignoreComments && isComment(curr));
	return curr;
}
 

/**
 * @param token
 * @return
 */
public static boolean isLiteralToken(final int token) {
	return token == ITerminalSymbols.TokenNameStringLiteral
			|| token == ITerminalSymbols.TokenNameCharacterLiteral
			|| token == ITerminalSymbols.TokenNameFloatingPointLiteral
			|| token == ITerminalSymbols.TokenNameIntegerLiteral
			|| token == ITerminalSymbols.TokenNameLongLiteral
			|| token == ITerminalSymbols.TokenNameDoubleLiteral;
}
 

@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));
	final PublicScanner scanner = createScanner();
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner.getCurrentTokenStartPosition();

				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				} else if (token == ITerminalSymbols.TokenNameIdentifier) {
					tokens.put(position,
							new FullToken(IDENTIFIER_TOKEN, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_BLOCK) {
					tokens.put(position, new FullToken(COMMENT_BLOCK, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_JAVADOC) {
					tokens.put(position, new FullToken(COMMENT_JAVADOC, ""));
				} else if (token == ITerminalSymbols.TokenNameCOMMENT_LINE) {
					tokens.put(position, new FullToken(COMMENT_LINE, ""));
				} else if (isLiteralToken(token)) {
					tokens.put(position, new FullToken(LITERAL_TOKEN, ""));
				} else {
					tokens.put(position,
							new FullToken(scanner.getCurrentTokenString(),
									""));
				}

			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 

private boolean isValidComment(String template) {
	IScanner scanner= ToolFactory.createScanner(true, false, false, false);
	scanner.setSource(template.toCharArray());
	try {
		int next= scanner.getNextToken();
		while (TokenScanner.isComment(next)) {
			next= scanner.getNextToken();
		}
		return next == ITerminalSymbols.TokenNameEOF;
	} catch (InvalidInputException e) {
	}
	return false;
}
 

@Test
public void testTokenTypes() {
	ITokenizer tokenizer = new JavaTokenizer();
	assertEquals(
			tokenizer.getTokenFromString("hello"),
			new ITokenizer.FullToken("hello", tokenizer.getIdentifierType()));
	assertEquals(
			tokenizer.getTokenFromString("{"),
			new ITokenizer.FullToken("{", Integer
					.toString(ITerminalSymbols.TokenNameLBRACE)));

}
 

@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	// TODO Duplicate Code
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position,
						new FullToken(nxtToken, Integer.toString(token)));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
 

@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	final PublicScanner scanner = prepareScanner(code);

	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				final int position = scanner
						.getCurrentTokenStartPosition();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				int i = 0;
				final List<String> cTokens = getConvertedToken(scanner,
						token);
				for (final String cToken : cTokens) {
					tokens.put(position + i, cToken);
					i++;
				}
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());
	}
	return tokens;
}
 

public List<WhitespaceAnnotatedToken> getTokensWithWidthData(
		final char[] code) {
	final List<WhitespaceAnnotatedToken> tokens = Lists.newArrayList();
	tokens.add(new WhitespaceAnnotatedToken(SENTENCE_START,
			SENTENCE_START, 0, 0));
	final PublicScanner scanner = prepareScanner(code);
	do {
		try {
			final int token = scanner.getNextToken();
			if (token == ITerminalSymbols.TokenNameEOF) {
				break;
			}
			for (final String cToken : getConvertedToken(scanner, token)) {
				final int currentPosition = scanner
						.getCurrentTokenStartPosition();
				final int currentLine = scanner
						.getLineNumber(currentPosition);
				final int lineStart = scanner.getLineStart(currentLine);
				tokens.add(new WhitespaceAnnotatedToken(cToken, "",
						currentPosition - lineStart, scanner
						.getCurrentTokenString().length()));
			}
		} catch (final InvalidInputException e) {
			LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
		}
	} while (!scanner.atEnd());
	tokens.add(new WhitespaceAnnotatedToken(SENTENCE_END, SENTENCE_END,
			0, 0));
	return tokens;
}
 

private int getLastSimpleNameStart(String reference) {
	fScanner.setSource(reference.toCharArray());
	int lastIdentifierStart= -1;
	try {
		int tokenType= fScanner.getNextToken();
		while (tokenType != ITerminalSymbols.TokenNameEOF) {
			if (tokenType == ITerminalSymbols.TokenNameIdentifier)
				lastIdentifierStart= fScanner.getCurrentTokenStartPosition();
			tokenType= fScanner.getNextToken();
		}
	} catch (InvalidInputException e) {
		JavaPlugin.log(e);
	}
	return lastIdentifierStart;
}