Java Code Examples for org.eclipse.wst.jsdt.internal.core.util.PublicScanner#getNextToken()

The following examples show how to use org.eclipse.wst.jsdt.internal.core.util.PublicScanner#getNextToken() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: JavascriptTokenizer.java    From api-mining with GNU General Public License v3.0 6 votes vote down vote up 
@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position, stripTokenIfNeeded(nxtToken));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
Example 2
Source File: JavascriptTokenizer.java    From tassal with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position, stripTokenIfNeeded(nxtToken));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
Example 3
Source File: JavascriptTokenizer.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up 
@Override
public SortedMap<Integer, String> tokenListWithPos(final char[] code) {
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, String> tokens = Maps.newTreeMap();
	tokens.put(-1, SENTENCE_START);
	tokens.put(Integer.MAX_VALUE, SENTENCE_END);
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position, stripTokenIfNeeded(nxtToken));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
Example 4
Source File: JavascriptTokenizer.java    From api-mining with GNU General Public License v3.0 5 votes vote down vote up 
@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	// TODO Duplicate Code
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position,
						new FullToken(nxtToken, Integer.toString(token)));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
Example 5
Source File: JavascriptTokenizer.java    From tassal with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	// TODO Duplicate Code
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position,
						new FullToken(nxtToken, Integer.toString(token)));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}
Example 6
Source File: JavascriptTokenizer.java    From codemining-core with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
@Override
public SortedMap<Integer, FullToken> fullTokenListWithPos(final char[] code) {
	// TODO Duplicate Code
	final PublicScanner scanner = prepareScanner();
	final SortedMap<Integer, FullToken> tokens = Maps.newTreeMap();
	tokens.put(-1, new FullToken(SENTENCE_START, SENTENCE_START));
	tokens.put(Integer.MAX_VALUE, new FullToken(SENTENCE_END, SENTENCE_END));
	scanner.setSource(code);
	while (!scanner.atEnd()) {
		do {
			try {
				final int token = scanner.getNextToken();
				if (token == ITerminalSymbols.TokenNameEOF) {
					break;
				}
				final String nxtToken = transformToken(token,
						scanner.getCurrentTokenString());
				final int position = scanner.getCurrentTokenStartPosition();
				tokens.put(position,
						new FullToken(nxtToken, Integer.toString(token)));
			} catch (final InvalidInputException e) {
				LOGGER.warning(ExceptionUtils.getFullStackTrace(e));
			}
		} while (!scanner.atEnd());

	}
	return tokens;
}