Java Code Examples for org.antlr.v4.runtime.CommonToken#setLine()

The following examples show how to use org.antlr.v4.runtime.CommonToken#setLine() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Formatter.java    From codebuff with BSD 2-Clause "Simplified" License 6 votes vote down vote up 
public static void wipeCharPositionInfoAndWhitespaceTokens(CodeBuffTokenStream tokens) {
	tokens.fill();
	CommonToken dummy = new CommonToken(Token.INVALID_TYPE, "");
	dummy.setChannel(Token.HIDDEN_CHANNEL);
	Token firstRealToken = tokens.getNextRealToken(-1);
	for (int i = 0; i<tokens.size(); i++) {
		if ( i==firstRealToken.getTokenIndex() ) continue; // don't wack first token
		CommonToken t = (CommonToken)tokens.get(i);
		if ( t.getText().matches("\\s+") ) {
			tokens.getTokens().set(i, dummy); // wack whitespace token so we can't use it during prediction
		}
		else {
			t.setLine(0);
			t.setCharPositionInLine(-1);
		}
	}
}
Example 2
Source File: PythonLexerBase.java    From depends with MIT License 5 votes vote down vote up 
private void emit(int tokenType, int channel, String text) {
    int charIndex = getCharIndex();
    CommonToken token = new CommonToken(_tokenFactorySourcePair, tokenType, channel, charIndex - text.length(), charIndex);
    token.setLine(getLine());
    token.setCharPositionInLine(getCharPositionInLine());
    token.setText(text);

    emit(token);
}
Example 3
Source File: StaticScope.java    From trygve with GNU General Public License v2.0 5 votes vote down vote up 
private static void reinitializeObject(final Type objectType, final StaticScope objectsScope) {
	final CommonToken objectToken = new CommonToken(0);
	objectToken.setLine(157239);
	final ClassDeclaration objectClass = new ClassDeclaration("Object", objectsScope, null, objectToken);
	globalScope_.declareClass(objectClass);
	objectClass.setType(objectType);
	objectsScope.setDeclaration(objectClass);
	
	typeDeclarationList_.add(objectClass);
}
Example 4
Source File: Formatter.java    From codebuff with BSD 2-Clause "Simplified" License 4 votes vote down vote up 
public void processToken(int indexIntoRealTokens, int tokenIndexInStream, boolean collectAnalysis) {
	CommonToken curToken = (CommonToken)testDoc.tokens.get(tokenIndexInStream);
	String tokText = curToken.getText();
	TerminalNode node = tokenToNodeMap.get(curToken);

	int[] features = getFeatures(testDoc, tokenIndexInStream);
	int[] featuresForAlign = new int[features.length];
	System.arraycopy(features, 0, featuresForAlign, 0, features.length);

	int injectNL_WS = wsClassifier.classify(k, features, Trainer.MAX_WS_CONTEXT_DIFF_THRESHOLD);

	injectNL_WS = emitCommentsToTheLeft(tokenIndexInStream, injectNL_WS);

	int newlines = 0;
	int ws = 0;
	if ( (injectNL_WS&0xFF)==CAT_INJECT_NL ) {
		newlines = Trainer.unnlcat(injectNL_WS);
	}
	else if ( (injectNL_WS&0xFF)==CAT_INJECT_WS ) {
		ws = Trainer.unwscat(injectNL_WS);
	}

	if ( newlines==0 && ws==0 && cannotJoin(realTokens.get(indexIntoRealTokens-1), curToken) ) { // failsafe!
		ws = 1;
	}

	int alignOrIndent = CAT_ALIGN;

	if ( newlines>0 ) {
		output.append(Tool.newlines(newlines));
		line+=newlines;
		charPosInLine = 0;

		// getFeatures() doesn't know what line curToken is on. If \n, we need to find exemplars that start a line
		featuresForAlign[INDEX_FIRST_ON_LINE] = 1; // use \n prediction to match exemplars for alignment

		alignOrIndent = hposClassifier.classify(k, featuresForAlign, MAX_ALIGN_CONTEXT_DIFF_THRESHOLD);

		if ( (alignOrIndent&0xFF)==CAT_ALIGN_WITH_ANCESTOR_CHILD ) {
			align(alignOrIndent, node);
		}
		else if ( (alignOrIndent&0xFF)==CAT_INDENT_FROM_ANCESTOR_CHILD ) {
			indent(alignOrIndent, node);
		}
		else if ( (alignOrIndent&0xFF)==CAT_ALIGN ) {
			List<Token> tokensOnPreviousLine = getTokensOnPreviousLine(testDoc.tokens, tokenIndexInStream, line);
			if ( tokensOnPreviousLine.size()>0 ) {
				Token firstTokenOnPrevLine = tokensOnPreviousLine.get(0);
				int indentCol = firstTokenOnPrevLine.getCharPositionInLine();
				charPosInLine = indentCol;
				output.append(Tool.spaces(indentCol));
			}
		}
		else if ( (alignOrIndent&0xFF)==CAT_INDENT ) {
			indent(alignOrIndent, node);
		}
	}
	else {
		// inject whitespace instead of \n?
		output.append(Tool.spaces(ws));
		charPosInLine += ws;
	}

	// update Token object with position information now that we are about
	// to emit it.
	curToken.setLine(line);
	curToken.setCharPositionInLine(charPosInLine);

	TokenPositionAnalysis tokenPositionAnalysis =
		getTokenAnalysis(features, featuresForAlign, tokenIndexInStream, injectNL_WS, alignOrIndent, collectAnalysis);

	analysis.set(tokenIndexInStream, tokenPositionAnalysis);

	int n = tokText.length();
	tokenPositionAnalysis.charIndexStart = output.length();
	tokenPositionAnalysis.charIndexStop = tokenPositionAnalysis.charIndexStart + n - 1;

	// emit
	output.append(tokText);
	charPosInLine += n;
}