org.antlr.v4.runtime.misc.Utils Java Examples

public static void main(String[] args) throws Exception {
		LangDescriptor[] languages = new LangDescriptor[] {
			JAVA_DESCR,
			JAVA8_DESCR,
			JAVA_GUAVA_DESCR,
			JAVA8_GUAVA_DESCR,
			ANTLR4_DESCR,
			SQLITE_CLEAN_DESCR,
			TSQL_CLEAN_DESCR,
			SQLITE_NOISY_DESCR,
			TSQL_NOISY_DESCR,
//			QUORUM_DESCR,
		};
		List<String> corpusDirs = map(languages, l -> l.corpusDir);
		String[] dirs = corpusDirs.toArray(new String[languages.length]);
		String python = testAllLanguages(languages, dirs, "leave_one_out.pdf");
		String fileName = "python/src/leave_one_out.py";
		Utils.writeFile(fileName, python);
		System.out.println("wrote python code to "+fileName);
	}
 

public static void format(LangDescriptor language,
                          String testFileName,
                          String outputFileName)
	throws Exception
{
	// load all files up front
	List<String> allFiles = getFilenames(new File(language.corpusDir), language.fileRegex);
	List<InputDocument> documents = load(allFiles, language);
	// if in corpus, don't include in corpus
	final String path = new File(testFileName).getAbsolutePath();
	List<InputDocument> others = filter(documents, d -> !d.fileName.equals(path));
	InputDocument testDoc = parse(testFileName, language);
	Corpus corpus = new Corpus(others, language);
	corpus.train();

	Formatter formatter = new Formatter(corpus, language.indentSize, Formatter.DEFAULT_K,
	                                    FEATURES_INJECT_WS, FEATURES_HPOS);
	String output = formatter.format(testDoc, false);

	if ( outputFileName!=null ) {
		Utils.writeFile(outputFileName, output);
	}
	else {
		System.out.print(output);
	}
}
 

/** Print out a whole tree in LISP form. Arg nodeTextProvider is used on the
 *  node payloads to get the text for the nodes.
 *
 *  @since 4.5.1
 */
public static String toStringTree(Tree t, TreeTextProvider nodeTextProvider) {
	if ( t==null ) return "null";
	String s = Utils.escapeWhitespace(nodeTextProvider.getText(t), false);
	if ( t.getChildCount()==0 ) return s;
	StringBuilder buf = new StringBuilder();
	buf.append("(");
	s = Utils.escapeWhitespace(nodeTextProvider.getText(t), false);
	buf.append(s);
	buf.append(' ');
	for (int i = 0; i<t.getChildCount(); i++) {
		if ( i>0 ) buf.append(' ');
		buf.append(toStringTree(t.getChild(i), nodeTextProvider));
	}
	buf.append(")");
	return buf.toString();
}
 

public static void main(String[] args) throws Exception {
	LangDescriptor[] languages = new LangDescriptor[] {
		JAVA_DESCR,
		JAVA8_DESCR,
		JAVA_GUAVA_DESCR,
	};
	List<String> corpusDirs = map(languages, l -> l.corpusDir);
	String[] dirs = corpusDirs.toArray(new String[languages.length]);
	String python = testAllLanguages(languages, dirs, "all_java_leave_one_out.pdf");
	String fileName = "python/src/all_java_leave_one_out.py";
	Utils.writeFile(fileName, python);
	System.out.println("wrote python code to "+fileName);
}
 

public static void main(String[] args) throws Exception {
	LangDescriptor[] languages = new LangDescriptor[] {
		SQLITE_NOISY_DESCR,
		SQLITE_CLEAN_DESCR,
		TSQL_NOISY_DESCR,
		TSQL_CLEAN_DESCR,
	};
	List<String> corpusDirs = map(languages, l -> l.corpusDir);
	String[] dirs = corpusDirs.toArray(new String[languages.length]);
	String python = testAllLanguages(languages, dirs, "all_sql_leave_one_out.pdf");
	String fileName = "python/src/all_sql_leave_one_out.py";
	Utils.writeFile(fileName, python);
	System.out.println("wrote python code to "+fileName);
}
 

public static void writePython(LangDescriptor[] languages, List<Integer> ks, Float[][] medians) throws IOException {
	StringBuilder data = new StringBuilder();
	StringBuilder plot = new StringBuilder();
	for (int i = 0; i<languages.length; i++) {
		LangDescriptor language = languages[i];
		List<Float> filteredMedians = BuffUtils.filter(Arrays.asList(medians[i]), m -> m!=null);
		data.append(language.name+'='+filteredMedians+'\n');
		plot.append(String.format("ax.plot(ks, %s, label=\"%s\", marker='%s', color='%s')\n",
		                          language.name, language.name,
		                          nameToGraphMarker.get(language.name),
								  nameToGraphColor.get(language.name)));
	}

	String python =
		"#\n"+
		"# AUTO-GENERATED FILE. DO NOT EDIT\n" +
		"# CodeBuff %s '%s'\n" +
		"#\n"+
		"import numpy as np\n"+
		"import matplotlib.pyplot as plt\n\n" +
		"%s\n" +
		"ks = %s\n"+
		"fig = plt.figure()\n"+
		"ax = plt.subplot(111)\n"+
		"%s"+
		"ax.tick_params(axis='both', which='major', labelsize=18)\n" +
		"ax.set_xlabel(\"$k$ nearest neighbors\", fontsize=20)\n"+
		"ax.set_ylabel(\"Median error rate\", fontsize=20)\n" +
		"#ax.set_title(\"k Nearest Neighbors vs\\nLeave-one-out Validation Error Rate\")\n"+
		"plt.legend(fontsize=18)\n\n" +
		"fig.savefig('images/vary_k.pdf', format='pdf')\n"+
		"plt.show()\n";
	String code = String.format(python, Tool.version, new Date(), data, ks, plot);

	String fileName = "python/src/vary_k.py";
	Utils.writeFile(fileName, code);
	System.out.println("wrote python code to "+fileName);
}
 

/**
 * @see org.antlr.v4.runtime.tree.Trees.toStringTree(Tree, List<String>)
 */
public static String toStringTree(final Tree t, final List<String> ruleNames, final int depth) {
	String s = Utils.escapeWhitespace(Trees.getNodeText(t, ruleNames), false);

	if (t.getChildCount() == 0) {
		return s;
	}

	final StringBuilder buf = new StringBuilder();

	if (depth > 0) {
		buf.append(NEWLINE);
	}

	buf.append(indent(depth));
	buf.append("(");
	s = Utils.escapeWhitespace(Trees.getNodeText(t, ruleNames), false);
	buf.append(s);
	buf.append(' ');

	for (int i = 0; i < t.getChildCount(); i++) {
		if (i > 0) {
			buf.append(' ');
		}

		buf.append(toStringTree(t.getChild(i), ruleNames, depth + 1));
	}

	buf.append(")");

	return buf.toString();
}
 

/**
 * Display syntax errors, hints in tooltips if under the cursor
 */
public static void showTooltips(EditorMouseEvent event, Editor editor,
                                @NotNull PreviewState previewState, int offset) {
	if ( previewState.parsingResult==null ) return; // no results?

	// Turn off any tooltips if none under the cursor
	// find the highlighter associated with this offset
	List<RangeHighlighter> highlightersAtOffset = MyActionUtils.getRangeHighlightersAtOffset(editor, offset);
	if ( highlightersAtOffset.size()==0 ) {
		return;
	}

	List<String> msgList = new ArrayList<>();
	boolean foundDecisionEvent = false;
	for ( RangeHighlighter r : highlightersAtOffset ) {
		DecisionEventInfo eventInfo = r.getUserData(ProfilerPanel.DECISION_EVENT_INFO_KEY);
		String msg;
		if ( eventInfo!=null ) {
			// TODO: move decision event stuff to profiler?
			if ( eventInfo instanceof AmbiguityInfo ) {
				msg = "Ambiguous upon alts " + eventInfo.configs.getAlts().toString();
			} else if ( eventInfo instanceof ContextSensitivityInfo ) {
				msg = "Context-sensitive";
			} else if ( eventInfo instanceof LookaheadEventInfo ) {
				int k = eventInfo.stopIndex - eventInfo.startIndex + 1;
				msg = "Deepest lookahead k=" + k;
			} else if ( eventInfo instanceof PredicateEvalInfo ) {
				PredicateEvalInfo evalInfo = (PredicateEvalInfo) eventInfo;
				msg = ProfilerPanel.getSemanticContextDisplayString(evalInfo,
						previewState,
						evalInfo.semctx, evalInfo.predictedAlt,
						evalInfo.evalResult);
				msg = msg + (!evalInfo.fullCtx ? " (DFA)" : "");
			} else {
				msg = "Unknown decision event: " + eventInfo;
			}
			foundDecisionEvent = true;
		} else {
			// error tool tips
			SyntaxError errorUnderCursor = r.getUserData(SYNTAX_ERROR);
			msg = getErrorDisplayString(errorUnderCursor);
			if ( msg.length()>MAX_HINT_WIDTH ) {
				msg = msg.substring(0, MAX_HINT_WIDTH) + "...";
			}
			if ( msg.indexOf('<') >= 0 ) {
				msg = msg.replaceAll("<", "&lt;");
			}
		}
		msgList.add(msg);
	}
	String combinedMsg = Utils.join(msgList.iterator(), "\n");
	HintManagerImpl hintMgr = (HintManagerImpl) HintManager.getInstance();
	if ( foundDecisionEvent ) {
		showDecisionEventToolTip(editor, offset, hintMgr, combinedMsg);
	}
	else {
		showPreviewEditorErrorToolTip(editor, offset, hintMgr, combinedMsg);
	}
}
 

/**
	 * Show tokens/region associated with parse tree parent of this token
	 * if the alt-key is down and mouse movement occurs.
	 */
	public void showParseRegion(EditorMouseEvent event, Editor editor,
	                            PreviewState previewState, int offset) {
		Token tokenUnderCursor = ParsingUtils.getTokenUnderCursor(previewState, offset);
		if ( tokenUnderCursor==null ) {
			return;
		}

		ParseTree tree = previewState.parsingResult.tree;
		TerminalNode nodeWithToken =
			(TerminalNode) ParsingUtils.getParseTreeNodeWithToken(tree, tokenUnderCursor);
		if ( nodeWithToken==null ) {
			// hidden token
			return;
		}

		PreviewParser parser = (PreviewParser) previewState.parsingResult.parser;
		CommonTokenStream tokenStream = (CommonTokenStream) parser.getInputStream();
		ParserRuleContext parent = (ParserRuleContext) nodeWithToken.getParent();
		Interval tokenInterval = parent.getSourceInterval();
		Token startToken = tokenStream.get(tokenInterval.a);
		Token stopToken = tokenStream.get(tokenInterval.b);
		Interval sourceInterval =
			Interval.of(startToken.getStartIndex(), stopToken.getStopIndex()+1);

		List<String> stack = parser.getRuleInvocationStack(parent);
		Collections.reverse(stack);

		if ( stack.size()>MAX_STACK_DISPLAY ) {
			// collapse contiguous dups to handle left-recursive stacks
			List<Pair<String, Integer>> smaller = new ArrayList<>();
			int last = 0;
			smaller.add(new Pair<>(stack.get(0), 1)); // init to having first element, count of 1
			for (int i = 1; i<stack.size(); i++) {
				String s = stack.get(i);
				if ( smaller.get(last).a.equals(s) ) {
					smaller.set(last, new Pair<>(s, smaller.get(last).b + 1));
				}
				else {
					smaller.add(new Pair<>(s, 1));
					last++;
				}
			}
			stack = new ArrayList<>();
			for ( Pair<String, Integer> pair : smaller ) {
				if ( pair.b>1 ) {
					stack.add(pair.a + "^" + pair.b);
				} else {
					stack.add(pair.a);
				}
			}
		}
		String stackS = Utils.join(stack.toArray(), "\n");
		highlightAndOfferHint(editor, offset, sourceInterval,
		                      JBColor.BLUE, EffectType.ROUNDED_BOX, stackS);


		// Code for a balloon.

//		JBPopupFactory popupFactory = JBPopupFactory.getInstance();
//		BalloonBuilder builder =
//		    popupFactory.createHtmlTextBalloonBuilder(Utils.join(stack.toArray(), "<br>"),
//												  MessageType.INFO, null);
//		builder.setHideOnClickOutside(true);
//		Balloon balloon = builder.createBalloon();
//		MouseEvent mouseEvent = event.getMouseEvent();
//		Point point = mouseEvent.getPoint();
//		point.translate(10, -15);
//		RelativePoint where = new RelativePoint(mouseEvent.getComponent(), point);
//		balloon.show(where, Balloon.Position.above);
	}
 

@Override
	public void actionPerformed(AnActionEvent e) {
		LOG.info("actionPerformed GenerateLexerRulesForLiteralsAction");
		final Project project = e.getProject();

		final PsiFile psiFile = e.getData(LangDataKeys.PSI_FILE);
		if (psiFile == null) {
			return;
		}
		String inputText = psiFile.getText();
		ParsingResult results = ParsingUtils.parseANTLRGrammar(inputText);

		final Parser parser = results.parser;
		final ParseTree tree = results.tree;
		Collection<ParseTree> literalNodes = XPath.findAll(tree, "//ruleBlock//STRING_LITERAL", parser);
		LinkedHashMap<String, String> lexerRules = new LinkedHashMap<String, String>();
		for (ParseTree node : literalNodes) {
			String literal = node.getText();
			String ruleText = String.format("%s : %s ;",
											RefactorUtils.getLexerRuleNameFromLiteral(literal), literal);
			lexerRules.put(literal, ruleText);
		}

		// remove those already defined
		String lexerRulesXPath = "//lexerRule";
		String treePattern = "<TOKEN_REF> : <STRING_LITERAL>;";
		ParseTreePattern p = parser.compileParseTreePattern(treePattern, ANTLRv4Parser.RULE_lexerRule);
		List<ParseTreeMatch> matches = p.findAll(tree, lexerRulesXPath);

		for (ParseTreeMatch match : matches) {
			ParseTree lit = match.get("STRING_LITERAL");
			if (lexerRules.containsKey(lit.getText())) { // we have rule for this literal already
				lexerRules.remove(lit.getText());
			}
		}

		final LiteralChooser chooser =
			new LiteralChooser(project, new ArrayList<String>(lexerRules.values()));
		chooser.show();
		List<String> selectedElements = chooser.getSelectedElements();
		// chooser disposed automatically.

		final Editor editor = e.getData(PlatformDataKeys.EDITOR);
		final Document doc = editor.getDocument();
		final CommonTokenStream tokens = (CommonTokenStream) parser.getTokenStream();
//		System.out.println(selectedElements);
		if (selectedElements != null) {
			String text = doc.getText();
			int cursorOffset = editor.getCaretModel().getOffset();
			// make sure it's not in middle of rule; put between.
//					System.out.println("offset "+cursorOffset);
			Collection<ParseTree> allRuleNodes = XPath.findAll(tree, "//ruleSpec", parser);
			for (ParseTree r : allRuleNodes) {
				Interval extent = r.getSourceInterval(); // token indexes
				int start = tokens.get(extent.a).getStartIndex();
				int stop = tokens.get(extent.b).getStopIndex();
//						System.out.println("rule "+r.getChild(0).getText()+": "+start+".."+stop);
				if (cursorOffset < start) {
					// before this rule, so must be between previous and this one
					cursorOffset = start; // put right before this rule
					break;
				}
				else if (cursorOffset >= start && cursorOffset <= stop) {
					// cursor in this rule
					cursorOffset = stop + 2; // put right before this rule (after newline)
					if (cursorOffset >= text.length()) {
						cursorOffset = text.length();
					}
					break;
				}
			}

			String allRules = Utils.join(selectedElements.iterator(), "\n");
			text =
				text.substring(0, cursorOffset) +
					"\n" + allRules + "\n" +
					text.substring(cursorOffset, text.length());
			MyPsiUtils.replacePsiFileFromText(project, psiFile, text);
		}
	}
 

/** Get a map from rule names to rule indexes. */
public static Map<String, Integer> createRuleIndexMap(String[] ruleNames) {
	return Utils.toMap(ruleNames);
}
 

/** Get a map from token names to token types. */
public static Map<String, Integer> createTokenTypeMap(String[] tokenNames) {
	return Utils.toMap(tokenNames);
}
 

@Override
public String toString() {
	return Utils.join(syntaxErrors.iterator(), "\n");
}
 

protected String getText(Tree tree) {
	String s = treeTextProvider.getText(tree);
	s = Utils.escapeWhitespace(s, true);
	return s;
}
 

public void text(Graphics g, String s, int x, int y) {
//		System.out.println("drawing '"+s+"' @ "+x+","+y);
		s = Utils.escapeWhitespace(s, true);
		g.drawString(s, x, y);
	}
 

protected String getText(Tree tree) {
	String s = treeTextProvider.getText(tree);
	s = Utils.escapeWhitespace(s, false);
	return s;
}
 

public static void main(String[] args) throws Exception {
		LangDescriptor[] languages = new LangDescriptor[] {
//			QUORUM_DESCR,
			ANTLR4_DESCR,
			JAVA_DESCR,
			JAVA8_DESCR,
			JAVA_GUAVA_DESCR,
			JAVA8_GUAVA_DESCR,
			SQLITE_CLEAN_DESCR,
			TSQL_CLEAN_DESCR,
		};

		int maxNumFiles = 30;
		int trials = 50;
		Map<String,float[]> results = new HashMap<>();
		for (LangDescriptor language : languages) {
			float[] medians = getMedianErrorRates(language, maxNumFiles, trials);
			results.put(language.name, medians);
		}
		String python =
			"#\n"+
			"# AUTO-GENERATED FILE. DO NOT EDIT\n" +
			"# CodeBuff <version> '<date>'\n" +
			"#\n"+
			"import numpy as np\n"+
			"import matplotlib.pyplot as plt\n\n" +
			"fig = plt.figure()\n"+
			"ax = plt.subplot(111)\n"+
			"N = <maxNumFiles>\n" +
			"sizes = range(1,N+1)\n" +
			"<results:{r |\n" +
			"<r> = [<rest(results.(r)); separator={,}>]\n"+
			"ax.plot(range(1,len(<r>)+1), <r>, label=\"<r>\", marker='<markers.(r)>', color='<colors.(r)>')\n" +
			"}>\n" +
			"ax.yaxis.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.5)\n" +
			"ax.set_xlabel(\"Number of training files in sample corpus subset\", fontsize=14)\n"+
			"ax.set_ylabel(\"Median Error rate for <trials> trials\", fontsize=14)\n" +
			"ax.set_title(\"Effect of Corpus size on Median Leave-one-out Validation Error Rate\")\n"+
			"plt.legend()\n" +
			"plt.tight_layout()\n" +
			"fig.savefig('images/subset_validator.pdf', format='pdf')\n"+
			"plt.show()\n";
		ST pythonST = new ST(python);
		pythonST.add("results", results);
		pythonST.add("markers", LeaveOneOutValidator.nameToGraphMarker);
		pythonST.add("colors", LeaveOneOutValidator.nameToGraphColor);
		pythonST.add("version", version);
		pythonST.add("date", new Date());
		pythonST.add("trials", trials);
		pythonST.add("maxNumFiles", maxNumFiles);
		List<String> corpusDirs = map(languages, l -> l.corpusDir);
		String[] dirs = corpusDirs.toArray(new String[languages.length]);
		String fileName = "python/src/subset_validator.py";
		Utils.writeFile(fileName, pythonST.render());
		System.out.println("wrote python code to "+fileName);
	}