org.apache.flink.client.program.OptimizerPlanEnvironment Java Examples

@Test
public void dumpWordCount() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		WordCount.main(new String[] {
				"--input", IN_FILE,
				"--output", OUT_FILE});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("WordCount failed with an exception");
	}
	dump(env.getPlan());
}
 

@Override
@ProcessComplete
public void complete() {
	try {
		super.complete();
	}
	catch(FlowException fe) {
		// check if we need to unwrap a ProgramAbortException
		Throwable t = fe.getCause();
		if (t instanceof OptimizerPlanEnvironment.ProgramAbortException) {
			throw (OptimizerPlanEnvironment.ProgramAbortException)t;
		}
		else {
			throw fe;
		}
	}
}
 

@Test
public void testEnsureStdoutStdErrIsRestored() throws Exception {
  PackagedProgram packagedProgram =
      PackagedProgram.newBuilder().setEntryPointClassName(getClass().getName()).build();
  // constructor changed between Flink 1.10.0 and 1.10.1 and will again change in 1.11
  OptimizerPlanEnvironment env = new OptimizerPlanEnvironment(new Configuration());
  try {
    // Flink will throw an error because no job graph will be generated by the main method
    env.getPipeline(packagedProgram, false);
    Assert.fail("This should have failed to create the Flink Plan.");
  } catch (ProgramInvocationException e) {
    // Test that Flink wasn't able to intercept the stdout/stderr and we printed to the regular
    // output instead
    MatcherAssert.assertThat(
        e.getMessage(),
        allOf(
            StringContains.containsString("System.out: (none)"),
            StringContains.containsString("System.err: (none)")));
  }
}
 

@Test
public void testEnsureStdoutStdErrIsRestored() throws Exception {
  PackagedProgram packagedProgram = new PackagedProgram(getClass());
  OptimizerPlanEnvironment env = new OptimizerPlanEnvironment(new Optimizer(new Configuration()));
  try {
    // Flink will throw an error because no job graph will be generated by the main method
    env.getOptimizedPlan(packagedProgram);
    Assert.fail("This should have failed to create the Flink Plan.");
  } catch (ProgramInvocationException e) {
    // Test that Flink wasn't able to intercept the stdout/stderr and we printed to the regular
    // output instead
    MatcherAssert.assertThat(
        e.getMessage(),
        allOf(
            StringContains.containsString("System.out: (none)"),
            StringContains.containsString("System.err: (none)")));
  }
}
 

@Test
public void dumpPageRank() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		PageRank.main(new String[]{
				"--pages", IN_FILE,
				"--links", IN_FILE,
				"--output", OUT_FILE,
				"--numPages", "10",
				"--iterations", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("PageRank failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpBulkIterationKMeans() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		ConnectedComponents.main(new String[] {
				"--vertices", IN_FILE,
				"--edges", IN_FILE,
				"--output", OUT_FILE,
				"--iterations", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("ConnectedComponents failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpWebLogAnalysis() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		WebLogAnalysis.main(new String[] {
				"--documents", IN_FILE,
				"--ranks", IN_FILE,
				"--visits", OUT_FILE,
				"--output", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("WebLogAnalysis failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpIterativeKMeans() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		KMeans.main(new String[] {
			"--points ", IN_FILE,
			"--centroids ", IN_FILE,
			"--output ", OUT_FILE,
			"--iterations", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("KMeans failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpTPCH3() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		TPCHQuery3.main(new String[] {
				"--lineitem", IN_FILE,
				"--customer", IN_FILE,
				"--orders", OUT_FILE,
				"--output", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("TPCH3 failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpWordCount() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		WordCount.main(new String[] {
				"--input", IN_FILE,
				"--output", OUT_FILE});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("WordCount failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpPageRank() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		// --pages <path> --links <path> --output <path> --numPages <n> --iterations <n>
		PageRank.main(new String[]{
				"--pages", IN_FILE,
				"--links", IN_FILE,
				"--output", OUT_FILE,
				"--numPages", "10",
				"--iterations", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("PageRank failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpBulkIterationKMeans() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		ConnectedComponents.main(new String[] {
				"--vertices", IN_FILE,
				"--edges", IN_FILE,
				"--output", OUT_FILE,
				"--iterations", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("ConnectedComponents failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpWebLogAnalysis() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		org.apache.flink.examples.java.relational.WebLogAnalysis.main(new String[] {
				"--documents", IN_FILE,
				"--ranks", IN_FILE,
				"--visits", OUT_FILE,
				"--output", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("WebLogAnalysis failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpIterativeKMeans() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		KMeans.main(new String[] {
			"--points ", IN_FILE,
			"--centroids ", IN_FILE,
			"--output ", OUT_FILE,
			"--iterations", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("KMeans failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpTPCH3() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		TPCHQuery3.main(new String[] {
				"--lineitem", IN_FILE,
				"--customer", IN_FILE,
				"--orders", OUT_FILE,
				"--output", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("TPCH3 failed with an exception");
	}
	dump(env.getPlan());
}
 

/**
 * Creates an execution environment that represents the context in which the
 * program is currently executed. If the program is invoked standalone, this
 * method returns a local execution environment, as returned by
 * {@link #createLocalEnvironment()}.
 *
 * @return The execution environment of the context in which the program is
 * executed.
 */
public static StreamExecutionEnvironment getExecutionEnvironment() {
	if (contextEnvironmentFactory != null) {
		return contextEnvironmentFactory.createExecutionEnvironment();
	}

	// because the streaming project depends on "flink-clients" (and not the other way around)
	// we currently need to intercept the data set environment and create a dependent stream env.
	// this should be fixed once we rework the project dependencies

	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	if (env instanceof ContextEnvironment) {
		return new StreamContextEnvironment((ContextEnvironment) env);
	} else if (env instanceof OptimizerPlanEnvironment || env instanceof PreviewPlanEnvironment) {
		return new StreamPlanEnvironment(env);
	} else {
		return createLocalEnvironment();
	}
}
 

@Test
public void dumpIterativeKMeans() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		KMeans.main(new String[] {
			"--points ", IN_FILE,
			"--centroids ", IN_FILE,
			"--output ", OUT_FILE,
			"--iterations", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("KMeans failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpWordCount() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		WordCount.main(new String[] {
				"--input", IN_FILE,
				"--output", OUT_FILE});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("WordCount failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpTPCH3() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		TPCHQuery3.main(new String[] {
				"--lineitem", IN_FILE,
				"--customer", IN_FILE,
				"--orders", OUT_FILE,
				"--output", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("TPCH3 failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpIterativeKMeans() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		KMeans.main(new String[] {
			"--points ", IN_FILE,
			"--centroids ", IN_FILE,
			"--output ", OUT_FILE,
			"--iterations", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("KMeans failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpWebLogAnalysis() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		org.apache.flink.examples.java.relational.WebLogAnalysis.main(new String[] {
				"--documents", IN_FILE,
				"--ranks", IN_FILE,
				"--visits", OUT_FILE,
				"--output", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("WebLogAnalysis failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpBulkIterationKMeans() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		ConnectedComponents.main(new String[] {
				"--vertices", IN_FILE,
				"--edges", IN_FILE,
				"--output", OUT_FILE,
				"--iterations", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("ConnectedComponents failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpPageRank() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		// --pages <path> --links <path> --output <path> --numPages <n> --iterations <n>
		PageRank.main(new String[]{
				"--pages", IN_FILE,
				"--links", IN_FILE,
				"--output", OUT_FILE,
				"--numPages", "10",
				"--iterations", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("PageRank failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpWordCount() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		WordCount.main(new String[] {
				"--input", IN_FILE,
				"--output", OUT_FILE});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("WordCount failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpTPCH3() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		TPCHQuery3.main(new String[] {
				"--lineitem", IN_FILE,
				"--customer", IN_FILE,
				"--orders", OUT_FILE,
				"--output", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("TPCH3 failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpWebLogAnalysis() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		WebLogAnalysis.main(new String[] {
				"--documents", IN_FILE,
				"--ranks", IN_FILE,
				"--visits", OUT_FILE,
				"--output", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("WebLogAnalysis failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpBulkIterationKMeans() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		ConnectedComponents.main(new String[] {
				"--vertices", IN_FILE,
				"--edges", IN_FILE,
				"--output", OUT_FILE,
				"--iterations", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("ConnectedComponents failed with an exception");
	}
	dump(env.getPlan());
}
 

@Test
public void dumpPageRank() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		PageRank.main(new String[]{
				"--pages", IN_FILE,
				"--links", IN_FILE,
				"--output", OUT_FILE,
				"--numPages", "10",
				"--iterations", "123"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("PageRank failed with an exception");
	}
	dump(env.getPlan());
}
 

@Override
public JobExecutionResult execute(String jobName) throws Exception {

	StreamGraph streamGraph = getStreamGraph();
	streamGraph.setJobName(jobName);

	transformations.clear();

	if (env instanceof OptimizerPlanEnvironment) {
		((OptimizerPlanEnvironment) env).setPlan(streamGraph);
	} else if (env instanceof PreviewPlanEnvironment) {
		((PreviewPlanEnvironment) env).setPreview(streamGraph.getStreamingPlanAsJSON());
	}

	throw new OptimizerPlanEnvironment.ProgramAbortException();
}
 

public static Plan getKMeansPlan() {
	// prepare the test environment
	PreviewPlanEnvironment env = new PreviewPlanEnvironment();
	env.setAsContext();
	try {
		kmeans(new String[]{IN_FILE, IN_FILE, OUT_FILE, "20"});
	} catch (OptimizerPlanEnvironment.ProgramAbortException pae) {
		// all good.
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("KMeans failed with an exception");
	}
	return env.getPlan();
}