Java Code Examples for org.apache.flink.api.java.ExecutionEnvironment#execute()

@Override
protected void testProgram() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Long> vertexIds = env.generateSequence(1, NUM_VERTICES);
	DataSet<String> edgeString = env.fromElements(ConnectedComponentsData.getRandomOddEvenEdges(NUM_EDGES, NUM_VERTICES, SEED).split("\n"));

	DataSet<Edge<Long, NullValue>> edges = edgeString.map(new EdgeParser());

	DataSet<Vertex<Long, Long>> initialVertices = vertexIds.map(new IdAssigner());

	Graph<Long, Long, NullValue> graph = Graph.fromDataSet(initialVertices, edges, env);

	DataSet<Vertex<Long, Long>> result = graph.run(new ConnectedComponents<>(100));

	result.writeAsCsv(resultPath, "\n", " ");
	env.execute();
}
 

@Test
public void testBulkIteration() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment();

		IterativeDataSet<Integer> iteration = env.fromElements(1).iterate(10);

		DataSet<Integer> result = iteration.closeWith(iteration.map(new AddSuperstepNumberMapper()));

		List<Integer> collected = new ArrayList<Integer>();
		result.output(new LocalCollectionOutputFormat<Integer>(collected));

		env.execute();

		assertEquals(1, collected.size());
		assertEquals(56, collected.get(0).intValue());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Override
protected void testProgram() throws Exception {

	final int numIters = 4;
	final double expectedFactor = (int) Math.pow(7, numIters);

	// this is an artificial program, it does not compute anything sensical
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	@SuppressWarnings("unchecked")
	DataSet<Tuple2<Long, Double>> initialData = env.fromElements(new Tuple2<Long, Double>(1L, 1.0), new Tuple2<Long, Double>(2L, 2.0),
														new Tuple2<Long, Double>(3L, 3.0), new Tuple2<Long, Double>(4L, 4.0),
														new Tuple2<Long, Double>(5L, 5.0), new Tuple2<Long, Double>(6L, 6.0));

	DataSet<Tuple2<Long, Double>> result = MultipleJoinsWithSolutionSetCompilerTest.constructPlan(initialData, numIters);

	List<Tuple2<Long, Double>> resultCollector = new ArrayList<Tuple2<Long, Double>>();
	result.output(new LocalCollectionOutputFormat<>(resultCollector));

	env.execute();

	for (Tuple2<Long, Double> tuple : resultCollector) {
		Assert.assertEquals(expectedFactor * tuple.f0, tuple.f1.doubleValue(), 0.0);
	}
}
 

@Test
public void testWithVertexAndEdgeLongValues() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> input = Graph.fromDataSet(
			SummarizationData.getVertices(env),
			SummarizationData.getEdges(env),
			env)
		.run(new TranslateVertexValues<>(new StringToLong()))
		.run(new TranslateEdgeValues<>(new StringToLong()));

	List<Vertex<Long, Summarization.VertexValue<Long>>> summarizedVertices = new ArrayList<>();
	List<Edge<Long, EdgeValue<Long>>> summarizedEdges = new ArrayList<>();

	Graph<Long, Summarization.VertexValue<Long>, EdgeValue<Long>> output =
		input.run(new Summarization<>());

	output.getVertices().output(new LocalCollectionOutputFormat<>(summarizedVertices));
	output.getEdges().output(new LocalCollectionOutputFormat<>(summarizedEdges));

	env.execute();

	validateVertices(SummarizationData.EXPECTED_VERTICES, summarizedVertices);
	validateEdges(SummarizationData.EXPECTED_EDGES_WITH_VALUES, summarizedEdges);
}
 

@Test
public void testConfigurableMapper() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	JobConf conf = new JobConf();
	conf.set("my.filterPrefix", "Hello");

	DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env);
	DataSet<Tuple2<IntWritable, Text>> hellos = ds.
			flatMap(new HadoopMapFunction<IntWritable, Text, IntWritable, Text>(new ConfigurableMapper(), conf));

	String resultPath = tempFolder.newFile().toURI().toString();

	hellos.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
	env.execute();

	String expected = "(2,Hello)\n" +
			"(3,Hello world)\n" +
			"(4,Hello world, how are you?)\n";

	compareResultsByLinesInMemory(expected, resultPath);
}
 

@Test
public void testStringSortingParallelism1() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<String> ds = CollectionDataSets.getStringDataSet(env);
	ds.writeAsText(resultPath).sortLocalOutput("*", Order.ASCENDING).setParallelism(1);

	env.execute();

	String expected = "Hello\n" +
			"Hello world\n" +
			"Hello world, how are you?\n" +
			"Hi\n" +
			"I am fine.\n" +
			"LOL\n" +
			"Luke Skywalker\n" +
			"Random comment\n";

	compareResultsByLinesInMemoryWithStrictOrder(expected, resultPath);
}
 

public static void main(String[] args) throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<String> input = env.fromElements("Please filter", "the words", "but not this");

	FilterFunction<String> filter = WordFilter::filter;

	DataSet<String> output = input.filter(filter);
	output.print();

	env.execute();
}
 

public static void main(String[] args) throws Exception {
	// set up the batch execution environment
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	/*
	 * Here, you can start creating your execution plan for Flink.
	 *
	 * Start with getting some data from the environment, like
	 * 	env.readTextFile(textPath);
	 *
	 * then, transform the resulting DataSet<String> using operations
	 * like
	 * 	.filter()
	 * 	.flatMap()
	 * 	.join()
	 * 	.coGroup()
	 *
	 * and many more.
	 * Have a look at the programming guide for the Java API:
	 *
	 * http://flink.apache.org/docs/latest/apis/batch/index.html
	 *
	 * and the examples
	 *
	 * http://flink.apache.org/docs/latest/apis/batch/examples.html
	 *
	 */

	// execute program
	env.execute("Flink Batch Java API Skeleton");
}
 

@Test
public void testDataDuplicatingMapper() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple2<IntWritable, Text>> ds = HadoopTestData.getKVPairDataSet(env);
	DataSet<Tuple2<IntWritable, Text>> duplicatingFlatMapDs = ds.
			flatMap(new HadoopMapFunction<IntWritable, Text, IntWritable, Text>(new DuplicatingMapper()));

	String resultPath = tempFolder.newFile().toURI().toString();

	duplicatingFlatMapDs.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
	env.execute();

	String expected = "(1,Hi)\n" + "(1,HI)\n" +
			"(2,Hello)\n" + "(2,HELLO)\n" +
			"(3,Hello world)\n" + "(3,HELLO WORLD)\n" +
			"(4,Hello world, how are you?)\n" + "(4,HELLO WORLD, HOW ARE YOU?)\n" +
			"(5,I am fine.)\n" + "(5,I AM FINE.)\n" +
			"(6,Luke Skywalker)\n" + "(6,LUKE SKYWALKER)\n" +
			"(7,Comment#1)\n" + "(7,COMMENT#1)\n" +
			"(8,Comment#2)\n" + "(8,COMMENT#2)\n" +
			"(9,Comment#3)\n" + "(9,COMMENT#3)\n" +
			"(10,Comment#4)\n" + "(10,COMMENT#4)\n" +
			"(11,Comment#5)\n" + "(11,COMMENT#5)\n" +
			"(12,Comment#6)\n" + "(12,COMMENT#6)\n" +
			"(13,Comment#7)\n" + "(13,COMMENT#7)\n" +
			"(14,Comment#8)\n" + "(14,COMMENT#8)\n" +
			"(15,Comment#9)\n" + "(15,COMMENT#9)\n" +
			"(16,Comment#10)\n" + "(16,COMMENT#10)\n" +
			"(17,Comment#11)\n" + "(17,COMMENT#11)\n" +
			"(18,Comment#12)\n" + "(18,COMMENT#12)\n" +
			"(19,Comment#13)\n" + "(19,COMMENT#13)\n" +
			"(20,Comment#14)\n" + "(20,COMMENT#14)\n" +
			"(21,Comment#15)\n" + "(21,COMMENT#15)\n";

	compareResultsByLinesInMemory(expected, resultPath);
}
 

public static void main(String[] args) throws Exception {
	// set up the execution environment
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	// get input data
	DataSet<String> text = StaticData.getDefaultTextLineDataSet(env);

	DataSet<Tuple2<String, Integer>> counts =
		// split up the lines in pairs (2-tuples) containing: (word,1)
		text.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
			@Override
			public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
				// normalize and split the line
				String[] tokens = value.toLowerCase().split("\\W+");

				// emit the pairs
				for (String token : tokens) {
					if (token.length() > 0) {
						out.collect(new Tuple2<String, Integer>(token, 1));
					}
				}
			}
		})
			// group by the tuple field "0" and sum up tuple field "1"
			.groupBy(0)
			.sum(1);

	// emit result
	counts.print();

	// execute program
	env.execute("WordCount Example");
}
 

@Test
public void testTupleSortingDualParallelism1() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	ds.writeAsCsv(resultPath)
		.sortLocalOutput(1, Order.DESCENDING).sortLocalOutput(0, Order.ASCENDING)
		.setParallelism(1);

	env.execute();

	String expected = "16,6,Comment#10\n" +
			"17,6,Comment#11\n" +
			"18,6,Comment#12\n" +
			"19,6,Comment#13\n" +
			"20,6,Comment#14\n" +
			"21,6,Comment#15\n" +
			"11,5,Comment#5\n" +
			"12,5,Comment#6\n" +
			"13,5,Comment#7\n" +
			"14,5,Comment#8\n" +
			"15,5,Comment#9\n" +
			"7,4,Comment#1\n" +
			"8,4,Comment#2\n" +
			"9,4,Comment#3\n" +
			"10,4,Comment#4\n" +
			"4,3,Hello world, how are you?\n" +
			"5,3,I am fine.\n" +
			"6,3,Luke Skywalker\n" +
			"2,2,Hello\n" +
			"3,2,Hello world\n" +
			"1,1,Hi\n";

	compareResultsByLinesInMemoryWithStrictOrder(expected, resultPath);
}
 

@Test
public void testDistributedCacheWithIterations() throws Exception{
	final String testString = "Et tu, Brute?";
	final String testName = "testing_caesar";

	final File folder = tempFolder.newFolder();
	final File resultFile = new File(folder, UUID.randomUUID().toString());

	String testPath = resultFile.toString();
	String resultPath = resultFile.toURI().toString();

	File tempFile = new File(testPath);
	try (FileWriter writer = new FileWriter(tempFile)) {
		writer.write(testString);
	}

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.registerCachedFile(resultPath, testName);

	IterativeDataSet<Long> solution = env.fromElements(1L).iterate(2);
	solution.closeWith(env.generateSequence(1, 2).filter(new RichFilterFunction<Long>() {
		@Override
		public void open(Configuration parameters) throws Exception{
			File file = getRuntimeContext().getDistributedCache().getFile(testName);
			BufferedReader reader = new BufferedReader(new FileReader(file));
			String output = reader.readLine();
			reader.close();
			assertEquals(output, testString);
		}

		@Override
		public boolean filter(Long value) throws Exception {
			return false;
		}
	}).withBroadcastSet(solution, "SOLUTION")).output(new DiscardingOutputFormat<Long>());
	env.execute();
}
 

@Override
protected void testProgram() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);

	DataSet<Tuple1<Long>> pipelinedSource = env.fromElements(new Tuple1<Long>(1L));

	DataSet<Tuple1<Long>> slowBlockingSource = env.generateSequence(0, 10).map(
			new MapFunction<Long, Tuple1<Long>>() {
				@Override
				public Tuple1<Long> map(Long value) throws Exception {
					Thread.sleep(200);

					return new Tuple1<Long>(value);
				}
			}
	);

	slowBlockingSource.join(slowBlockingSource)
			.where(0).equalTo(0).output(new DiscardingOutputFormat<Tuple2<Tuple1<Long>, Tuple1<Long>>>());

	// Join the slow blocking and the pipelined source. This test should verify that this works
	// w/o problems and the blocking result is not requested too early.
	pipelinedSource.join(slowBlockingSource)
			.where(0).equalTo(0)
			.output(new DiscardingOutputFormat<Tuple2<Tuple1<Long>, Tuple1<Long>>>());

	env.execute("Consume one pipelined and one blocking result test job");
}
 

@Test
public final void testIntersect() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	@SuppressWarnings("unchecked")
	List<Edge<Long, Long>> edges1 = new ArrayList<>();
	edges1.add(new Edge<>(1L, 3L, 12L));
	edges1.add(new Edge<>(1L, 3L, 13L)); // needs to be in the output
	edges1.add(new Edge<>(1L, 3L, 14L));

	@SuppressWarnings("unchecked")
	List<Edge<Long, Long>> edges2 = new ArrayList<>();
	edges2.add(new Edge<>(1L, 3L, 13L));

	Graph<Long, NullValue, Long> graph1 = Graph.fromCollection(edges1, env);
	Graph<Long, NullValue, Long> graph2 = Graph.fromCollection(edges2, env);

	Graph<Long, NullValue, Long> intersect = graph1.intersect(graph2, true);

	List<Vertex<Long, NullValue>> vertices = new ArrayList<>();
	List<Edge<Long, Long>> edges = new ArrayList<>();

	intersect.getVertices().output(new LocalCollectionOutputFormat<>(vertices));
	intersect.getEdges().output(new LocalCollectionOutputFormat<>(edges));

	env.execute();

	String expectedVertices = "1,(null)\n" + "3,(null)\n";

	String expectedEdges = "1,3,13\n";

	compareResultAsTuples(vertices, expectedVertices);
	compareResultAsTuples(edges, expectedEdges);
}
 

public static void tcph3(String[] args) throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(Integer.parseInt(args[0]));

	//order id, order status, order data, order prio, ship prio
	DataSet<Tuple5<Long, String, String, String, Integer>> orders =
			env.readCsvFile(args[1])
			.fieldDelimiter("|").lineDelimiter("\n")
			.includeFields("101011001").types(Long.class, String.class, String.class, String.class, Integer.class)
			.name(ORDERS);

	//order id, extended price
	DataSet<Tuple2<Long, Double>> lineItems =
			env.readCsvFile(args[2])
			.fieldDelimiter("|").lineDelimiter("\n")
			.includeFields("100001").types(Long.class, Double.class)
			.name(LINEITEM);

	DataSet<Tuple2<Long, Integer>> filterO = orders.flatMap(new FilterO()).name(MAPPER_NAME);

	DataSet<Tuple3<Long, Integer, Double>> joinLiO = filterO.join(lineItems).where(0).equalTo(0).with(new JoinLiO()).name(JOIN_NAME);

	DataSet<Tuple3<Long, Integer, Double>> aggLiO = joinLiO.groupBy(0, 1).reduceGroup(new AggLiO()).name(REDUCE_NAME);

	aggLiO.writeAsCsv(args[3], "\n", "|").name(SINK);

	env.execute();
}
 

@Override
protected void testProgram() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Long> data1 = env.generateSequence(1, 100);
	DataSet<Long> data2 = env.generateSequence(1, 100);

	IterativeDataSet<Long> firstIteration = data1.iterate(100);

	DataSet<Long> firstResult = firstIteration.closeWith(firstIteration.map(new IdMapper()));

	IterativeDataSet<Long> mainIteration = data2.map(new IdMapper()).iterate(100);

	DataSet<Long> joined = mainIteration.join(firstResult)
			.where(new IdKeyExtractor()).equalTo(new IdKeyExtractor())
			.with(new Joiner());

	DataSet<Long> mainResult = mainIteration.closeWith(joined);

	mainResult.output(new DiscardingOutputFormat<Long>());

	env.execute();
}
 

public static void main(String[] args) throws Exception {

		// Checking input parameters
		final ParameterTool params = ParameterTool.fromArgs(args);

		// set up execution environment
		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

		// make parameters available in the web interface
		env.getConfig().setGlobalJobParameters(params);

		// read input data
		DataSet<Edge> edges;
		if (params.has("edges")) {
			edges = env.readCsvFile(params.get("edges"))
					.fieldDelimiter(" ")
					.includeFields(true, true)
					.types(Integer.class, Integer.class)
					.map(new TupleEdgeConverter());
		} else {
			System.out.println("Executing EnumTriangles example with default edges data set.");
			System.out.println("Use --edges to specify file input.");
			edges = EnumTrianglesData.getDefaultEdgeDataSet(env);
		}

		// project edges by vertex id
		DataSet<Edge> edgesById = edges
				.map(new EdgeByIdProjector());

		DataSet<Triad> triangles = edgesById
				// build triads
				.groupBy(Edge.V1).sortGroup(Edge.V2, Order.ASCENDING).reduceGroup(new TriadBuilder())
				// filter triads
				.join(edgesById).where(Triad.V2, Triad.V3).equalTo(Edge.V1, Edge.V2).with(new TriadFilter());

		// emit result
		if (params.has("output")) {
			triangles.writeAsCsv(params.get("output"), "\n", ",");
			// execute program
			env.execute("Basic Triangle Enumeration Example");
		} else {
			System.out.println("Printing result to stdout. Use --output to specify output path.");
			triangles.print();
		}
	}
 

@Override
protected void testProgram() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Long> data1 = env.generateSequence(1, 100);
	DataSet<Long> data2 = env.generateSequence(1, 100);

	IterativeDataSet<Long> firstIteration = data1.iterate(100);

	DataSet<Long> firstResult = firstIteration.closeWith(firstIteration.map(new IdMapper()));

	IterativeDataSet<Long> mainIteration = data2.map(new IdMapper()).iterate(100);

	DataSet<Long> joined = mainIteration.join(firstResult)
			.where(new IdKeyExtractor()).equalTo(new IdKeyExtractor())
			.with(new Joiner());

	DataSet<Long> mainResult = mainIteration.closeWith(joined);

	mainResult.output(new DiscardingOutputFormat<Long>());

	env.execute();
}
 

@Override
protected void testProgram() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Integer, String>> ds = env.createInput(new LargeJoinDataGeneratorInputFormat(1000000));

	ds.join(ds).where(0).equalTo(1).with(new Joiner()).writeAsText(resultPath);

	env.execute("Local Selfjoin Test Job");
}
 

@Override
protected void testProgram() throws Exception {

	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 20).map(new Dupl());

	DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iter = input.iterateDelta(input, 20, 0);
	iter.closeWith(iter.getWorkset(), iter.getWorkset())
		.output(new LocalCollectionOutputFormat<Tuple2<Long, Long>>(result));

	env.execute();
}