Java Code Examples for org.apache.flink.api.java.ExecutionEnvironment#getExecutionEnvironment()

@Test
public void testCorrectnessOfDistinctOnAtomic() throws Exception {
	/*
	 * check correctness of distinct on Integers
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Integer> ds = CollectionDataSets.getIntegerDataSet(env);
	DataSet<Integer> reduceDs = ds.distinct();

	List<Integer> result = reduceDs.collect();

	String expected = "1\n2\n3\n4\n5";

	compareResultAsText(result, expected);
}
 

@Test
public void testReduceOnKeyedDatasetWithSelector() throws Exception {

	// set up the execution environment
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(4);

	// creates the input data and distributes them evenly among the available downstream tasks
	DataSet<Tuple3<String, Integer, Boolean>> input = createKeyedInput(env);

	List<Tuple3<String, Integer, Boolean>> actual = input
		.groupBy(new KeySelectorX())
		.reduceGroup(new KeyedCombReducer())
		.collect();
	String expected = "k1,6,true\nk2,4,true\n";

	compareResultAsTuples(actual, expected);
}
 

@Test
public void testFilterFromDataSet() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	tableEnv.createTemporaryView("DataSetTable", ds, $("x"), $("y"), $("z"));

	String sqlQuery = "SELECT x FROM DataSetTable WHERE z LIKE '%Hello%'";
	Table result = tableEnv.sqlQuery(sqlQuery);

	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();
	String expected = "2\n" + "3\n" + "4";
	compareResultAsText(results, expected);
}
 

@Test
@SuppressWarnings("serial")
public void testGroupByKeySelector4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	this.customTypeData.add(new CustomType());

	try {
		DataSet<CustomType> customDs = env.fromCollection(customTypeData);
		// should not work
		customDs.groupBy(
				new KeySelector<GroupingTest.CustomType, Tuple2<Integer, GroupingTest.CustomType>>() {
					@Override
					public Tuple2<Integer, CustomType> getKey(CustomType value) {
						return new Tuple2<Integer, CustomType>(value.myInt, value);
					}
				});
	} catch (Exception e) {
		Assert.fail();
	}
}
 

@Test(expected = InvalidProgramException.class)
public void testJoinKeyMixing4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<CustomType> ds2 = env.fromCollection(customTypeData);

	// should not work, more than one key field position
	ds1.join(ds2)
	.where(1, 3)
	.equalTo(
			new KeySelector<CustomType, Long>() {

					@Override
					public Long getKey(CustomType value) {
						return value.myLong;
					}
				}
			);
}
 

@Test
public void testStringBasedDefinitionOnGroupSortForTwoGroupingKeys() throws Exception {
	/*
	 * Test string-based definition on group sort, for two grouping keys
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);

	DataSet<Tuple2<Tuple2<Integer, Integer>, String>> ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env);
	// f0.f0 is first integer
	DataSet<String> reduceDs = ds.groupBy("f1").sortGroup("f0.f0", Order.DESCENDING).sortGroup("f0.f1", Order.DESCENDING).reduceGroup(new NestedTupleReducer());
	List<String> result = reduceDs.collect();

	String expected = "a--(2,1)-(1,3)-(1,2)-\n" +
			"b--(2,2)-\n" +
			"c--(4,9)-(3,6)-(3,3)-\n";

	compareResultAsText(result, expected);
}
 

@Test(expected = InvalidProgramException.class)
public void testCoGroupKeyAtomicInvalidExpression4() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<CustomType> ds1 = env.fromCollection(customTypeData);
	DataSet<Integer> ds2 = env.fromElements(0, 0, 1);

	ds1.coGroup(ds2).where("myInt").equalTo("*", "invalidKey");
}
 

@Test
public void testWithLessElements() throws Exception {
	/*
	 * Test joinWithEdges with the input DataSet passed as a parameter containing
	 * less elements than the edge DataSet, but of the same type
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	Graph<Long, Long, Long> res = graph.joinWithEdges(graph.getEdges().first(3)
		.map(new EdgeToTuple3Map<>()), new AddValuesMapper());

	DataSet<Edge<Long, Long>> data = res.getEdges();
	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "1,2,24\n" +
		"1,3,26\n" +
		"2,3,46\n" +
		"3,4,34\n" +
		"3,5,35\n" +
		"4,5,45\n" +
		"5,1,51\n";

	compareResultAsTuples(result, expectedResult);
}
 

@Test
public void testSemanticPropsWithKeySelector3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);

	GroupCombineOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> combineOp =
			tupleDs.groupBy(new DummyTestKeySelector())
					.combineGroup(new DummyGroupCombineFunction2())
						.withForwardedFields("0->4;1;1->3;2");

	SemanticProperties semProps = combineOp.getSemanticProperties();

	assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 1);
	assertTrue(semProps.getForwardingTargetFields(0, 2).contains(4));
	assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 2);
	assertTrue(semProps.getForwardingTargetFields(0, 3).contains(1));
	assertTrue(semProps.getForwardingTargetFields(0, 3).contains(3));
	assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
	assertTrue(semProps.getForwardingTargetFields(0, 4).contains(2));
	assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 0);

	assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
	assertTrue(semProps.getForwardingSourceField(0, 1) == 3);
	assertTrue(semProps.getForwardingSourceField(0, 2) == 4);
	assertTrue(semProps.getForwardingSourceField(0, 3) == 3);
	assertTrue(semProps.getForwardingSourceField(0, 4) == 2);

	assertTrue(semProps.getReadFields(0).size() == 3);
	assertTrue(semProps.getReadFields(0).contains(2));
	assertTrue(semProps.getReadFields(0).contains(5));
	assertTrue(semProps.getReadFields(0).contains(6));
}
 

private void testFullOuterStrategies(JoinHint hint) {

		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
		DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);

		// should work
		ds1.fullOuterJoin(ds2, hint)
				.where(0).equalTo(4)
				.with(new DummyJoin());
	}
 

@Test
public void testCustomPartitioningTupleReduce() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		
		DataSet<Tuple2<Integer, Integer>> data = env.fromElements(new Tuple2<Integer, Integer>(0, 0))
				.rebalance().setParallelism(4);
		
		data.groupBy(0).withPartitioner(new TestPartitionerInt())
			.reduce(new SelectOneReducer<Tuple2<Integer,Integer>>())
			.output(new DiscardingOutputFormat<Tuple2<Integer, Integer>>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
		SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
		
		assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
		assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
		assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testUDFJoinOnTuplesWithTupleReturningKeySelectors() throws Exception {
	/*
	 * UDF Join on tuples with tuple-returning key selectors
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
	DataSet<Tuple2<String, String>> joinDs =
			ds1.join(ds2)
					.where(new KeySelector3())
					.equalTo(new KeySelector4())
					.with(new T3T5FlatJoin());

	List<Tuple2<String, String>> result = joinDs.collect();

	String expected = "Hi,Hallo\n" +
			"Hello,Hallo Welt\n" +
			"Hello world,Hallo Welt wie gehts?\n" +
			"Hello world,ABC\n" +
			"I am fine.,HIJ\n" +
			"I am fine.,IJK\n";

	compareResultAsTuples(result, expected);
}
 

@Test
public void testMultipleIterations() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(100);
	
	DataSet<String> input = env.readTextFile(IN_FILE).name("source1");
	
	DataSet<String> reduced = input
			.map(new IdentityMapper<String>())
			.reduceGroup(new Top1GroupReducer<String>());
		
	IterativeDataSet<String> iteration1 = input.iterate(100);
	IterativeDataSet<String> iteration2 = input.iterate(20);
	IterativeDataSet<String> iteration3 = input.iterate(17);
	
	iteration1.closeWith(iteration1.map(new IdentityMapper<String>()).withBroadcastSet(reduced, "bc1"))
			.output(new DiscardingOutputFormat<String>());
	iteration2.closeWith(iteration2.reduceGroup(new Top1GroupReducer<String>()).withBroadcastSet(reduced, "bc2"))
			.output(new DiscardingOutputFormat<String>());
	iteration3.closeWith(iteration3.reduceGroup(new IdentityGroupReducer<String>()).withBroadcastSet(reduced, "bc3"))
			.output(new DiscardingOutputFormat<String>());
	
	Plan plan = env.createProgramPlan();
	
	try{
		compileNoStats(plan);
	}catch(Exception e){
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 

@Test
public void testRebalance() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
	ds.rebalance();
}
 

@Test
public void testOnTargetWithNoCommonKeys() throws Exception {
	/*
	 * Test joinWithEdgesOnTarget with the input DataSet containing different keys than the edge DataSet
	 * - the iterator becomes empty.
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
		TestGraphUtils.getLongLongEdgeData(env), env);

	Graph<Long, Long, Long> res = graph.joinWithEdgesOnTarget(TestGraphUtils.getLongLongTuple2TargetData(env),
		new DoubleValueMapper());

	DataSet<Edge<Long, Long>> data = res.getEdges();
	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "1,2,20\n" +
		"1,3,40\n" +
		"2,3,40\n" +
		"3,4,80\n" +
		"3,5,35\n" +
		"4,5,45\n" +
		"5,1,140\n";

	compareResultAsTuples(result, expectedResult);
}
 

@Test
public void testSemanticPropsWithKeySelector4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);

	GroupCombineOperator<Tuple5<Integer, Long, String, Long, Integer>, Tuple5<Integer, Long, String, Long, Integer>> combineOp =
			tupleDs.groupBy(new DummyTestKeySelector())
					.sortGroup(new DummyTestKeySelector(), Order.ASCENDING)
					.combineGroup(new DummyGroupCombineFunction2())
						.withForwardedFields("0->4;1;1->3;2");

	SemanticProperties semProps = combineOp.getSemanticProperties();

	assertTrue(semProps.getForwardingTargetFields(0, 0).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 1).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 2).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 3).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 4).size() == 1);
	assertTrue(semProps.getForwardingTargetFields(0, 4).contains(4));
	assertTrue(semProps.getForwardingTargetFields(0, 5).size() == 2);
	assertTrue(semProps.getForwardingTargetFields(0, 5).contains(1));
	assertTrue(semProps.getForwardingTargetFields(0, 5).contains(3));
	assertTrue(semProps.getForwardingTargetFields(0, 6).size() == 1);
	assertTrue(semProps.getForwardingTargetFields(0, 6).contains(2));
	assertTrue(semProps.getForwardingTargetFields(0, 7).size() == 0);
	assertTrue(semProps.getForwardingTargetFields(0, 8).size() == 0);

	assertTrue(semProps.getForwardingSourceField(0, 0) < 0);
	assertTrue(semProps.getForwardingSourceField(0, 1) == 5);
	assertTrue(semProps.getForwardingSourceField(0, 2) == 6);
	assertTrue(semProps.getForwardingSourceField(0, 3) == 5);
	assertTrue(semProps.getForwardingSourceField(0, 4) == 4);

	assertTrue(semProps.getReadFields(0).size() == 3);
	assertTrue(semProps.getReadFields(0).contains(4));
	assertTrue(semProps.getReadFields(0).contains(7));
	assertTrue(semProps.getReadFields(0).contains(8));
}
 

@Test
public void testAddEdgesInvalidVertices() throws Exception {
	/*
	 * Test addEdges() -- the source and target vertices do not exist in the graph
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	Graph<Long, Long, Long> graph = Graph.fromDataSet(TestGraphUtils.getLongLongVertexData(env),
			TestGraphUtils.getLongLongEdgeData(env), env);

	List<Edge<Long, Long>> edgesToBeAdded = new ArrayList<>();
	edgesToBeAdded.add(new Edge<>(6L, 1L, 61L));
	edgesToBeAdded.add(new Edge<>(7L, 1L, 71L));

	graph = graph.addEdges(edgesToBeAdded);

	DataSet<Edge<Long, Long>> data = graph.getEdges();
	List<Edge<Long, Long>> result = data.collect();

	expectedResult = "1,2,12\n" +
			"1,3,13\n" +
			"2,3,23\n" +
			"3,4,34\n" +
			"3,5,35\n" +
			"4,5,45\n" +
			"5,1,51\n";

	compareResultAsTuples(result, expectedResult);
}
 

public static void main(String[] args) throws Exception {

		if (!parseParameters(args)) {
			return;
		}

		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

		DataSet<Edge<Long, Double>> edges = getEdgesDataSet(env);

		Graph<Long, Double, Double> graph = Graph.fromDataSet(edges, new InitVertices(), env);

		// Execute the vertex-centric iteration
		Graph<Long, Double, Double> result = graph.runVertexCentricIteration(
				new SSSPComputeFunction(srcVertexId), new SSSPCombiner(),
				maxIterations);

		// Extract the vertices as the result
		DataSet<Vertex<Long, Double>> singleSourceShortestPaths = result.getVertices();

		// emit result
		if (fileOutput) {
			singleSourceShortestPaths.writeAsCsv(outputPath, "\n", ",");
			env.execute("Pregel Single Source Shortest Paths Example");
		} else {
			singleSourceShortestPaths.print();
		}

	}
 

@Test
public void testJoinKeyExpressionsNested() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<NestedCustomType> ds1 = env.fromCollection(customNestedTypeData);
	DataSet<NestedCustomType> ds2 = env.fromCollection(customNestedTypeData);

	// should work
	try {
		ds1.join(ds2).where("myInt").equalTo("myInt");
	} catch (Exception e) {
		Assert.fail();
	}
}
 

public static void main(String[] args) throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.fromElements(1, 2).output(new DiscardingOutputFormat<Integer>());
	env.execute().getJobID();
}