Java Code Examples for org.apache.flink.api.java.ExecutionEnvironment#fromCollection()

@Test
public void testPojoTwoOrder() {

	final ExecutionEnvironment env = ExecutionEnvironment
			.getExecutionEnvironment();
	DataSet<CustomType> pojoDs = env
			.fromCollection(pojoData);

	// should work
	try {
		pojoDs.writeAsText("/tmp/willNotHappen")
			.sortLocalOutput("myLong", Order.ASCENDING)
			.sortLocalOutput("myString", Order.DESCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}
 

public static DataSet<Tuple3<Tuple2<IntValue, IntValue>, StringValue, IntValue>> getGroupSortedNestedTupleDataSet2(ExecutionEnvironment env) {
	List<Tuple3<Tuple2<IntValue, IntValue>, StringValue, IntValue>> data = new ArrayList<>();

	data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(1), new IntValue(3)), new StringValue("a"), new IntValue(2)));
	data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(1), new IntValue(2)), new StringValue("a"), new IntValue(1)));
	data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(2), new IntValue(1)), new StringValue("a"), new IntValue(3)));
	data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(2), new IntValue(2)), new StringValue("b"), new IntValue(4)));
	data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(3), new IntValue(3)), new StringValue("c"), new IntValue(5)));
	data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(3), new IntValue(6)), new StringValue("c"), new IntValue(6)));
	data.add(new Tuple3<>(new Tuple2<IntValue, IntValue>(new IntValue(4), new IntValue(9)), new StringValue("c"), new IntValue(7)));

	TupleTypeInfo<Tuple3<Tuple2<IntValue, IntValue>, StringValue, IntValue>> type = new
		TupleTypeInfo<>(
			new TupleTypeInfo<Tuple2<IntValue, IntValue>>(ValueTypeInfo.INT_VALUE_TYPE_INFO, ValueTypeInfo.INT_VALUE_TYPE_INFO),
			ValueTypeInfo.STRING_VALUE_TYPE_INFO,
			ValueTypeInfo.INT_VALUE_TYPE_INFO
	);

	return env.fromCollection(data, type);
}
 

@Test
public void testSortPartitionWithKeySelector4() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs = env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// should work
	try {
		tupleDs.sortPartition(new KeySelector<Tuple4<Integer, Long, CustomType, Long[]>, Tuple2<Integer, Long>>() {
			@Override
			public Tuple2<Integer, Long> getKey(Tuple4<Integer, Long, CustomType, Long[]> value) throws Exception {
				return new Tuple2<>(value.f0, value.f1);
			}
		}, Order.ASCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}
 

public static DataSet<Tuple3<Long, Long, Long>> getLongLongLongTuple3Data(
		ExecutionEnvironment env) {
	List<Tuple3<Long, Long, Long>> tuples = new ArrayList<>();
	tuples.add(new Tuple3<>(1L, 2L, 12L));
	tuples.add(new Tuple3<>(1L, 3L, 13L));
	tuples.add(new Tuple3<>(2L, 3L, 23L));
	tuples.add(new Tuple3<>(3L, 4L, 34L));
	tuples.add(new Tuple3<>(3L, 6L, 36L));
	tuples.add(new Tuple3<>(4L, 6L, 46L));
	tuples.add(new Tuple3<>(6L, 1L, 61L));

	return env.fromCollection(tuples);
}
 

@Test(expected = IndexOutOfBoundsException.class)
public void testCrossProjection9() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);

	// should not work, index out of range
	ds1.cross(ds2)
		.projectSecond(5);
}
 

@Test
public void testRangePartitionOperatorPreservesFields() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

		DataSet<Tuple2<Long, Long>> data = env.fromCollection(Collections.singleton(new Tuple2<>(0L, 0L)));

		data.partitionByRange(1)
			.groupBy(1)
			.reduceGroup(new IdentityGroupReducerCombinable<Tuple2<Long,Long>>())
			.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());

		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);

		SinkPlanNode sink = op.getDataSinks().iterator().next();
		SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
		SingleInputPlanNode partitionNode = (SingleInputPlanNode)reducer.getInput().getSource();
		SingleInputPlanNode partitionIDRemover = (SingleInputPlanNode) partitionNode.getInput().getSource();

		assertEquals(ShipStrategyType.FORWARD, reducer.getInput().getShipStrategy());
		assertEquals(ShipStrategyType.FORWARD, partitionNode.getInput().getShipStrategy());
		assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitionIDRemover.getInput().getShipStrategy());

		SourcePlanNode sourcePlanNode = op.getDataSources().iterator().next();
		List<Channel> sourceOutgoingChannels = sourcePlanNode.getOutgoingChannels();
		assertEquals(2, sourceOutgoingChannels.size());
		assertEquals(ShipStrategyType.FORWARD, sourceOutgoingChannels.get(0).getShipStrategy());
		assertEquals(ShipStrategyType.FORWARD, sourceOutgoingChannels.get(1).getShipStrategy());
		assertEquals(DataExchangeMode.PIPELINED, sourceOutgoingChannels.get(0).getDataExchangeMode());
		assertEquals(DataExchangeMode.BATCH, sourceOutgoingChannels.get(1).getDataExchangeMode());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

public static DataSet<Vertex<Long, Double>> getDefaultVertexDataSet(ExecutionEnvironment env) {

		List<Vertex<Long, Double>> vertices = new ArrayList<>();
		vertices.add(new Vertex<>(1L, 6.0));
		vertices.add(new Vertex<>(2L, 2.0));
		vertices.add(new Vertex<>(3L, 3.0));
		vertices.add(new Vertex<>(4L, 1.0));
		vertices.add(new Vertex<>(5L, 0.0));

		return env.fromCollection(vertices);
	}
 

@Test(expected = InvalidProgramException.class)
public void testDistinctByKeyFields2() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Long> longDs = env.fromCollection(emptyLongData, BasicTypeInfo.LONG_TYPE_INFO);
	// should not work: distinct on basic type
	longDs.distinct(0);
}
 

@Test(expected = IndexOutOfBoundsException.class)
public void testCoGroupKeyFields5() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);

	// should not work, negative key field position
	ds1.coGroup(ds2).where(-1).equalTo(-1);
}
 

@Test(expected = InvalidProgramException.class)
public void testJoinKeyInvalidAtomic4() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
	DataSet<Integer> ds2 = env.fromElements(0, 0, 0);

	ds1.join(ds2).where(0).equalTo("invalidKey");
}
 

@Test
public void testTupleSingleOrderIdx() {

	final ExecutionEnvironment env = ExecutionEnvironment
			.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env
			.fromCollection(emptyTupleData, tupleTypeInfo);

	// should work
	try {
		tupleDs.writeAsText("/tmp/willNotHappen").sortLocalOutput(0, Order.ANY);
	} catch (Exception e) {
		Assert.fail();
	}
}
 

public static DataSet<Edge<String, Long>> getStringLongEdgeData(
		ExecutionEnvironment env) {
	List<Edge<String, Long>> edges = new ArrayList<>();
	edges.add(new Edge<>("1", "2", 12L));
	edges.add(new Edge<>("1", "3", 13L));
	edges.add(new Edge<>("2", "3", 23L));
	edges.add(new Edge<>("3", "4", 34L));
	edges.add(new Edge<>("3", "5", 35L));
	edges.add(new Edge<>("4", "5", 45L));
	edges.add(new Edge<>("5", "1", 51L));
	return env.fromCollection(edges);
}
 

public static DataSet<Edge<Long, Long>> getDisconnectedLongLongEdgeData(ExecutionEnvironment env) {
	List<Edge<Long, Long>> edges = new ArrayList<>();
	edges.add(new Edge<>(1L, 2L, 12L));
	edges.add(new Edge<>(1L, 3L, 13L));
	edges.add(new Edge<>(2L, 3L, 23L));
	edges.add(new Edge<>(4L, 5L, 45L));

	return env.fromCollection(edges);
}
 

@Test(expected = InvalidProgramException.class)
public void testGroupSortKeyFields5() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs =
			env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// should not work
	tupleDs.groupBy(0)
			.sortGroup(3, Order.ASCENDING);
}
 

public void testCrossProjection12() {

		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
		DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);

		// should work
		ds1.cross(ds2)
			.projectSecond(2)
			.projectFirst(1);
	}
 

private static DataSet<Tuple7<LongValue, IntValue, IntValue, LongValue, StringValue, IntValue, StringValue>> getSmallTuplebasedDataSetMatchingPojo(ExecutionEnvironment env) {
	List<Tuple7<LongValue, IntValue, IntValue, LongValue, StringValue, IntValue, StringValue>> data = new ArrayList<>();

	data.add(new Tuple7<>(new LongValue(10000L), new IntValue(10), new IntValue(100), new LongValue(1000L), new StringValue("One"), new IntValue(1), new StringValue("First")));
	data.add(new Tuple7<>(new LongValue(20000L), new IntValue(20), new IntValue(200), new LongValue(2000L), new StringValue("Two"), new IntValue(2), new StringValue("Second")));
	data.add(new Tuple7<>(new LongValue(30000L), new IntValue(30), new IntValue(300), new LongValue(3000L), new StringValue("Three"), new IntValue(3), new StringValue("Third")));

	return env.fromCollection(data);
}
 

@Test(expected = InvalidProgramException.class)
public void testGroupSortKeyFields4() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs =
			env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// should not work
	tupleDs.groupBy(0)
			.sortGroup(2, Order.ASCENDING);
}
 

public static DataSet<PojoContainingTupleAndWritable> getGroupSortedPojoContainingTupleAndWritable(ExecutionEnvironment env) {
	List<PojoContainingTupleAndWritable> data = new ArrayList<>();
	data.add(new PojoContainingTupleAndWritable(1, 10L, 100L)); // 1x
	data.add(new PojoContainingTupleAndWritable(2, 20L, 200L)); // 5x
	data.add(new PojoContainingTupleAndWritable(2, 20L, 201L));
	data.add(new PojoContainingTupleAndWritable(2, 30L, 200L));
	data.add(new PojoContainingTupleAndWritable(2, 30L, 600L));
	data.add(new PojoContainingTupleAndWritable(2, 30L, 400L));
	return env.fromCollection(data);
}
 

public void testCrossProjection12() {

		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds1 = env.fromCollection(emptyTupleData, tupleTypeInfo);
		DataSet<Tuple5<Integer, Long, String, Long, Integer>> ds2 = env.fromCollection(emptyTupleData, tupleTypeInfo);

		// should work
		ds1.cross(ds2)
			.projectSecond(2)
			.projectFirst(1);
	}
 

public static DataSet<Edge<Long, Long>> getLongLongEdgeData(
		ExecutionEnvironment env) {

	return env.fromCollection(getLongLongEdges());
}