org.apache.flink.api.java.tuple.Tuple4 Java Examples

@Test
public void testChiSquare() {

    Crosstab crossTable = new Crosstab();

    crossTable.data = new long[][]{
        {4L, 1L, 3L},
        {2L, 4L, 5L},
        {3L, 4L, 4L}
    };

    Tuple4 tuple4 = ChiSquareTest.test(Tuple2.of(0, crossTable));

    assertEquals(0, tuple4.f0);
    assertEquals(1.0, (double) tuple4.f1, 10e-4);
    assertEquals(0.0, (double) tuple4.f2, 10e-4);

}
 

@Override
public void invoke(
	T value, Context context) throws Exception {
	Long timestamp = context.timestamp();
	if (timestamp != null) {
		data.add(
			new Tuple4<>(
				context.currentWatermark(),
				context.currentProcessingTime(),
				context.timestamp(),
				value));
	} else {
		data.add(
			new Tuple4<>(
				context.currentWatermark(),
				context.currentProcessingTime(),
				null,
				value));

	}
}
 

@SuppressWarnings("unchecked, rawtypes")
private List<Tuple4<String, String, String, Object>> computeOuterJoin(ResettableMutableObjectIterator<Tuple2<String, String>> input1,
																	  ResettableMutableObjectIterator<Tuple2<String, Integer>> input2,
																	  OuterJoinType outerJoinType) throws Exception {
	input1.reset();
	input2.reset();
	AbstractMergeOuterJoinIterator iterator =
			createOuterJoinIterator(
					outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2,
					pairComp, this.memoryManager, this.ioManager, PAGES_FOR_BNLJN, this.parentTask
			);

	List<Tuple4<String, String, String, Object>> actual = new ArrayList<>();
	ListCollector<Tuple4<String, String, String, Object>> collector = new ListCollector<>(actual);
	while (iterator.callWithNextKey(new SimpleTupleJoinFunction(), collector)) ;
	iterator.close();

	return actual;
}
 

@Test
public void testSortPartitionWithKeySelector1() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs = env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// should work
	try {
		tupleDs.sortPartition(new KeySelector<Tuple4<Integer, Long, CustomType, Long[]>, Integer>() {
			@Override
			public Integer getKey(Tuple4<Integer, Long, CustomType, Long[]> value) throws Exception {
				return value.f0;
			}
		}, Order.ASCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}
 

@Ignore
@Test
public void testAsFromTupleToPojo() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	List<Tuple4<String, Integer, Double, String>> data = new ArrayList<>();
	data.add(new Tuple4<>("Rofl", 1, 1.0, "Hi"));
	data.add(new Tuple4<>("lol", 2, 1.0, "Hi"));
	data.add(new Tuple4<>("Test me", 4, 3.33, "Hello world"));

	Table table = tableEnv
		.fromDataSet(env.fromCollection(data), "q, w, e, r")
		.select("q as a, w as b, e as c, r as d");

	DataSet<SmallPojo2> ds = tableEnv.toDataSet(table, SmallPojo2.class);
	List<SmallPojo2> results = ds.collect();
	String expected = "Rofl,1,1.0,Hi\n" + "lol,2,1.0,Hi\n" + "Test me,4,3.33,Hello world\n";
	compareResultAsText(results, expected);
}
 

@Override
public void process(
        String id,
        Context ctx,
        Iterable<SensorReading> readings,
        Collector<Tuple4<String, Long, Long, Integer>> out) throws Exception {
    // count readings
    int cnt = 0;
    for (SensorReading r : readings) {
        cnt++;
    }
    // get current watermark
    long evalTime = ctx.currentWatermark();
    // emit result
    out.collect(Tuple4.of(id, ctx.window().getEnd(), evalTime, cnt));
}
 

@Test
public void testGroupSortByKeyExpression3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs =
			env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// should work
	try {
		tupleDs.groupBy("f0")
				.sortGroup("f2.myString", Order.ASCENDING)
				.sortGroup("f1", Order.DESCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}
 

@Override
public void reduce(Iterable <Tuple4 <Long, Row, Row, Double>> iterable, Collector <Row> collector)
	throws Exception {
	Object id = null;
	List<Tuple2<Double, Row>> list = new ArrayList<>();

	for (Tuple4 <Long, Row, Row, Double> t : iterable) {
		if (null == id) {
			id = t.f1.getField(0);
		}
		list.add(Tuple2.of(t.f3, t.f2));
	}
	list.sort(new Comparator<Tuple2<Double, Row>>() {
		@Override
		public int compare(Tuple2<Double, Row> o1, Tuple2<Double, Row> o2) {
			return o1.f0.compareTo(o2.f0);
		}
	});
	long rank = 1L;
	for(int i = 0; i < Math.min(list.size(), topN); i++){
	    Tuple2<Double, Row> tuple = list.get(i);
              Row row = Row.of(id, tuple.f1.getField(0), tuple.f0, rank++);
              collector.collect(row);
          }
}
 

@SuppressWarnings("unchecked")
protected void testRightSideEmpty() throws Exception {
	CollectionIterator<Tuple2<String, String>> input1 = CollectionIterator.of(
			new Tuple2<>("Jack", "Engineering"),
			new Tuple2<>("Tim", "Sales"),
			new Tuple2<>("Zed", "HR")
	);
	CollectionIterator<Tuple2<String, Integer>> input2 = CollectionIterator.of();

	List<Tuple4<String, String, String, Object>> actualLeft = computeOuterJoin(input1, input2, OuterJoinType.LEFT);
	List<Tuple4<String, String, String, Object>> actualRight = computeOuterJoin(input1, input2, OuterJoinType.RIGHT);
	List<Tuple4<String, String, String, Object>> actualFull = computeOuterJoin(input1, input2, OuterJoinType.FULL);

	List<Tuple4<String, String, String, Object>> expected = Arrays.asList(
			new Tuple4<String, String, String, Object>("Jack", "Engineering", null, null),
			new Tuple4<String, String, String, Object>("Tim", "Sales", null, null),
			new Tuple4<String, String, String, Object>("Zed", "HR", null, null)
	);

	Assert.assertEquals(expected, actualLeft);
	Assert.assertEquals(expected, actualFull);
	Assert.assertEquals(Collections.<Tuple4<String,String,String,Object>>emptyList(), actualRight);
}
 

@Override
public void reduce(Iterable<Tuple3<T, T, FloatValue>> values, Collector<Tuple4<IntValue, T, T, FloatValue>> out)
		throws Exception {
	int groupCount = 0;
	int groupSpans = 1;

	groupSpansValue.setValue(groupSpans);

	for (Tuple3<T, T, FloatValue> edge : values) {
		output.f1 = edge.f0;
		output.f2 = edge.f1;
		output.f3 = edge.f2;

		out.collect(output);

		if (++groupCount == GROUP_SIZE) {
			groupCount = 0;
			groupSpansValue.setValue(++groupSpans);
		}
	}
}
 

/**
 * Intermediate constructor for creating auxiliary fields.
 */
private RowComparator(
	int arity,
	int[] keyPositions,
	NullAwareComparator<Object>[] comparators,
	TypeSerializer<Object>[] serializers,
	Tuple4<int[], Integer, Integer, Boolean> auxiliaryFields) {

	this(
		arity,
		keyPositions,
		comparators,
		serializers,
		auxiliaryFields.f0,
		auxiliaryFields.f1,
		auxiliaryFields.f2,
		auxiliaryFields.f3);
}
 

@SuppressWarnings("unchecked")
protected void testLeftSideEmpty() throws Exception {
	CollectionIterator<Tuple2<String, String>> input1 = CollectionIterator.of();
	CollectionIterator<Tuple2<String, Integer>> input2 = CollectionIterator.of(
			new Tuple2<>("Allison", 100),
			new Tuple2<>("Jack", 200),
			new Tuple2<>("Zed", 150),
			new Tuple2<>("Zed", 250)
	);

	List<Tuple4<String, String, String, Object>> actualLeft = computeOuterJoin(input1, input2, OuterJoinType.LEFT);
	List<Tuple4<String, String, String, Object>> actualRight = computeOuterJoin(input1, input2, OuterJoinType.RIGHT);
	List<Tuple4<String, String, String, Object>> actualFull = computeOuterJoin(input1, input2, OuterJoinType.FULL);

	List<Tuple4<String, String, String, Object>> expected = Arrays.asList(
			new Tuple4<String, String, String, Object>(null, null, "Allison", 100),
			new Tuple4<String, String, String, Object>(null, null, "Jack", 200),
			new Tuple4<String, String, String, Object>(null, null, "Zed", 150),
			new Tuple4<String, String, String, Object>(null, null, "Zed", 250)
	);

	Assert.assertEquals(Collections.<Tuple4<String,String,String,Object>>emptyList(), actualLeft);
	Assert.assertEquals(expected, actualRight);
	Assert.assertEquals(expected, actualFull);
}
 

@Test
public void testGroupSortByKeyExpression3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs =
			env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// should work
	try {
		tupleDs.groupBy("f0")
				.sortGroup("f2.myString", Order.ASCENDING)
				.sortGroup("f1", Order.DESCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}
 

@SuppressWarnings("unchecked, rawtypes")
private List<Tuple4<String, String, String, Object>> computeOuterJoin(ResettableMutableObjectIterator<Tuple2<String, String>> input1,
																	  ResettableMutableObjectIterator<Tuple2<String, Integer>> input2,
																	  OuterJoinType outerJoinType) throws Exception {
	input1.reset();
	input2.reset();
	AbstractMergeOuterJoinIterator iterator =
			createOuterJoinIterator(
					outerJoinType, input1, input2, serializer1, comparator1, serializer2, comparator2,
					pairComp, this.memoryManager, this.ioManager, PAGES_FOR_BNLJN, this.parentTask
			);

	List<Tuple4<String, String, String, Object>> actual = new ArrayList<>();
	ListCollector<Tuple4<String, String, String, Object>> collector = new ListCollector<>(actual);
	while (iterator.callWithNextKey(new SimpleTupleJoinFunction(), collector)) ;
	iterator.close();

	return actual;
}
 

@Test
public void testSortPartitionWithKeySelector4() {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs = env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// should work
	try {
		tupleDs.sortPartition(new KeySelector<Tuple4<Integer, Long, CustomType, Long[]>, Tuple2<Integer, Long>>() {
			@Override
			public Tuple2<Integer, Long> getKey(Tuple4<Integer, Long, CustomType, Long[]> value) throws Exception {
				return new Tuple2<>(value.f0, value.f1);
			}
		}, Order.ASCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}
 

/**
 * Intermediate constructor for creating auxiliary fields.
 */
private RowComparator(
	int arity,
	int[] keyPositions,
	NullAwareComparator<Object>[] comparators,
	TypeSerializer<Object>[] serializers,
	Tuple4<int[], Integer, Integer, Boolean> auxiliaryFields) {

	this(
		arity,
		keyPositions,
		comparators,
		serializers,
		auxiliaryFields.f0,
		auxiliaryFields.f1,
		auxiliaryFields.f2,
		auxiliaryFields.f3);
}
 

public int compact() {
    int tempPoint = 0;
    this.iteratorArray.initializePoint();
    while (this.iteratorArray.getPoint() != -1) {
        if (tempPoint != this.iteratorArray.getPoint()) {
            Tuple4<Float, Double, Double, Float> nextBlock = getData();
            putData(tempPoint, nextBlock);
        }
        tempPoint++;
        advance();
    }
    return tempPoint;
}
 

@Test(expected = InvalidProgramException.class)
public void testSortPartitionWithPositionKeys3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs = env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// must not work
	tupleDs.sortPartition(2, Order.ASCENDING);
}
 

@Override
public void flatMap(Tuple4<IntValue, T, T, IntValue> value, Collector<Tuple4<IntValue, T, T, IntValue>> out)
		throws Exception {
	int spans = value.f0.getValue();

	for (int idx = 0; idx < spans; idx++) {
		value.f0.setValue(idx);
		out.collect(value);
	}
}
 

@Override
public void join(Vertex<K, VV> vertex, Edge<K, EV> edge, Collector<Tuple4<K, K, VV, EV>> collector)
		throws Exception {

	collector.collect(new Tuple4<>(edge.getSource(), edge.getTarget(), vertex.getValue(),
			edge.getValue()));
}
 

private DataSet<Tuple5<KT, KB, EV, VVT, VVB>> joinEdgeWithVertices() {
	return edges
		.join(topVertices, JoinHint.REPARTITION_HASH_SECOND)
		.where(0)
		.equalTo(0)
		.projectFirst(0, 1, 2)
		.<Tuple4<KT, KB, EV, VVT>>projectSecond(1)
			.name("Edge with vertex")
		.join(bottomVertices, JoinHint.REPARTITION_HASH_SECOND)
		.where(1)
		.equalTo(0)
		.projectFirst(0, 1, 2, 3)
		.<Tuple5<KT, KB, EV, VVT, VVB>>projectSecond(1)
			.name("Edge with vertices");
}
 

@Test
public void testStreamSchemaWithTuple() {
    TypeInformation<Tuple4<Integer,Long,String,Double>> typeInfo = TypeInformation.of(new TypeHint<Tuple4<Integer,Long,String,Double>>() {});
    StreamSchema<Tuple4<Integer,Long,String,Double>> schema = new StreamSchema<>(typeInfo, "id", "timestamp", "name", "price");
    assertEquals(Tuple4.class, schema.getTypeInfo().getTypeClass());
    assertEquals(4, schema.getFieldIndexes().length);
    assertEquals(Tuple4.class, schema.getTypeInfo().getTypeClass());
}
 

/**
 * Verify that we can correctly query watermark, processing time and the timestamp from the
 * context.
 */
@Test
public void testTimeQuerying() throws Exception {

	BufferingQueryingSink<String> bufferingSink = new BufferingQueryingSink<>();

	StreamSink<String> operator = new StreamSink<>(bufferingSink);

	OneInputStreamOperatorTestHarness<String, Object> testHarness =
			new OneInputStreamOperatorTestHarness<>(operator);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(17));
	testHarness.setProcessingTime(12);
	testHarness.processElement(new StreamRecord<>("Hello", 12L));

	testHarness.processWatermark(new Watermark(42));
	testHarness.setProcessingTime(15);
	testHarness.processElement(new StreamRecord<>("Ciao", 13L));

	testHarness.processWatermark(new Watermark(42));
	testHarness.setProcessingTime(15);
	testHarness.processElement(new StreamRecord<>("Ciao"));

	assertThat(bufferingSink.data.size(), is(3));

	assertThat(bufferingSink.data,
		contains(
			new Tuple4<>(17L, 12L, 12L, "Hello"),
			new Tuple4<>(42L, 15L, 13L, "Ciao"),
			new Tuple4<>(42L, 15L, null, "Ciao")));

	testHarness.close();
}
 

/**
 * Verify that we can correctly query watermark, processing time and the timestamp from the
 * context.
 */
@Test
public void testTimeQuerying() throws Exception {

	BufferingQueryingSink<String> bufferingSink = new BufferingQueryingSink<>();

	StreamSink<String> operator = new StreamSink<>(bufferingSink);

	OneInputStreamOperatorTestHarness<String, Object> testHarness =
			new OneInputStreamOperatorTestHarness<>(operator);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(17));
	testHarness.setProcessingTime(12);
	testHarness.processElement(new StreamRecord<>("Hello", 12L));

	testHarness.processWatermark(new Watermark(42));
	testHarness.setProcessingTime(15);
	testHarness.processElement(new StreamRecord<>("Ciao", 13L));

	testHarness.processWatermark(new Watermark(42));
	testHarness.setProcessingTime(15);
	testHarness.processElement(new StreamRecord<>("Ciao"));

	assertThat(bufferingSink.data.size(), is(3));

	assertThat(bufferingSink.data,
		contains(
			new Tuple4<>(17L, 12L, 12L, "Hello"),
			new Tuple4<>(42L, 15L, 13L, "Ciao"),
			new Tuple4<>(42L, 15L, null, "Ciao")));

	testHarness.close();
}
 

@Override
public void join(Vertex<K, VV> vertex, Edge<K, EV> edge, Collector<Tuple4<K, K, VV, EV>> collector)
		throws Exception {

	collector.collect(new Tuple4<>(edge.getSource(), edge.getTarget(), vertex.getValue(),
			edge.getValue()));
}
 

@Override
public void reduce(Iterable<Edge<T, Tuple2<ET, LongValue>>> values, Collector<Tuple4<IntValue, T, T, IntValue>> out)
		throws Exception {
	int groupCount = 0;
	int groupSpans = 1;

	groupSpansValue.setValue(groupSpans);

	for (Edge<T, Tuple2<ET, LongValue>> edge : values) {
		long degree = edge.f2.f1.getValue();
		if (degree > Integer.MAX_VALUE) {
			throw new RuntimeException("Degree overflows IntValue");
		}

		// group span, u, v, d(v)
		output.f1 = edge.f0;
		output.f2 = edge.f1;
		output.f3.setValue((int) degree);

		out.collect(output);

		if (++groupCount == groupSize) {
			groupCount = 0;
			groupSpansValue.setValue(++groupSpans);
		}
	}
}
 

/**
 * Maps items' ids to strings in association rules.
 *
 * @param rules     A dataset of: antecedent, consequent, support count, [lift, support, confidence].
 * @param itemIndex A dataset which is a mapping from items' names to indices.
 * @return A dataset of association rules that is for output.
 */
private static DataSet<Row> rulesIndexToString(DataSet<Tuple4<int[], int[], Integer, double[]>> rules,
                                               DataSet<Tuple2<String, Integer>> itemIndex) {
    return rules
        .map(new RichMapFunction<Tuple4<int[], int[], Integer, double[]>, Row>() {
            transient String[] itemNames;

            @Override
            public void open(Configuration parameters) throws Exception {
                List<Tuple2<String, Integer>> bc = getRuntimeContext().getBroadcastVariable("itemIndex");
                itemNames = new String[bc.size() + 1];
                bc.forEach(t -> {
                    itemNames[t.f1] = t.f0;
                });
            }

            @Override
            public Row map(Tuple4<int[], int[], Integer, double[]> value) throws Exception {
                Tuple3<String, Long, Long> antecedent = encodeSequence(value.f0, this.itemNames);
                Tuple3<String, Long, Long> consequent = encodeSequence(value.f1, this.itemNames);
                return Row.of(antecedent.f0 + RULE_SEPARATOR + consequent.f0,
                    antecedent.f2 + consequent.f2, value.f3[0], value.f3[1], value.f2.longValue());
            }
        })
        .withBroadcastSet(itemIndex, "itemIndex")
        .name("rulesIndexToString");
}
 

/**
 * Verify that we can correctly query watermark, processing time and the timestamp from the
 * context.
 */
@Test
public void testTimeQuerying() throws Exception {

	BufferingQueryingSink<String> bufferingSink = new BufferingQueryingSink<>();

	StreamSink<String> operator = new StreamSink<>(bufferingSink);

	OneInputStreamOperatorTestHarness<String, Object> testHarness =
			new OneInputStreamOperatorTestHarness<>(operator);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(17));
	testHarness.setProcessingTime(12);
	testHarness.processElement(new StreamRecord<>("Hello", 12L));

	testHarness.processWatermark(new Watermark(42));
	testHarness.setProcessingTime(15);
	testHarness.processElement(new StreamRecord<>("Ciao", 13L));

	testHarness.processWatermark(new Watermark(42));
	testHarness.setProcessingTime(15);
	testHarness.processElement(new StreamRecord<>("Ciao"));

	assertThat(bufferingSink.data.size(), is(3));

	assertThat(bufferingSink.data,
		contains(
			new Tuple4<>(17L, 12L, 12L, "Hello"),
			new Tuple4<>(42L, 15L, 13L, "Ciao"),
			new Tuple4<>(42L, 15L, null, "Ciao")));

	testHarness.close();
}
 

@Override
public void reduce(Iterable<Tuple4<T, T, Degrees, LongValue>> values, Collector<Tuple3<T, Degrees, LongValue>> out)
		throws Exception {
	Tuple4<T, T, Degrees, LongValue> value = values.iterator().next();

	output.f0 = value.f0;
	output.f1 = value.f2;
	output.f2 = value.f3;

	out.collect(output);
}
 

@Test
public void testCustomPartitioningTupleGroupReduceSorted2() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		
		DataSet<Tuple4<Integer,Integer,Integer, Integer>> data = env.fromElements(new Tuple4<Integer,Integer,Integer,Integer>(0, 0, 0, 0))
				.rebalance().setParallelism(4);
		
		data.groupBy(0).withPartitioner(new TestPartitionerInt())
			.sortGroup(1, Order.ASCENDING)
			.sortGroup(2, Order.DESCENDING)
			.reduceGroup(new IdentityGroupReducerCombinable<Tuple4<Integer,Integer,Integer,Integer>>())
			.output(new DiscardingOutputFormat<Tuple4<Integer, Integer, Integer, Integer>>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
		SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
		
		assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
		assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
		assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}