org.apache.flink.streaming.api.TimeDomain Java Examples

@Test
public void testTimerStateRestorable() throws Exception {
	Path path = new Path(folder.newFolder().toURI());

	OperatorSubtaskState state;
	KeyedStateBootstrapOperator<Long, Long> bootstrapOperator = new KeyedStateBootstrapOperator<>(0L, path, new TimerBootstrapFunction());
	try (KeyedOneInputStreamOperatorTestHarness<Long, Long, TaggedOperatorSubtaskState> harness = getHarness(bootstrapOperator)) {
		processElements(harness, 1L, 2L, 3L);
		state = getState(bootstrapOperator, harness);
	}

	KeyedProcessOperator<Long, Long, Tuple3<Long, Long, TimeDomain>> procOperator = new KeyedProcessOperator<>(new SimpleProcessFunction());
	try (KeyedOneInputStreamOperatorTestHarness<Long, Long, Tuple3<Long, Long, TimeDomain>> harness = getHarness(procOperator, state)) {
		harness.processWatermark(EVENT_TIMER);
		harness.setProcessingTime(PROC_TIMER);

		assertHarnessOutput(harness,
			Tuple3.of(1L, EVENT_TIMER, TimeDomain.EVENT_TIME),
			Tuple3.of(2L, EVENT_TIMER, TimeDomain.EVENT_TIME),
			Tuple3.of(3L, EVENT_TIMER, TimeDomain.EVENT_TIME),
			Tuple3.of(1L, PROC_TIMER, TimeDomain.PROCESSING_TIME),
			Tuple3.of(2L, PROC_TIMER, TimeDomain.PROCESSING_TIME),
			Tuple3.of(3L, PROC_TIMER, TimeDomain.PROCESSING_TIME));
	}
}
 

@Override
public void processElement(Integer value, Context ctx, Collector<String> out) throws Exception {
	final TimerService timerService = ctx.timerService();
	final ValueState<Integer> state = getRuntimeContext().getState(this.state);
	if (state.value() == null) {
		out.collect("INPUT:" + value);
		state.update(value);
		if (expectedTimeDomain.equals(TimeDomain.EVENT_TIME)) {
			timerService.registerEventTimeTimer(timerService.currentWatermark() + 5);
		} else {
			timerService.registerProcessingTimeTimer(timerService.currentProcessingTime() + 5);
		}
	} else {
		state.clear();
		if (expectedTimeDomain.equals(TimeDomain.EVENT_TIME)) {
			timerService.deleteEventTimeTimer(timerService.currentWatermark() + 4);
		} else {
			timerService.deleteProcessingTimeTimer(timerService.currentProcessingTime() + 4);
		}
	}
}
 

@Override
public void processElement(Integer value, Context ctx, Collector<String> out) throws Exception {
	final TimerService timerService = ctx.timerService();
	final ValueState<Integer> state = getRuntimeContext().getState(this.state);
	if (state.value() == null) {
		out.collect("INPUT:" + value);
		state.update(value);
		if (expectedTimeDomain.equals(TimeDomain.EVENT_TIME)) {
			timerService.registerEventTimeTimer(timerService.currentWatermark() + 5);
		} else {
			timerService.registerProcessingTimeTimer(timerService.currentProcessingTime() + 5);
		}
	} else {
		state.clear();
		if (expectedTimeDomain.equals(TimeDomain.EVENT_TIME)) {
			timerService.deleteEventTimeTimer(timerService.currentWatermark() + 4);
		} else {
			timerService.deleteProcessingTimeTimer(timerService.currentProcessingTime() + 4);
		}
	}
}
 

@Override
public void onEventTime(InternalTimer<K, VoidNamespace> timer) throws Exception {
	collector.setAbsoluteTimestamp(timer.getTimestamp());
	onTimerContext.timeDomain = TimeDomain.EVENT_TIME;
	onTimerContext.timer = timer;
	userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector);
	onTimerContext.timeDomain = null;
	onTimerContext.timer = null;
}
 

@Override
public void onEventTime(InternalTimer<K, VoidNamespace> timer) throws Exception {
	collector.setAbsoluteTimestamp(timer.getTimestamp());
	onTimerContext.timeDomain = TimeDomain.EVENT_TIME;
	onTimerContext.timer = timer;
	userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector);
	onTimerContext.timeDomain = null;
	onTimerContext.timer = null;
}
 

@Override
public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception {
	if (TimeDomain.EVENT_TIME.equals(ctx.timeDomain())) {
		out.collect("EVENT:1777");
	} else {
		out.collect("PROC:1777");
	}
}
 

@Override
public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception {
	if (TimeDomain.EVENT_TIME.equals(ctx.timeDomain())) {
		out.collect("EVENT:1777");
	} else {
		out.collect("PROC:1777");
	}
}
 

@Override
public void onTimer(
		long timestamp,
		OnTimerContext ctx,
		Collector<String> out) throws Exception {
	assertEquals(TimeDomain.PROCESSING_TIME, ctx.timeDomain());
	out.collect("STATE:" + getRuntimeContext().getState(state).value());
}
 

@Override
public void onTimer(
		long timestamp,
		OnTimerContext ctx,
		Collector<String> out) throws Exception {
	if (TimeDomain.EVENT_TIME.equals(ctx.timeDomain())) {
		out.collect("EVENT:1777");
	} else {
		out.collect("PROC:1777");
	}
}
 

@Override
public void onProcessingTime(InternalTimer<KS, VoidNamespace> timer) throws Exception {
	collector.eraseTimestamp();
	onTimerContext.timeDomain = TimeDomain.PROCESSING_TIME;
	onTimerContext.timer = timer;
	userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector);
	onTimerContext.timeDomain = null;
	onTimerContext.timer = null;
}
 

/**
 * Verifies that we don't have leakage between different keys.
 */
@Test
public void testProcessingTimeTimerWithState() throws Exception {

	KeyedProcessOperator<Integer, Integer, String> operator =
			new KeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.PROCESSING_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(1);
	testHarness.processElement(new StreamRecord<>(17)); // should set timer for 6
	testHarness.processElement(new StreamRecord<>(13)); // should set timer for 6

	testHarness.setProcessingTime(2);
	testHarness.processElement(new StreamRecord<>(13)); // should delete timer
	testHarness.processElement(new StreamRecord<>(42)); // should set timer for 7

	testHarness.setProcessingTime(6);
	testHarness.setProcessingTime(7);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("INPUT:17"));
	expectedOutput.add(new StreamRecord<>("INPUT:13"));
	expectedOutput.add(new StreamRecord<>("INPUT:42"));
	expectedOutput.add(new StreamRecord<>("STATE:17"));
	expectedOutput.add(new StreamRecord<>("STATE:42"));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

@Test
public void testTimestampAndWatermarkQuerying() throws Exception {

	ProcessOperator<Integer, String> operator =
			new ProcessOperator<>(new QueryingProcessFunction(TimeDomain.EVENT_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new OneInputStreamOperatorTestHarness<>(operator);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(17));
	testHarness.processElement(new StreamRecord<>(5, 12L));

	testHarness.processWatermark(new Watermark(42));
	testHarness.processElement(new StreamRecord<>(6, 13L));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(17L));
	expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L));
	expectedOutput.add(new Watermark(42L));
	expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

@Override
public void processElement(Integer value, Context ctx, Collector<String> out) throws Exception {
	if (expectedTimeDomain.equals(TimeDomain.EVENT_TIME)) {
		out.collect(value + "TIME:" + ctx.timerService().currentWatermark() + " TS:" + ctx.timestamp());
	} else {
		out.collect(value + "TIME:" + ctx.timerService().currentProcessingTime() + " TS:" + ctx.timestamp());
	}
}
 

@Override
public void onTimer(
		long timestamp,
		OnTimerContext ctx,
		Collector<String> out) throws Exception {
	if (TimeDomain.EVENT_TIME.equals(ctx.timeDomain())) {
		out.collect("EVENT:1777");
	} else {
		out.collect("PROC:1777");
	}
}
 

@Override
public void processElement(Integer value, Context ctx, Collector<String> out) throws Exception {
	out.collect("INPUT:" + value);
	getRuntimeContext().getState(state).update(value);
	if (timeDomain.equals(TimeDomain.EVENT_TIME)) {
		ctx.timerService().registerEventTimeTimer(ctx.timerService().currentWatermark() + 5);
	} else {
		ctx.timerService().registerProcessingTimeTimer(ctx.timerService().currentProcessingTime() + 5);
	}
}
 

@Override
public void onEventTime(InternalTimer<KS, VoidNamespace> timer) throws Exception {
	collector.setAbsoluteTimestamp(timer.getTimestamp());
	onTimerContext.timeDomain = TimeDomain.EVENT_TIME;
	onTimerContext.timer = timer;
	userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector);
	onTimerContext.timeDomain = null;
	onTimerContext.timer = null;
}
 

@Override
public void onTimer(
		long timestamp,
		OnTimerContext ctx,
		Collector<String> out) throws Exception {
	if (TimeDomain.EVENT_TIME.equals(ctx.timeDomain())) {
		out.collect("EVENT:1777");
	} else {
		out.collect("PROC:1777");
	}
}
 

@Test
public void testTimestampAndWatermarkQuerying() throws Exception {

	LegacyKeyedProcessOperator<Integer, Integer, String> operator =
			new LegacyKeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(17));
	testHarness.processElement(new StreamRecord<>(5, 12L));

	testHarness.processWatermark(new Watermark(42));
	testHarness.processElement(new StreamRecord<>(6, 13L));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(17L));
	expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L));
	expectedOutput.add(new Watermark(42L));
	expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

@Override
public void onTimer(
		long timestamp,
		OnTimerContext ctx,
		Collector<String> out) throws Exception {
	if (TimeDomain.EVENT_TIME.equals(ctx.timeDomain())) {
		out.collect("EVENT:1777");
	} else {
		out.collect("PROC:1777");
	}
}
 

@Override
public void onProcessingTime(InternalTimer<K, VoidNamespace> timer) throws Exception {
	collector.eraseTimestamp();
	onTimerContext.timeDomain = TimeDomain.PROCESSING_TIME;
	onTimerContext.timer = timer;
	userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector);
	onTimerContext.timeDomain = null;
	onTimerContext.timer = null;
}
 

@Override
public void onTimer(
		long timestamp,
		OnTimerContext ctx,
		Collector<String> out) throws Exception {
	assertEquals(TimeDomain.PROCESSING_TIME, ctx.timeDomain());
	out.collect("STATE:" + getRuntimeContext().getState(state).value());
}
 

/**
 * Verifies that we don't have leakage between different keys.
 */
@Test
public void testEventTimeTimerWithState() throws Exception {

	LegacyKeyedProcessOperator<Integer, Integer, String> operator =
			new LegacyKeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.EVENT_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(1));
	testHarness.processElement(new StreamRecord<>(17, 0L)); // should set timer for 6

	testHarness.processWatermark(new Watermark(2));
	testHarness.processElement(new StreamRecord<>(42, 1L)); // should set timer for 7

	testHarness.processWatermark(new Watermark(6));
	testHarness.processWatermark(new Watermark(7));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(1L));
	expectedOutput.add(new StreamRecord<>("INPUT:17", 0L));
	expectedOutput.add(new Watermark(2L));
	expectedOutput.add(new StreamRecord<>("INPUT:42", 1L));
	expectedOutput.add(new StreamRecord<>("STATE:17", 6L));
	expectedOutput.add(new Watermark(6L));
	expectedOutput.add(new StreamRecord<>("STATE:42", 7L));
	expectedOutput.add(new Watermark(7L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

/**
 * Verifies that we don't have leakage between different keys.
 */
@Test
public void testProcessingTimeTimerWithState() throws Exception {

	LegacyKeyedProcessOperator<Integer, Integer, String> operator =
			new LegacyKeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.PROCESSING_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);

	testHarness.setup();
	testHarness.open();

	testHarness.setProcessingTime(1);
	testHarness.processElement(new StreamRecord<>(17)); // should set timer for 6

	testHarness.setProcessingTime(2);
	testHarness.processElement(new StreamRecord<>(42)); // should set timer for 7

	testHarness.setProcessingTime(6);
	testHarness.setProcessingTime(7);

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new StreamRecord<>("INPUT:17"));
	expectedOutput.add(new StreamRecord<>("INPUT:42"));
	expectedOutput.add(new StreamRecord<>("STATE:17"));
	expectedOutput.add(new StreamRecord<>("STATE:42"));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

@Test
public void testTimestampAndWatermarkQuerying() throws Exception {

	ProcessOperator<Integer, String> operator =
			new ProcessOperator<>(new QueryingProcessFunction(TimeDomain.EVENT_TIME));

	OneInputStreamOperatorTestHarness<Integer, String> testHarness =
			new OneInputStreamOperatorTestHarness<>(operator);

	testHarness.setup();
	testHarness.open();

	testHarness.processWatermark(new Watermark(17));
	testHarness.processElement(new StreamRecord<>(5, 12L));

	testHarness.processWatermark(new Watermark(42));
	testHarness.processElement(new StreamRecord<>(6, 13L));

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	expectedOutput.add(new Watermark(17L));
	expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L));
	expectedOutput.add(new Watermark(42L));
	expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.close();
}
 

@Override
public void onTimer(
		long timestamp,
		OnTimerContext ctx,
		Collector<String> out) throws Exception {
	assertEquals(TimeDomain.EVENT_TIME, ctx.timeDomain());
	out.collect("STATE:" + getRuntimeContext().getState(state).value());
}
 

@Override
public void onProcessingTime(InternalTimer<K, VoidNamespace> timer) throws Exception {
	collector.eraseTimestamp();
	onTimerContext.timeDomain = TimeDomain.PROCESSING_TIME;
	onTimerContext.timer = timer;
	userFunction.onTimer(timer.getTimestamp(), onTimerContext, collector);
	onTimerContext.timeDomain = null;
	onTimerContext.timer = null;
}
 

@Override
public void onTimer(
		long timestamp,
		OnTimerContext ctx,
		Collector<String> out) throws Exception {
	assertEquals(TimeDomain.EVENT_TIME, ctx.timeDomain());
	out.collect("STATE:" + getRuntimeContext().getState(state).value());
}
 

@Override
public void onTimer(
		long timestamp,
		OnTimerContext ctx,
		Collector<String> out) throws Exception {
	assertEquals(expectedKey, ctx.getCurrentKey());

	if (TimeDomain.EVENT_TIME.equals(ctx.timeDomain())) {
		out.collect("EVENT:1777");
	} else {
		out.collect("PROC:1777");
	}
}
 

@Override
public void onTimer(
		long timestamp,
		OnTimerContext ctx,
		Collector<String> out) throws Exception {
	assertEquals(TimeDomain.EVENT_TIME, ctx.timeDomain());
	out.collect("STATE:" + getRuntimeContext().getState(state).value());
}
 

@Override
public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception {
	if (TimeDomain.EVENT_TIME.equals(ctx.timeDomain())) {
		out.collect("EVENT:1777");
	} else {
		out.collect("PROC:1777");
	}
}