Java Code Examples for org.apache.flink.streaming.api.graph.StreamConfig#setStreamOperator()

@Test
public void testLifeCycleCancel() throws Exception {
	ACTUAL_ORDER_TRACKING.clear();

	Configuration taskManagerConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(new Configuration());
	MockSourceFunction srcFun = new MockSourceFunction();
	cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, false));
	cfg.setOperatorID(new OperatorID());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = StreamTaskTest.createTask(SourceStreamTask.class, cfg, taskManagerConfig);

	task.startTaskThread();
	LifecycleTrackingStreamSource.runStarted.await();

	// this should cancel the task even though it is blocked on runFinished
	task.cancelExecution();

	// wait for clean termination
	task.getExecutingThread().join();
	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
	assertEquals(EXPECTED_CALL_ORDER_CANCEL_RUNNING, ACTUAL_ORDER_TRACKING);
}
 

/**
 * This test verifies that open() and close() are correctly called by the StreamTask.
 */
@Test
@SuppressWarnings("unchecked")
public void testOpenClose() throws Exception {
	final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(
			SourceStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamSource<String, ?> sourceOperator = new StreamSource<>(new OpenCloseTestSource());
	streamConfig.setStreamOperator(sourceOperator);
	streamConfig.setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskCompletion();

	assertTrue("RichFunction methods where not called.", OpenCloseTestSource.closeCalled);

	List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
	Assert.assertEquals(10, resultElements.size());
}
 

@VisibleForTesting
StreamConfig getConfig(OperatorID operatorID, StateBackend stateBackend, StreamOperator<TaggedOperatorSubtaskState> operator) {
	final StreamConfig config;
	if (keyType == null) {
		config = new BoundedStreamConfig();
	} else {
		TypeSerializer<?> keySerializer = keyType.createSerializer(dataSet.getExecutionEnvironment().getConfig());
		config = new BoundedStreamConfig(keySerializer, originalKeySelector);
	}

	config.setStreamOperator(operator);
	config.setOperatorName(operatorID.toHexString());
	config.setOperatorID(operatorID);
	config.setStateBackend(stateBackend);
	return config;
}
 

@Test
public void testQuiesceTimerServiceAfterOpClose() throws Exception {

	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			2, 2,
			BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	streamConfig.setStreamOperator(new TestOperator());
	streamConfig.setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	SystemProcessingTimeService timeService = (SystemProcessingTimeService)
			testHarness.getTimerService();

	// verify that the timer service is running
	Assert.assertTrue(timeService.isAlive());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();
	timeService.shutdownService();
}
 

@Test
public void testQuiesceTimerServiceAfterOpClose() throws Exception {

	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			2, 2,
			BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	streamConfig.setStreamOperator(new TestOperator());
	streamConfig.setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	SystemProcessingTimeService timeService = (SystemProcessingTimeService)
			testHarness.getTask().getProcessingTimeService();

	// verify that the timer service is running
	Assert.assertTrue(timeService.isAlive());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();
	timeService.shutdownService();
}
 

/**
 * This test verifies that open() and close() are correctly called. This test also verifies
 * that timestamps of emitted elements are correct. {@link CoStreamMap} assigns the input
 * timestamp to emitted elements.
 */
@Test
public void testOpenCloseAndTimestamps() throws Exception {
	final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness =
			new TwoInputStreamTaskTestHarness<>(
					TwoInputStreamTask::new,
					BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<>(new TestOpenCloseMapFunction());
	streamConfig.setStreamOperator(coMapOperator);
	streamConfig.setOperatorID(new OperatorID());

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	testHarness.processElement(new StreamRecord<>("Hello", initialTime + 1), 0, 0);
	expectedOutput.add(new StreamRecord<>("Hello", initialTime + 1));

	// wait until the input is processed to ensure ordering of the output
	testHarness.waitForInputProcessing();

	testHarness.processElement(new StreamRecord<>(1337, initialTime + 2), 1, 0);

	expectedOutput.add(new StreamRecord<>("1337", initialTime + 2));

	testHarness.waitForInputProcessing();

	testHarness.endInput();

	testHarness.waitForTaskCompletion();

	Assert.assertTrue("RichFunction methods were not called.", TestOpenCloseMapFunction.closeCalled);

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

/**
 * Tests the checkpoint related metrics are registered into {@link TaskIOMetricGroup}
 * correctly while generating the {@link TwoInputStreamTask}.
 */
@Test
public void testCheckpointBarrierMetrics() throws Exception {
	final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness =
		new TwoInputStreamTaskTestHarness<>(
			TwoInputStreamTask::new,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.INT_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO);
	final StreamConfig streamConfig = testHarness.getStreamConfig();
	final CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<>(new IdentityMap());
	testHarness.setupOutputForSingletonOperatorChain();
	streamConfig.setStreamOperator(coMapOperator);

	final Map<String, Metric> metrics = new ConcurrentHashMap<>();
	final TaskMetricGroup taskMetricGroup = new StreamTaskTestHarness.TestTaskMetricGroup(metrics);
	final StreamMockEnvironment environment = testHarness.createEnvironment();
	environment.setTaskMetricGroup(taskMetricGroup);

	testHarness.invoke(environment);
	testHarness.waitForTaskRunning();

	assertThat(metrics, IsMapContaining.hasKey(MetricNames.CHECKPOINT_ALIGNMENT_TIME));
	assertThat(metrics, IsMapContaining.hasKey(MetricNames.CHECKPOINT_START_DELAY_TIME));

	testHarness.endInput();
	testHarness.waitForTaskCompletion();
}
 

@Test
public void testStateBackendLoadingAndClosing() throws Exception {
	Configuration taskManagerConfig = new Configuration();
	taskManagerConfig.setString(CheckpointingOptions.STATE_BACKEND, TestMemoryStateBackendFactory.class.getName());

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateKeySerializer(mock(TypeSerializer.class));
	cfg.setOperatorID(new OperatorID(4711L, 42L));
	TestStreamSource<Long, MockSourceFunction> streamSource = new TestStreamSource<>(new MockSourceFunction());
	cfg.setStreamOperator(streamSource);
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = createTask(StateBackendTestSource.class, cfg, taskManagerConfig);

	StateBackendTestSource.fail = false;
	task.startTaskThread();

	// wait for clean termination
	task.getExecutingThread().join();

	// ensure that the state backends and stream iterables are closed ...
	verify(TestStreamSource.operatorStateBackend).close();
	verify(TestStreamSource.keyedStateBackend).close();
	verify(TestStreamSource.rawOperatorStateInputs).close();
	verify(TestStreamSource.rawKeyedStateInputs).close();
	// ... and disposed
	verify(TestStreamSource.operatorStateBackend).dispose();
	verify(TestStreamSource.keyedStateBackend).dispose();

	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 

/**
 * This test verifies that open() and close() are correctly called. This test also verifies
 * that timestamps of emitted elements are correct. {@link StreamMap} assigns the input
 * timestamp to emitted elements.
 */
@Test
public void testOpenCloseAndTimestamps() throws Exception {
	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamMap<String, String> mapOperator = new StreamMap<String, String>(new TestOpenCloseMapFunction());
	streamConfig.setStreamOperator(mapOperator);
	streamConfig.setOperatorID(new OperatorID());

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	testHarness.processElement(new StreamRecord<String>("Hello", initialTime + 1));
	testHarness.processElement(new StreamRecord<String>("Ciao", initialTime + 2));
	expectedOutput.add(new StreamRecord<String>("Hello", initialTime + 1));
	expectedOutput.add(new StreamRecord<String>("Ciao", initialTime + 2));

	testHarness.waitForInputProcessing();

	testHarness.endInput();

	testHarness.waitForTaskCompletion();

	assertTrue("RichFunction methods where not called.", TestOpenCloseMapFunction.closeCalled);

	TestHarnessUtil.assertOutputEquals("Output was not correct.",
		expectedOutput,
		testHarness.getOutput());
}
 

@Test
public void testStateBackendLoadingAndClosing() throws Exception {
	Configuration taskManagerConfig = new Configuration();
	taskManagerConfig.setString(CheckpointingOptions.STATE_BACKEND, TestMemoryStateBackendFactory.class.getName());

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateKeySerializer(mock(TypeSerializer.class));
	cfg.setOperatorID(new OperatorID(4711L, 42L));
	TestStreamSource<Long, MockSourceFunction> streamSource = new TestStreamSource<>(new MockSourceFunction());
	cfg.setStreamOperator(streamSource);
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = createTask(StateBackendTestSource.class, cfg, taskManagerConfig);

	StateBackendTestSource.fail = false;
	task.startTaskThread();

	// wait for clean termination
	task.getExecutingThread().join();

	// ensure that the state backends and stream iterables are closed ...
	verify(TestStreamSource.operatorStateBackend).close();
	verify(TestStreamSource.keyedStateBackend).close();
	verify(TestStreamSource.rawOperatorStateInputs).close();
	verify(TestStreamSource.rawKeyedStateInputs).close();
	// ... and disposed
	verify(TestStreamSource.operatorStateBackend).dispose();
	verify(TestStreamSource.keyedStateBackend).dispose();

	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 

/**
 * This test ensures that the SourceStreamTask properly serializes checkpointing
 * and element emission. This also verifies that there are no concurrent invocations
 * of the checkpoint method on the source operator.
 *
 * <p>The source emits elements and performs checkpoints. We have several checkpointer threads
 * that fire checkpoint requests at the source task.
 *
 * <p>If element emission and checkpointing are not in series the count of elements at the
 * beginning of a checkpoint and at the end of a checkpoint are not the same because the
 * source kept emitting elements while the checkpoint was ongoing.
 */
@Test
@SuppressWarnings("unchecked")
public void testCheckpointing() throws Exception {
	final int numElements = 100;
	final int numCheckpoints = 100;
	final int numCheckpointers = 1;
	final int checkpointInterval = 5; // in ms
	final int sourceCheckpointDelay = 1000; // how many random values we sum up in storeCheckpoint
	final int sourceReadDelay = 1; // in ms

	ExecutorService executor = Executors.newFixedThreadPool(10);
	try {
		final TupleTypeInfo<Tuple2<Long, Integer>> typeInfo = new TupleTypeInfo<>(BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);

		final StreamTaskTestHarness<Tuple2<Long, Integer>> testHarness = new StreamTaskTestHarness<>(
				SourceStreamTask::new, typeInfo);
		testHarness.setupOutputForSingletonOperatorChain();

		StreamConfig streamConfig = testHarness.getStreamConfig();
		StreamSource<Tuple2<Long, Integer>, ?> sourceOperator = new StreamSource<>(new MockSource(numElements, sourceCheckpointDelay, sourceReadDelay));
		streamConfig.setStreamOperator(sourceOperator);
		streamConfig.setOperatorID(new OperatorID());

		// prepare the

		Future<Boolean>[] checkpointerResults = new Future[numCheckpointers];

		// invoke this first, so the tasks are actually running when the checkpoints are scheduled
		testHarness.invoke();
		testHarness.waitForTaskRunning();

		final StreamTask<Tuple2<Long, Integer>, ?> sourceTask = testHarness.getTask();

		for (int i = 0; i < numCheckpointers; i++) {
			checkpointerResults[i] = executor.submit(new Checkpointer(numCheckpoints, checkpointInterval, sourceTask));
		}

		testHarness.waitForTaskCompletion();

		// Get the result from the checkpointers, if these threw an exception it
		// will be rethrown here
		for (int i = 0; i < numCheckpointers; i++) {
			if (!checkpointerResults[i].isDone()) {
				checkpointerResults[i].cancel(true);
			}
			if (!checkpointerResults[i].isCancelled()) {
				checkpointerResults[i].get();
			}
		}

		List<Tuple2<Long, Integer>> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
		Assert.assertEquals(numElements, resultElements.size());
	}
	finally {
		executor.shutdown();
	}
}
 

@Test
@SuppressWarnings("unchecked")
public void testCheckpointsTriggeredBySource() throws Exception {
	// set up the basic test harness
	final StreamTaskTestHarness<Long> testHarness = new StreamTaskTestHarness<>(
			SourceStreamTask::new,
			BasicTypeInfo.LONG_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();
	testHarness.getExecutionConfig().setLatencyTrackingInterval(-1);

	final long numElements = 10;
	final long checkpointEvery = 3;

	// set up the source function
	ExternalCheckpointsSource source = new ExternalCheckpointsSource(numElements, checkpointEvery);
	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamSource<Long, ?> sourceOperator = new StreamSource<>(source);
	streamConfig.setStreamOperator(sourceOperator);
	streamConfig.setOperatorID(new OperatorID());

	// this starts the source thread
	testHarness.invoke();

	final StreamTask<Long, ?> sourceTask = testHarness.getTask();

	ready.await();

	// now send an external trigger that should be ignored
	assertTrue(sourceTask.triggerCheckpoint(new CheckpointMetaData(32, 829), CheckpointOptions.forCheckpointWithDefaultLocation(), false));

	// step by step let the source thread emit elements
	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 1L);
	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 2L);
	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 3L);

	verifyCheckpointBarrier(testHarness.getOutput(), 1L);

	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 4L);

	// now send an regular trigger command that should be ignored
	assertTrue(sourceTask.triggerCheckpoint(new CheckpointMetaData(34, 900), CheckpointOptions.forCheckpointWithDefaultLocation(), false));

	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 5L);
	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 6L);

	verifyCheckpointBarrier(testHarness.getOutput(), 2L);

	// let the remainder run

	for (long l = 7L, checkpoint = 3L; l <= numElements; l++) {
		sync.trigger();
		verifyNextElement(testHarness.getOutput(), l);

		if (l % checkpointEvery == 0) {
			verifyCheckpointBarrier(testHarness.getOutput(), checkpoint++);
		}
	}

	// done!
}
 

/**
 * This test ensures that the SourceStreamTask properly serializes checkpointing
 * and element emission. This also verifies that there are no concurrent invocations
 * of the checkpoint method on the source operator.
 *
 * <p>The source emits elements and performs checkpoints. We have several checkpointer threads
 * that fire checkpoint requests at the source task.
 *
 * <p>If element emission and checkpointing are not in series the count of elements at the
 * beginning of a checkpoint and at the end of a checkpoint are not the same because the
 * source kept emitting elements while the checkpoint was ongoing.
 */
@Test
@SuppressWarnings("unchecked")
public void testCheckpointing() throws Exception {
	final int numElements = 100;
	final int numCheckpoints = 100;
	final int numCheckpointers = 1;
	final int checkpointInterval = 5; // in ms
	final int sourceCheckpointDelay = 1000; // how many random values we sum up in storeCheckpoint
	final int sourceReadDelay = 1; // in ms

	ExecutorService executor = Executors.newFixedThreadPool(10);
	try {
		final TupleTypeInfo<Tuple2<Long, Integer>> typeInfo = new TupleTypeInfo<>(BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);

		final StreamTaskTestHarness<Tuple2<Long, Integer>> testHarness = new StreamTaskTestHarness<>(
				SourceStreamTask::new, typeInfo);
		testHarness.setupOutputForSingletonOperatorChain();

		StreamConfig streamConfig = testHarness.getStreamConfig();
		StreamSource<Tuple2<Long, Integer>, ?> sourceOperator = new StreamSource<>(new MockSource(numElements, sourceCheckpointDelay, sourceReadDelay));
		streamConfig.setStreamOperator(sourceOperator);
		streamConfig.setOperatorID(new OperatorID());

		// prepare the

		Future<Boolean>[] checkpointerResults = new Future[numCheckpointers];

		// invoke this first, so the tasks are actually running when the checkpoints are scheduled
		testHarness.invoke();
		testHarness.waitForTaskRunning();

		final StreamTask<Tuple2<Long, Integer>, ?> sourceTask = testHarness.getTask();

		for (int i = 0; i < numCheckpointers; i++) {
			checkpointerResults[i] = executor.submit(new Checkpointer(numCheckpoints, checkpointInterval, sourceTask));
		}

		testHarness.waitForTaskCompletion();

		// Get the result from the checkpointers, if these threw an exception it
		// will be rethrown here
		for (int i = 0; i < numCheckpointers; i++) {
			if (!checkpointerResults[i].isDone()) {
				checkpointerResults[i].cancel(true);
			}
			if (!checkpointerResults[i].isCancelled()) {
				checkpointerResults[i].get();
			}
		}

		List<Tuple2<Long, Integer>> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
		Assert.assertEquals(numElements, resultElements.size());
	}
	finally {
		executor.shutdown();
	}
}
 

@Test
public void testBroadcastCancelCheckpointMarkerOnAbortingFromCoordinator() throws Exception {
	OneInputStreamTaskTestHarness<String, String> testHarness =
		new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1,
			1,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();
	StreamConfig streamConfig = testHarness.getStreamConfig();
	streamConfig.setStreamOperator(new MapOperator());

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	MockEnvironment mockEnvironment = MockEnvironment.builder().build();
	SubtaskCheckpointCoordinator subtaskCheckpointCoordinator = new MockSubtaskCheckpointCoordinatorBuilder()
		.setEnvironment(mockEnvironment)
		.build();

	TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(1, 4096);
	ArrayList<Object> recordOrEvents = new ArrayList<>();
	StreamElementSerializer<String> stringStreamElementSerializer = new StreamElementSerializer<>(StringSerializer.INSTANCE);
	ResultPartitionWriter resultPartitionWriter = new RecordOrEventCollectingResultPartitionWriter<>(
		recordOrEvents, bufferProvider, stringStreamElementSerializer);
	mockEnvironment.addOutputs(Collections.singletonList(resultPartitionWriter));

	OneInputStreamTask<String, String> task = testHarness.getTask();
	OperatorChain<String, OneInputStreamOperator<String, String>> operatorChain = new OperatorChain<>(
		task, StreamTask.createRecordWriterDelegate(streamConfig, mockEnvironment));
	long checkpointId = 42L;
	// notify checkpoint aborted before execution.
	subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
	subtaskCheckpointCoordinator.checkpointState(
		new CheckpointMetaData(checkpointId, System.currentTimeMillis()),
		CheckpointOptions.forCheckpointWithDefaultLocation(),
		new CheckpointMetrics(),
		operatorChain,
		() -> true);

	assertEquals(1, recordOrEvents.size());
	Object recordOrEvent = recordOrEvents.get(0);
	// ensure CancelCheckpointMarker is broadcast downstream.
	assertTrue(recordOrEvent instanceof CancelCheckpointMarker);
	assertEquals(checkpointId, ((CancelCheckpointMarker) recordOrEvent).getCheckpointId());
	testHarness.endInput();
	testHarness.waitForTaskCompletion();
}
 

/**
 * This test verifies that checkpoint barriers and barrier buffers work correctly with
 * concurrent checkpoint barriers where one checkpoint is "overtaking" another checkpoint, i.e.
 * some inputs receive barriers from an earlier checkpoint, thereby blocking,
 * then all inputs receive barriers from a later checkpoint.
 */
@Test
@SuppressWarnings("unchecked")
public void testOvertakingCheckpointBarriers() throws Exception {
	final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness =
			new TwoInputStreamTaskTestHarness<>(
					TwoInputStreamTask::new,
					2, 2, new int[] {1, 2},
					BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<String, Integer, String>(new IdentityMap());
	streamConfig.setStreamOperator(coMapOperator);
	streamConfig.setOperatorID(new OperatorID());

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
	long initialTime = 0L;

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);

	// These elements should be buffered until we receive barriers from
	// all inputs
	testHarness.processElement(new StreamRecord<String>("Hello-0-0", initialTime), 0, 0);
	testHarness.processElement(new StreamRecord<String>("Ciao-0-0", initialTime), 0, 0);

	// These elements should be forwarded, since we did not yet receive a checkpoint barrier
	// on that input, only add to same input, otherwise we would not know the ordering
	// of the output since the Task might read the inputs in any order
	testHarness.processElement(new StreamRecord<Integer>(42, initialTime), 1, 1);
	testHarness.processElement(new StreamRecord<Integer>(1337, initialTime), 1, 1);
	expectedOutput.add(new StreamRecord<String>("42", initialTime));
	expectedOutput.add(new StreamRecord<String>("1337", initialTime));

	testHarness.waitForInputProcessing();
	// we should not yet see the barrier, only the two elements from non-blocked input
	TestHarnessUtil.assertOutputEquals("Output was not correct.",
			expectedOutput,
			testHarness.getOutput());

	// Now give a later barrier to all inputs, this should unblock the first channel,
	// thereby allowing the two blocked elements through
	testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);
	testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
	testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
	testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);

	expectedOutput.add(new CancelCheckpointMarker(0));
	expectedOutput.add(new StreamRecord<String>("Hello-0-0", initialTime));
	expectedOutput.add(new StreamRecord<String>("Ciao-0-0", initialTime));
	expectedOutput.add(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()));

	testHarness.waitForInputProcessing();

	TestHarnessUtil.assertOutputEquals("Output was not correct.",
			expectedOutput,
			testHarness.getOutput());

	// Then give the earlier barrier, these should be ignored
	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);

	testHarness.waitForInputProcessing();

	testHarness.endInput();

	testHarness.waitForTaskCompletion();

	TestHarnessUtil.assertOutputEquals("Output was not correct.",
			expectedOutput,
			testHarness.getOutput());
}
 

@Test
@SuppressWarnings("unchecked")
public void testCheckpointsTriggeredBySource() throws Exception {
	// set up the basic test harness
	final StreamTaskTestHarness<Long> testHarness = new StreamTaskTestHarness<>(
			SourceStreamTask::new,
			BasicTypeInfo.LONG_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();
	testHarness.getExecutionConfig().setLatencyTrackingInterval(-1);

	final long numElements = 10;
	final long checkpointEvery = 3;

	// set up the source function
	ExternalCheckpointsSource source = new ExternalCheckpointsSource(numElements, checkpointEvery);
	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamSource<Long, ?> sourceOperator = new StreamSource<>(source);
	streamConfig.setStreamOperator(sourceOperator);
	streamConfig.setOperatorID(new OperatorID());

	// this starts the source thread
	testHarness.invoke();

	final StreamTask<Long, ?> sourceTask = testHarness.getTask();

	ready.await();

	// now send an external trigger that should be ignored
	assertTrue(sourceTask.triggerCheckpoint(new CheckpointMetaData(32, 829), CheckpointOptions.forCheckpointWithDefaultLocation()));

	// step by step let the source thread emit elements
	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 1L);
	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 2L);
	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 3L);

	verifyCheckpointBarrier(testHarness.getOutput(), 1L);

	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 4L);

	// now send an regular trigger command that should be ignored
	assertTrue(sourceTask.triggerCheckpoint(new CheckpointMetaData(34, 900), CheckpointOptions.forCheckpointWithDefaultLocation()));

	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 5L);
	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 6L);

	verifyCheckpointBarrier(testHarness.getOutput(), 2L);

	// let the remainder run

	for (long l = 7L, checkpoint = 3L; l <= numElements; l++) {
		sync.trigger();
		verifyNextElement(testHarness.getOutput(), l);

		if (l % checkpointEvery == 0) {
			verifyCheckpointBarrier(testHarness.getOutput(), checkpoint++);
		}
	}

	// done!
}
 

private static Task createTask(
	StreamOperator<?> op,
	StateBackend backend,
	CheckpointResponder checkpointResponder) throws IOException {

	Configuration taskConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(taskConfig);
	cfg.setStreamOperator(op);
	cfg.setOperatorID(new OperatorID());
	cfg.setStateBackend(backend);

	ExecutionConfig executionConfig = new ExecutionConfig();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"test job name",
			new SerializedValue<>(executionConfig),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"test task name",
			1,
			11,
			TestStreamTask.class.getName(),
			taskConfig);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			checkpointResponder,
			new NoOpTaskOperatorEventGateway(),
			new TestGlobalAggregateManager(),
			TestingClassLoaderLease.newBuilder().build(),
			new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() },
				VoidPermanentBlobService.INSTANCE),
			new TestingTaskManagerRuntimeInfo(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
			new NoOpResultPartitionConsumableNotifier(),
			mock(PartitionProducerStateChecker.class),
			Executors.directExecutor());
}
 

@Test
public void testOpenCloseAndTimestamps() throws Exception {

	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();

	StreamMap<String, String> mapOperator = new StreamMap<>(new DummyMapFunction<String>());
	streamConfig.setStreamOperator(mapOperator);
	streamConfig.setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	final OneInputStreamTask<String, String> mapTask = testHarness.getTask();

	// first one spawns thread
	mapTask.getProcessingTimeService().registerTimer(System.currentTimeMillis(), new ProcessingTimeCallback() {
		@Override
		public void onProcessingTime(long timestamp) {
		}
	});

	assertEquals(1, StreamTask.TRIGGER_THREAD_GROUP.activeCount());

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	// thread needs to die in time
	long deadline = System.currentTimeMillis() + 4000;
	while (StreamTask.TRIGGER_THREAD_GROUP.activeCount() > 0 && System.currentTimeMillis() < deadline) {
		Thread.sleep(10);
	}

	assertEquals("Trigger timer thread did not properly shut down",
			0, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
}
 

/**
 * Tests that the StreamTask first closes all of its operators before setting its
 * state to not running (isRunning == false)
 *
 * <p>See FLINK-7430.
 */
@Test
public void testOperatorClosingBeforeStopRunning() throws Throwable {
	Configuration taskConfiguration = new Configuration();
	StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	streamConfig.setStreamOperator(new BlockingCloseStreamOperator());
	streamConfig.setOperatorID(new OperatorID());

	try (MockEnvironment mockEnvironment =
			new MockEnvironmentBuilder()
				.setTaskName("Test Task")
				.setMemorySize(32L * 1024L)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(1)
				.setTaskConfiguration(taskConfiguration)
				.build()) {
		StreamTask<Void, BlockingCloseStreamOperator> streamTask = new NoOpStreamTask<>(mockEnvironment);
		final AtomicReference<Throwable> atomicThrowable = new AtomicReference<>(null);

		CompletableFuture<Void> invokeFuture = CompletableFuture.runAsync(
			() -> {
				try {
					streamTask.invoke();
				} catch (Exception e) {
					atomicThrowable.set(e);
				}
			},
			TestingUtils.defaultExecutor());

		BlockingCloseStreamOperator.IN_CLOSE.await();

		// check that the StreamTask is not yet in isRunning == false
		assertTrue(streamTask.isRunning());

		// let the operator finish its close operation
		BlockingCloseStreamOperator.FINISH_CLOSE.trigger();

		// wait until the invoke is complete
		invokeFuture.get();

		// now the StreamTask should no longer be running
		assertFalse(streamTask.isRunning());

		// check if an exception occurred
		if (atomicThrowable.get() != null) {
			throw atomicThrowable.get();
		}
	}
}
 

/**
 *	Tests that the AsyncWaitOperator works together with chaining.
 */
@Test
public void testOperatorChainWithProcessingTime() throws Exception {

	JobVertex chainedVertex = createChainedVertex(false);

	final OneInputStreamTaskTestHarness<Integer, Integer> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			1, 1,
			BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	testHarness.taskConfig = chainedVertex.getConfiguration();

	final StreamConfig streamConfig = testHarness.getStreamConfig();
	final StreamConfig operatorChainStreamConfig = new StreamConfig(chainedVertex.getConfiguration());
	final AsyncWaitOperator<Integer, Integer> headOperator =
			operatorChainStreamConfig.getStreamOperator(AsyncWaitOperatorTest.class.getClassLoader());
	streamConfig.setStreamOperator(headOperator);

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	long initialTimestamp = 0L;

	testHarness.processElement(new StreamRecord<>(5, initialTimestamp));
	testHarness.processElement(new StreamRecord<>(6, initialTimestamp + 1L));
	testHarness.processElement(new StreamRecord<>(7, initialTimestamp + 2L));
	testHarness.processElement(new StreamRecord<>(8, initialTimestamp + 3L));
	testHarness.processElement(new StreamRecord<>(9, initialTimestamp + 4L));

	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
	expectedOutput.add(new StreamRecord<>(22, initialTimestamp));
	expectedOutput.add(new StreamRecord<>(26, initialTimestamp + 1L));
	expectedOutput.add(new StreamRecord<>(30, initialTimestamp + 2L));
	expectedOutput.add(new StreamRecord<>(34, initialTimestamp + 3L));
	expectedOutput.add(new StreamRecord<>(38, initialTimestamp + 4L));

	TestHarnessUtil.assertOutputEqualsSorted(
			"Test for chained operator with AsyncWaitOperator failed",
			expectedOutput,
			testHarness.getOutput(),
			new StreamRecordComparator());
}