Java Code Examples for org.apache.flink.streaming.api.graph.StreamConfig#setOperatorID()

private StreamTaskTestHarness<Long> getSourceStreamTaskTestHarness() {
	final StreamTaskTestHarness<Long> testHarness = new StreamTaskTestHarness<>(
			SourceStreamTask::new,
			BasicTypeInfo.LONG_TYPE_INFO);

	final LockStepSourceWithOneWmPerElement source = new LockStepSourceWithOneWmPerElement();

	testHarness.setupOutputForSingletonOperatorChain();
	testHarness.getExecutionConfig().setLatencyTrackingInterval(-1);

	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamSource<Long, ?> sourceOperator = new StreamSource<>(source);
	streamConfig.setStreamOperator(sourceOperator);
	streamConfig.setOperatorID(new OperatorID());
	return testHarness;
}
 

/**
 * This test verifies that open() and close() are correctly called by the StreamTask.
 */
@Test
@SuppressWarnings("unchecked")
public void testOpenClose() throws Exception {
	final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(
			SourceStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamSource<String, ?> sourceOperator = new StreamSource<>(new OpenCloseTestSource());
	streamConfig.setStreamOperator(sourceOperator);
	streamConfig.setOperatorID(new OperatorID());

	testHarness.invoke();
	testHarness.waitForTaskCompletion();

	assertTrue("RichFunction methods where not called.", OpenCloseTestSource.closeCalled);

	List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
	Assert.assertEquals(10, resultElements.size());
}
 

private static <T> void setupSourceOperator(
		StreamSource<T, ?> operator,
		ExecutionConfig executionConfig,
		Environment env,
		ProcessingTimeService timeProvider) {

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(TimeCharacteristic.EventTime);
	cfg.setOperatorID(new OperatorID());

	try {
		MockStreamTask mockTask = new MockStreamTaskBuilder(env)
			.setConfig(cfg)
			.setExecutionConfig(executionConfig)
			.setProcessingTimeService(timeProvider)
			.build();

		operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testStateBackendClosingOnFailure() throws Exception {
	Configuration taskManagerConfig = new Configuration();
	taskManagerConfig.setString(CheckpointingOptions.STATE_BACKEND, TestMemoryStateBackendFactory.class.getName());

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateKeySerializer(mock(TypeSerializer.class));
	cfg.setOperatorID(new OperatorID(4711L, 42L));
	TestStreamSource<Long, MockSourceFunction> streamSource = new TestStreamSource<>(new MockSourceFunction());
	cfg.setStreamOperator(streamSource);
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	try (NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
		Task task = createTask(StateBackendTestSource.class, shuffleEnvironment, cfg, taskManagerConfig);

		StateBackendTestSource.fail = true;
		task.startTaskThread();

		// wait for clean termination
		task.getExecutingThread().join();

		// ensure that the state backends and stream iterables are closed ...
		verify(TestStreamSource.operatorStateBackend).close();
		verify(TestStreamSource.keyedStateBackend).close();
		verify(TestStreamSource.rawOperatorStateInputs).close();
		verify(TestStreamSource.rawKeyedStateInputs).close();
		// ... and disposed
		verify(TestStreamSource.operatorStateBackend).dispose();
		verify(TestStreamSource.keyedStateBackend).dispose();

		assertEquals(ExecutionState.FAILED, task.getExecutionState());
	}
}
 

/**
 * Tests that the StreamTask first closes all of its operators before setting its
 * state to not running (isRunning == false)
 *
 * <p>See FLINK-7430.
 */
@Test
public void testOperatorClosingBeforeStopRunning() throws Throwable {
	BlockingCloseStreamOperator.resetLatches();
	Configuration taskConfiguration = new Configuration();
	StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	streamConfig.setStreamOperator(new BlockingCloseStreamOperator());
	streamConfig.setOperatorID(new OperatorID());

	try (MockEnvironment mockEnvironment =
			new MockEnvironmentBuilder()
				.setTaskName("Test Task")
				.setMemorySize(32L * 1024L)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(1)
				.setTaskConfiguration(taskConfiguration)
				.build()) {

		RunningTask<StreamTask<Void, BlockingCloseStreamOperator>> task = runTask(() -> new NoOpStreamTask<>(mockEnvironment));

		BlockingCloseStreamOperator.inClose.await();

		// check that the StreamTask is not yet in isRunning == false
		assertTrue(task.streamTask.isRunning());

		// let the operator finish its close operation
		BlockingCloseStreamOperator.finishClose.trigger();

		task.waitForTaskCompletion(false);

		// now the StreamTask should no longer be running
		assertFalse(task.streamTask.isRunning());
	}
}
 

@Test
public void testStateBackendLoadingAndClosing() throws Exception {
	Configuration taskManagerConfig = new Configuration();
	taskManagerConfig.setString(CheckpointingOptions.STATE_BACKEND, TestMemoryStateBackendFactory.class.getName());

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateKeySerializer(mock(TypeSerializer.class));
	cfg.setOperatorID(new OperatorID(4711L, 42L));
	TestStreamSource<Long, MockSourceFunction> streamSource = new TestStreamSource<>(new MockSourceFunction());
	cfg.setStreamOperator(streamSource);
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = createTask(StateBackendTestSource.class, cfg, taskManagerConfig);

	StateBackendTestSource.fail = false;
	task.startTaskThread();

	// wait for clean termination
	task.getExecutingThread().join();

	// ensure that the state backends and stream iterables are closed ...
	verify(TestStreamSource.operatorStateBackend).close();
	verify(TestStreamSource.keyedStateBackend).close();
	verify(TestStreamSource.rawOperatorStateInputs).close();
	verify(TestStreamSource.rawKeyedStateInputs).close();
	// ... and disposed
	verify(TestStreamSource.operatorStateBackend).dispose();
	verify(TestStreamSource.keyedStateBackend).dispose();

	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 

/**
 * This test verifies that open() and close() are correctly called. This test also verifies
 * that timestamps of emitted elements are correct. {@link StreamMap} assigns the input
 * timestamp to emitted elements.
 */
@Test
public void testOpenCloseAndTimestamps() throws Exception {
	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamMap<String, String> mapOperator = new StreamMap<String, String>(new TestOpenCloseMapFunction());
	streamConfig.setStreamOperator(mapOperator);
	streamConfig.setOperatorID(new OperatorID());

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	testHarness.processElement(new StreamRecord<String>("Hello", initialTime + 1));
	testHarness.processElement(new StreamRecord<String>("Ciao", initialTime + 2));
	expectedOutput.add(new StreamRecord<String>("Hello", initialTime + 1));
	expectedOutput.add(new StreamRecord<String>("Ciao", initialTime + 2));

	testHarness.waitForInputProcessing();

	testHarness.endInput();

	testHarness.waitForTaskCompletion();

	assertTrue("RichFunction methods where not called.", TestOpenCloseMapFunction.closeCalled);

	TestHarnessUtil.assertOutputEquals("Output was not correct.",
		expectedOutput,
		testHarness.getOutput());
}
 

@SuppressWarnings("unchecked")
private static <T> void setupSourceOperator(StreamSource<T, ?> operator,
											TimeCharacteristic timeChar,
											long watermarkInterval,
											final ProcessingTimeService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setProcessingTimeService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
}
 

@SuppressWarnings("unchecked")
private static <T> MockStreamTask setupSourceOperator(
		StreamSource<T, ?> operator,
		TimeCharacteristic timeChar,
		long watermarkInterval,
		final TimerService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setTimerService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	return mockTask;
}
 

/**
 * This test verifies that checkpoint barriers are correctly forwarded.
 */
@Test
public void testCheckpointBarriers() throws Exception {
	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			2, 2,
			BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamMap<String, String> mapOperator = new StreamMap<String, String>(new IdentityMap());
	streamConfig.setStreamOperator(mapOperator);
	streamConfig.setOperatorID(new OperatorID());

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
	long initialTime = 0L;

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);

	// These elements should be buffered until we receive barriers from
	// all inputs
	testHarness.processElement(new StreamRecord<String>("Hello-0-0", initialTime), 0, 0);
	testHarness.processElement(new StreamRecord<String>("Ciao-0-0", initialTime), 0, 0);

	// These elements should be forwarded, since we did not yet receive a checkpoint barrier
	// on that input, only add to same input, otherwise we would not know the ordering
	// of the output since the Task might read the inputs in any order
	testHarness.processElement(new StreamRecord<String>("Hello-1-1", initialTime), 1, 1);
	testHarness.processElement(new StreamRecord<String>("Ciao-1-1", initialTime), 1, 1);
	expectedOutput.add(new StreamRecord<String>("Hello-1-1", initialTime));
	expectedOutput.add(new StreamRecord<String>("Ciao-1-1", initialTime));

	testHarness.waitForInputProcessing();
	// we should not yet see the barrier, only the two elements from non-blocked input
	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);

	testHarness.waitForInputProcessing();

	// now we should see the barrier and after that the buffered elements
	expectedOutput.add(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()));
	expectedOutput.add(new StreamRecord<String>("Hello-0-0", initialTime));
	expectedOutput.add(new StreamRecord<String>("Ciao-0-0", initialTime));

	testHarness.endInput();

	testHarness.waitForTaskCompletion();

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing operation after
 * the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
	final Configuration taskConfiguration = new Configuration();
	final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();

	final StateBackend blockingStateBackend = new BlockingStateBackend();

	streamConfig.setStreamOperator(noOpStreamOperator);
	streamConfig.setOperatorID(new OperatorID());
	streamConfig.setStateBackend(blockingStateBackend);

	final long checkpointId = 0L;
	final long checkpointTimestamp = 0L;

	final JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Test Job",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		BlockingStreamTask.class.getName(),
		taskConfiguration);

	final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();

	final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	final Task task = new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		new MemoryManager(32L * 1024L, 1),
		new IOManagerAsync(),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		new BlobLibraryCacheManager(
			blobService.getPermanentBlobService(),
			FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
			new String[0]),
		mock(FileCache.class),
		taskManagerRuntimeInfo,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class),
		Executors.directExecutor());

	CompletableFuture<Void> taskRun = CompletableFuture.runAsync(
		() -> task.run(),
		TestingUtils.defaultExecutor());

	// wait until the stream task started running
	RUN_LATCH.await();

	// trigger a checkpoint
	task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), false);

	// wait until the task has completed execution
	taskRun.get();

	// check that no failure occurred
	if (task.getFailureCause() != null) {
		throw new Exception("Task failed", task.getFailureCause());
	}

	// check that we have entered the finished state
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 

@Test
@SuppressWarnings("unchecked")
public void testCheckpointsTriggeredBySource() throws Exception {
	// set up the basic test harness
	final StreamTaskTestHarness<Long> testHarness = new StreamTaskTestHarness<>(
			SourceStreamTask::new,
			BasicTypeInfo.LONG_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();
	testHarness.getExecutionConfig().setLatencyTrackingInterval(-1);

	final long numElements = 10;
	final long checkpointEvery = 3;

	// set up the source function
	ExternalCheckpointsSource source = new ExternalCheckpointsSource(numElements, checkpointEvery);
	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamSource<Long, ?> sourceOperator = new StreamSource<>(source);
	streamConfig.setStreamOperator(sourceOperator);
	streamConfig.setOperatorID(new OperatorID());

	// this starts the source thread
	testHarness.invoke();

	final StreamTask<Long, ?> sourceTask = testHarness.getTask();

	ready.await();

	// now send an external trigger that should be ignored
	assertTrue(sourceTask.triggerCheckpoint(new CheckpointMetaData(32, 829), CheckpointOptions.forCheckpointWithDefaultLocation()));

	// step by step let the source thread emit elements
	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 1L);
	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 2L);
	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 3L);

	verifyCheckpointBarrier(testHarness.getOutput(), 1L);

	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 4L);

	// now send an regular trigger command that should be ignored
	assertTrue(sourceTask.triggerCheckpoint(new CheckpointMetaData(34, 900), CheckpointOptions.forCheckpointWithDefaultLocation()));

	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 5L);
	sync.trigger();
	verifyNextElement(testHarness.getOutput(), 6L);

	verifyCheckpointBarrier(testHarness.getOutput(), 2L);

	// let the remainder run

	for (long l = 7L, checkpoint = 3L; l <= numElements; l++) {
		sync.trigger();
		verifyNextElement(testHarness.getOutput(), l);

		if (l % checkpointEvery == 0) {
			verifyCheckpointBarrier(testHarness.getOutput(), checkpoint++);
		}
	}

	// done!
}
 

private void configureChainedTestingStreamOperator(
	StreamConfig streamConfig,
	int numberChainedTasks) {

	Preconditions.checkArgument(numberChainedTasks >= 1, "The operator chain must at least " +
		"contain one operator.");

	TestingStreamOperator<Integer, Integer> previousOperator = new TestingStreamOperator<>();
	streamConfig.setStreamOperator(previousOperator);
	streamConfig.setOperatorID(new OperatorID(0L, 0L));

	// create the chain of operators
	Map<Integer, StreamConfig> chainedTaskConfigs = new HashMap<>(numberChainedTasks - 1);
	List<StreamEdge> outputEdges = new ArrayList<>(numberChainedTasks - 1);

	for (int chainedIndex = 1; chainedIndex < numberChainedTasks; chainedIndex++) {
		TestingStreamOperator<Integer, Integer> chainedOperator = new TestingStreamOperator<>();
		StreamConfig chainedConfig = new StreamConfig(new Configuration());
		chainedConfig.setStreamOperator(chainedOperator);
		chainedConfig.setOperatorID(new OperatorID(0L, chainedIndex));
		chainedTaskConfigs.put(chainedIndex, chainedConfig);

		StreamEdge outputEdge = new StreamEdge(
			new StreamNode(
				null,
				chainedIndex - 1,
				null,
				null,
				null,
				null,
				null,
				null
			),
			new StreamNode(
				null,
				chainedIndex,
				null,
				null,
				null,
				null,
				null,
				null
			),
			0,
			Collections.<String>emptyList(),
			null,
			null
		);

		outputEdges.add(outputEdge);
	}

	streamConfig.setChainedOutputs(outputEdges);
	streamConfig.setTransitiveChainedTaskConfigs(chainedTaskConfigs);
}
 

/**
 * This test verifies that checkpoint barriers and barrier buffers work correctly with
 * concurrent checkpoint barriers where one checkpoint is "overtaking" another checkpoint, i.e.
 * some inputs receive barriers from an earlier checkpoint, thereby blocking,
 * then all inputs receive barriers from a later checkpoint.
 */
@Test
public void testOvertakingCheckpointBarriers() throws Exception {
	final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(
			OneInputStreamTask::new,
			2, 2,
			BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	StreamMap<String, String> mapOperator = new StreamMap<String, String>(new IdentityMap());
	streamConfig.setStreamOperator(mapOperator);
	streamConfig.setOperatorID(new OperatorID());

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
	long initialTime = 0L;

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);

	// These elements should be buffered until we receive barriers from
	// all inputs
	testHarness.processElement(new StreamRecord<String>("Hello-0-0", initialTime), 0, 0);
	testHarness.processElement(new StreamRecord<String>("Ciao-0-0", initialTime), 0, 0);

	// These elements should be forwarded, since we did not yet receive a checkpoint barrier
	// on that input, only add to same input, otherwise we would not know the ordering
	// of the output since the Task might read the inputs in any order
	testHarness.processElement(new StreamRecord<String>("Hello-1-1", initialTime), 1, 1);
	testHarness.processElement(new StreamRecord<String>("Ciao-1-1", initialTime), 1, 1);
	expectedOutput.add(new StreamRecord<String>("Hello-1-1", initialTime));
	expectedOutput.add(new StreamRecord<String>("Ciao-1-1", initialTime));

	testHarness.waitForInputProcessing();
	// we should not yet see the barrier, only the two elements from non-blocked input
	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	// Now give a later barrier to all inputs, this should unblock the first channel,
	// thereby allowing the two blocked elements through
	testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);
	testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
	testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
	testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);

	expectedOutput.add(new CancelCheckpointMarker(0));
	expectedOutput.add(new StreamRecord<String>("Hello-0-0", initialTime));
	expectedOutput.add(new StreamRecord<String>("Ciao-0-0", initialTime));
	expectedOutput.add(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()));

	testHarness.waitForInputProcessing();

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());

	// Then give the earlier barrier, these should be ignored
	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);

	testHarness.waitForInputProcessing();

	testHarness.endInput();

	testHarness.waitForTaskCompletion();

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

/**
 * This test ensures that the SourceStreamTask properly serializes checkpointing
 * and element emission. This also verifies that there are no concurrent invocations
 * of the checkpoint method on the source operator.
 *
 * <p>The source emits elements and performs checkpoints. We have several checkpointer threads
 * that fire checkpoint requests at the source task.
 *
 * <p>If element emission and checkpointing are not in series the count of elements at the
 * beginning of a checkpoint and at the end of a checkpoint are not the same because the
 * source kept emitting elements while the checkpoint was ongoing.
 */
@Test
@SuppressWarnings("unchecked")
public void testCheckpointing() throws Exception {
	final int numElements = 100;
	final int numCheckpoints = 100;
	final int numCheckpointers = 1;
	final int checkpointInterval = 5; // in ms
	final int sourceCheckpointDelay = 1000; // how many random values we sum up in storeCheckpoint
	final int sourceReadDelay = 1; // in ms

	ExecutorService executor = Executors.newFixedThreadPool(10);
	try {
		final TupleTypeInfo<Tuple2<Long, Integer>> typeInfo = new TupleTypeInfo<>(BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);

		final StreamTaskTestHarness<Tuple2<Long, Integer>> testHarness = new StreamTaskTestHarness<>(
				SourceStreamTask::new, typeInfo);
		testHarness.setupOutputForSingletonOperatorChain();

		StreamConfig streamConfig = testHarness.getStreamConfig();
		StreamSource<Tuple2<Long, Integer>, ?> sourceOperator = new StreamSource<>(new MockSource(numElements, sourceCheckpointDelay, sourceReadDelay));
		streamConfig.setStreamOperator(sourceOperator);
		streamConfig.setOperatorID(new OperatorID());

		// prepare the

		Future<Boolean>[] checkpointerResults = new Future[numCheckpointers];

		// invoke this first, so the tasks are actually running when the checkpoints are scheduled
		testHarness.invoke();
		testHarness.waitForTaskRunning();

		final StreamTask<Tuple2<Long, Integer>, ?> sourceTask = testHarness.getTask();

		for (int i = 0; i < numCheckpointers; i++) {
			checkpointerResults[i] = executor.submit(new Checkpointer(numCheckpoints, checkpointInterval, sourceTask));
		}

		testHarness.waitForTaskCompletion();

		// Get the result from the checkpointers, if these threw an exception it
		// will be rethrown here
		for (int i = 0; i < numCheckpointers; i++) {
			if (!checkpointerResults[i].isDone()) {
				checkpointerResults[i].cancel(true);
			}
			if (!checkpointerResults[i].isCancelled()) {
				checkpointerResults[i].get();
			}
		}

		List<Tuple2<Long, Integer>> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
		Assert.assertEquals(numElements, resultElements.size());
	}
	finally {
		executor.shutdown();
	}
}
 

/**
 * Tests that the StreamTask first closes all of its operators before setting its
 * state to not running (isRunning == false)
 *
 * <p>See FLINK-7430.
 */
@Test
public void testOperatorClosingBeforeStopRunning() throws Throwable {
	Configuration taskConfiguration = new Configuration();
	StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	streamConfig.setStreamOperator(new BlockingCloseStreamOperator());
	streamConfig.setOperatorID(new OperatorID());

	try (MockEnvironment mockEnvironment =
			new MockEnvironmentBuilder()
				.setTaskName("Test Task")
				.setMemorySize(32L * 1024L)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(1)
				.setTaskConfiguration(taskConfiguration)
				.build()) {
		StreamTask<Void, BlockingCloseStreamOperator> streamTask = new NoOpStreamTask<>(mockEnvironment);
		final AtomicReference<Throwable> atomicThrowable = new AtomicReference<>(null);

		CompletableFuture<Void> invokeFuture = CompletableFuture.runAsync(
			() -> {
				try {
					streamTask.invoke();
				} catch (Exception e) {
					atomicThrowable.set(e);
				}
			},
			TestingUtils.defaultExecutor());

		BlockingCloseStreamOperator.IN_CLOSE.await();

		// check that the StreamTask is not yet in isRunning == false
		assertTrue(streamTask.isRunning());

		// let the operator finish its close operation
		BlockingCloseStreamOperator.FINISH_CLOSE.trigger();

		// wait until the invoke is complete
		invokeFuture.get();

		// now the StreamTask should no longer be running
		assertFalse(streamTask.isRunning());

		// check if an exception occurred
		if (atomicThrowable.get() != null) {
			throw atomicThrowable.get();
		}
	}
}
 

/**
 * This test verifies that checkpoint barriers and barrier buffers work correctly with
 * concurrent checkpoint barriers where one checkpoint is "overtaking" another checkpoint, i.e.
 * some inputs receive barriers from an earlier checkpoint, thereby blocking,
 * then all inputs receive barriers from a later checkpoint.
 */
@Test
public void testOvertakingCheckpointBarriers() throws Exception {

	final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness =
			new TwoInputStreamTaskTestHarness<>(
					TwoInputStreamTask::new,
					2, 2, new int[] {1, 2},
					BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);

	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<>(new IdentityMap());
	streamConfig.setStreamOperator(coMapOperator);
	streamConfig.setOperatorID(new OperatorID());

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
	long initialTime = 0L;

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);

	// These elements should be forwarded, since we did not yet receive a checkpoint barrier
	// on that input, only add to same input, otherwise we would not know the ordering
	// of the output since the Task might read the inputs in any order
	testHarness.processElement(new StreamRecord<>(42, initialTime), 1, 1);
	testHarness.processElement(new StreamRecord<>(1337, initialTime), 1, 1);
	expectedOutput.add(new StreamRecord<>("42", initialTime));
	expectedOutput.add(new StreamRecord<>("1337", initialTime));

	testHarness.waitForInputProcessing();
	// we should not yet see the barrier, only the two elements from non-blocked input
	TestHarnessUtil.assertOutputEquals("Output was not correct.",
			expectedOutput,
			testHarness.getOutput());

	// Now give a later barrier to all inputs, this should unblock the first channel
	testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
	testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);
	testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
	testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);

	expectedOutput.add(new CancelCheckpointMarker(0));
	expectedOutput.add(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()));

	testHarness.waitForInputProcessing();

	TestHarnessUtil.assertOutputEquals("Output was not correct.",
			expectedOutput,
			testHarness.getOutput());

	// Then give the earlier barrier, these should be ignored
	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);

	testHarness.waitForInputProcessing();

	testHarness.endInput();

	testHarness.waitForTaskCompletion();

	TestHarnessUtil.assertOutputEquals("Output was not correct.",
			expectedOutput,
			testHarness.getOutput());
}
 

private static Task createTask(
	StreamOperator<?> op,
	StateBackend backend,
	CheckpointResponder checkpointResponder) throws IOException {

	Configuration taskConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(taskConfig);
	cfg.setStreamOperator(op);
	cfg.setOperatorID(new OperatorID());
	cfg.setStateBackend(backend);

	ExecutionConfig executionConfig = new ExecutionConfig();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"test job name",
			new SerializedValue<>(executionConfig),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"test task name",
			1,
			11,
			TestStreamTask.class.getName(),
			taskConfig);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			checkpointResponder,
			new NoOpTaskOperatorEventGateway(),
			new TestGlobalAggregateManager(),
			TestingClassLoaderLease.newBuilder().build(),
			new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() },
				VoidPermanentBlobService.INSTANCE),
			new TestingTaskManagerRuntimeInfo(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
			new NoOpResultPartitionConsumableNotifier(),
			mock(PartitionProducerStateChecker.class),
			Executors.directExecutor());
}
 

/**
 * This test ensures that the SourceStreamTask properly serializes checkpointing
 * and element emission. This also verifies that there are no concurrent invocations
 * of the checkpoint method on the source operator.
 *
 * <p>The source emits elements and performs checkpoints. We have several checkpointer threads
 * that fire checkpoint requests at the source task.
 *
 * <p>If element emission and checkpointing are not in series the count of elements at the
 * beginning of a checkpoint and at the end of a checkpoint are not the same because the
 * source kept emitting elements while the checkpoint was ongoing.
 */
@Test
@SuppressWarnings("unchecked")
public void testCheckpointing() throws Exception {
	final int numElements = 100;
	final int numCheckpoints = 100;
	final int numCheckpointers = 1;
	final int checkpointInterval = 5; // in ms
	final int sourceCheckpointDelay = 1000; // how many random values we sum up in storeCheckpoint
	final int sourceReadDelay = 1; // in ms

	ExecutorService executor = Executors.newFixedThreadPool(10);
	try {
		final TupleTypeInfo<Tuple2<Long, Integer>> typeInfo = new TupleTypeInfo<>(BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);

		final StreamTaskTestHarness<Tuple2<Long, Integer>> testHarness = new StreamTaskTestHarness<>(
				SourceStreamTask::new, typeInfo);
		testHarness.setupOutputForSingletonOperatorChain();

		StreamConfig streamConfig = testHarness.getStreamConfig();
		StreamSource<Tuple2<Long, Integer>, ?> sourceOperator = new StreamSource<>(new MockSource(numElements, sourceCheckpointDelay, sourceReadDelay));
		streamConfig.setStreamOperator(sourceOperator);
		streamConfig.setOperatorID(new OperatorID());

		// prepare the

		Future<Boolean>[] checkpointerResults = new Future[numCheckpointers];

		// invoke this first, so the tasks are actually running when the checkpoints are scheduled
		testHarness.invoke();
		testHarness.waitForTaskRunning();

		final StreamTask<Tuple2<Long, Integer>, ?> sourceTask = testHarness.getTask();

		for (int i = 0; i < numCheckpointers; i++) {
			checkpointerResults[i] = executor.submit(new Checkpointer(numCheckpoints, checkpointInterval, sourceTask));
		}

		testHarness.waitForTaskCompletion();

		// Get the result from the checkpointers, if these threw an exception it
		// will be rethrown here
		for (int i = 0; i < numCheckpointers; i++) {
			if (!checkpointerResults[i].isDone()) {
				checkpointerResults[i].cancel(true);
			}
			if (!checkpointerResults[i].isCancelled()) {
				checkpointerResults[i].get();
			}
		}

		List<Tuple2<Long, Integer>> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
		Assert.assertEquals(numElements, resultElements.size());
	}
	finally {
		executor.shutdown();
	}
}
 

/**
 * This test verifies that checkpoint barriers are correctly forwarded.
 */
@Test
@SuppressWarnings("unchecked")
public void testCheckpointBarriers() throws Exception {
	if (isInputSelectable) {
		// In the case of selective reading, checkpoints are not currently supported, and we skip this test
		return;
	}

	final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness =
			new TwoInputStreamTaskTestHarness<String, Integer, String>(
					TwoInputStreamTask::new,
					2, 2, new int[] {1, 2},
					BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
	testHarness.setupOutputForSingletonOperatorChain();

	StreamConfig streamConfig = testHarness.getStreamConfig();
	CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<String, Integer, String>(new IdentityMap());
	streamConfig.setStreamOperator(coMapOperator);
	streamConfig.setOperatorID(new OperatorID());

	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
	long initialTime = 0L;

	testHarness.invoke();
	testHarness.waitForTaskRunning();

	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);

	// This element should be buffered since we received a checkpoint barrier on
	// this input
	testHarness.processElement(new StreamRecord<String>("Hello-0-0", initialTime), 0, 0);

	// This one should go through
	testHarness.processElement(new StreamRecord<String>("Ciao-0-0", initialTime), 0, 1);
	expectedOutput.add(new StreamRecord<String>("Ciao-0-0", initialTime));

	testHarness.waitForInputProcessing();

	// These elements should be forwarded, since we did not yet receive a checkpoint barrier
	// on that input, only add to same input, otherwise we would not know the ordering
	// of the output since the Task might read the inputs in any order
	testHarness.processElement(new StreamRecord<Integer>(11, initialTime), 1, 1);
	testHarness.processElement(new StreamRecord<Integer>(111, initialTime), 1, 1);
	expectedOutput.add(new StreamRecord<String>("11", initialTime));
	expectedOutput.add(new StreamRecord<String>("111", initialTime));

	testHarness.waitForInputProcessing();

	// Wait to allow input to end up in the output.
	// TODO Use count down latches instead as a cleaner solution
	for (int i = 0; i < 20; ++i) {
		if (testHarness.getOutput().size() >= expectedOutput.size()) {
			break;
		} else {
			Thread.sleep(100);
		}
	}

	// we should not yet see the barrier, only the two elements from non-blocked input
	TestHarnessUtil.assertOutputEquals("Output was not correct.",
		expectedOutput,
		testHarness.getOutput());

	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
	testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);

	testHarness.waitForInputProcessing();
	testHarness.endInput();
	testHarness.waitForTaskCompletion();

	// now we should see the barrier and after that the buffered elements
	expectedOutput.add(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()));
	expectedOutput.add(new StreamRecord<String>("Hello-0-0", initialTime));

	TestHarnessUtil.assertOutputEquals("Output was not correct.",
			expectedOutput,
			testHarness.getOutput());

	List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
	Assert.assertEquals(4, resultElements.size());
}