org.apache.flink.runtime.taskmanager.TaskManagerActions Java Examples

public JobManagerConnection(
			JobID jobID,
			ResourceID resourceID,
			JobMasterGateway jobMasterGateway,
			TaskManagerActions taskManagerActions,
			CheckpointResponder checkpointResponder,
			GlobalAggregateManager aggregateManager,
			LibraryCacheManager libraryCacheManager,
			ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
			PartitionProducerStateChecker partitionStateChecker) {
	this.jobID = Preconditions.checkNotNull(jobID);
	this.resourceID = Preconditions.checkNotNull(resourceID);
	this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
	this.taskManagerActions = Preconditions.checkNotNull(taskManagerActions);
	this.checkpointResponder = Preconditions.checkNotNull(checkpointResponder);
	this.aggregateManager = Preconditions.checkNotNull(aggregateManager);
	this.libraryCacheManager = Preconditions.checkNotNull(libraryCacheManager);
	this.resultPartitionConsumableNotifier = Preconditions.checkNotNull(resultPartitionConsumableNotifier);
	this.partitionStateChecker = Preconditions.checkNotNull(partitionStateChecker);
}
 

static void registerJobMasterConnection(
		JobTable jobTable,
		JobID jobId,
		RpcService testingRpcService,
		JobMasterGateway jobMasterGateway,
		TaskManagerActions taskManagerActions,
		Time timeout,
		MainThreadExecutable mainThreadExecutable) {
	mainThreadExecutable.runAsync(() -> {
		final JobTable.Job job = jobTable.getOrCreateJob(jobId, () -> TestingJobServices.newBuilder().build());
		job.connect(
			ResourceID.generate(),
			jobMasterGateway,
			taskManagerActions,
			new TestCheckpointResponder(),
			new TestGlobalAggregateManager(),
			new RpcResultPartitionConsumableNotifier(jobMasterGateway, testingRpcService.getExecutor(), timeout),
			TestingPartitionProducerStateChecker.newBuilder()
				.setPartitionProducerStateFunction((jobID, intermediateDataSetID, resultPartitionID) -> CompletableFuture.completedFuture(ExecutionState.RUNNING))
				.build());
	});
}
 

private EstablishedConnection(
	ResourceID resourceID,
	JobMasterGateway jobMasterGateway,
	TaskManagerActions taskManagerActions,
	CheckpointResponder checkpointResponder,
	GlobalAggregateManager globalAggregateManager,
	ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
	PartitionProducerStateChecker partitionStateChecker) {
	this.resourceID = Preconditions.checkNotNull(resourceID);
	this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
	this.taskManagerActions = Preconditions.checkNotNull(taskManagerActions);
	this.checkpointResponder = Preconditions.checkNotNull(checkpointResponder);
	this.globalAggregateManager = Preconditions.checkNotNull(globalAggregateManager);
	this.resultPartitionConsumableNotifier = Preconditions.checkNotNull(resultPartitionConsumableNotifier);
	this.partitionStateChecker = Preconditions.checkNotNull(partitionStateChecker);
}
 

@Override
public JobTable.Connection connect(
		ResourceID resourceId,
		JobMasterGateway jobMasterGateway,
		TaskManagerActions taskManagerActions,
		CheckpointResponder checkpointResponder,
		GlobalAggregateManager aggregateManager,
		ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
		PartitionProducerStateChecker partitionStateChecker) {
	verifyJobIsNotClosed();
	Preconditions.checkState(connection == null);

	connection = new EstablishedConnection(
		resourceId,
		jobMasterGateway,
		taskManagerActions,
		checkpointResponder,
		aggregateManager,
		resultPartitionConsumableNotifier,
		partitionStateChecker);
	resourceIdJobIdIndex.put(resourceId, jobId);

	return this;
}
 

static JobManagerConnection createJobManagerConnection(JobID jobId, JobMasterGateway jobMasterGateway, RpcService testingRpcService, TaskManagerActions taskManagerActions, Time timeout) {
	final LibraryCacheManager libraryCacheManager = mock(LibraryCacheManager.class);
	when(libraryCacheManager.getClassLoader(any(JobID.class))).thenReturn(ClassLoader.getSystemClassLoader());

	final PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	when(partitionProducerStateChecker.requestPartitionProducerState(any(), any(), any()))
		.thenReturn(CompletableFuture.completedFuture(ExecutionState.RUNNING));

	return new JobManagerConnection(
		jobId,
		ResourceID.generate(),
		jobMasterGateway,
		taskManagerActions,
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		libraryCacheManager,
		new RpcResultPartitionConsumableNotifier(jobMasterGateway, testingRpcService.getExecutor(), timeout),
		partitionProducerStateChecker);
}
 

public JobManagerConnection(
			JobID jobID,
			ResourceID resourceID,
			JobMasterGateway jobMasterGateway,
			TaskManagerActions taskManagerActions,
			CheckpointResponder checkpointResponder,
			GlobalAggregateManager aggregateManager,
			LibraryCacheManager libraryCacheManager,
			ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
			PartitionProducerStateChecker partitionStateChecker) {
	this.jobID = Preconditions.checkNotNull(jobID);
	this.resourceID = Preconditions.checkNotNull(resourceID);
	this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
	this.taskManagerActions = Preconditions.checkNotNull(taskManagerActions);
	this.checkpointResponder = Preconditions.checkNotNull(checkpointResponder);
	this.aggregateManager = Preconditions.checkNotNull(aggregateManager);
	this.libraryCacheManager = Preconditions.checkNotNull(libraryCacheManager);
	this.resultPartitionConsumableNotifier = Preconditions.checkNotNull(resultPartitionConsumableNotifier);
	this.partitionStateChecker = Preconditions.checkNotNull(partitionStateChecker);
}
 

/**
 * This test checks that cancel calls that are issued before the operator is
 * instantiated still lead to proper canceling.
 */
@Test
public void testEarlyCanceling() throws Exception {
	final StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setOperatorID(new OperatorID(4711L, 42L));
	cfg.setStreamOperator(new SlowlyDeserializingOperator());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	final TaskManagerActions taskManagerActions = spy(new NoOpTaskManagerActions());
	final Task task = createTask(SourceStreamTask.class, cfg, new Configuration(), taskManagerActions);

	final TaskExecutionState state = new TaskExecutionState(
		task.getJobID(), task.getExecutionId(), ExecutionState.RUNNING);

	task.startTaskThread();

	verify(taskManagerActions, timeout(2000L)).updateTaskExecutionState(eq(state));

	// send a cancel. because the operator takes a long time to deserialize, this should
	// hit the task before the operator is deserialized
	task.cancelExecution();

	task.getExecutingThread().join();

	assertFalse("Task did not cancel", task.getExecutingThread().isAlive());
	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
}
 

/**
 * This test checks that cancel calls that are issued before the operator is
 * instantiated still lead to proper canceling.
 */
@Test
public void testEarlyCanceling() throws Exception {
	final StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setOperatorID(new OperatorID(4711L, 42L));
	cfg.setStreamOperator(new SlowlyDeserializingOperator());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	final TaskManagerActions taskManagerActions = spy(new NoOpTaskManagerActions());
	try (NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
		final Task task =  new TestTaskBuilder(shuffleEnvironment)
			.setInvokable(SourceStreamTask.class)
			.setTaskConfig(cfg.getConfiguration())
			.setTaskManagerActions(taskManagerActions)
			.build();

		final TaskExecutionState state = new TaskExecutionState(
			task.getJobID(), task.getExecutionId(), ExecutionState.RUNNING);

		task.startTaskThread();

		verify(taskManagerActions, timeout(2000L)).updateTaskExecutionState(eq(state));

		// send a cancel. because the operator takes a long time to deserialize, this should
		// hit the task before the operator is deserialized
		task.cancelExecution();

		task.getExecutingThread().join();

		assertFalse("Task did not cancel", task.getExecutingThread().isAlive());
		assertEquals(ExecutionState.CANCELED, task.getExecutionState());
	}
}
 

private JobTable.Connection associateWithJobManager(
		JobTable.Job job,
		ResourceID resourceID,
		JobMasterGateway jobMasterGateway) {
	checkNotNull(resourceID);
	checkNotNull(jobMasterGateway);

	TaskManagerActions taskManagerActions = new TaskManagerActionsImpl(jobMasterGateway);

	CheckpointResponder checkpointResponder = new RpcCheckpointResponder(jobMasterGateway);
	GlobalAggregateManager aggregateManager = new RpcGlobalAggregateManager(jobMasterGateway);

	ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = new RpcResultPartitionConsumableNotifier(
		jobMasterGateway,
		getRpcService().getExecutor(),
		taskManagerConfiguration.getTimeout());

	PartitionProducerStateChecker partitionStateChecker = new RpcPartitionStateChecker(jobMasterGateway);

	registerQueryableState(job.getJobId(), jobMasterGateway);

	return job.connect(
		resourceID,
		jobMasterGateway,
		taskManagerActions,
		checkpointResponder,
		aggregateManager,
		resultPartitionConsumableNotifier,
		partitionStateChecker);
}
 

public static Task createTask(
	Class<? extends AbstractInvokable> invokable,
	StreamConfig taskConfig,
	Configuration taskManagerConfig,
	TaskManagerActions taskManagerActions) throws Exception {
	return createTask(invokable, taskConfig, taskManagerConfig, new TestTaskStateManager(), taskManagerActions);
}
 

private JobManagerConnection associateWithJobManager(
		JobID jobID,
		ResourceID resourceID,
		JobMasterGateway jobMasterGateway) {
	checkNotNull(jobID);
	checkNotNull(resourceID);
	checkNotNull(jobMasterGateway);

	TaskManagerActions taskManagerActions = new TaskManagerActionsImpl(jobMasterGateway);

	CheckpointResponder checkpointResponder = new RpcCheckpointResponder(jobMasterGateway);
	GlobalAggregateManager aggregateManager = new RpcGlobalAggregateManager(jobMasterGateway);

	final LibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(
		blobCacheService.getPermanentBlobService(),
		taskManagerConfiguration.getClassLoaderResolveOrder(),
		taskManagerConfiguration.getAlwaysParentFirstLoaderPatterns());

	ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = new RpcResultPartitionConsumableNotifier(
		jobMasterGateway,
		getRpcService().getExecutor(),
		taskManagerConfiguration.getTimeout());

	PartitionProducerStateChecker partitionStateChecker = new RpcPartitionStateChecker(jobMasterGateway);

	registerQueryableState(jobID, jobMasterGateway);

	return new JobManagerConnection(
		jobID,
		resourceID,
		jobMasterGateway,
		taskManagerActions,
		checkpointResponder,
		aggregateManager,
		libraryCacheManager,
		resultPartitionConsumableNotifier,
		partitionStateChecker);
}
 

private JobManagerConnection associateWithJobManager(
		JobID jobID,
		ResourceID resourceID,
		JobMasterGateway jobMasterGateway) {
	checkNotNull(jobID);
	checkNotNull(resourceID);
	checkNotNull(jobMasterGateway);

	TaskManagerActions taskManagerActions = new TaskManagerActionsImpl(jobMasterGateway);

	CheckpointResponder checkpointResponder = new RpcCheckpointResponder(jobMasterGateway);
	GlobalAggregateManager aggregateManager = new RpcGlobalAggregateManager(jobMasterGateway);

	final LibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(
		blobCacheService.getPermanentBlobService(),
		taskManagerConfiguration.getClassLoaderResolveOrder(),
		taskManagerConfiguration.getAlwaysParentFirstLoaderPatterns());

	ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = new RpcResultPartitionConsumableNotifier(
		jobMasterGateway,
		getRpcService().getExecutor(),
		taskManagerConfiguration.getTimeout());

	PartitionProducerStateChecker partitionStateChecker = new RpcPartitionStateChecker(jobMasterGateway);

	registerQueryableState(jobID, jobMasterGateway);

	return new JobManagerConnection(
		jobID,
		resourceID,
		jobMasterGateway,
		taskManagerActions,
		checkpointResponder,
		aggregateManager,
		libraryCacheManager,
		resultPartitionConsumableNotifier,
		partitionStateChecker);
}
 

/**
 * This test checks that cancel calls that are issued before the operator is
 * instantiated still lead to proper canceling.
 */
@Test
public void testEarlyCanceling() throws Exception {
	final StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setOperatorID(new OperatorID(4711L, 42L));
	cfg.setStreamOperator(new SlowlyDeserializingOperator());
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	final TaskManagerActions taskManagerActions = spy(new NoOpTaskManagerActions());
	final Task task = createTask(SourceStreamTask.class, cfg, new Configuration(), taskManagerActions);

	final TaskExecutionState state = new TaskExecutionState(
		task.getJobID(), task.getExecutionId(), ExecutionState.RUNNING);

	task.startTaskThread();

	verify(taskManagerActions, timeout(2000L)).updateTaskExecutionState(eq(state));

	// send a cancel. because the operator takes a long time to deserialize, this should
	// hit the task before the operator is deserialized
	task.cancelExecution();

	task.getExecutingThread().join();

	assertFalse("Task did not cancel", task.getExecutingThread().isAlive());
	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
}
 

public static Task createTask(
	Class<? extends AbstractInvokable> invokable,
	StreamConfig taskConfig,
	Configuration taskManagerConfig,
	TaskManagerActions taskManagerActions) throws Exception {
	return createTask(invokable, taskConfig, taskManagerConfig, new TestTaskStateManager(), taskManagerActions);
}
 

@Override
public TaskManagerActions getTaskManagerActions() {
	return verifyContainsEstablishedConnection().getTaskManagerActions();
}
 

private static Task createTask(
	StreamOperator<?> op,
	StateBackend backend,
	CheckpointResponder checkpointResponder) throws IOException {

	Configuration taskConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(taskConfig);
	cfg.setStreamOperator(op);
	cfg.setOperatorID(new OperatorID());
	cfg.setStateBackend(backend);

	ExecutionConfig executionConfig = new ExecutionConfig();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"test job name",
			new SerializedValue<>(executionConfig),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"test task name",
			1,
			11,
			TestStreamTask.class.getName(),
			taskConfig);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			checkpointResponder,
			new NoOpTaskOperatorEventGateway(),
			new TestGlobalAggregateManager(),
			TestingClassLoaderLease.newBuilder().build(),
			new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() },
				VoidPermanentBlobService.INSTANCE),
			new TestingTaskManagerRuntimeInfo(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
			new NoOpResultPartitionConsumableNotifier(),
			mock(PartitionProducerStateChecker.class),
			Executors.directExecutor());
}
 

public TaskManagerActions getTaskManagerActions() {
	return taskManagerActions;
}
 

private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {

		ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
		PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
		Executor executor = mock(Executor.class);
		ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

		TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

		JobInformation jobInformation = new JobInformation(
				new JobID(),
				"Job Name",
				new SerializedValue<>(new ExecutionConfig()),
				new Configuration(),
				Collections.emptyList(),
				Collections.emptyList());

		TaskInformation taskInformation = new TaskInformation(
				new JobVertexID(),
				"Test Task",
				1,
				1,
				invokableClass.getName(),
				new Configuration());

		return new Task(
				jobInformation,
				taskInformation,
				new ExecutionAttemptID(),
				new AllocationID(),
				0,
				0,
				Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
				Collections.<InputGateDeploymentDescriptor>emptyList(),
				0,
				mock(MemoryManager.class),
				mock(IOManager.class),
				shuffleEnvironment,
				new KvStateService(new KvStateRegistry(), null, null),
				mock(BroadcastVariableManager.class),
				new TaskEventDispatcher(),
				ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
				new TestTaskStateManager(),
				mock(TaskManagerActions.class),
				mock(InputSplitProvider.class),
				mock(CheckpointResponder.class),
				new NoOpTaskOperatorEventGateway(),
				new TestGlobalAggregateManager(),
				TestingClassLoaderLease.newBuilder().build(),
				mock(FileCache.class),
				new TestingTaskManagerRuntimeInfo(),
				taskMetricGroup,
				consumableNotifier,
				partitionProducerStateChecker,
				executor);
	}
 

/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing operation after
 * the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
	final Configuration taskConfiguration = new Configuration();
	final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();

	final StateBackend blockingStateBackend = new BlockingStateBackend();

	streamConfig.setStreamOperator(noOpStreamOperator);
	streamConfig.setOperatorID(new OperatorID());
	streamConfig.setStateBackend(blockingStateBackend);

	final long checkpointId = 0L;
	final long checkpointTimestamp = 0L;

	final JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Test Job",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		BlockingStreamTask.class.getName(),
		taskConfiguration);

	final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();

	final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	final Task task = new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		MemoryManagerBuilder.newBuilder().setMemorySize(32L * 1024L).build(),
		new IOManagerAsync(),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new NoOpTaskOperatorEventGateway(),
		new TestGlobalAggregateManager(),
		TestingClassLoaderLease.newBuilder().build(),
		mock(FileCache.class),
		taskManagerRuntimeInfo,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class),
		Executors.directExecutor());

	CompletableFuture<Void> taskRun = CompletableFuture.runAsync(
		() -> task.run(),
		TestingUtils.defaultExecutor());

	// wait until the stream task started running
	RUN_LATCH.await();

	// trigger a checkpoint
	task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), false);

	// wait until the task has completed execution
	taskRun.get();

	// check that no failure occurred
	if (task.getFailureCause() != null) {
		throw new Exception("Task failed", task.getFailureCause());
	}

	// check that we have entered the finished state
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 

/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing operation after
 * the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
	final Configuration taskConfiguration = new Configuration();
	final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();

	final StateBackend blockingStateBackend = new BlockingStateBackend();

	streamConfig.setStreamOperator(noOpStreamOperator);
	streamConfig.setOperatorID(new OperatorID());
	streamConfig.setStateBackend(blockingStateBackend);

	final long checkpointId = 0L;
	final long checkpointTimestamp = 0L;

	final JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Test Job",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		BlockingStreamTask.class.getName(),
		taskConfiguration);

	final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();

	TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
	final NetworkEnvironment networkEnv = mock(NetworkEnvironment.class);
	when(networkEnv.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mock(TaskKvStateRegistry.class));
	when(networkEnv.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	final Task task = new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		new MemoryManager(32L * 1024L, 1),
		new IOManagerAsync(),
		networkEnv,
		mock(BroadcastVariableManager.class),
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		new BlobLibraryCacheManager(
			blobService.getPermanentBlobService(),
			FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
			new String[0]),
		mock(FileCache.class),
		taskManagerRuntimeInfo,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class),
		Executors.directExecutor());

	CompletableFuture<Void> taskRun = CompletableFuture.runAsync(
		() -> task.run(),
		TestingUtils.defaultExecutor());

	// wait until the stream task started running
	RUN_LATCH.await();

	// trigger a checkpoint
	task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation());

	// wait until the task has completed execution
	taskRun.get();

	// check that no failure occurred
	if (task.getFailureCause() != null) {
		throw new Exception("Task failed", task.getFailureCause());
	}

	// check that we have entered the finished state
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 

public TaskManagerActions getTaskManagerActions() {
	return taskManagerActions;
}
 

public static Task createTask(
	Class<? extends AbstractInvokable> invokable,
	StreamConfig taskConfig,
	Configuration taskManagerConfig) throws Exception {
	return createTask(invokable, taskConfig, taskManagerConfig, new TestTaskStateManager(), mock(TaskManagerActions.class));
}
 

public TaskManagerActions getTaskManagerActions() {
	return taskManagerActions;
}
 

private static Task createTask(
	StreamOperator<?> op,
	StateBackend backend,
	CheckpointResponder checkpointResponder) throws IOException {

	Configuration taskConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(taskConfig);
	cfg.setStreamOperator(op);
	cfg.setOperatorID(new OperatorID());
	cfg.setStateBackend(backend);

	ExecutionConfig executionConfig = new ExecutionConfig();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"test job name",
			new SerializedValue<>(executionConfig),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"test task name",
			1,
			11,
			TestStreamTask.class.getName(),
			taskConfig);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			checkpointResponder,
			new TestGlobalAggregateManager(),
			blobService,
			new BlobLibraryCacheManager(
				blobService.getPermanentBlobService(),
				FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
				new String[0]),
			new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() },
				blobService.getPermanentBlobService()),
			new TestingTaskManagerRuntimeInfo(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
			new NoOpResultPartitionConsumableNotifier(),
			mock(PartitionProducerStateChecker.class),
			Executors.directExecutor());
}
 

public static Task createTask(
		Class<? extends AbstractInvokable> invokable,
		StreamConfig taskConfig,
		Configuration taskManagerConfig,
		TestTaskStateManager taskStateManager,
		TaskManagerActions taskManagerActions) throws Exception {

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(StreamTaskTest.class.getClassLoader());

	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);

	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokable.getName(),
		taskConfig.getConfiguration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		taskStateManager,
		taskManagerActions,
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		libCache,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig, new String[] {System.getProperty("java.io.tmpdir")}),
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 

public static Task createTask(
		Class<? extends AbstractInvokable> invokable,
		StreamConfig taskConfig,
		Configuration taskManagerConfig,
		TestTaskStateManager taskStateManager,
		TaskManagerActions taskManagerActions) throws Exception {

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(StreamTaskTest.class.getClassLoader());

	ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();

	NetworkEnvironment network = mock(NetworkEnvironment.class);
	when(network.getResultPartitionManager()).thenReturn(partitionManager);
	when(network.getDefaultIOMode()).thenReturn(IOManager.IOMode.SYNC);
	when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class)))
			.thenReturn(mock(TaskKvStateRegistry.class));
	when(network.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Job Name",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		invokable.getName(),
		taskConfig.getConfiguration());

	return new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		mock(MemoryManager.class),
		mock(IOManager.class),
		network,
		mock(BroadcastVariableManager.class),
		taskStateManager,
		taskManagerActions,
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		libCache,
		mock(FileCache.class),
		new TestingTaskManagerRuntimeInfo(taskManagerConfig, new String[] {System.getProperty("java.io.tmpdir")}),
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		consumableNotifier,
		partitionProducerStateChecker,
		executor);
}
 

public static Task createTask(
	Class<? extends AbstractInvokable> invokable,
	StreamConfig taskConfig,
	Configuration taskManagerConfig) throws Exception {
	return createTask(invokable, taskConfig, taskManagerConfig, new TestTaskStateManager(), mock(TaskManagerActions.class));
}
 

private static Task createTask(
	StreamOperator<?> op,
	StateBackend backend,
	CheckpointResponder checkpointResponder,
	boolean failOnCheckpointErrors) throws IOException {

	Configuration taskConfig = new Configuration();
	StreamConfig cfg = new StreamConfig(taskConfig);
	cfg.setStreamOperator(op);
	cfg.setOperatorID(new OperatorID());
	cfg.setStateBackend(backend);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setFailTaskOnCheckpointError(failOnCheckpointErrors);

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"test job name",
			new SerializedValue<>(executionConfig),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"test task name",
			1,
			11,
			TestStreamTask.class.getName(),
			taskConfig);

	TaskKvStateRegistry mockKvRegistry = mock(TaskKvStateRegistry.class);
	TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();
	NetworkEnvironment network = mock(NetworkEnvironment.class);
	when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mockKvRegistry);
	when(network.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			network,
			mock(BroadcastVariableManager.class),
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			checkpointResponder,
			new TestGlobalAggregateManager(),
			blobService,
			new BlobLibraryCacheManager(
				blobService.getPermanentBlobService(),
				FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
				new String[0]),
			new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() },
				blobService.getPermanentBlobService()),
			new TestingTaskManagerRuntimeInfo(),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
			new NoOpResultPartitionConsumableNotifier(),
			mock(PartitionProducerStateChecker.class),
			Executors.directExecutor());
}
 

private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {
	BlobCacheService blobService =
			new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(ClassLoader.getSystemClassLoader());

	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"Job Name",
			new SerializedValue<>(new ExecutionConfig()),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"Test Task",
			1,
			1,
			invokableClass.getName(),
			new Configuration());

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			mock(CheckpointResponder.class),
			new TestGlobalAggregateManager(),
			blobService,
			libCache,
			mock(FileCache.class),
			new TestingTaskManagerRuntimeInfo(),
			taskMetricGroup,
			consumableNotifier,
			partitionProducerStateChecker,
			executor);
}
 

/**
 * FLINK-6833
 *
 * <p>Tests that a finished stream task cannot be failed by an asynchronous checkpointing operation after
 * the stream task has stopped running.
 */
@Test
public void testConcurrentAsyncCheckpointCannotFailFinishedStreamTask() throws Exception {
	final Configuration taskConfiguration = new Configuration();
	final StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	final NoOpStreamOperator<Long> noOpStreamOperator = new NoOpStreamOperator<>();

	final StateBackend blockingStateBackend = new BlockingStateBackend();

	streamConfig.setStreamOperator(noOpStreamOperator);
	streamConfig.setOperatorID(new OperatorID());
	streamConfig.setStateBackend(blockingStateBackend);

	final long checkpointId = 0L;
	final long checkpointTimestamp = 0L;

	final JobInformation jobInformation = new JobInformation(
		new JobID(),
		"Test Job",
		new SerializedValue<>(new ExecutionConfig()),
		new Configuration(),
		Collections.emptyList(),
		Collections.emptyList());

	final TaskInformation taskInformation = new TaskInformation(
		new JobVertexID(),
		"Test Task",
		1,
		1,
		BlockingStreamTask.class.getName(),
		taskConfiguration);

	final TaskManagerRuntimeInfo taskManagerRuntimeInfo = new TestingTaskManagerRuntimeInfo();

	final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	BlobCacheService blobService =
		new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	final Task task = new Task(
		jobInformation,
		taskInformation,
		new ExecutionAttemptID(),
		new AllocationID(),
		0,
		0,
		Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
		Collections.<InputGateDeploymentDescriptor>emptyList(),
		0,
		new MemoryManager(32L * 1024L, 1),
		new IOManagerAsync(),
		shuffleEnvironment,
		new KvStateService(new KvStateRegistry(), null, null),
		mock(BroadcastVariableManager.class),
		new TaskEventDispatcher(),
		new TestTaskStateManager(),
		mock(TaskManagerActions.class),
		mock(InputSplitProvider.class),
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		blobService,
		new BlobLibraryCacheManager(
			blobService.getPermanentBlobService(),
			FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
			new String[0]),
		mock(FileCache.class),
		taskManagerRuntimeInfo,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		new NoOpResultPartitionConsumableNotifier(),
		mock(PartitionProducerStateChecker.class),
		Executors.directExecutor());

	CompletableFuture<Void> taskRun = CompletableFuture.runAsync(
		() -> task.run(),
		TestingUtils.defaultExecutor());

	// wait until the stream task started running
	RUN_LATCH.await();

	// trigger a checkpoint
	task.triggerCheckpointBarrier(checkpointId, checkpointTimestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), false);

	// wait until the task has completed execution
	taskRun.get();

	// check that no failure occurred
	if (task.getFailureCause() != null) {
		throw new Exception("Task failed", task.getFailureCause());
	}

	// check that we have entered the finished state
	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}