org.apache.flink.runtime.testtasks.BlockingNoOpInvokable Java Examples

/**
 * Tests that the TaskManager sends a proper exception back to the sender if the submit task
 * message fails.
 */
@Test(timeout = TEST_TIMEOUT)
public void testSubmitTaskFailure() throws Exception {
	final ExecutionAttemptID eid = new ExecutionAttemptID();

	final TaskDeploymentDescriptor tdd = createTestTaskDeploymentDescriptor(
		"test task",
		eid,
		BlockingNoOpInvokable.class,
		0); // this will make the submission fail because the number of key groups must be >= 1

	try (TaskSubmissionTestEnvironment env =
		new TaskSubmissionTestEnvironment.Builder(jobId)
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		TaskSlotTable taskSlotTable = env.getTaskSlotTable();

		taskSlotTable.allocateSlot(0, jobId, tdd.getAllocationId(), Time.seconds(60));
		tmGateway.submitTask(tdd, env.getJobMasterId(), timeout).get();
	} catch (Exception e) {
		assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
	}
}
 

/**
 * Tests that the TaskManager sends a proper exception back to the sender if the submit task
 * message fails.
 */
@Test(timeout = 10000L)
public void testSubmitTaskFailure() throws Exception {
	final ExecutionAttemptID eid = new ExecutionAttemptID();

	final TaskDeploymentDescriptor tdd = createTestTaskDeploymentDescriptor(
		"test task",
		eid,
		BlockingNoOpInvokable.class,
		0); // this will make the submission fail because the number of key groups must be >= 1

	try (TaskSubmissionTestEnvironment env =
		new TaskSubmissionTestEnvironment.Builder(jobId)
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		TaskSlotTable taskSlotTable = env.getTaskSlotTable();

		taskSlotTable.allocateSlot(0, jobId, tdd.getAllocationId(), Time.seconds(60));
		tmGateway.submitTask(tdd, env.getJobMasterId(), timeout).get();
	} catch (Exception e) {
		assertThat(e.getCause(), instanceOf(IllegalArgumentException.class));
	}
}
 

@Nonnull
private JobGraph createJobGraph(TestCase testCase, int numTasks) {
	JobVertex source = new JobVertex("Source");
	if (testCase == TestCase.JOB_FAILS) {
		source.setInvokableClass(FailingBlockingInvokable.class);
	} else if (testCase == TestCase.JOB_IS_CANCELLED) {
		source.setInvokableClass(BlockingNoOpInvokable.class);
	} else {
		source.setInvokableClass(NoOpInvokable.class);
	}
	source.setParallelism(numTasks);

	return new JobGraph(new JobID(0, testCase.ordinal()), "BlobCleanupTest", source);
}
 

@Test
public void testTriggerSavepointWithCheckpointingDisabled() throws Exception {
	// Config
	final int numTaskManagers = 1;
	final int numSlotsPerTaskManager = 1;

	final Configuration config = new Configuration();

	final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(numTaskManagers)
			.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
			.build());
	cluster.before();
	final ClusterClient<?> client = cluster.getClusterClient();

	final JobVertex vertex = new JobVertex("Blocking vertex");
	vertex.setInvokableClass(BlockingNoOpInvokable.class);
	vertex.setParallelism(1);

	final JobGraph graph = new JobGraph(vertex);

	try {
		client.setDetached(true);
		client.submitJob(graph, SavepointITCase.class.getClassLoader());

		client.triggerSavepoint(graph.getJobID(), null).get();

		fail();
	} catch (ExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, IllegalStateException.class).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, graph.getJobID().toString()).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, "is not a streaming job").isPresent());
	} finally {
		cluster.after();
	}
}
 

@Nonnull
public JobGraph createKvJobGraph() {
	final JobVertex vertex1 = new JobVertex("v1");
	vertex1.setParallelism(4);
	vertex1.setMaxParallelism(16);
	vertex1.setInvokableClass(BlockingNoOpInvokable.class);

	final JobVertex vertex2 = new JobVertex("v2");
	vertex2.setParallelism(4);
	vertex2.setMaxParallelism(16);
	vertex2.setInvokableClass(BlockingNoOpInvokable.class);

	return new JobGraph(vertex1, vertex2);
}
 

private void setupAndRunHandleJobsWhenNotEnoughSlots(ScheduleMode scheduleMode) throws Exception {
	final JobVertex vertex = new JobVertex("Test Vertex");
	vertex.setParallelism(2);
	vertex.setMaxParallelism(2);
	vertex.setInvokableClass(BlockingNoOpInvokable.class);

	final JobGraph jobGraph = new JobGraph("Test Job", vertex);
	jobGraph.setScheduleMode(scheduleMode);

	runHandleJobsWhenNotEnoughSlots(jobGraph);
}
 

@Test
public void testTriggerSavepointWithCheckpointingDisabled() throws Exception {
	// Config
	final int numTaskManagers = 1;
	final int numSlotsPerTaskManager = 1;

	final Configuration config = new Configuration();

	final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(numTaskManagers)
			.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
			.build());
	cluster.before();
	final ClusterClient<?> client = cluster.getClusterClient();

	final JobVertex vertex = new JobVertex("Blocking vertex");
	vertex.setInvokableClass(BlockingNoOpInvokable.class);
	vertex.setParallelism(1);

	final JobGraph graph = new JobGraph(vertex);

	try {
		ClientUtils.submitJob(client, graph);

		client.triggerSavepoint(graph.getJobID(), null).get();

		fail();
	} catch (ExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, IllegalStateException.class).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, graph.getJobID().toString()).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, "is not a streaming job").isPresent());
	} finally {
		cluster.after();
	}
}
 

@Nonnull
public JobGraph createKvJobGraph() {
	final JobVertex vertex1 = new JobVertex("v1");
	vertex1.setParallelism(4);
	vertex1.setMaxParallelism(16);
	vertex1.setInvokableClass(BlockingNoOpInvokable.class);

	final JobVertex vertex2 = new JobVertex("v2");
	vertex2.setParallelism(4);
	vertex2.setMaxParallelism(16);
	vertex2.setInvokableClass(BlockingNoOpInvokable.class);

	return new JobGraph(vertex1, vertex2);
}
 

private void setupAndRunHandleJobsWhenNotEnoughSlots(ScheduleMode scheduleMode) throws Exception {
	final JobVertex vertex = new JobVertex("Test Vertex");
	vertex.setParallelism(2);
	vertex.setMaxParallelism(2);
	vertex.setInvokableClass(BlockingNoOpInvokable.class);

	final JobGraph jobGraph = new JobGraph("Test Job", vertex);
	jobGraph.setScheduleMode(scheduleMode);

	runHandleJobsWhenNotEnoughSlots(jobGraph);
}
 

@Test
public void testTriggerSavepointWithCheckpointingDisabled() throws Exception {
	// Config
	final int numTaskManagers = 1;
	final int numSlotsPerTaskManager = 1;

	final Configuration config = new Configuration();

	final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(numTaskManagers)
			.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
			.build());
	cluster.before();
	final ClusterClient<?> client = cluster.getClusterClient();

	final JobVertex vertex = new JobVertex("Blocking vertex");
	vertex.setInvokableClass(BlockingNoOpInvokable.class);
	vertex.setParallelism(1);

	final JobGraph graph = new JobGraph(vertex);

	try {
		client.setDetached(true);
		client.submitJob(graph, SavepointITCase.class.getClassLoader());

		client.triggerSavepoint(graph.getJobID(), null).get();

		fail();
	} catch (ExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, IllegalStateException.class).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, graph.getJobID().toString()).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, "is not a streaming job").isPresent());
	} finally {
		cluster.after();
	}
}
 

private static JobGraph createJobWithoutBackPressure() {
	final JobGraph jobGraph = new JobGraph(TEST_JOB_ID, "Test Job");

	TEST_JOB_VERTEX.setInvokableClass(BlockingNoOpInvokable.class);
	TEST_JOB_VERTEX.setParallelism(JOB_PARALLELISM);

	jobGraph.addVertex(TEST_JOB_VERTEX);
	return jobGraph;
}
 

@Nonnull
public JobGraph createKvJobGraph() {
	final JobVertex vertex1 = new JobVertex("v1");
	vertex1.setParallelism(4);
	vertex1.setMaxParallelism(16);
	vertex1.setInvokableClass(BlockingNoOpInvokable.class);

	final JobVertex vertex2 = new JobVertex("v2");
	vertex2.setParallelism(4);
	vertex2.setMaxParallelism(16);
	vertex2.setInvokableClass(BlockingNoOpInvokable.class);

	return new JobGraph(vertex1, vertex2);
}
 

private void setupAndRunHandleJobsWhenNotEnoughSlots(ScheduleMode scheduleMode) throws Exception {
	final JobVertex vertex = new JobVertex("Test Vertex");
	vertex.setParallelism(2);
	vertex.setMaxParallelism(2);
	vertex.setInvokableClass(BlockingNoOpInvokable.class);

	final JobGraph jobGraph = new JobGraph("Test Job", vertex);
	jobGraph.setScheduleMode(scheduleMode);

	runHandleJobsWhenNotEnoughSlots(jobGraph);
}
 

/**
 * Tests that we can cancel the task of the TaskManager given that we've submitted it.
 */
@Test(timeout = 10000L)
public void testTaskSubmissionAndCancelling() throws Exception {
	final ExecutionAttemptID eid1 = new ExecutionAttemptID();
	final ExecutionAttemptID eid2 = new ExecutionAttemptID();

	final TaskDeploymentDescriptor tdd1 = createTestTaskDeploymentDescriptor("test task", eid1, BlockingNoOpInvokable.class);
	final TaskDeploymentDescriptor tdd2 = createTestTaskDeploymentDescriptor("test task", eid2, BlockingNoOpInvokable.class);

	final CompletableFuture<Void> task1RunningFuture = new CompletableFuture<>();
	final CompletableFuture<Void> task2RunningFuture = new CompletableFuture<>();
	final CompletableFuture<Void> task1CanceledFuture = new CompletableFuture<>();

	try (TaskSubmissionTestEnvironment env =
		new TaskSubmissionTestEnvironment.Builder(jobId)
			.setSlotSize(2)
			.addTaskManagerActionListener(eid1, ExecutionState.RUNNING, task1RunningFuture)
			.addTaskManagerActionListener(eid2, ExecutionState.RUNNING, task2RunningFuture)
			.addTaskManagerActionListener(eid1, ExecutionState.CANCELED, task1CanceledFuture)
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		TaskSlotTable taskSlotTable = env.getTaskSlotTable();

		taskSlotTable.allocateSlot(0, jobId, tdd1.getAllocationId(), Time.seconds(60));
		tmGateway.submitTask(tdd1, env.getJobMasterId(), timeout).get();
		task1RunningFuture.get();

		taskSlotTable.allocateSlot(1, jobId, tdd2.getAllocationId(), Time.seconds(60));
		tmGateway.submitTask(tdd2, env.getJobMasterId(), timeout).get();
		task2RunningFuture.get();

		assertSame(taskSlotTable.getTask(eid1).getExecutionState(), ExecutionState.RUNNING);
		assertSame(taskSlotTable.getTask(eid2).getExecutionState(), ExecutionState.RUNNING);

		tmGateway.cancelTask(eid1, timeout);
		task1CanceledFuture.get();

		assertSame(taskSlotTable.getTask(eid1).getExecutionState(), ExecutionState.CANCELED);
		assertSame(taskSlotTable.getTask(eid2).getExecutionState(), ExecutionState.RUNNING);
	}
}
 

/**
 * Tests that the TaskManager fails the task if the partition update fails.
 */
@Test
public void testUpdateTaskInputPartitionsFailure() throws Exception {
	final ExecutionAttemptID eid = new ExecutionAttemptID();

	final TaskDeploymentDescriptor tdd = createTestTaskDeploymentDescriptor("test task", eid, BlockingNoOpInvokable.class);

	final CompletableFuture<Void> taskRunningFuture = new CompletableFuture<>();
	final CompletableFuture<Void> taskFailedFuture = new CompletableFuture<>();
	final ShuffleEnvironment<?, ?> shuffleEnvironment = mock(ShuffleEnvironment.class, Mockito.RETURNS_MOCKS);

	try (TaskSubmissionTestEnvironment env =
		new TaskSubmissionTestEnvironment.Builder(jobId)
			.setShuffleEnvironment(shuffleEnvironment)
			.setSlotSize(1)
			.addTaskManagerActionListener(eid, ExecutionState.RUNNING, taskRunningFuture)
			.addTaskManagerActionListener(eid, ExecutionState.FAILED, taskFailedFuture)
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		TaskSlotTable taskSlotTable = env.getTaskSlotTable();

		taskSlotTable.allocateSlot(0, jobId, tdd.getAllocationId(), Time.seconds(60));
		tmGateway.submitTask(tdd, env.getJobMasterId(), timeout).get();
		taskRunningFuture.get();

		final ResourceID producerLocation = env.getTaskExecutor().getResourceID();
		NettyShuffleDescriptor shuffleDescriptor =
			createRemoteWithIdAndLocation(new IntermediateResultPartitionID(), producerLocation);
		final PartitionInfo partitionUpdate = new PartitionInfo(new IntermediateDataSetID(), shuffleDescriptor);
		doThrow(new IOException()).when(shuffleEnvironment).updatePartitionInfo(eid, partitionUpdate);

		final CompletableFuture<Acknowledge> updateFuture = tmGateway.updatePartitions(
			eid,
			Collections.singletonList(partitionUpdate),
			timeout);

		updateFuture.get();
		taskFailedFuture.get();
		Task task = taskSlotTable.getTask(tdd.getExecutionAttemptId());
		assertThat(task.getExecutionState(), is(ExecutionState.FAILED));
		assertThat(task.getFailureCause(), instanceOf(IOException.class));
	}
}
 

/**
 * Tests that the TaskManager sends a proper exception back to the sender if the trigger stack
 * trace message fails.
 */
@Test
public void testUpdateTaskInputPartitionsFailure() throws Exception {
	ActorGateway jobManager = null;
	ActorGateway taskManager = null;

	try {

		final ExecutionAttemptID executionAttemptId = new ExecutionAttemptID();

		ActorRef jm = system.actorOf(Props.create(SimpleJobManager.class, LEADER_SESSION_ID));
		jobManager = new AkkaActorGateway(jm, LEADER_SESSION_ID);

		highAvailabilityServices.setJobMasterLeaderRetriever(
			HighAvailabilityServices.DEFAULT_JOB_ID,
			new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

		taskManager = TestingUtils.createTaskManager(
			system,
			highAvailabilityServices,
			new Configuration(),
			true,
			true);

		TaskDeploymentDescriptor tdd = createTaskDeploymentDescriptor(
			new JobID(),
			"test job",
			new JobVertexID(),
			executionAttemptId,
			new SerializedValue<>(new ExecutionConfig()),
			"test task",
			1,
			0,
			1,
			0,
			new Configuration(),
			new Configuration(),
			BlockingNoOpInvokable.class.getName(),
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			0);

		Future<Object> submitResponse = taskManager.ask(new SubmitTask(tdd), timeout);

		Await.result(submitResponse, timeout);

		Future<Object> partitionUpdateResponse = taskManager.ask(
			new TaskMessages.UpdateTaskSinglePartitionInfo(
				executionAttemptId,
				new IntermediateDataSetID(),
				new InputChannelDeploymentDescriptor(new ResultPartitionID(), ResultPartitionLocation.createLocal())),
			timeout);

		try {
			Await.result(partitionUpdateResponse, timeout);

			fail("The update task input partitions message should have failed.");
		} catch (Exception e) {
			// expected
		}
	} finally {
		TestingUtils.stopActor(jobManager);
		TestingUtils.stopActor(taskManager);
	}
}
 

/**
 * Tests that we can cancel the task of the TaskManager given that we've submitted it.
 */
@Test(timeout = TEST_TIMEOUT)
public void testTaskSubmissionAndCancelling() throws Exception {
	final ExecutionAttemptID eid1 = new ExecutionAttemptID();
	final ExecutionAttemptID eid2 = new ExecutionAttemptID();

	final TaskDeploymentDescriptor tdd1 = createTestTaskDeploymentDescriptor("test task", eid1, BlockingNoOpInvokable.class);
	final TaskDeploymentDescriptor tdd2 = createTestTaskDeploymentDescriptor("test task", eid2, BlockingNoOpInvokable.class);

	final CompletableFuture<Void> task1RunningFuture = new CompletableFuture<>();
	final CompletableFuture<Void> task2RunningFuture = new CompletableFuture<>();
	final CompletableFuture<Void> task1CanceledFuture = new CompletableFuture<>();

	try (TaskSubmissionTestEnvironment env =
		new TaskSubmissionTestEnvironment.Builder(jobId)
			.setSlotSize(2)
			.addTaskManagerActionListener(eid1, ExecutionState.RUNNING, task1RunningFuture)
			.addTaskManagerActionListener(eid2, ExecutionState.RUNNING, task2RunningFuture)
			.addTaskManagerActionListener(eid1, ExecutionState.CANCELED, task1CanceledFuture)
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		TaskSlotTable<Task> taskSlotTable = env.getTaskSlotTable();

		taskSlotTable.allocateSlot(0, jobId, tdd1.getAllocationId(), Time.seconds(60));
		tmGateway.submitTask(tdd1, env.getJobMasterId(), timeout).get();
		task1RunningFuture.get();

		taskSlotTable.allocateSlot(1, jobId, tdd2.getAllocationId(), Time.seconds(60));
		tmGateway.submitTask(tdd2, env.getJobMasterId(), timeout).get();
		task2RunningFuture.get();

		assertSame(taskSlotTable.getTask(eid1).getExecutionState(), ExecutionState.RUNNING);
		assertSame(taskSlotTable.getTask(eid2).getExecutionState(), ExecutionState.RUNNING);

		tmGateway.cancelTask(eid1, timeout);
		task1CanceledFuture.get();

		assertSame(taskSlotTable.getTask(eid1).getExecutionState(), ExecutionState.CANCELED);
		assertSame(taskSlotTable.getTask(eid2).getExecutionState(), ExecutionState.RUNNING);
	}
}
 

/**
 * Tests that the TaskManager fails the task if the partition update fails.
 */
@Test
public void testUpdateTaskInputPartitionsFailure() throws Exception {
	final ExecutionAttemptID eid = new ExecutionAttemptID();

	final TaskDeploymentDescriptor tdd = createTestTaskDeploymentDescriptor("test task", eid, BlockingNoOpInvokable.class);

	final CompletableFuture<Void> taskRunningFuture = new CompletableFuture<>();
	final CompletableFuture<Void> taskFailedFuture = new CompletableFuture<>();
	final ShuffleEnvironment<?, ?> shuffleEnvironment = mock(ShuffleEnvironment.class, Mockito.RETURNS_MOCKS);

	try (TaskSubmissionTestEnvironment env =
		new TaskSubmissionTestEnvironment.Builder(jobId)
			.setShuffleEnvironment(shuffleEnvironment)
			.setSlotSize(1)
			.addTaskManagerActionListener(eid, ExecutionState.RUNNING, taskRunningFuture)
			.addTaskManagerActionListener(eid, ExecutionState.FAILED, taskFailedFuture)
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		TaskSlotTable<Task> taskSlotTable = env.getTaskSlotTable();

		taskSlotTable.allocateSlot(0, jobId, tdd.getAllocationId(), Time.seconds(60));
		tmGateway.submitTask(tdd, env.getJobMasterId(), timeout).get();
		taskRunningFuture.get();

		final ResourceID producerLocation = env.getTaskExecutor().getResourceID();
		NettyShuffleDescriptor shuffleDescriptor =
			createRemoteWithIdAndLocation(new IntermediateResultPartitionID(), producerLocation);
		final PartitionInfo partitionUpdate = new PartitionInfo(new IntermediateDataSetID(), shuffleDescriptor);
		doThrow(new IOException()).when(shuffleEnvironment).updatePartitionInfo(eid, partitionUpdate);

		final CompletableFuture<Acknowledge> updateFuture = tmGateway.updatePartitions(
			eid,
			Collections.singletonList(partitionUpdate),
			timeout);

		updateFuture.get();
		taskFailedFuture.get();
		Task task = taskSlotTable.getTask(tdd.getExecutionAttemptId());
		assertThat(task.getExecutionState(), is(ExecutionState.FAILED));
		assertThat(task.getFailureCause(), instanceOf(IOException.class));
	}
}
 

/**
 * Tests that the TaskManager sends a proper exception back to the sender if the stop task
 * message fails.
 */
@Test
public void testStopTaskFailure() throws Exception {
	ActorGateway jobManager = null;
	ActorGateway taskManager = null;

	try {
		final ExecutionAttemptID executionAttemptId = new ExecutionAttemptID();

		ActorRef jm = system.actorOf(Props.create(SimpleJobManager.class, LEADER_SESSION_ID));
		jobManager = new AkkaActorGateway(jm, LEADER_SESSION_ID);

		highAvailabilityServices.setJobMasterLeaderRetriever(
			HighAvailabilityServices.DEFAULT_JOB_ID,
			new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

		taskManager = TestingUtils.createTaskManager(
			system,
			highAvailabilityServices,
			new Configuration(),
			true,
			true);

		TaskDeploymentDescriptor tdd = createTaskDeploymentDescriptor(
			new JobID(),
			"test job",
			new JobVertexID(),
			executionAttemptId,
			new SerializedValue<>(new ExecutionConfig()),
			"test task",
			1,
			0,
			1,
			0,
			new Configuration(),
			new Configuration(),
			BlockingNoOpInvokable.class.getName(),
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			0);

		Future<Object> submitResponse = taskManager.ask(new SubmitTask(tdd), timeout);

		Await.result(submitResponse, timeout);

		final Future<Object> taskRunning = taskManager.ask(new TestingTaskManagerMessages.NotifyWhenTaskIsRunning(executionAttemptId), timeout);

		Await.result(taskRunning, timeout);

		Future<Object> stopResponse = taskManager.ask(new StopTask(executionAttemptId), timeout);

		try {
			Await.result(stopResponse, timeout);

			fail("The stop task message should have failed.");
		} catch (UnsupportedOperationException e) {
			// expected
		}
	} finally {
		TestingUtils.stopActor(jobManager);
		TestingUtils.stopActor(taskManager);
	}
}