org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy Java Examples

private ExecutionGraph createExecutionGraph(final PartitionTracker partitionTracker, final JobVertex... vertices) throws Exception {
	final ExecutionGraph executionGraph = ExecutionGraphBuilder.buildGraph(
		null,
		new JobGraph(new JobID(), "test job", vertices),
		new Configuration(),
		scheduledExecutorService,
		mainThreadExecutor.getMainThreadExecutor(),
		new TestingSlotProvider(ignored -> CompletableFuture.completedFuture(new TestingLogicalSlotBuilder().createTestingLogicalSlot())),
		ExecutionGraphPartitionReleaseTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		AkkaUtils.getDefaultTimeout(),
		log,
		NettyShuffleMaster.INSTANCE,
		partitionTracker);

	executionGraph.start(mainThreadExecutor.getMainThreadExecutor());
	mainThreadExecutor.execute(executionGraph::scheduleForExecution);

	return executionGraph;
}
 

@Test
public void testNoManualRestart() throws Exception {
	NoRestartStrategy restartStrategy = new NoRestartStrategy();
	ExecutionGraph eg = createSimpleExecutionGraph(
		restartStrategy, new SimpleSlotProvider(TEST_JOB_ID, NUM_TASKS), createJobGraph());

	eg.getAllExecutionVertices().iterator().next().fail(new Exception("Test Exception"));

	completeCanceling(eg);

	assertEquals(JobStatus.FAILED, eg.getState());

	// This should not restart the graph.
	eg.restart(eg.getGlobalModVersion());

	assertEquals(JobStatus.FAILED, eg.getState());
}
 

public static ExecutionJobVertex getExecutionVertex(
		JobVertexID id, ScheduledExecutorService executor) 
	throws Exception {

	JobVertex ajv = new JobVertex("TestVertex", id);
	ajv.setInvokableClass(mock(AbstractInvokable.class).getClass());

	ExecutionGraph graph = new ExecutionGraph(
		executor,
		executor,
		new JobID(), 
		"test job", 
		new Configuration(),
		new SerializedValue<>(new ExecutionConfig()),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
		new TestingSlotProvider(ignored -> new CompletableFuture<>()));

	graph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());

	return spy(new ExecutionJobVertex(graph, ajv, 1, AkkaUtils.getDefaultTimeout()));
}
 

/**
 * Tests that when a task fail, and restart strategy doesn't support restarting, the job will go to failed.
 */
@Test
public void testNoRestart() throws Exception {
	final JobGraph jobGraph = createBatchJobGraph();
	final NoRestartStrategy restartStrategy = new NoRestartStrategy();
	final ExecutionGraph eg = createExecutionGraph(jobGraph, restartStrategy);

	final ExecutionVertex ev = eg.getAllExecutionVertices().iterator().next();

	ev.fail(new Exception("Test Exception"));

	for (ExecutionVertex evs : eg.getAllExecutionVertices()) {
		evs.getCurrentExecutionAttempt().completeCancelling();
	}

	manualMainThreadExecutor.triggerAll();

	assertEquals(JobStatus.FAILED, eg.getState());
}
 

private LegacyScheduler createLegacyScheduler(JobGraph jobGraph, SlotPool slotPool, ComponentMainThreadExecutor mainThreadExecutor, Time slotRequestTimeout) throws Exception {
	final Scheduler scheduler = createScheduler(slotPool, mainThreadExecutor);
	final LegacyScheduler legacyScheduler = new LegacyScheduler(
		LOG,
		jobGraph,
		VoidBackPressureStatsTracker.INSTANCE,
		TestingUtils.defaultExecutor(),
		new Configuration(),
		scheduler,
		TestingUtils.defaultExecutor(),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		TestingUtils.TIMEOUT(),
		new NoRestartStrategy.NoRestartStrategyFactory(),
		VoidBlobWriter.getInstance(),
		UnregisteredMetricGroups.createUnregisteredJobManagerJobMetricGroup(),
		slotRequestTimeout,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);

	legacyScheduler.setMainThreadExecutor(mainThreadExecutor);

	return legacyScheduler;
}
 

private ExecutionGraph createExecutionGraph(JobGraph jobGraph) throws JobException, JobExecutionException {
	// configure the pipelined failover strategy
	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setString(
			JobManagerOptions.EXECUTION_FAILOVER_STRATEGY,
			FailoverStrategyLoader.PIPELINED_REGION_RESTART_STRATEGY_NAME);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobManagerConfig,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		PipelinedFailoverRegionBuildingTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		1000,
		VoidBlobWriter.getInstance(),
		timeout,
		log);
}
 

private ExecutionGraph createExecutionGraph(JobGraph jobGraph, SlotProvider slotProvider, Time timeout) throws Exception {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		new Configuration(),
		executor,
		executor,
		slotProvider,
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		1,
		VoidBlobWriter.getInstance(),
		timeout,
		log);
}
 

private ExecutionGraph createExecutionGraph(JobGraph jobGraph, SlotProvider slotProvider, Time timeout) throws Exception {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		new Configuration(),
		executor,
		executor,
		slotProvider,
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);
}
 

private ExecutionGraph createExecutionGraph(JobGraph jobGraph) throws JobException, JobExecutionException {
	// configure the pipelined failover strategy
	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setString(
			JobManagerOptions.EXECUTION_FAILOVER_STRATEGY,
			FailoverStrategyLoader.LEGACY_PIPELINED_REGION_RESTART_STRATEGY_NAME);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobManagerConfig,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		PipelinedFailoverRegionBuildingTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);
}
 

private ExecutionGraph createExecutionGraph(JobGraph jobGraph, SlotProvider slotProvider, Time timeout) throws Exception {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		new Configuration(),
		executor,
		executor,
		slotProvider,
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpJobMasterPartitionTracker.INSTANCE);
}
 

@Test
public void testNoManualRestart() throws Exception {
	NoRestartStrategy restartStrategy = new NoRestartStrategy();
	Tuple2<ExecutionGraph, Instance> executionGraphInstanceTuple = createExecutionGraph(restartStrategy);
	ExecutionGraph eg = executionGraphInstanceTuple.f0;

	eg.getAllExecutionVertices().iterator().next().fail(new Exception("Test Exception"));

	completeCanceling(eg);

	assertEquals(JobStatus.FAILED, eg.getState());

	// This should not restart the graph.
	eg.restart(eg.getGlobalModVersion());

	assertEquals(JobStatus.FAILED, eg.getState());
}
 

private ExecutionGraph createExecutionGraph(final JobMasterPartitionTracker partitionTracker, final JobVertex... vertices) throws Exception {
	final ExecutionGraph executionGraph = ExecutionGraphBuilder.buildGraph(
		null,
		new JobGraph(new JobID(), "test job", vertices),
		new Configuration(),
		scheduledExecutorService,
		mainThreadExecutor.getMainThreadExecutor(),
		new TestingSlotProvider(ignored -> CompletableFuture.completedFuture(new TestingLogicalSlotBuilder().createTestingLogicalSlot())),
		ExecutionGraphPartitionReleaseTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		AkkaUtils.getDefaultTimeout(),
		log,
		NettyShuffleMaster.INSTANCE,
		partitionTracker);

	executionGraph.start(mainThreadExecutor.getMainThreadExecutor());
	mainThreadExecutor.execute(executionGraph::scheduleForExecution);

	return executionGraph;
}
 

/**
 * Tests that a slot release will atomically release the assigned {@link Execution}.
 */
@Test
public void testSlotReleaseAtomicallyReleasesExecution() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();
	final SingleLogicalSlot slot = ExecutionGraphSchedulingTest.createSingleLogicalSlot(
		slotOwner,
		new SimpleAckingTaskManagerGateway(),
		new SlotRequestId());
	final CompletableFuture<LogicalSlot> slotFuture = CompletableFuture.completedFuture(slot);

	final CountDownLatch slotRequestLatch = new CountDownLatch(1);
	final TestingSlotProvider slotProvider = new TestingSlotProvider(slotRequestId -> {
		slotRequestLatch.countDown();
		return slotFuture;
	});
	final ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		new JobID(),
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	final Execution execution = executionGraph.getJobVertex(jobVertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt();

	executionGraph.start(testMainThreadUtil.getMainThreadExecutor());
	testMainThreadUtil.execute(executionGraph::scheduleForExecution);

	// wait until the slot has been requested
	slotRequestLatch.await();

	testMainThreadUtil.execute(() -> {
		assertThat(execution.getAssignedResource(), is(sameInstance(slot)));

		slot.release(new FlinkException("Test exception"));

		assertThat(execution.getReleaseFuture().isDone(), is(true));
	});
}
 

/**
 * Tests that a slot release will atomically release the assigned {@link Execution}.
 */
@Test
public void testSlotReleaseAtomicallyReleasesExecution() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();
	final SingleLogicalSlot slot = ExecutionGraphSchedulingTest.createSingleLogicalSlot(
		slotOwner,
		new SimpleAckingTaskManagerGateway(),
		new SlotRequestId());
	final CompletableFuture<LogicalSlot> slotFuture = CompletableFuture.completedFuture(slot);

	final CountDownLatch slotRequestLatch = new CountDownLatch(1);
	final TestingSlotProvider slotProvider = new TestingSlotProvider(slotRequestId -> {
		slotRequestLatch.countDown();
		return slotFuture;
	});
	final ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	final Execution execution = executionGraph.getJobVertex(jobVertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt();

	executionGraph.start(testMainThreadUtil.getMainThreadExecutor());
	testMainThreadUtil.execute(executionGraph::scheduleForExecution);

	// wait until the slot has been requested
	slotRequestLatch.await();

	testMainThreadUtil.execute(() -> {
		assertThat(execution.getAssignedResource(), is(sameInstance(slot)));

		slot.release(new FlinkException("Test exception"));

		assertThat(execution.getReleaseFuture().isDone(), is(true));
	});
}
 

/**
 * Tests that the task restore state is nulled after the {@link Execution} has been
 * deployed. See FLINK-9693.
 */
@Test
public void testTaskRestoreStateIsNulledAfterDeployment() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();
	final JobVertexID jobVertexId = jobVertex.getID();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();
	final ProgrammedSlotProvider slotProvider = createProgrammedSlotProvider(
		1,
		Collections.singleton(jobVertexId),
		slotOwner);

	ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		new JobID(),
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertexId);

	ExecutionVertex executionVertex = executionJobVertex.getTaskVertices()[0];

	final Execution execution = executionVertex.getCurrentExecutionAttempt();

	final JobManagerTaskRestore taskRestoreState = new JobManagerTaskRestore(1L, new TaskStateSnapshot());
	execution.setInitialState(taskRestoreState);

	assertThat(execution.getTaskRestore(), is(notNullValue()));

	// schedule the execution vertex and wait for its deployment
	executionVertex.scheduleForExecution(slotProvider, false, LocationPreferenceConstraint.ANY, Collections.emptySet()).get();

	assertThat(execution.getTaskRestore(), is(nullValue()));
}
 

private ExecutionGraph createExecutionGraph(Configuration configuration) throws Exception {
	final ScheduledExecutorService executor = TestingUtils.defaultExecutor();

	final JobID jobId = new JobID();
	final JobGraph jobGraph = new JobGraph(jobId, "test");
	jobGraph.setSnapshotSettings(
		new JobCheckpointingSettings(
			Collections.<JobVertexID>emptyList(),
			Collections.<JobVertexID>emptyList(),
			Collections.<JobVertexID>emptyList(),
			new CheckpointCoordinatorConfiguration(
				100,
				10 * 60 * 1000,
				0,
				1,
				CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
				false),
			null));

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		configuration,
		executor,
		executor,
		new ProgrammedSlotProvider(1),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		1,
		blobWriter,
		timeout,
		LoggerFactory.getLogger(getClass()));
}
 

private ExecutionGraph getDummyExecutionGraph() throws Exception {
	return new ExecutionGraph(
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		new JobID(),
		"Test Job Sample Name",
		new Configuration(),
		new SerializedValue<>(new ExecutionConfig()),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
		new TestingSlotProvider(ignored -> new CompletableFuture<>()));
}
 

/**
 * Verifies that building an {@link ExecutionGraph} from a {@link JobGraph} with
 * parallelism higher than the maximum parallelism fails.
 */
@Test
public void testExecutionGraphConstructionFailsRescaleDopExceedMaxParallelism() throws Exception {

	final Configuration config = new Configuration();

	final int initialParallelism = 1;
	final int maxParallelism = 10;
	final JobVertex[] jobVertices = createVerticesForSimpleBipartiteJobGraph(initialParallelism,  maxParallelism);
	final JobGraph jobGraph = new JobGraph(jobVertices);

	for (JobVertex jv : jobVertices) {
		jv.setParallelism(maxParallelism + 1);
	}

	try {
		// this should fail since we set the parallelism to maxParallelism + 1
		ExecutionGraphBuilder.buildGraph(
			null,
			jobGraph,
			config,
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			new TestingSlotProvider(ignore -> new CompletableFuture<>()),
			Thread.currentThread().getContextClassLoader(),
			new StandaloneCheckpointRecoveryFactory(),
			AkkaUtils.getDefaultTimeout(),
			new NoRestartStrategy(),
			new UnregisteredMetricsGroup(),
			VoidBlobWriter.getInstance(),
			AkkaUtils.getDefaultTimeout(),
			TEST_LOGGER);

		fail("Building the ExecutionGraph with a parallelism higher than the max parallelism should fail.");
	} catch (JobException e) {
		// expected, ignore
	}
}
 

private ExecutionGraph createExecutionGraph(Configuration configuration) throws Exception {
	final ScheduledExecutorService executor = TestingUtils.defaultExecutor();

	final JobID jobId = new JobID();
	final JobGraph jobGraph = new JobGraph(jobId, "test");
	jobGraph.setSnapshotSettings(
		new JobCheckpointingSettings(
			Collections.<JobVertexID>emptyList(),
			Collections.<JobVertexID>emptyList(),
			Collections.<JobVertexID>emptyList(),
			new CheckpointCoordinatorConfiguration(
				100,
				10 * 60 * 1000,
				0,
				1,
				CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
				false,
				false,
				0),
			null));

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		configuration,
		executor,
		executor,
		new ProgrammedSlotProvider(1),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		blobWriter,
		timeout,
		LoggerFactory.getLogger(getClass()),
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);
}
 

/**
 * Tests that when a task fail, and restart strategy doesn't support restarting, the job will go to failed
 * @throws Exception
 */
@Test
public void testNoManualRestart() throws Exception {
	NoRestartStrategy restartStrategy = new NoRestartStrategy();
	ExecutionGraph eg = createSingleRegionExecutionGraph(restartStrategy);

	ExecutionVertex ev = eg.getAllExecutionVertices().iterator().next();

	ev.fail(new Exception("Test Exception"));

	for (ExecutionVertex evs : eg.getAllExecutionVertices()) {
		evs.getCurrentExecutionAttempt().completeCancelling();
	}
	assertEquals(JobStatus.FAILED, eg.getState());
}
 

/**
 * Tests that when a task fail, and restart strategy doesn't support restarting, the job will go to failed.
 * @throws Exception if fail to create the single region execution graph.
 */
@Test
public void testNoManualRestart() throws Exception {
	NoRestartStrategy restartStrategy = new NoRestartStrategy();
	ExecutionGraph eg = createSingleRegionExecutionGraph(restartStrategy);

	ExecutionVertex ev = eg.getAllExecutionVertices().iterator().next();

	ev.fail(new Exception("Test Exception"));

	for (ExecutionVertex evs : eg.getAllExecutionVertices()) {
		evs.getCurrentExecutionAttempt().completeCancelling();
	}
	assertEquals(JobStatus.FAILED, eg.getState());
}
 

private ExecutionGraph getDummyExecutionGraph() throws Exception {
	return new ExecutionGraph(
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		new JobID(),
		"Test Job Sample Name",
		new Configuration(),
		new SerializedValue<>(new ExecutionConfig()),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
		new TestingSlotProvider(ignored -> new CompletableFuture<>()));
}
 

/**
 * Verifies that building an {@link ExecutionGraph} from a {@link JobGraph} with
 * parallelism higher than the maximum parallelism fails.
 */
@Test
public void testExecutionGraphConstructionFailsRescaleDopExceedMaxParallelism() throws Exception {

	final Configuration config = new Configuration();

	final int initialParallelism = 1;
	final int maxParallelism = 10;
	final JobVertex[] jobVertices = createVerticesForSimpleBipartiteJobGraph(initialParallelism,  maxParallelism);
	final JobGraph jobGraph = new JobGraph(jobVertices);

	for (JobVertex jv : jobVertices) {
		jv.setParallelism(maxParallelism + 1);
	}

	try {
		// this should fail since we set the parallelism to maxParallelism + 1
		ExecutionGraphBuilder.buildGraph(
			null,
			jobGraph,
			config,
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			new TestingSlotProvider(ignore -> new CompletableFuture<>()),
			Thread.currentThread().getContextClassLoader(),
			new StandaloneCheckpointRecoveryFactory(),
			AkkaUtils.getDefaultTimeout(),
			new NoRestartStrategy(),
			new UnregisteredMetricsGroup(),
			VoidBlobWriter.getInstance(),
			AkkaUtils.getDefaultTimeout(),
			TEST_LOGGER,
			NettyShuffleMaster.INSTANCE,
			NoOpPartitionTracker.INSTANCE);

		fail("Building the ExecutionGraph with a parallelism higher than the max parallelism should fail.");
	} catch (JobException e) {
		// expected, ignore
	}
}
 

/**
 * Tests that the task restore state is nulled after the {@link Execution} has been
 * deployed. See FLINK-9693.
 */
@Test
public void testTaskRestoreStateIsNulledAfterDeployment() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();
	final JobVertexID jobVertexId = jobVertex.getID();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();
	final ProgrammedSlotProvider slotProvider = createProgrammedSlotProvider(
		1,
		Collections.singleton(jobVertexId),
		slotOwner);

	ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		new JobID(),
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertexId);

	ExecutionVertex executionVertex = executionJobVertex.getTaskVertices()[0];

	final Execution execution = executionVertex.getCurrentExecutionAttempt();

	final JobManagerTaskRestore taskRestoreState = new JobManagerTaskRestore(1L, new TaskStateSnapshot());
	execution.setInitialState(taskRestoreState);

	assertThat(execution.getTaskRestore(), is(notNullValue()));

	// schedule the execution vertex and wait for its deployment
	executionVertex.scheduleForExecution(
		executionGraph.getSlotProviderStrategy(),
		LocationPreferenceConstraint.ANY,
		Collections.emptySet())
		.get();

	assertThat(execution.getTaskRestore(), is(nullValue()));
}
 

/**
 * Tests that a slot release will atomically release the assigned {@link Execution}.
 */
@Test
public void testSlotReleaseAtomicallyReleasesExecution() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();
	final SingleLogicalSlot slot = ExecutionGraphSchedulingTest.createSingleLogicalSlot(
		slotOwner,
		new SimpleAckingTaskManagerGateway(),
		new SlotRequestId());
	final CompletableFuture<LogicalSlot> slotFuture = CompletableFuture.completedFuture(slot);

	final CountDownLatch slotRequestLatch = new CountDownLatch(1);
	final TestingSlotProvider slotProvider = new TestingSlotProvider(slotRequestId -> {
		slotRequestLatch.countDown();
		return slotFuture;
	});
	final ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		new JobID(),
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	final Execution execution = executionGraph.getJobVertex(jobVertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt();

	executionGraph.start(testMainThreadUtil.getMainThreadExecutor());
	testMainThreadUtil.execute(executionGraph::scheduleForExecution);

	// wait until the slot has been requested
	slotRequestLatch.await();

	testMainThreadUtil.execute(() -> {
		assertThat(execution.getAssignedResource(), is(sameInstance(slot)));

		slot.release(new FlinkException("Test exception"));

		assertThat(execution.getReleaseFuture().isDone(), is(true));
	});
}
 

/**
 * Verifies that building an {@link ExecutionGraph} from a {@link JobGraph} with
 * parallelism higher than the maximum parallelism fails.
 */
@Test
public void testExecutionGraphConstructionFailsRescaleDopExceedMaxParallelism() throws Exception {

	final Configuration config = new Configuration();

	final int initialParallelism = 1;
	final int maxParallelism = 10;
	final JobVertex[] jobVertices = createVerticesForSimpleBipartiteJobGraph(initialParallelism,  maxParallelism);
	final JobGraph jobGraph = new JobGraph(jobVertices);

	for (JobVertex jv : jobVertices) {
		jv.setParallelism(maxParallelism + 1);
	}

	try {
		// this should fail since we set the parallelism to maxParallelism + 1
		ExecutionGraphBuilder.buildGraph(
			null,
			jobGraph,
			config,
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			new TestingSlotProvider(ignore -> new CompletableFuture<>()),
			Thread.currentThread().getContextClassLoader(),
			new StandaloneCheckpointRecoveryFactory(),
			AkkaUtils.getDefaultTimeout(),
			new NoRestartStrategy(),
			new UnregisteredMetricsGroup(),
			VoidBlobWriter.getInstance(),
			AkkaUtils.getDefaultTimeout(),
			TEST_LOGGER,
			NettyShuffleMaster.INSTANCE,
			NoOpJobMasterPartitionTracker.INSTANCE);

		fail("Building the ExecutionGraph with a parallelism higher than the max parallelism should fail.");
	} catch (JobException e) {
		// expected, ignore
	}
}
 

/**
 * Tests that the task restore state is nulled after the {@link Execution} has been
 * deployed. See FLINK-9693.
 */
@Test
public void testTaskRestoreStateIsNulledAfterDeployment() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();
	final JobVertexID jobVertexId = jobVertex.getID();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();
	final ProgrammedSlotProvider slotProvider = createProgrammedSlotProvider(
		1,
		Collections.singleton(jobVertexId),
		slotOwner);

	ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertexId);

	ExecutionVertex executionVertex = executionJobVertex.getTaskVertices()[0];

	final Execution execution = executionVertex.getCurrentExecutionAttempt();

	final JobManagerTaskRestore taskRestoreState = new JobManagerTaskRestore(1L, new TaskStateSnapshot());
	execution.setInitialState(taskRestoreState);

	assertThat(execution.getTaskRestore(), is(notNullValue()));

	// schedule the execution vertex and wait for its deployment
	executionVertex.scheduleForExecution(
		executionGraph.getSlotProviderStrategy(),
		LocationPreferenceConstraint.ANY,
		Collections.emptySet())
		.get();

	assertThat(execution.getTaskRestore(), is(nullValue()));
}
 

private ExecutionGraph createExecutionGraphAndEnableCheckpointing(
		CheckpointIDCounter counter,
		CompletedCheckpointStore store) throws Exception {
	final Time timeout = Time.days(1L);
	ExecutionGraph executionGraph = new ExecutionGraph(
		new DummyJobInformation(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		timeout,
		new NoRestartStrategy(),
		new RestartAllStrategy.Factory(),
		new TestingSlotProvider(slotRequestId -> CompletableFuture.completedFuture(new TestingLogicalSlot())),
		ClassLoader.getSystemClassLoader(),
		VoidBlobWriter.getInstance(),
		timeout);

	executionGraph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());

	executionGraph.enableCheckpointing(
			100,
			100,
			100,
			1,
			CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
			Collections.emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			counter,
			store,
			new MemoryStateBackend(),
			CheckpointStatsTrackerTest.createTestTracker());

	JobVertex jobVertex = new JobVertex("MockVertex");
	jobVertex.setInvokableClass(AbstractInvokable.class);
	executionGraph.attachJobGraph(Collections.singletonList(jobVertex));
	executionGraph.setQueuedSchedulingAllowed(true);

	return executionGraph;
}
 

@BeforeClass
public static void setupExecutionGraph() throws Exception {
	// -------------------------------------------------------------------------------------------------------------
	// Setup
	// -------------------------------------------------------------------------------------------------------------

	JobVertexID v1ID = new JobVertexID();
	JobVertexID v2ID = new JobVertexID();

	JobVertex v1 = new JobVertex("v1", v1ID);
	JobVertex v2 = new JobVertex("v2", v2ID);

	v1.setParallelism(1);
	v2.setParallelism(2);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	List<JobVertex> vertices = new ArrayList<>(Arrays.asList(v1, v2));

	ExecutionConfig config = new ExecutionConfig();

	config.setExecutionMode(ExecutionMode.BATCH_FORCED);
	config.setRestartStrategy(new RestartStrategies.NoRestartStrategyConfiguration());
	config.setParallelism(4);
	config.enableObjectReuse();
	config.setGlobalJobParameters(new TestJobParameters());

	runtimeGraph = new ExecutionGraph(
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		new JobID(),
		"test job",
		new Configuration(),
		new SerializedValue<>(config),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
		mock(SlotProvider.class));

	runtimeGraph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());

	runtimeGraph.attachJobGraph(vertices);

	List<ExecutionJobVertex> jobVertices = new ArrayList<>();
	jobVertices.add(runtimeGraph.getJobVertex(v1ID));
	jobVertices.add(runtimeGraph.getJobVertex(v2ID));

	CheckpointStatsTracker statsTracker = new CheckpointStatsTracker(
			0,
			jobVertices,
			mock(CheckpointCoordinatorConfiguration.class),
			new UnregisteredMetricsGroup());

	runtimeGraph.enableCheckpointing(
		100,
		100,
		100,
		1,
		CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<MasterTriggerRestoreHook<?>>emptyList(),
		new StandaloneCheckpointIDCounter(),
		new StandaloneCompletedCheckpointStore(1),
		new MemoryStateBackend(),
		statsTracker);

	runtimeGraph.setJsonPlan("{}");

	runtimeGraph.getJobVertex(v2ID).getTaskVertices()[0].getCurrentExecutionAttempt().fail(new RuntimeException("This exception was thrown on purpose."));
}
 

/**
 * Tests that the slot is released in case of a execution cancellation when having
 * a slot assigned and being in state SCHEDULED.
 */
@Test
public void testSlotReleaseOnExecutionCancellationInScheduled() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();
	final JobVertexID jobVertexId = jobVertex.getID();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();

	final LogicalSlot slot = createTestingLogicalSlot(slotOwner);

	final ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(1);
	slotProvider.addSlot(jobVertexId, 0, CompletableFuture.completedFuture(slot));

	ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		new JobID(),
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	executionGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertexId);

	final Execution execution = executionJobVertex.getTaskVertices()[0].getCurrentExecutionAttempt();

	CompletableFuture<Execution> allocationFuture = execution.allocateResourcesForExecution(
		executionGraph.getSlotProviderStrategy(),
		LocationPreferenceConstraint.ALL,
		Collections.emptySet());

	assertTrue(allocationFuture.isDone());

	assertEquals(ExecutionState.SCHEDULED, execution.getState());

	assertEquals(slot, execution.getAssignedResource());

	// cancelling the execution should move it into state CANCELED
	execution.cancel();
	assertEquals(ExecutionState.CANCELED, execution.getState());

	assertEquals(slot, slotOwner.getReturnedSlotFuture().get());
}