org.apache.flink.runtime.jobmaster.slotpool.SlotPool Java Examples

@Before
public void setup() throws Exception {
	final JobID jobId = new JobID();
	final SlotPool slotPool = new SlotPoolImpl(jobId);
	final TestingScheduler testingScheduler = new TestingScheduler(
		new HashMap<>(16),
		LocationPreferenceSlotSelectionStrategy.INSTANCE,
		slotPool);

	testingSlotProvider = new TestingSlotPoolSlotProvider(slotPool, testingScheduler);

	final JobMasterId jobMasterId = JobMasterId.generate();
	final String jobManagerAddress = "localhost";
	ComponentMainThreadExecutor executor = TestingComponentMainThreadExecutorServiceAdapter.forMainThread();
	slotPool.start(jobMasterId, jobManagerAddress, executor);
	testingScheduler.start(executor);
}
 

private static Scheduler createSchedulerWithSlots(SlotPool slotPool, TaskManagerLocation taskManagerLocation, int numSlots) throws Exception {
	final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	setupSlotPool(slotPool);
	Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
	scheduler.start(mainThreadExecutor);
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS);
	for (int i = 0; i < numSlots; i++) {
		final AllocationID allocationId = new AllocationID();
		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY);
		slotOffers.add(slotOffer);
	}

	slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

	return scheduler;
}
 

/**
 * Tests that it is possible to fail a graph via a call to
 * {@link ExecutionGraph#failGlobal(Throwable)} after cancellation.
 */
@Test
public void testFailExecutionGraphAfterCancel() throws Exception {
	try (SlotPool slotPool = createSlotPoolImpl()) {
		ExecutionGraph eg = TestingExecutionGraphBuilder
			.newBuilder()
			.setRestartStrategy(new InfiniteDelayRestartStrategy())
			.setJobGraph(createJobGraphToCancel())
			.setSlotProvider(createSchedulerWithSlots(slotPool, new LocalTaskManagerLocation(), 2))
			.build();

		startAndScheduleExecutionGraph(eg);

		// Fail right after cancel (for example with concurrent slot release)
		eg.cancel();
		assertEquals(JobStatus.CANCELLING, eg.getState());

		eg.failGlobal(new Exception("Test Exception"));
		assertEquals(JobStatus.FAILING, eg.getState());

		Execution execution = eg.getAllExecutionVertices().iterator().next().getCurrentExecutionAttempt();

		execution.completeCancelling();
		assertEquals(JobStatus.RESTARTING, eg.getState());
	}
}
 

@Test
public void testRestartAutomatically() throws Exception {
	try (SlotPool slotPool = createSlotPoolImpl()) {
		ExecutionGraph executionGraph = TestingExecutionGraphBuilder
			.newBuilder()
			.setJobGraph(createJobGraph())
			.setRestartStrategy(TestRestartStrategy.directExecuting())
			.setSlotProvider(createSchedulerWithSlots(slotPool))
			.build();

		startAndScheduleExecutionGraph(executionGraph);

		executionGraph.getAllExecutionVertices().iterator().next().fail(new Exception("Test Exception"));

		assertEquals(JobStatus.FAILING, executionGraph.getState());

		for (ExecutionVertex vertex : executionGraph.getAllExecutionVertices()) {
			vertex.getCurrentExecutionAttempt().completeCancelling();
		}

		assertEquals(JobStatus.RUNNING, executionGraph.getState());
		finishAllVertices(executionGraph);
		assertEquals(JobStatus.FINISHED, executionGraph.getState());
	}

}
 

private static Scheduler createSchedulerWithSlots(
		int numSlots,
		SlotPool slotPool,
		TaskManagerLocation taskManagerLocation) throws Exception {
	final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	final String jobManagerAddress = "foobar";
	final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	slotPool.start(JobMasterId.generate(), jobManagerAddress, mainThreadExecutor);
	slotPool.connectToResourceManager(resourceManagerGateway);
	Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
	scheduler.start(mainThreadExecutor);

	CompletableFuture.runAsync(() -> slotPool.registerTaskManager(taskManagerLocation.getResourceID()), mainThreadExecutor).join();

	final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS);
	for (int i = 0; i < numSlots; i++) {
		final AllocationID allocationId = new AllocationID();
		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY);
		slotOffers.add(slotOffer);
	}

	CompletableFuture.runAsync(() -> slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers), mainThreadExecutor).join();

	return scheduler;
}
 

@Before
public void setup() throws Exception {
	final JobID jobId = new JobID();
	final SlotPool slotPool = new TestingSlotPoolImpl(jobId);
	final TestingScheduler testingScheduler = new TestingScheduler(
		new HashMap<>(16),
		LocationPreferenceSlotSelectionStrategy.INSTANCE,
		slotPool);

	testingSlotProvider = new TestingSlotPoolSlotProvider(slotPool, testingScheduler);

	final JobMasterId jobMasterId = JobMasterId.generate();
	final String jobManagerAddress = "localhost";
	ComponentMainThreadExecutor executor = ComponentMainThreadExecutorServiceAdapter.forMainThread();
	slotPool.start(jobMasterId, jobManagerAddress, executor);
	testingScheduler.start(executor);
}
 

private static Scheduler createSchedulerWithSlots(
	int numSlots, SlotPool slotPool, TaskManagerLocation taskManagerLocation) throws Exception {

	final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	setupSlotPool(slotPool);
	Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.INSTANCE, slotPool);
	scheduler.start(mainThreadExecutor);
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS);
	for (int i = 0; i < numSlots; i++) {
		final AllocationID allocationId = new AllocationID();
		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY);
		slotOffers.add(slotOffer);
	}

	slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

	return scheduler;
}
 

/**
 * Tests that it is possible to fail a graph via a call to
 * {@link ExecutionGraph#failGlobal(Throwable)} after cancellation.
 */
@Test
public void testFailExecutionGraphAfterCancel() throws Exception {
	try (SlotPool slotPool = createSlotPoolImpl()) {
		ExecutionGraph eg = TestingExecutionGraphBuilder.newBuilder()
			.setRestartStrategy(new InfiniteDelayRestartStrategy())
			.setJobGraph(createJobGraphToCancel())
			.setNumberOfTasks(2)
			.buildAndScheduleForExecution(slotPool);

		// Fail right after cancel (for example with concurrent slot release)
		eg.cancel();
		assertEquals(JobStatus.CANCELLING, eg.getState());

		eg.failGlobal(new Exception("Test Exception"));
		assertEquals(JobStatus.FAILING, eg.getState());

		Execution execution = eg.getAllExecutionVertices().iterator().next().getCurrentExecutionAttempt();

		execution.completeCancelling();
		assertEquals(JobStatus.RESTARTING, eg.getState());
	}
}
 

/**
 * Tests that a graph is not restarted after cancellation via a call to
 * {@link ExecutionGraph#failGlobal(Throwable)}. This can happen when a slot is
 * released concurrently with cancellation.
 */
@Test
public void testFailExecutionAfterCancel() throws Exception {
	try (SlotPool slotPool = createSlotPoolImpl()) {
		ExecutionGraph eg = TestingExecutionGraphBuilder.newBuilder()
			.setRestartStrategy(new InfiniteDelayRestartStrategy())
			.setJobGraph(createJobGraphToCancel())
			.setNumberOfTasks(2)
			.buildAndScheduleForExecution(slotPool);

		// Fail right after cancel (for example with concurrent slot release)
		eg.cancel();

		for (ExecutionVertex v : eg.getAllExecutionVertices()) {
			v.getCurrentExecutionAttempt().fail(new Exception("Test Exception"));
		}

		assertEquals(JobStatus.CANCELED, eg.getTerminationFuture().get());

		Execution execution = eg.getAllExecutionVertices().iterator().next().getCurrentExecutionAttempt();

		execution.completeCancelling();
		assertEquals(JobStatus.CANCELED, eg.getState());
	}
}
 

@Test
public void testNoRestartOnSuppressException() throws Exception {
	try (SlotPool slotPool = createSlotPoolImpl()) {
		ExecutionGraph eg = TestingExecutionGraphBuilder.newBuilder()
			.setRestartStrategy(new FixedDelayRestartStrategy(Integer.MAX_VALUE, 0))
			.buildAndScheduleForExecution(slotPool);

		// Fail with unrecoverable Exception
		eg.getAllExecutionVertices().iterator().next().fail(
			new SuppressRestartsException(new Exception("Test Exception")));

		assertEquals(JobStatus.FAILING, eg.getState());

		completeCanceling(eg);

		eg.waitUntilTerminal();
		assertEquals(JobStatus.FAILED, eg.getState());

		RestartStrategy restartStrategy = eg.getRestartStrategy();
		assertTrue(restartStrategy instanceof FixedDelayRestartStrategy);

		assertEquals(0, ((FixedDelayRestartStrategy) restartStrategy).getCurrentRestartAttempt());
	}

}
 

@Test
public void testTaskFailingWhileGlobalFailing() throws Exception {
	try (SlotPool slotPool = createSlotPoolImpl()) {
		final ExecutionGraph graph = TestingExecutionGraphBuilder.newBuilder()
			.setRestartStrategy(new InfiniteDelayRestartStrategy())
			.setFailoverStrategyFactory(new TestFailoverStrategy.Factory())
			.buildAndScheduleForExecution(slotPool);
		final TestFailoverStrategy failoverStrategy = (TestFailoverStrategy) graph.getFailoverStrategy();

		// switch all tasks to running
		for (ExecutionVertex vertex : graph.getVerticesTopologically().iterator().next().getTaskVertices()) {
			vertex.getCurrentExecutionAttempt().switchToRunning();
		}

		graph.failGlobal(new Exception("test"));

		graph.getAllExecutionVertices().iterator().next().fail(new Exception("Test task failure"));

		// no local failover should happen when in global failover cancelling
		assertEquals(0, failoverStrategy.getLocalFailoverCount());
	}

}
 

@Test
public void testCancelWhileRestarting() throws Exception {
	// We want to manually control the restart and delay
	try (SlotPool slotPool = createSlotPoolImpl()) {
		TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
		final ExecutionGraph executionGraph = TestingExecutionGraphBuilder.newBuilder()
			.setRestartStrategy(new InfiniteDelayRestartStrategy())
			.setTaskManagerLocation(taskManagerLocation)
			.buildAndScheduleForExecution(slotPool);

		// Release the TaskManager and wait for the job to restart
		slotPool.releaseTaskManager(taskManagerLocation.getResourceID(), new Exception("Test Exception"));
		assertEquals(JobStatus.RESTARTING, executionGraph.getState());

		// Canceling needs to abort the restart
		executionGraph.cancel();

		assertEquals(JobStatus.CANCELED, executionGraph.getState());

		// The restart has been aborted
		executionGraph.restart(executionGraph.getGlobalModVersion());

		assertEquals(JobStatus.CANCELED, executionGraph.getState());
	}

}
 

private LegacyScheduler createLegacyScheduler(JobGraph jobGraph, SlotPool slotPool, ComponentMainThreadExecutor mainThreadExecutor, Time slotRequestTimeout) throws Exception {
	final Scheduler scheduler = createScheduler(slotPool, mainThreadExecutor);
	final LegacyScheduler legacyScheduler = new LegacyScheduler(
		LOG,
		jobGraph,
		VoidBackPressureStatsTracker.INSTANCE,
		TestingUtils.defaultExecutor(),
		new Configuration(),
		scheduler,
		TestingUtils.defaultExecutor(),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		TestingUtils.TIMEOUT(),
		new NoRestartStrategy.NoRestartStrategyFactory(),
		VoidBlobWriter.getInstance(),
		UnregisteredMetricGroups.createUnregisteredJobManagerJobMetricGroup(),
		slotRequestTimeout,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);

	legacyScheduler.setMainThreadExecutor(mainThreadExecutor);

	return legacyScheduler;
}
 

public TestingScheduler(
	@Nonnull Map<SlotSharingGroupId, SlotSharingManager> slotSharingManagersMap,
	@Nonnull SlotSelectionStrategy slotSelectionStrategy,
	@Nonnull SlotPool slotPoolGateway) {

	super(slotSelectionStrategy, slotPoolGateway, slotSharingManagersMap);
	this.slotSharingManagersMap = slotSharingManagersMap;
}
 

/**
 * SlotPool#failAllocation should not fail with a {@link java.util.ConcurrentModificationException}
 * if there is a concurrent scheduling operation. See FLINK-13421.
 */
@Test
public void slotPoolExecutionGraph_ConcurrentSchedulingAndAllocationFailure_ShouldNotFailWithConcurrentModificationException() throws Exception {
	final SlotSharingGroup group = new SlotSharingGroup();
	final JobVertex vertex1 = createNoOpVertex("vertex1", 1);
	vertex1.setSlotSharingGroup(group);
	final JobVertex vertex2 = createNoOpVertex("vertex2", 3);
	vertex2.setSlotSharingGroup(group);
	final JobVertex vertex3 = createNoOpVertex("vertex3", 1);
	vertex3.setSlotSharingGroup(group);
	vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

	try (SlotPool slotPool = createSlotPoolImpl()) {
		final SlotProvider slots = createSchedulerWithSlots(slotPool, new LocalTaskManagerLocation(), 2);

		final AllocationID allocationId = slotPool.getAvailableSlotsInformation().iterator().next().getAllocationId();

		final JobGraph jobGraph = new JobGraph(TEST_JOB_ID, "Test Job", vertex1, vertex2, vertex3);
		jobGraph.setScheduleMode(ScheduleMode.EAGER);
		final ExecutionGraph eg = TestingExecutionGraphBuilder
			.newBuilder()
			.setJobGraph(jobGraph)
			.setSlotProvider(slots)
			.setAllocationTimeout(Time.minutes(60))
			.build();

		startAndScheduleExecutionGraph(eg);

		slotPool.failAllocation(
			allocationId,
			new Exception("test exception"));

		eg.waitUntilTerminal();
	}
}
 

/**
 * Tests that a suspend call while restarting a job, will abort the restarting.
 */
@Test
public void testSuspendWhileRestarting() throws Exception {
	TestRestartStrategy controllableRestartStrategy = TestRestartStrategy.manuallyTriggered();
	try (SlotPool slotPool = createSlotPoolImpl()) {
		TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
		ExecutionGraph eg = TestingExecutionGraphBuilder
			.newBuilder()
			.setJobGraph(createJobGraph())
			.setRestartStrategy(controllableRestartStrategy)
			.setSlotProvider(createSchedulerWithSlots(slotPool, taskManagerLocation))
			.build();

		startAndScheduleExecutionGraph(eg);

		// Release the TaskManager and wait for the job to restart
		slotPool.releaseTaskManager(taskManagerLocation.getResourceID(), new Exception("Test Exception"));

		assertEquals(1, controllableRestartStrategy.getNumberOfQueuedActions());

		assertEquals(JobStatus.RESTARTING, eg.getState());

		eg.suspend(new Exception("Test exception"));

		assertEquals(JobStatus.SUSPENDED, eg.getState());

		controllableRestartStrategy.triggerAll().join();

		assertEquals(JobStatus.SUSPENDED, eg.getState());
	}
}
 

private TestingSlotPoolSlotProvider(SlotPool slotPool, TestingScheduler testingScheduler) {
	this.slotPool = Preconditions.checkNotNull(slotPool);

	this.scheduler = testingScheduler;

	this.numberOfLocalizedAssignments = new AtomicInteger();
	this.numberOfNonLocalizedAssignments = new AtomicInteger();
	this.numberOfUnconstrainedAssignments = new AtomicInteger();
	this.numberOfHostLocalizedAssignments = new AtomicInteger();
}
 

/**
 * Tests that a graph is not restarted after cancellation via a call to
 * {@link ExecutionGraph#failGlobal(Throwable)}. This can happen when a slot is
 * released concurrently with cancellation.
 */
@Test
public void testFailExecutionAfterCancel() throws Exception {
	try (SlotPool slotPool = createSlotPoolImpl()) {
		ExecutionGraph eg = TestingExecutionGraphBuilder.newBuilder()
			.setRestartStrategy(new InfiniteDelayRestartStrategy())
			.setJobGraph(createJobGraphToCancel())
			.setSlotProvider(createSchedulerWithSlots(slotPool, new LocalTaskManagerLocation(), 2))
			.build();

		startAndScheduleExecutionGraph(eg);

		// Fail right after cancel (for example with concurrent slot release)
		eg.cancel();

		for (ExecutionVertex v : eg.getAllExecutionVertices()) {
			v.getCurrentExecutionAttempt().fail(new Exception("Test Exception"));
		}

		assertEquals(JobStatus.CANCELED, eg.getTerminationFuture().get());

		Execution execution = eg.getAllExecutionVertices().iterator().next().getCurrentExecutionAttempt();

		execution.completeCancelling();
		assertEquals(JobStatus.CANCELED, eg.getState());
	}
}
 

@Test
public void testNoRestartOnSuppressException() throws Exception {
	try (SlotPool slotPool = createSlotPoolImpl()) {
		ExecutionGraph eg = TestingExecutionGraphBuilder
			.newBuilder()
			.setJobGraph(createJobGraph())
			.setRestartStrategy(new FixedDelayRestartStrategy(Integer.MAX_VALUE, 0))
			.setSlotProvider(createSchedulerWithSlots(slotPool))
			.build();

		startAndScheduleExecutionGraph(eg);

		// Fail with unrecoverable Exception
		eg.getAllExecutionVertices().iterator().next().fail(
			new SuppressRestartsException(new Exception("Test Exception")));

		assertEquals(JobStatus.FAILING, eg.getState());

		completeCanceling(eg);

		eg.waitUntilTerminal();
		assertEquals(JobStatus.FAILED, eg.getState());

		RestartStrategy restartStrategy = eg.getRestartStrategy();
		assertTrue(restartStrategy instanceof FixedDelayRestartStrategy);

		assertEquals(0, ((FixedDelayRestartStrategy) restartStrategy).getCurrentRestartAttempt());
	}

}
 

@Test
public void testTaskFailingWhileGlobalFailing() throws Exception {
	try (SlotPool slotPool = createSlotPoolImpl()) {
		final ExecutionGraph graph = TestingExecutionGraphBuilder
			.newBuilder()
			.setRestartStrategy(new InfiniteDelayRestartStrategy())
			.setFailoverStrategyFactory(new TestFailoverStrategy.Factory())
			.setJobGraph(createJobGraph())
			.setSlotProvider(createSchedulerWithSlots(slotPool))
			.build();

		startAndScheduleExecutionGraph(graph);

		final TestFailoverStrategy failoverStrategy = (TestFailoverStrategy) graph.getFailoverStrategy();

		// switch all tasks to running
		for (ExecutionVertex vertex : graph.getVerticesTopologically().iterator().next().getTaskVertices()) {
			vertex.getCurrentExecutionAttempt().switchToRunning();
		}

		graph.failGlobal(new Exception("test"));

		graph.getAllExecutionVertices().iterator().next().fail(new Exception("Test task failure"));

		// no local failover should happen when in global failover cancelling
		assertEquals(0, failoverStrategy.getLocalFailoverCount());
	}

}
 

@Test
public void testFailWhileCanceling() throws Exception {
	try (SlotPool slotPool = createSlotPoolImpl()) {
		final ExecutionGraph graph = TestingExecutionGraphBuilder
			.newBuilder()
			.setJobGraph(createJobGraph())
			.setSlotProvider(createSchedulerWithSlots(slotPool))
			.build();

		startAndScheduleExecutionGraph(graph);

		assertEquals(JobStatus.RUNNING, graph.getState());
		switchAllTasksToRunning(graph);

		graph.cancel();

		assertEquals(JobStatus.CANCELLING, graph.getState());

		graph.failGlobal(new Exception("test"));

		assertEquals(JobStatus.FAILING, graph.getState());

		// let all tasks finish cancelling
		completeCanceling(graph);

		assertEquals(JobStatus.FAILED, graph.getState());
	}

}
 

@Test
public void testCancelWhileFailing() throws Exception {
	try (SlotPool slotPool = createSlotPoolImpl()) {
		final ExecutionGraph graph = TestingExecutionGraphBuilder
			.newBuilder()
			.setJobGraph(createJobGraph())
			.setRestartStrategy(new InfiniteDelayRestartStrategy())
			.setSlotProvider(createSchedulerWithSlots(slotPool))
			.build();

		startAndScheduleExecutionGraph(graph);

		assertEquals(JobStatus.RUNNING, graph.getState());

		// switch all tasks to running
		for (ExecutionVertex vertex : graph.getVerticesTopologically().iterator().next().getTaskVertices()) {
			vertex.getCurrentExecutionAttempt().switchToRunning();
		}

		graph.failGlobal(new Exception("test"));

		assertEquals(JobStatus.FAILING, graph.getState());

		graph.cancel();

		assertEquals(JobStatus.CANCELLING, graph.getState());

		// let all tasks finish cancelling
		completeCanceling(graph);

		assertEquals(JobStatus.CANCELED, graph.getState());
	}

}
 

@Test
public void testFailWhileRestarting() throws Exception {
	try (SlotPool slotPool = createSlotPoolImpl()) {
		TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
		final ExecutionGraph executionGraph = TestingExecutionGraphBuilder
			.newBuilder()
			.setJobGraph(createJobGraph())
			.setRestartStrategy(new InfiniteDelayRestartStrategy())
			.setSlotProvider(createSchedulerWithSlots(slotPool, taskManagerLocation))
			.build();

		startAndScheduleExecutionGraph(executionGraph);

		// Release the TaskManager and wait for the job to restart
		slotPool.releaseTaskManager(taskManagerLocation.getResourceID(), new Exception("Test Exception"));

		assertEquals(JobStatus.RESTARTING, executionGraph.getState());

		// If we fail when being in RESTARTING, then we should try to restart again
		final long globalModVersion = executionGraph.getGlobalModVersion();
		final Exception testException = new Exception("Test exception");
		executionGraph.failGlobal(testException);

		assertNotEquals(globalModVersion, executionGraph.getGlobalModVersion());
		assertEquals(JobStatus.RESTARTING, executionGraph.getState());
		assertEquals(testException, executionGraph.getFailureCause()); // we should have updated the failure cause

		// but it should fail when sending a SuppressRestartsException
		executionGraph.failGlobal(new SuppressRestartsException(new Exception("Suppress restart exception")));

		assertEquals(JobStatus.FAILED, executionGraph.getState());

		// The restart has been aborted
		executionGraph.restart(executionGraph.getGlobalModVersion());

		assertEquals(JobStatus.FAILED, executionGraph.getState());
	}
}
 

@Test
public void testCancelWhileRestarting() throws Exception {
	// We want to manually control the restart and delay
	try (SlotPool slotPool = createSlotPoolImpl()) {
		TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
		final ExecutionGraph executionGraph = TestingExecutionGraphBuilder
			.newBuilder()
			.setJobGraph(createJobGraph())
			.setRestartStrategy(new InfiniteDelayRestartStrategy())
			.setSlotProvider(createSchedulerWithSlots(slotPool, taskManagerLocation))
			.build();

		startAndScheduleExecutionGraph(executionGraph);

		// Release the TaskManager and wait for the job to restart
		slotPool.releaseTaskManager(taskManagerLocation.getResourceID(), new Exception("Test Exception"));
		assertEquals(JobStatus.RESTARTING, executionGraph.getState());

		// Canceling needs to abort the restart
		executionGraph.cancel();

		assertEquals(JobStatus.CANCELED, executionGraph.getState());

		// The restart has been aborted
		executionGraph.restart(executionGraph.getGlobalModVersion());

		assertEquals(JobStatus.CANCELED, executionGraph.getState());
	}

}
 

public TestingScheduler(
	@Nonnull Map<SlotSharingGroupId, SlotSharingManager> slotSharingManagersMap,
	@Nonnull SlotSelectionStrategy slotSelectionStrategy,
	@Nonnull SlotPool slotPoolGateway) {

	super(slotSelectionStrategy, slotPoolGateway, slotSharingManagersMap);
	this.slotSharingManagersMap = slotSharingManagersMap;
}
 

@Nonnull
private SlotProvider createSlotProvider(SlotPool slotPool, ComponentMainThreadExecutor mainThreadExecutor) {
	final SchedulerImpl scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
	scheduler.start(mainThreadExecutor);

	return scheduler;
}
 

private TestingSlotPoolSlotProvider(SlotPool slotPool, TestingScheduler testingScheduler) {
	this.slotPool = Preconditions.checkNotNull(slotPool);

	this.scheduler = testingScheduler;

	this.numberOfLocalizedAssignments = new AtomicInteger();
	this.numberOfNonLocalizedAssignments = new AtomicInteger();
	this.numberOfUnconstrainedAssignments = new AtomicInteger();
	this.numberOfHostLocalizedAssignments = new AtomicInteger();
}
 

@Nonnull
private SchedulerImpl createScheduler(SlotPool slotPool, ComponentMainThreadExecutor mainThreadExecutor) {
	final SchedulerImpl scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.INSTANCE, slotPool);
	scheduler.start(mainThreadExecutor);

	return scheduler;
}
 

private ExecutionGraph buildAndScheduleForExecution(SlotPool slotPool) throws Exception {
	final Scheduler scheduler = createSchedulerWithSlots(tasksNum, slotPool, taskManagerLocation);
	final ExecutionGraph eg = createSimpleExecutionGraph(
		restartStrategy,
		failoverStrategyFactory,
		scheduler,
		jobGraph);

	assertEquals(JobStatus.CREATED, eg.getState());

	eg.scheduleForExecution();
	assertEquals(JobStatus.RUNNING, eg.getState());

	return eg;
}
 

@Test
public void failGlobalIfExecutionIsStillRunning_failingAnExecutionTwice_ShouldTriggerOnlyOneFailover() throws Exception {
	JobVertex sender = ExecutionGraphTestUtils.createJobVertex("Task1", 1, NoOpInvokable.class);
	JobVertex receiver = ExecutionGraphTestUtils.createJobVertex("Task2", 1, NoOpInvokable.class);
	JobGraph jobGraph = new JobGraph("Pointwise job", sender, receiver);

	try (SlotPool slotPool = createSlotPoolImpl()) {
		ExecutionGraph eg = TestingExecutionGraphBuilder.newBuilder()
			.setRestartStrategy(new TestRestartStrategy(1, false))
			.setJobGraph(jobGraph)
			.setNumberOfTasks(2)
			.buildAndScheduleForExecution(slotPool);

		Iterator<ExecutionVertex> executionVertices = eg.getAllExecutionVertices().iterator();

		Execution finishedExecution = executionVertices.next().getCurrentExecutionAttempt();
		Execution failedExecution = executionVertices.next().getCurrentExecutionAttempt();

		finishedExecution.markFinished();

		failedExecution.fail(new Exception("Test Exception"));
		failedExecution.completeCancelling();

		assertEquals(JobStatus.RUNNING, eg.getState());

		// At this point all resources have been assigned
		for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
			assertNotNull("No assigned resource (test instability).", vertex.getCurrentAssignedResource());
			vertex.getCurrentExecutionAttempt().switchToRunning();
		}

		// fail global with old finished execution, this should not affect the execution
		eg.failGlobalIfExecutionIsStillRunning(new Exception("This should have no effect"), finishedExecution.getAttemptId());

		assertThat(eg.getState(), is(JobStatus.RUNNING));

		// the state of the finished execution should have not changed since it is terminal
		assertThat(finishedExecution.getState(), is(ExecutionState.FINISHED));
	}
}