Java Code Examples for org.apache.flink.runtime.leaderelection.TestingLeaderElectionService#notLeader()

public void revokeLeadership() {
	assert(currentLeaderId != null);
	assert(0 <= currentLeaderIndex &&  currentLeaderIndex < leaderElectionServices.size());

	TestingLeaderElectionService testingLeaderElectionService = leaderElectionServices.get(currentLeaderIndex);

	testingLeaderElectionService.notLeader();

	currentLeaderIndex = -1;
	currentLeaderId = null;
}
 

public void revokeLeadership() {
	assert(currentLeaderId != null);
	assert(0 <= currentLeaderIndex &&  currentLeaderIndex < leaderElectionServices.size());

	TestingLeaderElectionService testingLeaderElectionService = leaderElectionServices.get(currentLeaderIndex);

	testingLeaderElectionService.notLeader();

	currentLeaderIndex = -1;
	currentLeaderId = null;
}
 

public void revokeLeadership() {
	assert(currentLeaderId != null);
	assert(0 <= currentLeaderIndex &&  currentLeaderIndex < leaderElectionServices.size());

	TestingLeaderElectionService testingLeaderElectionService = leaderElectionServices.get(currentLeaderIndex);

	testingLeaderElectionService.notLeader();

	currentLeaderIndex = -1;
	currentLeaderId = null;
}
 

/**
 * Tests that interleaved granting and revoking of the leadership won't interfere
 * with the job recovery and the resulting internal state of the Dispatcher.
 */
@Test
public void testGrantingRevokingLeadership() throws Exception {
	final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
	final JobGraph nonEmptyJobGraph = createNonEmptyJobGraph();
	final SubmittedJobGraph submittedJobGraph = new SubmittedJobGraph(nonEmptyJobGraph);

	final OneShotLatch enterGetJobIdsLatch = new OneShotLatch();
	final OneShotLatch proceedGetJobIdsLatch = new OneShotLatch();
	highAvailabilityServices.setSubmittedJobGraphStore(new BlockingSubmittedJobGraphStore(submittedJobGraph, enterGetJobIdsLatch, proceedGetJobIdsLatch));
	final TestingLeaderElectionService dispatcherLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);

	final BlockingQueue<DispatcherId> fencingTokens = new ArrayBlockingQueue<>(2);

	final HATestingDispatcher dispatcher = createDispatcherWithObservableFencingTokens(highAvailabilityServices, fencingTokens);

	dispatcher.start();

	try {
		// wait until the election service has been started
		dispatcherLeaderElectionService.getStartFuture().get();

		final UUID leaderId = UUID.randomUUID();
		dispatcherLeaderElectionService.isLeader(leaderId);

		dispatcherLeaderElectionService.notLeader();

		final DispatcherId firstFencingToken = fencingTokens.take();

		assertThat(firstFencingToken, equalTo(NULL_FENCING_TOKEN));

		enterGetJobIdsLatch.await();
		proceedGetJobIdsLatch.trigger();

		assertThat(dispatcher.getNumberJobs(timeout).get(), is(0));

	} finally {
		RpcUtils.terminateRpcEndpoint(dispatcher, timeout);
	}
}
 

/**
 * Tests that all JobManagerRunner are terminated if the leadership of the
 * Dispatcher is revoked.
 */
@Test
public void testRevokeLeadershipTerminatesJobManagerRunners() throws Exception {

	final TestingLeaderElectionService leaderElectionService = new TestingLeaderElectionService();
	final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServicesBuilder()
		.setDispatcherLeaderElectionService(leaderElectionService)
		.build();

	final ArrayBlockingQueue<DispatcherId> fencingTokens = new ArrayBlockingQueue<>(2);
	final HATestingDispatcher dispatcher = createDispatcherWithObservableFencingTokens(
		highAvailabilityServices,
		fencingTokens);

	dispatcher.start();

	try {
		// grant leadership and submit a single job
		final DispatcherId expectedDispatcherId = DispatcherId.generate();

		leaderElectionService.isLeader(expectedDispatcherId.toUUID()).get();

		assertThat(fencingTokens.take(), is(equalTo(expectedDispatcherId)));

		final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

		final CompletableFuture<Acknowledge> submissionFuture = dispatcherGateway.submitJob(createNonEmptyJobGraph(), timeout);

		submissionFuture.get();

		assertThat(dispatcher.getNumberJobs(timeout).get(), is(1));

		// revoke the leadership --> this should stop all running JobManagerRunners
		leaderElectionService.notLeader();

		assertThat(fencingTokens.take(), is(equalTo(NULL_FENCING_TOKEN)));

		assertThat(dispatcher.getNumberJobs(timeout).get(), is(0));
	} finally {
		RpcUtils.terminateRpcEndpoint(dispatcher, timeout);
	}
}
 

/**
 * Tests that a Dispatcher does not remove the JobGraph from the submitted job graph store
 * when losing leadership and recovers it when regaining leadership.
 */
@Test
public void testJobRecoveryWhenChangingLeadership() throws Exception {
	final InMemorySubmittedJobGraphStore submittedJobGraphStore = new InMemorySubmittedJobGraphStore();

	final CompletableFuture<JobID> recoveredJobFuture = new CompletableFuture<>();
	submittedJobGraphStore.setRecoverJobGraphFunction((jobID, jobIDSubmittedJobGraphMap) -> {
		recoveredJobFuture.complete(jobID);
		return jobIDSubmittedJobGraphMap.get(jobID);
	});

	final TestingLeaderElectionService leaderElectionService = new TestingLeaderElectionService();

	final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServicesBuilder()
		.setSubmittedJobGraphStore(submittedJobGraphStore)
		.setDispatcherLeaderElectionService(leaderElectionService)
		.build();

	final ArrayBlockingQueue<DispatcherId> fencingTokens = new ArrayBlockingQueue<>(2);
	final HATestingDispatcher dispatcher = createDispatcherWithObservableFencingTokens(
		highAvailabilityServices,
		fencingTokens);

	dispatcher.start();

	try {
		// grant leadership and submit a single job
		final DispatcherId expectedDispatcherId = DispatcherId.generate();
		leaderElectionService.isLeader(expectedDispatcherId.toUUID()).get();

		assertThat(fencingTokens.take(), is(equalTo(expectedDispatcherId)));

		final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

		final JobGraph jobGraph = createNonEmptyJobGraph();
		final CompletableFuture<Acknowledge> submissionFuture = dispatcherGateway.submitJob(jobGraph, timeout);

		submissionFuture.get();

		final JobID jobId = jobGraph.getJobID();
		assertThat(submittedJobGraphStore.contains(jobId), is(true));

		// revoke the leadership --> this should stop all running JobManagerRunners
		leaderElectionService.notLeader();

		assertThat(fencingTokens.take(), is(equalTo(NULL_FENCING_TOKEN)));

		assertThat(submittedJobGraphStore.contains(jobId), is(true));

		assertThat(recoveredJobFuture.isDone(), is(false));

		// re-grant leadership
		leaderElectionService.isLeader(DispatcherId.generate().toUUID());

		assertThat(recoveredJobFuture.get(), is(equalTo(jobId)));
	} finally {
		RpcUtils.terminateRpcEndpoint(dispatcher, timeout);
	}
}
 

/**
 * Tests that the {@link Dispatcher} releases a locked {@link SubmittedJobGraph} if it
 * lost the leadership.
 */
@Test
public void testSubmittedJobGraphRelease() throws Exception {
	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final CuratorFramework otherClient = ZooKeeperUtils.startCuratorFramework(configuration);

	try (final TestingHighAvailabilityServices testingHighAvailabilityServices = new TestingHighAvailabilityServices()) {
		testingHighAvailabilityServices.setSubmittedJobGraphStore(ZooKeeperUtils.createSubmittedJobGraphs(client, configuration));

		final ZooKeeperSubmittedJobGraphStore otherSubmittedJobGraphStore = ZooKeeperUtils.createSubmittedJobGraphs(
			otherClient,
			configuration);

		otherSubmittedJobGraphStore.start(NoOpSubmittedJobGraphListener.INSTANCE);

		final TestingLeaderElectionService leaderElectionService = new TestingLeaderElectionService();
		testingHighAvailabilityServices.setDispatcherLeaderElectionService(leaderElectionService);

		final TestingDispatcher dispatcher = createDispatcher(
			testingHighAvailabilityServices,
			new TestingJobManagerRunnerFactory(new CompletableFuture<>(), new CompletableFuture<>(), CompletableFuture.completedFuture(null)));

		dispatcher.start();

		try {
			final DispatcherId expectedLeaderId = DispatcherId.generate();
			leaderElectionService.isLeader(expectedLeaderId.toUUID()).get();

			final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

			final JobGraph nonEmptyJobGraph = DispatcherHATest.createNonEmptyJobGraph();
			final CompletableFuture<Acknowledge> submissionFuture = dispatcherGateway.submitJob(nonEmptyJobGraph, TIMEOUT);
			submissionFuture.get();

			Collection<JobID> jobIds = otherSubmittedJobGraphStore.getJobIds();

			final JobID jobId = nonEmptyJobGraph.getJobID();
			assertThat(jobIds, Matchers.contains(jobId));

			leaderElectionService.notLeader();

			// wait for the job to properly terminate
			final CompletableFuture<Void> jobTerminationFuture = dispatcher.getJobTerminationFuture(jobId, TIMEOUT);
			jobTerminationFuture.get();

			// recover the job
			final SubmittedJobGraph submittedJobGraph = otherSubmittedJobGraphStore.recoverJobGraph(jobId);

			assertThat(submittedJobGraph, is(notNullValue()));

			// check that the other submitted job graph store can remove the job graph after the original leader
			// has lost its leadership
			otherSubmittedJobGraphStore.removeJobGraph(jobId);

			jobIds = otherSubmittedJobGraphStore.getJobIds();

			assertThat(jobIds, Matchers.not(Matchers.contains(jobId)));
		} finally {
			RpcUtils.terminateRpcEndpoint(dispatcher, TIMEOUT);
			client.close();
			otherClient.close();
		}
	}
}
 

/**
 * Tests that a standby Dispatcher does not interfere with the clean up of a completed
 * job.
 */
@Test
public void testStandbyDispatcherJobExecution() throws Exception {
	try (final TestingHighAvailabilityServices haServices1 = new TestingHighAvailabilityServices();
		final TestingHighAvailabilityServices haServices2 = new TestingHighAvailabilityServices();
		final CuratorFramework curatorFramework = ZooKeeperUtils.startCuratorFramework(configuration)) {

		final ZooKeeperSubmittedJobGraphStore submittedJobGraphStore1 = ZooKeeperUtils.createSubmittedJobGraphs(curatorFramework, configuration);
		haServices1.setSubmittedJobGraphStore(submittedJobGraphStore1);
		final TestingLeaderElectionService leaderElectionService1 = new TestingLeaderElectionService();
		haServices1.setDispatcherLeaderElectionService(leaderElectionService1);

		final ZooKeeperSubmittedJobGraphStore submittedJobGraphStore2 = ZooKeeperUtils.createSubmittedJobGraphs(curatorFramework, configuration);
		haServices2.setSubmittedJobGraphStore(submittedJobGraphStore2);
		final TestingLeaderElectionService leaderElectionService2 = new TestingLeaderElectionService();
		haServices2.setDispatcherLeaderElectionService(leaderElectionService2);

		final CompletableFuture<JobGraph> jobGraphFuture = new CompletableFuture<>();
		final CompletableFuture<ArchivedExecutionGraph> resultFuture = new CompletableFuture<>();
		final TestingDispatcher dispatcher1 = createDispatcher(
			haServices1,
			new TestingJobManagerRunnerFactory(jobGraphFuture, resultFuture, CompletableFuture.completedFuture(null)));

		final TestingDispatcher dispatcher2 = createDispatcher(
			haServices2,
			new TestingJobManagerRunnerFactory(new CompletableFuture<>(), new CompletableFuture<>(), CompletableFuture.completedFuture(null)));

		try {
			dispatcher1.start();
			dispatcher2.start();

			leaderElectionService1.isLeader(UUID.randomUUID()).get();
			final DispatcherGateway dispatcherGateway1 = dispatcher1.getSelfGateway(DispatcherGateway.class);

			final JobGraph jobGraph = DispatcherHATest.createNonEmptyJobGraph();

			dispatcherGateway1.submitJob(jobGraph, TIMEOUT).get();

			final CompletableFuture<JobResult> jobResultFuture = dispatcherGateway1.requestJobResult(jobGraph.getJobID(), TIMEOUT);

			jobGraphFuture.get();

			// complete the job
			resultFuture.complete(new ArchivedExecutionGraphBuilder().setJobID(jobGraph.getJobID()).setState(JobStatus.FINISHED).build());

			final JobResult jobResult = jobResultFuture.get();

			assertThat(jobResult.isSuccess(), is(true));

			// wait for the completion of the job
			dispatcher1.getJobTerminationFuture(jobGraph.getJobID(), TIMEOUT).get();

			// change leadership
			leaderElectionService1.notLeader();
			leaderElectionService2.isLeader(UUID.randomUUID()).get();

			// Dispatcher 2 should not recover any jobs
			final DispatcherGateway dispatcherGateway2 = dispatcher2.getSelfGateway(DispatcherGateway.class);
			assertThat(dispatcherGateway2.listJobs(TIMEOUT).get(), is(empty()));
		} finally {
			RpcUtils.terminateRpcEndpoint(dispatcher1, TIMEOUT);
			RpcUtils.terminateRpcEndpoint(dispatcher2, TIMEOUT);
		}
	}
}
 

/**
 * Tests that interleaved granting and revoking of the leadership won't interfere
 * with the job recovery and the resulting internal state of the Dispatcher.
 */
@Test
public void testGrantingRevokingLeadership() throws Exception {
	final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
	final JobGraph nonEmptyJobGraph = createNonEmptyJobGraph();
	final SubmittedJobGraph submittedJobGraph = new SubmittedJobGraph(nonEmptyJobGraph);

	final OneShotLatch enterGetJobIdsLatch = new OneShotLatch();
	final OneShotLatch proceedGetJobIdsLatch = new OneShotLatch();
	highAvailabilityServices.setSubmittedJobGraphStore(new BlockingSubmittedJobGraphStore(submittedJobGraph, enterGetJobIdsLatch, proceedGetJobIdsLatch));
	final TestingLeaderElectionService dispatcherLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);

	final BlockingQueue<DispatcherId> fencingTokens = new ArrayBlockingQueue<>(2);

	final HATestingDispatcher dispatcher = createDispatcherWithObservableFencingTokens(highAvailabilityServices, fencingTokens);

	dispatcher.start();

	try {
		// wait until the election service has been started
		dispatcherLeaderElectionService.getStartFuture().get();

		final UUID leaderId = UUID.randomUUID();
		dispatcherLeaderElectionService.isLeader(leaderId);

		dispatcherLeaderElectionService.notLeader();

		final DispatcherId firstFencingToken = fencingTokens.take();

		assertThat(firstFencingToken, equalTo(NULL_FENCING_TOKEN));

		enterGetJobIdsLatch.await();
		proceedGetJobIdsLatch.trigger();

		assertThat(dispatcher.getNumberJobs(timeout).get(), is(0));

	} finally {
		RpcUtils.terminateRpcEndpoint(dispatcher, timeout);
	}
}
 

/**
 * Tests that all JobManagerRunner are terminated if the leadership of the
 * Dispatcher is revoked.
 */
@Test
public void testRevokeLeadershipTerminatesJobManagerRunners() throws Exception {

	final TestingLeaderElectionService leaderElectionService = new TestingLeaderElectionService();
	final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServicesBuilder()
		.setDispatcherLeaderElectionService(leaderElectionService)
		.build();

	final ArrayBlockingQueue<DispatcherId> fencingTokens = new ArrayBlockingQueue<>(2);
	final HATestingDispatcher dispatcher = createDispatcherWithObservableFencingTokens(
		highAvailabilityServices,
		fencingTokens);

	dispatcher.start();

	try {
		// grant leadership and submit a single job
		final DispatcherId expectedDispatcherId = DispatcherId.generate();

		leaderElectionService.isLeader(expectedDispatcherId.toUUID()).get();

		assertThat(fencingTokens.take(), is(equalTo(expectedDispatcherId)));

		final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

		final CompletableFuture<Acknowledge> submissionFuture = dispatcherGateway.submitJob(createNonEmptyJobGraph(), timeout);

		submissionFuture.get();

		assertThat(dispatcher.getNumberJobs(timeout).get(), is(1));

		// revoke the leadership --> this should stop all running JobManagerRunners
		leaderElectionService.notLeader();

		assertThat(fencingTokens.take(), is(equalTo(NULL_FENCING_TOKEN)));

		assertThat(dispatcher.getNumberJobs(timeout).get(), is(0));
	} finally {
		RpcUtils.terminateRpcEndpoint(dispatcher, timeout);
	}
}
 

/**
 * Tests that a Dispatcher does not remove the JobGraph from the submitted job graph store
 * when losing leadership and recovers it when regaining leadership.
 */
@Test
public void testJobRecoveryWhenChangingLeadership() throws Exception {
	final InMemorySubmittedJobGraphStore submittedJobGraphStore = new InMemorySubmittedJobGraphStore();

	final CompletableFuture<JobID> recoveredJobFuture = new CompletableFuture<>();
	submittedJobGraphStore.setRecoverJobGraphFunction((jobID, jobIDSubmittedJobGraphMap) -> {
		recoveredJobFuture.complete(jobID);
		return jobIDSubmittedJobGraphMap.get(jobID);
	});

	final TestingLeaderElectionService leaderElectionService = new TestingLeaderElectionService();

	final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServicesBuilder()
		.setSubmittedJobGraphStore(submittedJobGraphStore)
		.setDispatcherLeaderElectionService(leaderElectionService)
		.build();

	final ArrayBlockingQueue<DispatcherId> fencingTokens = new ArrayBlockingQueue<>(2);
	final HATestingDispatcher dispatcher = createDispatcherWithObservableFencingTokens(
		highAvailabilityServices,
		fencingTokens);

	dispatcher.start();

	try {
		// grant leadership and submit a single job
		final DispatcherId expectedDispatcherId = DispatcherId.generate();
		leaderElectionService.isLeader(expectedDispatcherId.toUUID()).get();

		assertThat(fencingTokens.take(), is(equalTo(expectedDispatcherId)));

		final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

		final JobGraph jobGraph = createNonEmptyJobGraph();
		final CompletableFuture<Acknowledge> submissionFuture = dispatcherGateway.submitJob(jobGraph, timeout);

		submissionFuture.get();

		final JobID jobId = jobGraph.getJobID();
		assertThat(submittedJobGraphStore.contains(jobId), is(true));

		// revoke the leadership --> this should stop all running JobManagerRunners
		leaderElectionService.notLeader();

		assertThat(fencingTokens.take(), is(equalTo(NULL_FENCING_TOKEN)));

		assertThat(submittedJobGraphStore.contains(jobId), is(true));

		assertThat(recoveredJobFuture.isDone(), is(false));

		// re-grant leadership
		leaderElectionService.isLeader(DispatcherId.generate().toUUID());

		assertThat(recoveredJobFuture.get(), is(equalTo(jobId)));
	} finally {
		RpcUtils.terminateRpcEndpoint(dispatcher, timeout);
	}
}
 

/**
 * Tests that the {@link Dispatcher} releases a locked {@link SubmittedJobGraph} if it
 * lost the leadership.
 */
@Test
public void testSubmittedJobGraphRelease() throws Exception {
	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final CuratorFramework otherClient = ZooKeeperUtils.startCuratorFramework(configuration);

	try (final TestingHighAvailabilityServices testingHighAvailabilityServices = new TestingHighAvailabilityServices()) {
		testingHighAvailabilityServices.setSubmittedJobGraphStore(ZooKeeperUtils.createSubmittedJobGraphs(client, configuration));

		final ZooKeeperSubmittedJobGraphStore otherSubmittedJobGraphStore = ZooKeeperUtils.createSubmittedJobGraphs(
			otherClient,
			configuration);

		otherSubmittedJobGraphStore.start(NoOpSubmittedJobGraphListener.INSTANCE);

		final TestingLeaderElectionService leaderElectionService = new TestingLeaderElectionService();
		testingHighAvailabilityServices.setDispatcherLeaderElectionService(leaderElectionService);

		final TestingDispatcher dispatcher = createDispatcher(
			testingHighAvailabilityServices,
			new TestingJobManagerRunnerFactory(new CompletableFuture<>(), new CompletableFuture<>(), CompletableFuture.completedFuture(null)));

		dispatcher.start();

		try {
			final DispatcherId expectedLeaderId = DispatcherId.generate();
			leaderElectionService.isLeader(expectedLeaderId.toUUID()).get();

			final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

			final JobGraph nonEmptyJobGraph = DispatcherHATest.createNonEmptyJobGraph();
			final CompletableFuture<Acknowledge> submissionFuture = dispatcherGateway.submitJob(nonEmptyJobGraph, TIMEOUT);
			submissionFuture.get();

			Collection<JobID> jobIds = otherSubmittedJobGraphStore.getJobIds();

			final JobID jobId = nonEmptyJobGraph.getJobID();
			assertThat(jobIds, Matchers.contains(jobId));

			leaderElectionService.notLeader();

			// wait for the job to properly terminate
			final CompletableFuture<Void> jobTerminationFuture = dispatcher.getJobTerminationFuture(jobId, TIMEOUT);
			jobTerminationFuture.get();

			// recover the job
			final SubmittedJobGraph submittedJobGraph = otherSubmittedJobGraphStore.recoverJobGraph(jobId);

			assertThat(submittedJobGraph, is(notNullValue()));

			// check that the other submitted job graph store can remove the job graph after the original leader
			// has lost its leadership
			otherSubmittedJobGraphStore.removeJobGraph(jobId);

			jobIds = otherSubmittedJobGraphStore.getJobIds();

			assertThat(jobIds, Matchers.not(Matchers.contains(jobId)));
		} finally {
			RpcUtils.terminateRpcEndpoint(dispatcher, TIMEOUT);
			client.close();
			otherClient.close();
		}
	}
}
 

/**
 * Tests that a standby Dispatcher does not interfere with the clean up of a completed
 * job.
 */
@Test
public void testStandbyDispatcherJobExecution() throws Exception {
	try (final TestingHighAvailabilityServices haServices1 = new TestingHighAvailabilityServices();
		final TestingHighAvailabilityServices haServices2 = new TestingHighAvailabilityServices();
		final CuratorFramework curatorFramework = ZooKeeperUtils.startCuratorFramework(configuration)) {

		final ZooKeeperSubmittedJobGraphStore submittedJobGraphStore1 = ZooKeeperUtils.createSubmittedJobGraphs(curatorFramework, configuration);
		haServices1.setSubmittedJobGraphStore(submittedJobGraphStore1);
		final TestingLeaderElectionService leaderElectionService1 = new TestingLeaderElectionService();
		haServices1.setDispatcherLeaderElectionService(leaderElectionService1);

		final ZooKeeperSubmittedJobGraphStore submittedJobGraphStore2 = ZooKeeperUtils.createSubmittedJobGraphs(curatorFramework, configuration);
		haServices2.setSubmittedJobGraphStore(submittedJobGraphStore2);
		final TestingLeaderElectionService leaderElectionService2 = new TestingLeaderElectionService();
		haServices2.setDispatcherLeaderElectionService(leaderElectionService2);

		final CompletableFuture<JobGraph> jobGraphFuture = new CompletableFuture<>();
		final CompletableFuture<ArchivedExecutionGraph> resultFuture = new CompletableFuture<>();
		final TestingDispatcher dispatcher1 = createDispatcher(
			haServices1,
			new TestingJobManagerRunnerFactory(jobGraphFuture, resultFuture, CompletableFuture.completedFuture(null)));

		final TestingDispatcher dispatcher2 = createDispatcher(
			haServices2,
			new TestingJobManagerRunnerFactory(new CompletableFuture<>(), new CompletableFuture<>(), CompletableFuture.completedFuture(null)));

		try {
			dispatcher1.start();
			dispatcher2.start();

			leaderElectionService1.isLeader(UUID.randomUUID()).get();
			final DispatcherGateway dispatcherGateway1 = dispatcher1.getSelfGateway(DispatcherGateway.class);

			final JobGraph jobGraph = DispatcherHATest.createNonEmptyJobGraph();

			dispatcherGateway1.submitJob(jobGraph, TIMEOUT).get();

			final CompletableFuture<JobResult> jobResultFuture = dispatcherGateway1.requestJobResult(jobGraph.getJobID(), TIMEOUT);

			jobGraphFuture.get();

			// complete the job
			resultFuture.complete(new ArchivedExecutionGraphBuilder().setJobID(jobGraph.getJobID()).setState(JobStatus.FINISHED).build());

			final JobResult jobResult = jobResultFuture.get();

			assertThat(jobResult.isSuccess(), is(true));

			// wait for the completion of the job
			dispatcher1.getJobTerminationFuture(jobGraph.getJobID(), TIMEOUT).get();

			// change leadership
			leaderElectionService1.notLeader();
			leaderElectionService2.isLeader(UUID.randomUUID()).get();

			// Dispatcher 2 should not recover any jobs
			final DispatcherGateway dispatcherGateway2 = dispatcher2.getSelfGateway(DispatcherGateway.class);
			assertThat(dispatcherGateway2.listJobs(TIMEOUT).get(), is(empty()));
		} finally {
			RpcUtils.terminateRpcEndpoint(dispatcher1, TIMEOUT);
			RpcUtils.terminateRpcEndpoint(dispatcher2, TIMEOUT);
		}
	}
}
 

/**
 * See FLINK-11665.
 */
@Test
public void testResourceCleanupUnderLeadershipChange() throws Exception {
	final TestingRpcService rpcService = testingRpcServiceResource.getTestingRpcService();
	final TestingLeaderElectionService dispatcherLeaderElectionService = new TestingLeaderElectionService();

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	try (final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServicesBuilder()
			.setRunningJobsRegistry(new ZooKeeperRunningJobsRegistry(client, configuration))
			.setDispatcherLeaderElectionService(dispatcherLeaderElectionService)
			.setJobMasterLeaderRetrieverFunction(jobId -> ZooKeeperUtils.createLeaderRetrievalService(client, configuration))
			.build()) {

		final PartialDispatcherServices partialDispatcherServices = new PartialDispatcherServices(
			configuration,
			highAvailabilityServices,
			CompletableFuture::new,
			blobServer,
			new TestingHeartbeatServices(),
			UnregisteredMetricGroups::createUnregisteredJobManagerMetricGroup,
			new MemoryArchivedExecutionGraphStore(),
			fatalErrorHandler,
			VoidHistoryServerArchivist.INSTANCE,
			null);

		final JobGraph jobGraph = createJobGraphWithBlobs();

		final DefaultDispatcherRunnerFactory defaultDispatcherRunnerFactory = DefaultDispatcherRunnerFactory.createSessionRunner(SessionDispatcherFactory.INSTANCE);

		try (final DispatcherRunner dispatcherRunner = createDispatcherRunner(
			rpcService,
			dispatcherLeaderElectionService,
			() -> createZooKeeperJobGraphStore(client),
			partialDispatcherServices,
			defaultDispatcherRunnerFactory)) {

			// initial run
			DispatcherGateway dispatcherGateway = grantLeadership(dispatcherLeaderElectionService);

			LOG.info("Initial job submission {}.", jobGraph.getJobID());
			dispatcherGateway.submitJob(jobGraph, TESTING_TIMEOUT).get();

			dispatcherLeaderElectionService.notLeader();

			// recovering submitted jobs
			LOG.info("Re-grant leadership first time.");
			dispatcherGateway = grantLeadership(dispatcherLeaderElectionService);

			LOG.info("Cancel recovered job {}.", jobGraph.getJobID());
			// cancellation of the job should remove everything
			final CompletableFuture<JobResult> jobResultFuture = dispatcherGateway.requestJobResult(jobGraph.getJobID(), TESTING_TIMEOUT);
			dispatcherGateway.cancelJob(jobGraph.getJobID(), TESTING_TIMEOUT).get();

			// a successful cancellation should eventually remove all job information
			final JobResult jobResult = jobResultFuture.get();

			assertThat(jobResult.getApplicationStatus(), is(ApplicationStatus.CANCELED));

			dispatcherLeaderElectionService.notLeader();

			// check that the job has been removed from ZooKeeper
			final ZooKeeperJobGraphStore submittedJobGraphStore = createZooKeeperJobGraphStore(client);

			CommonTestUtils.waitUntilCondition(() -> submittedJobGraphStore.getJobIds().isEmpty(), Deadline.fromNow(VERIFICATION_TIMEOUT), 20L);
		}
	}

	// check resource clean up
	assertThat(clusterHaStorageDir.listFiles(), is(emptyArray()));
}