org.apache.flink.runtime.util.ZooKeeperUtils Java Examples

@Override
public MesosWorkerStore createMesosWorkerStore(Configuration configuration, Executor executor) throws Exception {
	RetrievableStateStorageHelper<MesosWorkerStore.Worker> stateStorageHelper =
		ZooKeeperUtils.createFileSystemStateStorage(configuration, "mesosWorkerStore");

	ZooKeeperStateHandleStore<MesosWorkerStore.Worker> zooKeeperStateHandleStore = zooKeeperUtilityFactory.createZooKeeperStateHandleStore(
		"/workers",
		stateStorageHelper);

	ZooKeeperSharedValue frameworkId = zooKeeperUtilityFactory.createSharedValue("/frameworkId", new byte[0]);
	ZooKeeperSharedCount totalTaskCount = zooKeeperUtilityFactory.createSharedCount("/taskCount", 0);

	return new ZooKeeperMesosWorkerStore(
		zooKeeperStateHandleStore,
		frameworkId,
		totalTaskCount);
}
 

/**
 * Tests that checkpoints are discarded when the completed checkpoint store is shut
 * down with a globally terminal state.
 */
@Test
public void testDiscardingCheckpointsAtShutDown() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		checkpointStore.shutdown(JobStatus.FINISHED);

		// verify that the checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 

public static HighAvailabilityServices createAvailableOrEmbeddedServices(
	Configuration config,
	Executor executor) throws Exception {
	HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(config);

	switch (highAvailabilityMode) {
		case NONE:
			return new EmbeddedHaServices(executor);

		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(config),
				executor,
				config,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(config, executor);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 

@Before
public void before() throws Exception {
	testingServer = new TestingServer();

	config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
	config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());

	CuratorFramework client = ZooKeeperUtils.startCuratorFramework(config);

	highAvailabilityServices = new ZooKeeperHaServices(
		client,
		TestingUtils.defaultExecutor(),
		config,
		new VoidBlobStore());
}
 

/**
 * Tests that checkpoints are discarded when the completed checkpoint store is shut
 * down with a globally terminal state.
 */
@Test
public void testDiscardingCheckpointsAtShutDown() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		checkpointStore.shutdown(JobStatus.FINISHED);

		// verify that the checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 

@Before
public void before() throws Exception {
	testingServer = new TestingServer();

	config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
	config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());

	CuratorFramework client = ZooKeeperUtils.startCuratorFramework(config);

	highAvailabilityServices = new ZooKeeperHaServices(
		client,
		TestingUtils.defaultExecutor(),
		config,
		new VoidBlobStore());
}
 

public static HighAvailabilityServices createAvailableOrEmbeddedServices(
	Configuration config,
	Executor executor) throws Exception {
	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(config);

	switch (highAvailabilityMode) {
		case NONE:
			return new EmbeddedHaServices(executor);

		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(config),
				executor,
				config,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(config, executor);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 

@Override
public MesosWorkerStore createMesosWorkerStore(Configuration configuration, Executor executor) throws Exception {
	RetrievableStateStorageHelper<MesosWorkerStore.Worker> stateStorageHelper =
		ZooKeeperUtils.createFileSystemStateStorage(configuration, "mesosWorkerStore");

	ZooKeeperStateHandleStore<MesosWorkerStore.Worker> zooKeeperStateHandleStore = zooKeeperUtilityFactory.createZooKeeperStateHandleStore(
		"/workers",
		stateStorageHelper);

	ZooKeeperSharedValue frameworkId = zooKeeperUtilityFactory.createSharedValue("/frameworkId", new byte[0]);
	ZooKeeperSharedCount totalTaskCount = zooKeeperUtilityFactory.createSharedCount("/taskCount", 0);

	return new ZooKeeperMesosWorkerStore(
		zooKeeperStateHandleStore,
		frameworkId,
		totalTaskCount);
}
 

/**
 * Tests that checkpoints are discarded when the completed checkpoint store is shut
 * down with a globally terminal state.
 */
@Test
public void testDiscardingCheckpointsAtShutDown() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		checkpointStore.shutdown(JobStatus.FINISHED);

		// verify that the checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 

public static HighAvailabilityServices createAvailableOrEmbeddedServices(
	Configuration config,
	Executor executor) throws Exception {
	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(config);

	switch (highAvailabilityMode) {
		case NONE:
			return new EmbeddedHaServices(executor);

		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(config),
				executor,
				config,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(config, executor);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 

/**
 * Tests that the function of ZookeeperRegistry, setJobRunning(), setJobFinished(), isJobRunning()
 */
@Test
public void testZooKeeperRegistry() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	final HighAvailabilityServices zkHaService = new ZooKeeperHaServices(
			ZooKeeperUtils.startCuratorFramework(configuration),
		Executors.directExecutor(),
		configuration,
		new VoidBlobStore());

	final RunningJobsRegistry zkRegistry = zkHaService.getRunningJobsRegistry();

	try {
		JobID jobID = JobID.generate();
		assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.setJobRunning(jobID);
		assertEquals(JobSchedulingStatus.RUNNING, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.setJobFinished(jobID);
		assertEquals(JobSchedulingStatus.DONE, zkRegistry.getJobSchedulingStatus(jobID));

		zkRegistry.clearJob(jobID);
		assertEquals(JobSchedulingStatus.PENDING, zkRegistry.getJobSchedulingStatus(jobID));
	} finally {
		zkHaService.close();
	}
}
 

private void runCleanupTest(
		Configuration configuration,
		TestingBlobStoreService blobStoreService,
		ThrowingConsumer<ZooKeeperHaServices, Exception> zooKeeperHaServicesConsumer) throws Exception {
	try (ZooKeeperHaServices zooKeeperHaServices = new ZooKeeperHaServices(
		ZooKeeperUtils.startCuratorFramework(configuration),
		Executors.directExecutor(),
		configuration,
		blobStoreService)) {

		// create some Zk services to trigger the generation of paths
		final LeaderRetrievalService resourceManagerLeaderRetriever = zooKeeperHaServices.getResourceManagerLeaderRetriever();
		final LeaderElectionService resourceManagerLeaderElectionService = zooKeeperHaServices.getResourceManagerLeaderElectionService();
		final RunningJobsRegistry runningJobsRegistry = zooKeeperHaServices.getRunningJobsRegistry();

		final TestingListener listener = new TestingListener();
		resourceManagerLeaderRetriever.start(listener);
		resourceManagerLeaderElectionService.start(new TestingContender("foobar", resourceManagerLeaderElectionService));
		final JobID jobId = new JobID();
		runningJobsRegistry.setJobRunning(jobId);

		listener.waitForNewLeader(2000L);

		resourceManagerLeaderRetriever.stop();
		resourceManagerLeaderElectionService.stop();
		runningJobsRegistry.clearJob(jobId);

		zooKeeperHaServicesConsumer.accept(zooKeeperHaServices);
	}
}
 

@Test
public void testDeleteAllShouldRemoveAllPaths() throws Exception {
	final ZooKeeperStateHandleStore<Long> zkStore = new ZooKeeperStateHandleStore<>(
		ZooKeeperUtils.useNamespaceAndEnsurePath(ZOOKEEPER.getClient(), "/path"),
		new LongStateStorage());

	zkStore.addAndLock("/state", 1L);
	zkStore.deleteChildren();

	assertThat(zkStore.getAllPaths(), is(empty()));
}
 

/**
 * Tests that the ZooKeeperLeaderElection/RetrievalService return both the correct URL.
 */
@Test
public void testZooKeeperLeaderElectionRetrieval() throws Exception {
	ZooKeeperLeaderElectionService leaderElectionService = null;
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	try {
		leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client, configuration);
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
		TestingListener listener = new TestingListener();

		leaderElectionService.start(contender);
		leaderRetrievalService.start(listener);

		contender.waitForLeader(timeout);

		assertTrue(contender.isLeader());
		assertEquals(leaderElectionService.getLeaderSessionID(), contender.getLeaderSessionID());

		listener.waitForNewLeader(timeout);

		assertEquals(TEST_URL, listener.getAddress());
		assertEquals(leaderElectionService.getLeaderSessionID(), listener.getLeaderSessionID());

	} finally {
		if (leaderElectionService != null) {
			leaderElectionService.stop();
		}

		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}
	}
}
 

@Before
public void before() {
	try {
		testingServer = new TestingServer();
	} catch (Exception e) {
		throw new RuntimeException("Could not start ZooKeeper testing cluster.", e);
	}

	configuration = new Configuration();

	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	client = ZooKeeperUtils.startCuratorFramework(configuration);
}
 

/**
 * Tests that we fail with an exception if the job cannot be removed from the
 * ZooKeeperSubmittedJobGraphStore.
 *
 * <p>Tests that a close ZooKeeperSubmittedJobGraphStore no longer holds any locks.
 */
@Test
public void testJobGraphRemovalFailureAndLockRelease() throws Exception {
	try (final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration)) {
		final TestingRetrievableStateStorageHelper<SubmittedJobGraph> stateStorage = new TestingRetrievableStateStorageHelper<>();
		final ZooKeeperSubmittedJobGraphStore submittedJobGraphStore = createSubmittedJobGraphStore(client, stateStorage);
		submittedJobGraphStore.start(null);
		final ZooKeeperSubmittedJobGraphStore otherSubmittedJobGraphStore = createSubmittedJobGraphStore(client, stateStorage);
		otherSubmittedJobGraphStore.start(null);

		final SubmittedJobGraph jobGraph = new SubmittedJobGraph(new JobGraph());
		submittedJobGraphStore.putJobGraph(jobGraph);

		final SubmittedJobGraph recoveredJobGraph = otherSubmittedJobGraphStore.recoverJobGraph(jobGraph.getJobId());

		assertThat(recoveredJobGraph, is(notNullValue()));

		try {
			otherSubmittedJobGraphStore.removeJobGraph(recoveredJobGraph.getJobId());
			fail("It should not be possible to remove the JobGraph since the first store still has a lock on it.");
		} catch (Exception ignored) {
			// expected
		}

		submittedJobGraphStore.stop();

		// now we should be able to delete the job graph
		otherSubmittedJobGraphStore.removeJobGraph(recoveredJobGraph.getJobId());

		assertThat(otherSubmittedJobGraphStore.recoverJobGraph(recoveredJobGraph.getJobId()), is(nullValue()));

		otherSubmittedJobGraphStore.stop();
	}
}
 

/**
 * Tests that the ZooKeeperLeaderElection/RetrievalService return both the correct URL.
 */
@Test
public void testZooKeeperLeaderElectionRetrieval() throws Exception {
	ZooKeeperLeaderElectionService leaderElectionService = null;
	ZooKeeperLeaderRetrievalService leaderRetrievalService = null;

	try {
		leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client, configuration);
		leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(client, configuration);

		TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
		TestingListener listener = new TestingListener();

		leaderElectionService.start(contender);
		leaderRetrievalService.start(listener);

		contender.waitForLeader(timeout);

		assertTrue(contender.isLeader());
		assertEquals(leaderElectionService.getLeaderSessionID(), contender.getLeaderSessionID());

		listener.waitForNewLeader(timeout);

		assertEquals(TEST_URL, listener.getAddress());
		assertEquals(leaderElectionService.getLeaderSessionID(), listener.getLeaderSessionID());

	} finally {
		if (leaderElectionService != null) {
			leaderElectionService.stop();
		}

		if (leaderRetrievalService != null) {
			leaderRetrievalService.stop();
		}
	}
}
 

@Override
public void setup() throws Exception {
	try {
		testingServer = new TestingServer();
	} catch (Exception e) {
		throw new RuntimeException("Could not start ZooKeeper testing cluster.", e);
	}

	configuration = new Configuration();

	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	client = ZooKeeperUtils.startCuratorFramework(configuration);
}
 

@Before
public void before() {
	try {
		testingServer = new TestingServer();
	} catch (Exception e) {
		throw new RuntimeException("Could not start ZooKeeper testing cluster.", e);
	}

	configuration = new Configuration();

	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	client = ZooKeeperUtils.startCuratorFramework(configuration);
}
 

/**
 * FLINK-6612
 *
 * Tests that lock nodes will be released if the client dies.
 */
@Test
public void testLockCleanupWhenClientTimesOut() throws Exception {
	LongStateStorage longStateStorage = new LongStateStorage();

	Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, ZOOKEEPER.getConnectString());
	configuration.setInteger(HighAvailabilityOptions.ZOOKEEPER_SESSION_TIMEOUT, 100);
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_ROOT, "timeout");

	try (CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
		CuratorFramework client2 = ZooKeeperUtils.startCuratorFramework(configuration)) {

		ZooKeeperStateHandleStore<Long> zkStore = new ZooKeeperStateHandleStore<>(
			client,
			longStateStorage);

		final String path = "/state";

		zkStore.addAndLock(path, 42L);

		// this should delete all ephemeral nodes
		client.close();

		Stat stat = client2.checkExists().forPath(path);

		// check that our state node still exists
		assertNotNull(stat);

		Collection<String> children = client2.getChildren().forPath(path);

		// check that the lock node has been released
		assertEquals(0, children.size());
	}
}
 

@Test
public void testDeleteAllShouldRemoveAllPaths() throws Exception {
	final ZooKeeperStateHandleStore<Long> zkStore = new ZooKeeperStateHandleStore<>(
		ZooKeeperUtils.useNamespaceAndEnsurePath(ZOOKEEPER.getClient(), "/path"),
		new LongStateStorage());

	zkStore.addAndLock("/state", 1L);
	zkStore.deleteChildren();

	assertThat(zkStore.getAllPaths(), is(empty()));
}
 

/**
 * Tests that subsumed checkpoints are discarded.
 */
@Test
public void testDiscardingSubsumedCheckpoints() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint2 = CompletedCheckpointStoreTest.createCheckpoint(1, sharedStateRegistry);
		checkpointStore.addCheckpoint(checkpoint2);
		final List<CompletedCheckpoint> allCheckpoints = checkpointStore.getAllCheckpoints();
		assertThat(allCheckpoints, Matchers.contains(checkpoint2));
		assertThat(allCheckpoints, Matchers.not(Matchers.contains(checkpoint1)));

		// verify that the subsumed checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 

/**
 * Tests that we fail with an exception if the job cannot be removed from the
 * ZooKeeperSubmittedJobGraphStore.
 *
 * <p>Tests that a close ZooKeeperSubmittedJobGraphStore no longer holds any locks.
 */
@Test
public void testJobGraphRemovalFailureAndLockRelease() throws Exception {
	try (final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration)) {
		final TestingRetrievableStateStorageHelper<SubmittedJobGraph> stateStorage = new TestingRetrievableStateStorageHelper<>();
		final ZooKeeperSubmittedJobGraphStore submittedJobGraphStore = createSubmittedJobGraphStore(client, stateStorage);
		submittedJobGraphStore.start(null);
		final ZooKeeperSubmittedJobGraphStore otherSubmittedJobGraphStore = createSubmittedJobGraphStore(client, stateStorage);
		otherSubmittedJobGraphStore.start(null);

		final SubmittedJobGraph jobGraph = new SubmittedJobGraph(new JobGraph());
		submittedJobGraphStore.putJobGraph(jobGraph);

		final SubmittedJobGraph recoveredJobGraph = otherSubmittedJobGraphStore.recoverJobGraph(jobGraph.getJobId());

		assertThat(recoveredJobGraph, is(notNullValue()));

		try {
			otherSubmittedJobGraphStore.removeJobGraph(recoveredJobGraph.getJobId());
			fail("It should not be possible to remove the JobGraph since the first store still has a lock on it.");
		} catch (Exception ignored) {
			// expected
		}

		submittedJobGraphStore.stop();

		// now we should be able to delete the job graph
		otherSubmittedJobGraphStore.removeJobGraph(recoveredJobGraph.getJobId());

		assertThat(otherSubmittedJobGraphStore.recoverJobGraph(recoveredJobGraph.getJobId()), is(nullValue()));

		otherSubmittedJobGraphStore.stop();
	}
}
 

@Override
public void setup() throws Exception {
	try {
		testingServer = new TestingServer();
	} catch (Exception e) {
		throw new RuntimeException("Could not start ZooKeeper testing cluster.", e);
	}

	configuration = new Configuration();

	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
	configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	client = ZooKeeperUtils.startCuratorFramework(configuration);
}
 

private void runCleanupTest(
		Configuration configuration,
		TestingBlobStoreService blobStoreService,
		ThrowingConsumer<ZooKeeperHaServices, Exception> zooKeeperHaServicesConsumer) throws Exception {
	try (ZooKeeperHaServices zooKeeperHaServices = new ZooKeeperHaServices(
		ZooKeeperUtils.startCuratorFramework(configuration),
		Executors.directExecutor(),
		configuration,
		blobStoreService)) {

		// create some Zk services to trigger the generation of paths
		final LeaderRetrievalService resourceManagerLeaderRetriever = zooKeeperHaServices.getResourceManagerLeaderRetriever();
		final LeaderElectionService resourceManagerLeaderElectionService = zooKeeperHaServices.getResourceManagerLeaderElectionService();
		final RunningJobsRegistry runningJobsRegistry = zooKeeperHaServices.getRunningJobsRegistry();

		final TestingListener listener = new TestingListener();
		resourceManagerLeaderRetriever.start(listener);
		resourceManagerLeaderElectionService.start(new TestingContender("foobar", resourceManagerLeaderElectionService));
		final JobID jobId = new JobID();
		runningJobsRegistry.setJobRunning(jobId);

		listener.waitForNewLeader(2000L);

		resourceManagerLeaderRetriever.stop();
		resourceManagerLeaderElectionService.stop();
		runningJobsRegistry.clearJob(jobId);

		zooKeeperHaServicesConsumer.accept(zooKeeperHaServices);
	}
}
 

@Override
public CompletedCheckpointStore createCheckpointStore(JobID jobId, int maxNumberOfCheckpointsToRetain, ClassLoader userClassLoader)
		throws Exception {

	return ZooKeeperUtils.createCompletedCheckpoints(client, config, jobId,
			maxNumberOfCheckpointsToRetain, executor);
}
 

@Override
protected ZooKeeperCompletedCheckpointStore createCompletedCheckpoints(int maxNumberOfCheckpointsToRetain) throws Exception {
	final ZooKeeperStateHandleStore<CompletedCheckpoint> checkpointsInZooKeeper = ZooKeeperUtils.createZooKeeperStateHandleStore(
		ZOOKEEPER.getClient(),
		CHECKPOINT_PATH,
		new TestingRetrievableStateStorageHelper<>());

	return new ZooKeeperCompletedCheckpointStore(
		maxNumberOfCheckpointsToRetain,
		checkpointsInZooKeeper,
		Executors.directExecutor());
}
 

public ZooKeeperUtilityFactory(Configuration configuration, String path) throws Exception {
	Preconditions.checkNotNull(path, "path");

	root = ZooKeeperUtils.startCuratorFramework(configuration);

	root.newNamespaceAwareEnsurePath(path).ensure(root.getZookeeperClient());
	facade = root.usingNamespace(ZooKeeperUtils.generateZookeeperPath(root.getNamespace(), path));
}
 

@Nonnull
private ZooKeeperCompletedCheckpointStore createZooKeeperCheckpointStore(CuratorFramework client) throws Exception {
	final ZooKeeperStateHandleStore<CompletedCheckpoint> checkpointsInZooKeeper = ZooKeeperUtils.createZooKeeperStateHandleStore(
		client,
		"/checkpoints",
		new TestingRetrievableStateStorageHelper<>());

	return new ZooKeeperCompletedCheckpointStore(
		1,
		checkpointsInZooKeeper,
		Executors.directExecutor());
}
 

/**
 * Returns the port range for the common {@link RpcService}.
 *
 * @param configuration to extract the port range from
 * @return Port range for the common {@link RpcService}
 */
protected String getRPCPortRange(Configuration configuration) {
	if (ZooKeeperUtils.isZooKeeperRecoveryMode(configuration)) {
		return configuration.getString(HighAvailabilityOptions.HA_JOB_MANAGER_PORT_RANGE);
	} else {
		return String.valueOf(configuration.getInteger(JobManagerOptions.PORT));
	}
}