org.apache.flink.runtime.jobmanager.HighAvailabilityMode Java Examples

@Test
public void testStaticJobIdWithHa() throws Throwable {
	final JobID testJobID = new JobID(0, 2);

	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, testJobID.toHexString());
	configurationUnderTest.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final CompletableFuture<JobID> submittedJobId = new CompletableFuture<>();

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> {
				submittedJobId.complete(jobGraph.getJobID());
				return CompletableFuture.completedFuture(Acknowledge.get());
			})
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.FINISHED))
			.setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createSuccessfulJobResult(jobId)));

	final CompletableFuture<Void> applicationFuture =
			runApplication(dispatcherBuilder, configurationUnderTest, 1);

	applicationFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	assertThat(submittedJobId.get(TIMEOUT_SECONDS, TimeUnit.SECONDS), is(new JobID(0L, 2L)));
}
 

/**
 * For non-HA cluster, {@link JobManagerOptions#ADDRESS} has be set to Kubernetes service name on client side. See
 * {@link KubernetesClusterDescriptor#deployClusterInternal}. So the TaskManager will use service address to contact
 * with JobManager.
 * For HA cluster, {@link JobManagerOptions#ADDRESS} will be set to the pod ip address. The TaskManager use Zookeeper
 * or other high-availability service to find the address of JobManager.
 *
 * @return Updated configuration
 */
static Configuration loadConfiguration() {
	final String configDir = System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR);
	Preconditions.checkNotNull(
		configDir,
		"Flink configuration directory (%s) in environment should not be null!",
		ConfigConstants.ENV_FLINK_CONF_DIR);

	final Configuration configuration = GlobalConfiguration.loadConfiguration(configDir);

	if (HighAvailabilityMode.isHighAvailabilityModeActivated(configuration)) {
		final String ipAddress = System.getenv().get(Constants.ENV_FLINK_POD_IP_ADDRESS);
		Preconditions.checkState(
			ipAddress != null,
			"JobManager ip address environment variable %s not set",
			Constants.ENV_FLINK_POD_IP_ADDRESS);
		configuration.setString(JobManagerOptions.ADDRESS, ipAddress);
		configuration.setString(RestOptions.ADDRESS, ipAddress);
	}

	return configuration;
}
 

public static HighAvailabilityServices createAvailableOrEmbeddedServices(
	Configuration config,
	Executor executor) throws Exception {
	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(config);

	switch (highAvailabilityMode) {
		case NONE:
			return new EmbeddedHaServices(executor);

		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(config),
				executor,
				config,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(config, executor);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 

@Test
public void testDeployHighAvailabilitySessionCluster() throws ClusterDeploymentException {
	flinkConfig.setString(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.toString());
	final ClusterClient<String> clusterClient = deploySessionCluster().getClusterClient();
	checkClusterClient(clusterClient);

	final Container jmContainer = kubeClient
		.apps()
		.deployments()
		.list()
		.getItems()
		.get(0)
		.getSpec()
		.getTemplate()
		.getSpec()
		.getContainers()
		.get(0);
	assertTrue(
		"Environment " + ENV_FLINK_POD_IP_ADDRESS + " should be set.",
		jmContainer.getEnv().stream()
			.map(EnvVar::getName)
			.collect(Collectors.toList())
			.contains(ENV_FLINK_POD_IP_ADDRESS));

	clusterClient.close();
}
 

/**
 * Creates the high-availability services for the TaskManagers participating in
 * a Flink YARN application.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forYarnTaskManager(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnPreConfiguredMasterNonHaServices(
				flinkConfig,
				hadoopConfig,
				HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 

/**
 * Creates the high-availability services for a single-job Flink YARN application, to be
 * used in the Application Master that runs both ResourceManager and JobManager.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forSingleJobAppMaster(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 

public static HighAvailabilityServices createAvailableOrEmbeddedServices(
	Configuration config,
	Executor executor) throws Exception {
	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(config);

	switch (highAvailabilityMode) {
		case NONE:
			return new EmbeddedHaServices(executor);

		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(config),
				executor,
				config,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(config, executor);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 

public static ClientHighAvailabilityServices createClientHAService(Configuration configuration) throws Exception {
	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final String webMonitorAddress = getWebMonitorAddress(configuration, AddressResolution.TRY_ADDRESS_RESOLUTION);
			return new StandaloneClientHAServices(webMonitorAddress);
		case ZOOKEEPER:
			final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
			return new ZooKeeperClientHAServices(client, configuration);
		case FACTORY_CLASS:
			return createCustomClientHAServices(configuration);
		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}
 

@VisibleForTesting
CompletableFuture<Void> fixJobIdAndRunApplicationAsync(
		final DispatcherGateway dispatcher,
		final ScheduledExecutor scheduledExecutor) {

	final Optional<String> configuredJobId =
			configuration.getOptional(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID);

	if (!HighAvailabilityMode.isHighAvailabilityModeActivated(configuration) && !configuredJobId.isPresent()) {
		return runApplicationAsync(dispatcher, scheduledExecutor, false);
	}

	if (!configuredJobId.isPresent()) {
		configuration.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, ZERO_JOB_ID.toHexString());
	}
	return runApplicationAsync(dispatcher, scheduledExecutor, true);
}
 

public static HighAvailabilityServices createAvailableOrEmbeddedServices(
	Configuration config,
	Executor executor) throws Exception {
	HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(config);

	switch (highAvailabilityMode) {
		case NONE:
			return new EmbeddedHaServices(executor);

		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(config),
				executor,
				config,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(config, executor);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 

@Test
public void testJobIdDefaultsToZeroWithHa() throws Throwable {
	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final CompletableFuture<JobID> submittedJobId = new CompletableFuture<>();

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> {
				submittedJobId.complete(jobGraph.getJobID());
				return CompletableFuture.completedFuture(Acknowledge.get());
			})
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.FINISHED))
			.setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createSuccessfulJobResult(jobId)));

	final CompletableFuture<Void> applicationFuture =
			runApplication(dispatcherBuilder, configurationUnderTest, 1);

	applicationFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	assertThat(submittedJobId.get(TIMEOUT_SECONDS, TimeUnit.SECONDS), is(new JobID(0L, 0L)));
}
 

/**
 * Creates the high-availability services for the TaskManagers participating in
 * a Flink YARN application.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forYarnTaskManager(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnPreConfiguredMasterNonHaServices(
				flinkConfig,
				hadoopConfig,
				HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 

/**
 * Creates the high-availability services for a single-job Flink YARN application, to be
 * used in the Application Master that runs both ResourceManager and JobManager.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forSingleJobAppMaster(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 

@Test(expected = Exception.class)
public void testCustomHAServicesFactoryNotDefined() throws Exception {
	Configuration config = new Configuration();

	Executor executor = Executors.directExecutor();

	config.setString(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.FACTORY_CLASS.name().toLowerCase());

	// expect
	HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(config, executor);
}
 

@Test
public void testOnlyOneJobIsAllowedWithHa() throws Throwable {
	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final CompletableFuture<Void> applicationFuture =
			runApplication(configurationUnderTest, 2);

	assertException(applicationFuture, FlinkRuntimeException.class);
}
 

@Test
public void testOnlyOneJobAllowedWithStaticJobIdAndHa() throws Throwable {
	final JobID testJobID = new JobID(0, 2);

	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, testJobID.toHexString());
	configurationUnderTest.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final CompletableFuture<Void> applicationFuture =
			runApplication(configurationUnderTest, 2);

	assertException(applicationFuture, FlinkRuntimeException.class);
}
 

/**
 * Creates a {@link MesosServices} instance depending on the high availability settings.
 *
 * @param configuration containing the high availability settings
 * @param hostname the hostname to advertise to remote clients
 * @return a mesos services instance
 * @throws Exception if the mesos services instance could not be created
 */
public static MesosServices createMesosServices(Configuration configuration, String hostname) throws Exception {

	ActorSystem localActorSystem = AkkaUtils.createLocalActorSystem(configuration);

	MesosArtifactServer artifactServer = createArtifactServer(configuration, hostname);

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			return new StandaloneMesosServices(localActorSystem, artifactServer);

		case ZOOKEEPER:
			final String zkMesosRootPath = configuration.getString(
				HighAvailabilityOptions.HA_ZOOKEEPER_MESOS_WORKERS_PATH);

			ZooKeeperUtilityFactory zooKeeperUtilityFactory = new ZooKeeperUtilityFactory(
				configuration,
				zkMesosRootPath);

			return new ZooKeeperMesosServices(localActorSystem, artifactServer, zooKeeperUtilityFactory);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 

@Test
public void testDisableInternalService() throws IOException {
	this.flinkConfig.setString(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final List<HasMetadata> resources = this.internalServiceDecorator.buildAccompanyingKubernetesResources();
	assertEquals(0, resources.size());
}
 

@Nonnull
private static JobID createJobIdForCluster(Configuration globalConfiguration) {
  if (HighAvailabilityMode.isHighAvailabilityModeActivated(globalConfiguration)) {
    return ZERO_JOB_ID;
  } else {
    return JobID.generate();
  }
}
 

@Nonnull
private static JobID createJobIdForCluster(Configuration globalConfiguration) {
  if (HighAvailabilityMode.isHighAvailabilityModeActivated(globalConfiguration)) {
    return ZERO_JOB_ID;
  } else {
    return JobID.generate();
  }
}
 

@Nonnull
private static JobID createJobIdForCluster(Configuration globalConfiguration) {
	if (HighAvailabilityMode.isHighAvailabilityModeActivated(globalConfiguration)) {
		return ZERO_JOB_ID;
	} else {
		return JobID.generate();
	}
}
 

@Nonnull
private static JobID createJobIdForCluster(Configuration globalConfiguration) {
	if (HighAvailabilityMode.isHighAvailabilityModeActivated(globalConfiguration)) {
		return ZERO_JOB_ID;
	} else {
		return JobID.generate();
	}
}
 

@Test(expected = Exception.class)
public void testCustomHAServicesFactoryNotDefined() throws Exception {
	Configuration config = new Configuration();

	Executor executor = Mockito.mock(Executor.class);

	config.setString(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.FACTORY_CLASS.name().toLowerCase());

	// expect
	HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(config, executor);
}
 

/**
 * Creates a {@link MesosServices} instance depending on the high availability settings.
 *
 * @param configuration containing the high availability settings
 * @param hostname the hostname to advertise to remote clients
 * @return a mesos services instance
 * @throws Exception if the mesos services instance could not be created
 */
public static MesosServices createMesosServices(Configuration configuration, String hostname) throws Exception {

	ActorSystem localActorSystem = AkkaUtils.createLocalActorSystem(configuration);

	MesosArtifactServer artifactServer = createArtifactServer(configuration, hostname);

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			return new StandaloneMesosServices(localActorSystem, artifactServer);

		case ZOOKEEPER:
			final String zkMesosRootPath = configuration.getString(
				HighAvailabilityOptions.HA_ZOOKEEPER_MESOS_WORKERS_PATH);

			ZooKeeperUtilityFactory zooKeeperUtilityFactory = new ZooKeeperUtilityFactory(
				configuration,
				zkMesosRootPath);

			return new ZooKeeperMesosServices(localActorSystem, artifactServer, zooKeeperUtilityFactory);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 

/**
 * Creates a {@link MesosServices} instance depending on the high availability settings.
 *
 * @param configuration containing the high availability settings
 * @param hostname the hostname to advertise to remote clients
 * @return a mesos services instance
 * @throws Exception if the mesos services instance could not be created
 */
public static MesosServices createMesosServices(Configuration configuration, String hostname) throws Exception {

	ActorSystem localActorSystem = AkkaUtils.createLocalActorSystem(configuration);

	MesosArtifactServer artifactServer = createArtifactServer(configuration, hostname);

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			return new StandaloneMesosServices(localActorSystem, artifactServer);

		case ZOOKEEPER:
			final String zkMesosRootPath = configuration.getString(
				HighAvailabilityOptions.HA_ZOOKEEPER_MESOS_WORKERS_PATH);

			ZooKeeperUtilityFactory zooKeeperUtilityFactory = new ZooKeeperUtilityFactory(
				configuration,
				zkMesosRootPath);

			return new ZooKeeperMesosServices(localActorSystem, artifactServer, zooKeeperUtilityFactory);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 

@Test(expected = Exception.class)
public void testCustomHAServicesFactoryNotDefined() throws Exception {
	Configuration config = new Configuration();

	Executor executor = Mockito.mock(Executor.class);

	config.setString(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.FACTORY_CLASS.name().toLowerCase());

	// expect
	HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(config, executor);
}
 

/**
 * Returns whether {@link HighAvailabilityMode#ZOOKEEPER} is configured.
 */
public static boolean isZooKeeperRecoveryMode(Configuration flinkConf) {
	return HighAvailabilityMode.fromConfig(flinkConf).equals(HighAvailabilityMode.ZOOKEEPER);
}
 

/**
 * Returns whether {@link HighAvailabilityMode#ZOOKEEPER} is configured.
 */
public static boolean isZooKeeperRecoveryMode(Configuration flinkConf) {
	return HighAvailabilityMode.fromConfig(flinkConf).equals(HighAvailabilityMode.ZOOKEEPER);
}
 

public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				JobMaster.JOB_MANAGER_NAME,
				addressResolution,
				configuration);
			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				ResourceManager.RESOURCE_MANAGER_NAME,
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				Dispatcher.DISPATCHER_NAME,
				addressResolution,
				configuration);

			final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
				"%s must be set",
				RestOptions.ADDRESS.key());
			final int port = configuration.getInteger(RestOptions.PORT);
			final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
			final String protocol = enableSSL ? "https://" : "http://";

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				jobManagerRpcUrl,
				String.format("%s%s:%s", protocol, address, port));
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}
 

public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				AkkaRpcServiceUtils.createWildcardName(ResourceManager.RESOURCE_MANAGER_NAME),
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				AkkaRpcServiceUtils.createWildcardName(Dispatcher.DISPATCHER_NAME),
				addressResolution,
				configuration);
			final String webMonitorAddress = getWebMonitorAddress(
				configuration,
				addressResolution);

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				webMonitorAddress);
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}