org.apache.flink.configuration.HighAvailabilityOptions Java Examples

/**
 * A test where the connection fails twice and then the get operation succeeds
 * (with high availability set, job-related job).
 */
@Test
public void testBlobFetchRetriesHa() throws IOException {
	final Configuration config = new Configuration();
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH,
		temporaryFolder.newFolder().getPath());

	BlobStoreService blobStoreService = null;

	try {
		blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

		testBlobFetchRetries(config, blobStoreService, new JobID(), PERMANENT_BLOB);
	} finally {
		if (blobStoreService != null) {
			blobStoreService.closeAndCleanupAllData();
		}
	}
}
 

/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed corrupted JARs are
 * recognised during the download via a BLOB cache.
 */
@Test
public void testBlobCacheCorruptedFile() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobCacheCorruptionTest
			.testGetFailsFromCorruptFile(new JobID(), config, blobStoreService, exception);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
 

/**
 * Override configuration settings by specified command line options.
 *
 * @param commandLine containing the overriding values
 * @return Effective configuration with the overridden configuration settings
 */
protected Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) throws FlinkException {
	final Configuration resultingConfiguration = new Configuration(configuration);

	if (commandLine.hasOption(addressOption.getOpt())) {
		String addressWithPort = commandLine.getOptionValue(addressOption.getOpt());
		InetSocketAddress jobManagerAddress = ClientUtils.parseHostPortAddress(addressWithPort);
		setJobManagerAddressInConfig(resultingConfiguration, jobManagerAddress);
	}

	if (commandLine.hasOption(zookeeperNamespaceOption.getOpt())) {
		String zkNamespace = commandLine.getOptionValue(zookeeperNamespaceOption.getOpt());
		resultingConfiguration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zkNamespace);
	}

	return resultingConfiguration;
}
 

private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}
 

private static HighAvailabilityServices createCustomHAServices(Configuration config, Executor executor) throws FlinkException {
	final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
	final String haServicesClassName = config.getString(HighAvailabilityOptions.HA_MODE);

	final HighAvailabilityServicesFactory highAvailabilityServicesFactory = InstantiationUtil.instantiate(
		haServicesClassName,
		HighAvailabilityServicesFactory.class,
		classLoader);

	try {
		return highAvailabilityServicesFactory.createHAServices(config, executor);
	} catch (Exception e) {
		throw new FlinkException(
			String.format(
				"Could not create the ha services from the instantiated HighAvailabilityServicesFactory %s.",
				haServicesClassName),
			e);
	}
}
 

/**
 * Checks the GET operation fails when the downloaded file (from {@link BlobServer} or HA store)
 * is corrupt, i.e. its content's hash does not match the {@link BlobKey}'s hash.
 */
@Test
public void testGetFailsFromCorruptFile() throws IOException {

	final Configuration config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getPath());

	BlobStoreService blobStoreService = null;

	try {
		blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

		testGetFailsFromCorruptFile(config, blobStoreService, exception);
	} finally {
		if (blobStoreService != null) {
			blobStoreService.closeAndCleanupAllData();
		}
	}
}
 

/**
 * Checks the GET operation fails when the downloaded file (from {@link BlobServer} or HA store)
 * is corrupt, i.e. its content's hash does not match the {@link BlobKey}'s hash.
 *
 * @param jobId
 * 		job ID or <tt>null</tt> if job-unrelated
 * @param blobType
 * 		whether the BLOB should become permanent or transient
 * @param corruptOnHAStore
 * 		whether the file should be corrupt in the HA store (<tt>true</tt>, required
 * 		<tt>highAvailability</tt> to be set) or on the {@link BlobServer}'s local store
 * 		(<tt>false</tt>)
 */
private void testGetFailsFromCorruptFile(final JobID jobId, BlobKey.BlobType blobType,
		boolean corruptOnHAStore) throws IOException {

	final Configuration config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getPath());

	BlobStoreService blobStoreService = null;

	try {
		blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

		testGetFailsFromCorruptFile(jobId, blobType, corruptOnHAStore, config,
			blobStoreService, exception);
	} finally {
		if (blobStoreService != null) {
			blobStoreService.closeAndCleanupAllData();
		}
	}
}
 

@Test
public void testResumeFromYarnIDZookeeperNamespaceOverride() throws Exception {
	final Configuration configuration = new Configuration();
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final String overrideZkNamespace = "my_cluster";

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true);

	final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	final String clusterId = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertEquals(overrideZkNamespace, clusterId);
}
 

/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer when talking to the {@link org.apache.flink.runtime.blob.BlobServer} directly.
 */
@Test
public void testBlobServerRecovery() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobServerRecoveryTest.testBlobServerRecovery(config, blobStoreService);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
 

/**
 * Tests that checkpoints are discarded when the completed checkpoint store is shut
 * down with a globally terminal state.
 */
@Test
public void testDiscardingCheckpointsAtShutDown() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		checkpointStore.shutdown(JobStatus.FINISHED);

		// verify that the checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 

/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer when uploaded via a BLOB cache.
 */
@Test
public void testBlobCacheRecovery() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobCacheRecoveryTest.testBlobCacheRecovery(config, blobStoreService);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
 

@Test
public void testJobIdDefaultsToZeroWithHa() throws Throwable {
	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final CompletableFuture<JobID> submittedJobId = new CompletableFuture<>();

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> {
				submittedJobId.complete(jobGraph.getJobID());
				return CompletableFuture.completedFuture(Acknowledge.get());
			})
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.FINISHED))
			.setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createSuccessfulJobResult(jobId)));

	final CompletableFuture<Void> applicationFuture =
			runApplication(dispatcherBuilder, configurationUnderTest, 1);

	applicationFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	assertThat(submittedJobId.get(TIMEOUT_SECONDS, TimeUnit.SECONDS), is(new JobID(0L, 0L)));
}
 

/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer when talking to the {@link org.apache.flink.runtime.blob.BlobServer} directly.
 */
@Test
public void testBlobServerRecovery() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobServerRecoveryTest.testBlobServerRecovery(config, blobStoreService);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
 

/**
 * Tests HA mode configuration.
 */
@Test
public void testFromConfig() throws Exception {
	Configuration config = new Configuration();

	// Check default
	assertEquals(DEFAULT_HA_MODE, HighAvailabilityMode.fromConfig(config));

	// Check not equals default
	config.setString(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name().toLowerCase());
	assertEquals(HighAvailabilityMode.ZOOKEEPER, HighAvailabilityMode.fromConfig(config));

	// Check factory class
	config.setString(HighAvailabilityOptions.HA_MODE, "factory.class.FQN");
	assertEquals(HighAvailabilityMode.FACTORY_CLASS, HighAvailabilityMode.fromConfig(config));
}
 

/**
 * Tests that checkpoints are discarded when the completed checkpoint store is shut
 * down with a globally terminal state.
 */
@Test
public void testDiscardingCheckpointsAtShutDown() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		checkpointStore.shutdown(JobStatus.FINISHED);

		// verify that the checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 

@Test
public void testResumeFromYarnIDZookeeperNamespaceOverride() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();

	final String overrideZkNamespace = "my_cluster";

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true);
	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);

	final YarnClusterDescriptor clusterDescriptor = (YarnClusterDescriptor) clientFactory.createClusterDescriptor(executorConfig);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();
	final String clusterId = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertEquals(overrideZkNamespace, clusterId);
}
 

@Before
public void before() throws Exception {
	testingServer = new TestingServer();

	config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
	config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());

	CuratorFramework client = ZooKeeperUtils.startCuratorFramework(config);

	highAvailabilityServices = new ZooKeeperHaServices(
		client,
		TestingUtils.defaultExecutor(),
		config,
		new VoidBlobStore());
}
 

@Nonnull
private YarnClusterDescriptor setupYarnClusterDescriptor() {
	final Configuration flinkConfiguration = new Configuration();
	flinkConfiguration.setString(YarnConfigOptions.APPLICATION_ATTEMPTS, "10");
	flinkConfiguration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
	flinkConfiguration.setString(HighAvailabilityOptions.HA_STORAGE_PATH, storageDir);
	flinkConfiguration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
	flinkConfiguration.setInteger(HighAvailabilityOptions.ZOOKEEPER_SESSION_TIMEOUT, 1000);

	flinkConfiguration.setString(ConfigConstants.RESTART_STRATEGY, "fixed-delay");
	flinkConfiguration.setInteger(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, Integer.MAX_VALUE);

	final int minMemory = 100;
	flinkConfiguration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, minMemory);

	return createYarnClusterDescriptor(flinkConfiguration);
}
 

/**
 * Override configuration settings by specified command line options.
 *
 * @param commandLine containing the overriding values
 * @return Effective configuration with the overridden configuration settings
 */
protected Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) throws FlinkException {
	final Configuration resultingConfiguration = new Configuration(configuration);

	if (commandLine.hasOption(addressOption.getOpt())) {
		String addressWithPort = commandLine.getOptionValue(addressOption.getOpt());
		InetSocketAddress jobManagerAddress = ClientUtils.parseHostPortAddress(addressWithPort);
		setJobManagerAddressInConfig(resultingConfiguration, jobManagerAddress);
	}

	if (commandLine.hasOption(zookeeperNamespaceOption.getOpt())) {
		String zkNamespace = commandLine.getOptionValue(zookeeperNamespaceOption.getOpt());
		resultingConfiguration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zkNamespace);
	}

	return resultingConfiguration;
}
 

/**
 * Tests HA mode configuration.
 */
@Test
public void testFromConfig() throws Exception {
	Configuration config = new Configuration();

	// Check default
	assertEquals(DEFAULT_HA_MODE, HighAvailabilityMode.fromConfig(config));

	// Check not equals default
	config.setString(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name().toLowerCase());
	assertEquals(HighAvailabilityMode.ZOOKEEPER, HighAvailabilityMode.fromConfig(config));

	// Check factory class
	config.setString(HighAvailabilityOptions.HA_MODE, "factory.class.FQN");
	assertEquals(HighAvailabilityMode.FACTORY_CLASS, HighAvailabilityMode.fromConfig(config));
}
 

private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}
 

/**
 * Checks the GET operation fails when the downloaded file (from {@link BlobServer} or HA store)
 * is corrupt, i.e. its content's hash does not match the {@link BlobKey}'s hash.
 */
@Test
public void testGetFailsFromCorruptFile() throws IOException {

	final Configuration config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getPath());

	BlobStoreService blobStoreService = null;

	try {
		blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

		testGetFailsFromCorruptFile(config, blobStoreService, exception);
	} finally {
		if (blobStoreService != null) {
			blobStoreService.closeAndCleanupAllData();
		}
	}
}
 

@Test
public void testCreateCustomHAServices() throws Exception {
	Configuration config = new Configuration();

	HighAvailabilityServices haServices = Mockito.mock(HighAvailabilityServices.class);
	TestHAFactory.haServices = haServices;

	Executor executor = Mockito.mock(Executor.class);

	config.setString(HighAvailabilityOptions.HA_MODE, TestHAFactory.class.getName());

	// when
	HighAvailabilityServices actualHaServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(config, executor);

	// then
	assertSame(haServices, actualHaServices);

	// when
	actualHaServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(config, executor,
		HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
	// then
	assertSame(haServices, actualHaServices);
}
 

/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer.
 */
@Test
public void testBlobCacheRecovery() throws Exception {
	Configuration config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getPath());

	BlobStoreService blobStoreService = null;

	try {
		blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

		testBlobCacheRecovery(config, blobStoreService);
	} finally {
		if (blobStoreService != null) {
			blobStoreService.closeAndCleanupAllData();
		}
	}
}
 

/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer when talking to the {@link org.apache.flink.runtime.blob.BlobServer} directly.
 */
@Test
public void testBlobServerRecovery() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobServerRecoveryTest.testBlobServerRecovery(config, blobStoreService);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
 

/**
 * Checks the GET operation fails when the downloaded file (from {@link BlobServer} or HA store)
 * is corrupt, i.e. its content's hash does not match the {@link BlobKey}'s hash.
 *
 * @param jobId
 * 		job ID or <tt>null</tt> if job-unrelated
 * @param blobType
 * 		whether the BLOB should become permanent or transient
 * @param corruptOnHAStore
 * 		whether the file should be corrupt in the HA store (<tt>true</tt>, required
 * 		<tt>highAvailability</tt> to be set) or on the {@link BlobServer}'s local store
 * 		(<tt>false</tt>)
 */
private void testGetFailsFromCorruptFile(final JobID jobId, BlobKey.BlobType blobType,
		boolean corruptOnHAStore) throws IOException {

	final Configuration config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getPath());

	BlobStoreService blobStoreService = null;

	try {
		blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

		testGetFailsFromCorruptFile(jobId, blobType, corruptOnHAStore, config,
			blobStoreService, exception);
	} finally {
		if (blobStoreService != null) {
			blobStoreService.closeAndCleanupAllData();
		}
	}
}
 

/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer.
 */
@Test
public void testBlobCacheRecovery() throws Exception {
	Configuration config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getPath());

	BlobStoreService blobStoreService = null;

	try {
		blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

		testBlobCacheRecovery(config, blobStoreService);
	} finally {
		if (blobStoreService != null) {
			blobStoreService.closeAndCleanupAllData();
		}
	}
}
 

@Before
public void setup() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, true);
	configuration.setString(HighAvailabilityOptions.HA_MODE, TestingHAFactory.class.getName());

	checkpointPath = new Path(TEMPORARY_FOLDER.newFolder().toURI());
	cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(configuration)
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(1).build());
	cluster.before();

	NormalMap.reset();
	DeclineSink.reset();
	TestingCompletedCheckpointStore.reset();
}
 

private static Configuration startSecureFlinkClusterWithRecoveryModeEnabled() {
	try {
		LOG.info("Starting Flink and ZK in secure mode");

		dfs.mkdirs(new Path("/flink/checkpoints"));
		dfs.mkdirs(new Path("/flink/recovery"));

		final Configuration result = new Configuration();

		result.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, false);
		result.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, 3);
		result.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
		result.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
		result.setString(HighAvailabilityOptions.HA_ZOOKEEPER_CHECKPOINTS_PATH, hdfsURI + "/flink/checkpoints");
		result.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI + "/flink/recovery");
		result.setString("state.backend.fs.checkpointdir", hdfsURI + "/flink/checkpoints");

		SecureTestEnvironment.populateFlinkSecureConfigurations(result);

		return result;
	} catch (Exception e) {
		throw new RuntimeException(e);
	}
}
 

/**
 * Creates a {@link ZooKeeperLeaderElectionService} instance.
 *
 * @param client        The {@link CuratorFramework} ZooKeeper client to use
 * @param configuration {@link Configuration} object containing the configuration values
 * @param pathSuffix    The path suffix which we want to append
 * @return {@link ZooKeeperLeaderElectionService} instance.
 */
public static ZooKeeperLeaderElectionService createLeaderElectionService(
		final CuratorFramework client,
		final Configuration configuration,
		final String pathSuffix) {
	final String latchPath = configuration.getString(
		HighAvailabilityOptions.HA_ZOOKEEPER_LATCH_PATH) + pathSuffix;
	final String leaderPath = configuration.getString(
		HighAvailabilityOptions.HA_ZOOKEEPER_LEADER_PATH) + pathSuffix;

	return new ZooKeeperLeaderElectionService(client, latchPath, leaderPath);
}