Java Code Examples for org.apache.flink.configuration.Configuration#set()

@Test
public void testConfigurationWithDeprecatedOptions() {
	final Configuration configuration = new Configuration();

	final Duration refusedRegistrationBackoff = Duration.ofMinutes(42L);
	final Duration registrationMaxBackoff = Duration.ofSeconds(1L);
	final Duration initialRegistrationBackoff = Duration.ofHours(1337L);

	configuration.set(TaskManagerOptions.REFUSED_REGISTRATION_BACKOFF, refusedRegistrationBackoff);
	configuration.set(TaskManagerOptions.REGISTRATION_MAX_BACKOFF, registrationMaxBackoff);
	configuration.set(TaskManagerOptions.INITIAL_REGISTRATION_BACKOFF, initialRegistrationBackoff);

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	assertThat(retryingRegistrationConfiguration.getInitialRegistrationTimeoutMillis(), is(ClusterOptions.INITIAL_REGISTRATION_TIMEOUT.defaultValue()));
	assertThat(retryingRegistrationConfiguration.getRefusedDelayMillis(), is(ClusterOptions.REFUSED_REGISTRATION_DELAY.defaultValue()));
	assertThat(retryingRegistrationConfiguration.getMaxRegistrationTimeoutMillis(), is(ClusterOptions.MAX_REGISTRATION_TIMEOUT.defaultValue()));
}
 

@Test
public void testParsePythonFiles() {
	// Skip this test on Windows as we can not control the Window Driver letters.
	Assume.assumeFalse(OperatingSystem.isWindows());

	Configuration config = new Configuration();
	Map<String, String> pythonFiles = new HashMap<>();
	pythonFiles.put("python_file_{SHA256_0}", "test_file1.py");
	pythonFiles.put("python_file_{SHA256_1}", "test_file2.py");
	config.set(PythonDependencyUtils.PYTHON_FILES, pythonFiles);
	PythonDependencyInfo dependencyInfo = PythonDependencyInfo.create(new PythonConfig(config), distributedCache);

	Map<String, String> expected = new HashMap<>();
	expected.put("/distributed_cache/file0", "test_file1.py");
	expected.put("/distributed_cache/file1", "test_file2.py");
	assertEquals(expected, dependencyInfo.getPythonFiles());
}
 

@BeforeClass
public static void setUp() throws Exception {

	Configuration config = new Configuration();

	// we need to use the "filesystem" state backend to ensure FLINK-2543 is not happening again.
	config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// Savepoint path
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// required as we otherwise run out of memory
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("80m"));

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(2)
			.setNumberSlotsPerTaskManager(2)
			.setConfiguration(config)
			.build());

	miniClusterResource.before();
}
 

private Configuration getConfiguration() throws Exception {
	// Flink configuration
	final Configuration config = new Configuration();

	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, DEFAULT_PARALLELISM);

	UUID id = UUID.randomUUID();
	final File checkpointDir = TEMP_FOLDER.newFolder("checkpoints_" + id).getAbsoluteFile();
	final File savepointDir = TEMP_FOLDER.newFolder("savepoints_" + id).getAbsoluteFile();

	if (!checkpointDir.exists() || !savepointDir.exists()) {
		throw new Exception("Test setup failed: failed to create (temporary) directories.");
	}

	LOG.info("Created temporary checkpoint directory: " + checkpointDir + ".");
	LOG.info("Created savepoint directory: " + savepointDir + ".");

	config.setString(CheckpointingOptions.STATE_BACKEND, "memory");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
	config.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, MemorySize.ZERO);
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());
	config.setLong(HeartbeatManagerOptions.HEARTBEAT_INTERVAL, 300L);

	return config;
}
 

@Test
public void testOnlyOneJobAllowedWithStaticJobIdAndHa() throws Throwable {
	final JobID testJobID = new JobID(0, 2);

	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, testJobID.toHexString());
	configurationUnderTest.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final CompletableFuture<Void> applicationFuture =
			runApplication(configurationUnderTest, 2);

	assertException(applicationFuture, FlinkRuntimeException.class);
}
 

/**
 * Verify that we pick the first valid security context.
 */
@Test
public void testSecurityContextShouldPickFirstIfBothCompatible() throws Exception {
	Configuration testFlinkConf = new Configuration();

	testFlinkConf.set(
			SecurityOptions.SECURITY_CONTEXT_FACTORY_CLASSES,
			Lists.newArrayList(
					AnotherCompatibleTestSecurityContextFactory.class.getCanonicalName(),
					TestSecurityContextFactory.class.getCanonicalName()));

	SecurityConfiguration testSecurityConf = new SecurityConfiguration(testFlinkConf);

	SecurityUtils.install(testSecurityConf);
	assertEquals(
			AnotherCompatibleTestSecurityContextFactory.TestSecurityContext.class,
			SecurityUtils.getInstalledContext().getClass());

	SecurityUtils.uninstall();
	assertEquals(NoOpSecurityContext.class, SecurityUtils.getInstalledContext().getClass());

	testFlinkConf.set(
			SecurityOptions.SECURITY_CONTEXT_FACTORY_CLASSES,
			Lists.newArrayList(
					TestSecurityContextFactory.class.getCanonicalName(),
					AnotherCompatibleTestSecurityContextFactory.class.getCanonicalName()));

	testSecurityConf = new SecurityConfiguration(testFlinkConf);

	SecurityUtils.install(testSecurityConf);
	assertEquals(
			TestSecurityContextFactory.TestSecurityContext.class,
			SecurityUtils.getInstalledContext().getClass());

	SecurityUtils.uninstall();
	assertEquals(NoOpSecurityContext.class, SecurityUtils.getInstalledContext().getClass());
}
 

private static Configuration getConfiguration() {
	verifyJvmOptions();
	Configuration config = new Configuration();
	config.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
	config.setString(AkkaOptions.ASK_TIMEOUT, TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT());
	config.set(TaskManagerOptions.MEMORY_SEGMENT_SIZE, MemorySize.parse("4096"));
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 2048);

	return config;
}
 

@Test
public void testPythonExec() {
	Configuration config = new Configuration();
	config.set(PythonOptions.PYTHON_EXECUTABLE, "/usr/local/bin/python3");
	PythonConfig pythonConfig = new PythonConfig(config);
	assertThat(pythonConfig.getPythonExec(), is(equalTo("/usr/local/bin/python3")));
}
 

private static Configuration getConfig() {
	Configuration config = new Configuration();
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
	config.setBoolean(WebOptions.SUBMIT_ENABLE, false);
	return config;
}
 

@Override
protected void configWithFineGrainedOptions(Configuration configuration, MemorySize totalFlinkMemorySize) {
	MemorySize heapSize = new MemorySize(totalFlinkMemorySize.getBytes() / 2);
	MemorySize offHeapSize = totalFlinkMemorySize.subtract(heapSize);
	configuration.set(JobManagerOptions.JVM_HEAP_MEMORY, heapSize);
	configuration.set(JobManagerOptions.OFF_HEAP_MEMORY, offHeapSize);
}
 

private static void adjustNetworkMemoryForLocalExecution(Configuration config) {
	if (!config.contains(TaskManagerOptions.NETWORK_MEMORY_MIN) &&
		config.contains(TaskManagerOptions.NETWORK_MEMORY_MAX)) {
		config.set(TaskManagerOptions.NETWORK_MEMORY_MIN, config.get(TaskManagerOptions.NETWORK_MEMORY_MAX));
	}
	if (!config.contains(TaskManagerOptions.NETWORK_MEMORY_MAX) &&
		config.contains(TaskManagerOptions.NETWORK_MEMORY_MIN)) {
		config.set(TaskManagerOptions.NETWORK_MEMORY_MAX, config.get(TaskManagerOptions.NETWORK_MEMORY_MIN));
	}
	setConfigOptionToDefaultIfNotSet(config, TaskManagerOptions.NETWORK_MEMORY_MIN, DEFAULT_SHUFFLE_MEMORY_SIZE);
	setConfigOptionToDefaultIfNotSet(config, TaskManagerOptions.NETWORK_MEMORY_MAX, DEFAULT_SHUFFLE_MEMORY_SIZE);
}
 

/**
 * Tests that the TaskManagerRunner startup fails synchronously when the memory configuration is wrong.
 */
@Test(expected = IllegalConfigurationException.class)
public void testMemoryConfigWrong() throws Exception {
	Configuration cfg = createFlinkConfiguration();

	// something invalid
	cfg.set(TaskManagerOptions.NETWORK_MEMORY_MIN, MemorySize.parse("100m"));
	cfg.set(TaskManagerOptions.NETWORK_MEMORY_MAX, MemorySize.parse("10m"));
	startTaskManager(
		cfg,
		rpcService,
		highAvailabilityServices);
}
 

@Test
public void testConfigNetworkMemoryRangeFailure() {
	final MemorySize networkMin = MemorySize.parse("200m");
	final MemorySize networkMax = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MAX, networkMax);
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MIN, networkMin);

	validateFailInAllConfigurations(conf);
}
 

@Test
public void testConfigTotalProcessMemorySize() {
	MemorySize totalProcessMemorySize = MemorySize.parse("2g");

	Configuration conf = new Configuration();
	conf.set(options.getTotalProcessMemoryOption(), totalProcessMemorySize);

	T processSpec = processSpecFromConfig(conf);
	assertThat(processSpec.getTotalProcessMemorySize(), is(totalProcessMemorySize));
}
 

@Test
public void testConfigTaskOffheapMemory() {
	final MemorySize taskOffHeapSize = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.TASK_OFF_HEAP_MEMORY, taskOffHeapSize);

	validateInAllConfigurations(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getTaskOffHeapSize(), is(taskOffHeapSize)));
}
 

@Test
public void testConfigFrameworkOffHeapMemory() {
	final MemorySize frameworkOffHeapSize = MemorySize.parse("10m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.FRAMEWORK_OFF_HEAP_MEMORY, frameworkOffHeapSize);

	validateInAllConfigurations(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getFrameworkOffHeapMemorySize(), is(frameworkOffHeapSize)));
}
 

@Test
public void testPythonFrameworkMemorySize() {
	Configuration config = new Configuration();
	config.set(PythonOptions.PYTHON_FRAMEWORK_MEMORY_SIZE, "100mb");
	PythonConfig pythonConfig = new PythonConfig(config);
	assertThat(pythonConfig.getPythonFrameworkMemorySize(), is(equalTo("100mb")));
}
 

@Test
public void testConfigJvmHeapMemory() {
	MemorySize jvmHeapSize = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeapSize);

	JobManagerProcessSpec jobManagerProcessSpec = JobManagerProcessUtils.processSpecFromConfig(conf);
	assertThat(jobManagerProcessSpec.getJvmHeapMemorySize(), is(jvmHeapSize));
}
 

@Test
public void testConfigBothNewOptionAndLegacyHeapSize() {
	MemorySize newOptionValue = MemorySize.parse("1g");
	MemorySize legacyHeapSize = MemorySize.parse("2g");

	Configuration conf = new Configuration();
	conf.set(getNewOptionForLegacyHeapOption(), newOptionValue);
	conf.set(legacyMemoryOptions.getHeap(), legacyHeapSize);

	testConfigLegacyHeapMemory(conf, newOptionValue);
}
 

public static Configuration getConfiguration() {
	Configuration config = new Configuration();
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("12m"));
	return config;
}