Java Code Examples for org.apache.flink.configuration.Configuration#getOptional()

@Test
public void testPythonFiles() {
	final Configuration configuration = new Configuration();
	final Optional<String> defaultPythonFiles = configuration.getOptional(PythonOptions.PYTHON_FILES);
	assertThat(defaultPythonFiles, is(equalTo(Optional.empty())));

	final String expectedPythonFiles = "tmp_dir/test1.py,tmp_dir/test2.py";
	configuration.set(PythonOptions.PYTHON_FILES, expectedPythonFiles);

	final String actualPythonFiles = configuration.get(PythonOptions.PYTHON_FILES);
	assertThat(actualPythonFiles, is(equalTo(expectedPythonFiles)));
}
 

@Test
public void testPythonRequirements() {
	final Configuration configuration = new Configuration();
	final Optional<String> defaultPythonRequirements = configuration.getOptional(PythonOptions.PYTHON_REQUIREMENTS);
	assertThat(defaultPythonRequirements, is(equalTo(Optional.empty())));

	final String expectedPythonRequirements = "tmp_dir/requirements.txt#tmp_dir/cache";
	configuration.set(PythonOptions.PYTHON_REQUIREMENTS, expectedPythonRequirements);

	final String actualPythonRequirements = configuration.get(PythonOptions.PYTHON_REQUIREMENTS);
	assertThat(actualPythonRequirements, is(equalTo(expectedPythonRequirements)));
}
 

@Test
public void testPythonArchives() {
	final Configuration configuration = new Configuration();
	final Optional<String> defaultPythonArchives = configuration.getOptional(PythonOptions.PYTHON_ARCHIVES);
	assertThat(defaultPythonArchives, is(equalTo(Optional.empty())));

	final String expectedPythonArchives = "tmp_dir/py37.zip#venv,tmp_dir/data.zip";
	configuration.set(PythonOptions.PYTHON_ARCHIVES, expectedPythonArchives);

	final String actualPythonArchives = configuration.get(PythonOptions.PYTHON_ARCHIVES);
	assertThat(actualPythonArchives, is(equalTo(expectedPythonArchives)));
}
 

@Test
public void testPythonExecutable() {
	final Configuration configuration = new Configuration();
	final Optional<String> defaultPythonExecutable = configuration.getOptional(PythonOptions.PYTHON_EXECUTABLE);
	assertThat(defaultPythonExecutable, is(equalTo(Optional.empty())));

	final String expectedPythonExecutable = "venv/py37/bin/python";
	configuration.set(PythonOptions.PYTHON_EXECUTABLE, expectedPythonExecutable);

	final String actualPythonExecutable = configuration.get(PythonOptions.PYTHON_EXECUTABLE);
	assertThat(actualPythonExecutable, is(equalTo(expectedPythonExecutable)));
}
 

@Test
public void testPythonClientExecutable() {
	final Configuration configuration = new Configuration();
	final Optional<String> defaultPythonClientExecutable =
		configuration.getOptional(PythonOptions.PYTHON_CLIENT_EXECUTABLE);
	assertThat(defaultPythonClientExecutable, is(equalTo(Optional.empty())));

	final String expectedPythonClientExecutable = "tmp_dir/test1.py,tmp_dir/test2.py";
	configuration.set(PythonOptions.PYTHON_CLIENT_EXECUTABLE, expectedPythonClientExecutable);

	final String actualPythonClientExecutable = configuration.get(PythonOptions.PYTHON_CLIENT_EXECUTABLE);
	assertThat(actualPythonClientExecutable, is(equalTo(expectedPythonClientExecutable)));
}
 

/**
 * Get the map of resource name and amount of all of enabled external resources.
 */
public static Map<String, Long> getExternalResourceAmountMap(Configuration config) {
	final Set<String> resourceSet = getExternalResourceSet(config);
	LOG.info("Enabled external resources: {}", resourceSet);

	if (resourceSet.isEmpty()) {
		return Collections.emptyMap();
	}

	final Map<String, Long> externalResourceAmountMap = new HashMap<>();
	for (String resourceName: resourceSet) {
		final ConfigOption<Long> amountOption =
			key(ExternalResourceOptions.getAmountConfigOptionForResource(resourceName))
				.longType()
				.noDefaultValue();
		final Optional<Long> amountOpt = config.getOptional(amountOption);
		if (!amountOpt.isPresent()) {
			LOG.warn("The amount of the {} should be configured. Will ignore that resource.", resourceName);
		} else if (amountOpt.get() <= 0) {
			LOG.warn("The amount of the {} should be positive while finding {}. Will ignore that resource.", amountOpt.get(), resourceName);
		} else {
			externalResourceAmountMap.put(resourceName, amountOpt.get());
		}
	}

	return externalResourceAmountMap;
}
 

/**
 * Get the external resources map. The key should be used for deployment specific container request,
 * and values should be the amount of that resource.
 *
 * @param config Configurations
 * @param suffix suffix of config option for deployment specific configuration key
 * @return external resources map, map the amount to the configuration key for deployment specific container request
 */
public static Map<String, Long> getExternalResources(Configuration config, String suffix) {
	final Set<String> resourceSet = getExternalResourceSet(config);
	LOG.info("Enabled external resources: {}", resourceSet);

	if (resourceSet.isEmpty()) {
		return Collections.emptyMap();
	}

	final Map<String, Long> externalResourceConfigs = new HashMap<>();
	for (String resourceName: resourceSet) {
		final ConfigOption<Long> amountOption =
			key(ExternalResourceOptions.getAmountConfigOptionForResource(resourceName))
				.longType()
				.noDefaultValue();
		final ConfigOption<String> configKeyOption =
			key(ExternalResourceOptions.getSystemConfigKeyConfigOptionForResource(resourceName, suffix))
				.stringType()
				.noDefaultValue();
		final String configKey = config.getString(configKeyOption);
		final Optional<Long> amountOpt = config.getOptional(amountOption);

		if (StringUtils.isNullOrWhitespaceOnly(configKey)) {
			LOG.warn("Could not find valid {} for {}. Will ignore that resource.", configKeyOption.key(), resourceName);
			continue;
		}
		if (!amountOpt.isPresent()) {
			LOG.warn("The amount of the {} should be configured. Will ignore that resource.", resourceName);
			continue;
		} else if (amountOpt.get() <= 0) {
			LOG.warn("The amount of the {} should be positive while finding {}. Will ignore that resource.", amountOpt.get(), resourceName);
			continue;
		}

		if (externalResourceConfigs.put(configKey, amountOpt.get()) != null) {
			LOG.warn("Duplicate config key {} occurred for external resources, the one named {} with amount {} will overwrite the value.", configKey, resourceName, amountOpt);
		} else {
			LOG.info("Add external resource config for {} with key {} value {}.", resourceName, configKey, amountOpt);
		}
	}

	return externalResourceConfigs;
}