Java Code Examples for org.apache.flink.configuration.Configuration#removeConfig()

/**
 * Returns a flink configuration object with the given values.
 *
 * @param javaMemMB
 * 		total JVM memory to use (in megabytes)
 * @param useOffHeap
 * 		whether to use off-heap memory (<tt>true</tt>) or not (<tt>false</tt>)
 * @param netBufMemFrac
 * 		fraction of JVM memory to use for network buffers
 * @param netBufMemMin
 * 		minimum memory size for network buffers (in bytes)
 * @param netBufMemMax
 * 		maximum memory size for network buffers (in bytes)
 * @param managedMemSizeMB
 * 		amount of managed memory (in megabytes)
 * @param managedMemFrac
 * 		fraction of free memory to use for managed memory (if <tt>managedMemSizeMB</tt> is
 * 		<tt>-1</tt>)
 *
 * @return flink configuration
 */
private static Configuration getConfig(
		final int javaMemMB, final boolean useOffHeap, final float netBufMemFrac,
		final long netBufMemMin, final long netBufMemMax, final int managedMemSizeMB,
		final float managedMemFrac) {

	Configuration config = new Configuration();

	config.setLong(KEY_TASKM_MEM_SIZE, javaMemMB);
	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, useOffHeap);

	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, netBufMemFrac);
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(netBufMemMin));
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(netBufMemMax));

	if (managedMemSizeMB == 0) {
		config.removeConfig(TaskManagerOptions.MANAGED_MEMORY_SIZE);
	} else {
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemSizeMB + "m");
	}
	config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, managedMemFrac);

	return config;
}
 

private void setRestartStrategy(Configuration conf) {
	RestartStrategyConfiguration restartStrategy = environment.getExecution().getRestartStrategy();
	if (restartStrategy instanceof NoRestartStrategyConfiguration) {
		conf.set(RestartStrategyOptions.RESTART_STRATEGY, "none");
	} else if (restartStrategy instanceof FixedDelayRestartStrategyConfiguration) {
		conf.set(RestartStrategyOptions.RESTART_STRATEGY, "fixed-delay");
		FixedDelayRestartStrategyConfiguration fixedDelay = ((FixedDelayRestartStrategyConfiguration) restartStrategy);
		conf.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS,
				fixedDelay.getRestartAttempts());
		conf.set(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_DELAY,
				Duration.ofMillis(fixedDelay.getDelayBetweenAttemptsInterval().toMilliseconds()));
	} else if (restartStrategy instanceof FailureRateRestartStrategyConfiguration) {
		conf.set(RestartStrategyOptions.RESTART_STRATEGY, "failure-rate");
		FailureRateRestartStrategyConfiguration failureRate = (FailureRateRestartStrategyConfiguration) restartStrategy;
		conf.set(RestartStrategyOptions.RESTART_STRATEGY_FAILURE_RATE_MAX_FAILURES_PER_INTERVAL,
				failureRate.getMaxFailureRate());
		conf.set(RestartStrategyOptions.RESTART_STRATEGY_FAILURE_RATE_FAILURE_RATE_INTERVAL,
				Duration.ofMillis(failureRate.getFailureInterval().toMilliseconds()));
		conf.set(RestartStrategyOptions.RESTART_STRATEGY_FAILURE_RATE_DELAY,
				Duration.ofMillis(failureRate.getDelayBetweenAttemptsInterval().toMilliseconds()));
	} else if (restartStrategy instanceof FallbackRestartStrategyConfiguration) {
		// default is FallbackRestartStrategyConfiguration
		// see ExecutionConfig.restartStrategyConfiguration
		conf.removeConfig(RestartStrategyOptions.RESTART_STRATEGY);
	}
}
 

/**
 * Clones the given configuration and resets instance specific config options.
 *
 * @param configuration to clone
 * @return Cloned configuration with reset instance specific config options
 */
public static Configuration cloneConfiguration(Configuration configuration) {
	final Configuration clonedConfiguration = new Configuration(configuration);

	if (clonedConfiguration.getBoolean(USE_LOCAL_DEFAULT_TMP_DIRS)){
		clonedConfiguration.removeConfig(CoreOptions.TMP_DIRS);
		clonedConfiguration.removeConfig(USE_LOCAL_DEFAULT_TMP_DIRS);
	}

	return clonedConfiguration;
}
 

/**
 * Clones the given configuration and resets instance specific config options.
 *
 * @param configuration to clone
 * @return Cloned configuration with reset instance specific config options
 */
public static Configuration cloneConfiguration(Configuration configuration) {
	final Configuration clonedConfiguration = new Configuration(configuration);

	if (clonedConfiguration.getBoolean(USE_LOCAL_DEFAULT_TMP_DIRS)){
		clonedConfiguration.removeConfig(CoreOptions.TMP_DIRS);
		clonedConfiguration.removeConfig(USE_LOCAL_DEFAULT_TMP_DIRS);
	}

	return clonedConfiguration;
}
 

/**
 * Test for {@link TaskManagerServices#calculateHeapSizeMB(long, Configuration)} with some
 * manually calculated scenarios.
 */
@Test
public void calculateHeapSizeMB() throws Exception {
	Configuration config = new Configuration();
	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(64L << 20)); // 64MB
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(1L << 30)); // 1GB

	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, false);
	assertEquals(900, TaskManagerServices.calculateHeapSizeMB(1000, config));

	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, false);
	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.2f);
	assertEquals(800, TaskManagerServices.calculateHeapSizeMB(1000, config));

	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.6f);
	assertEquals(400, TaskManagerServices.calculateHeapSizeMB(1000, config));

	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, true);
	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "10m"); // 10MB
	assertEquals(890, TaskManagerServices.calculateHeapSizeMB(1000, config));

	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.6f);
	assertEquals(390, TaskManagerServices.calculateHeapSizeMB(1000, config));

	config.removeConfig(TaskManagerOptions.MANAGED_MEMORY_SIZE); // use fraction of given memory
	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, 0.1f); // 10%
	assertEquals(810, TaskManagerServices.calculateHeapSizeMB(1000, config));
}
 

/**
 * Get properties map for the cluster-side after removal of some keys.
 */
private Map<String, String> getClusterSidePropertiesMap(Configuration flinkConfig) {
	final Configuration clusterSideConfig = flinkConfig.clone();
	// Remove some configuration options that should not be taken to cluster side.
	clusterSideConfig.removeConfig(KubernetesConfigOptions.KUBE_CONFIG_FILE);
	clusterSideConfig.removeConfig(DeploymentOptionsInternal.CONF_DIR);
	return clusterSideConfig.toMap();
}
 

/**
 * Clones the given configuration and resets instance specific config options.
 *
 * @param configuration to clone
 * @return Cloned configuration with reset instance specific config options
 */
public static Configuration cloneConfiguration(Configuration configuration) {
	final Configuration clonedConfiguration = new Configuration(configuration);

	if (clonedConfiguration.getBoolean(USE_LOCAL_DEFAULT_TMP_DIRS)){
		clonedConfiguration.removeConfig(CoreOptions.TMP_DIRS);
		clonedConfiguration.removeConfig(USE_LOCAL_DEFAULT_TMP_DIRS);
	}

	return clonedConfiguration;
}
 

private Configuration createActiveResourceManagerConfiguration(Configuration originalConfiguration) {
	final Configuration copiedConfig = new Configuration(originalConfiguration);
	// In active mode, it's depend on the ResourceManager to set the ResourceID of TaskManagers.
	copiedConfig.removeConfig(TaskManagerOptions.TASK_MANAGER_RESOURCE_ID);
	return TaskExecutorProcessUtils.getConfigurationMapLegacyTaskManagerHeapSizeToConfigOption(
		copiedConfig, TaskManagerOptions.TOTAL_PROCESS_MEMORY);
}
 

/**
 * Get the configuration for standalone ResourceManager, overwrite invalid configs.
 *
 * @param configuration configuration object
 * @return the configuration for standalone ResourceManager
 */
private static Configuration getConfigurationWithoutMaxSlotNumberIfSet(Configuration configuration) {
	final Configuration copiedConfig = new Configuration(configuration);
	// The max slot limit should not take effect for standalone cluster, we overwrite the configure in case user
	// sets this value by mistake.
	if (copiedConfig.removeConfig(ResourceManagerOptions.MAX_SLOT_NUM)) {
		LOG.warn("Config option {} will be ignored in standalone mode.", ResourceManagerOptions.MAX_SLOT_NUM.key());
	}
	return copiedConfig;
}