Java Code Examples for org.apache.flink.configuration.Configuration#contains()

private JobExecutorService createJobExecutorService(Configuration configuration) throws Exception {
	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(
			configuration.getInteger(
				ConfigConstants.LOCAL_NUMBER_TASK_MANAGER,
				ConfigConstants.DEFAULT_LOCAL_NUMBER_TASK_MANAGER))
		.setRpcServiceSharing(RpcServiceSharing.SHARED)
		.setNumSlotsPerTaskManager(
			configuration.getInteger(
				TaskManagerOptions.NUM_TASK_SLOTS, 1))
		.build();

	final MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration);
	miniCluster.start();

	configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

	return miniCluster;
}
 

@Override
public JobManagerFlinkMemory deriveFromRequiredFineGrainedOptions(Configuration config) {
	MemorySize jvmHeapMemorySize = ProcessMemoryUtils.getMemorySizeFromConfig(config, JobManagerOptions.JVM_HEAP_MEMORY);
	MemorySize offHeapMemorySize = ProcessMemoryUtils.getMemorySizeFromConfig(config, JobManagerOptions.OFF_HEAP_MEMORY);

	if (config.contains(JobManagerOptions.TOTAL_FLINK_MEMORY)) {
		// derive network memory from total flink memory, and check against network min/max
		MemorySize totalFlinkMemorySize = ProcessMemoryUtils.getMemorySizeFromConfig(config, JobManagerOptions.TOTAL_FLINK_MEMORY);
		if (config.contains(JobManagerOptions.OFF_HEAP_MEMORY)) {
			// off-heap memory is explicitly set by user
			sanityCheckTotalFlinkMemory(totalFlinkMemorySize, jvmHeapMemorySize, totalFlinkMemorySize);
		} else {
			// off-heap memory is not explicitly set by user, derive it from Total Flink Memory and JVM Heap
			offHeapMemorySize = deriveOffHeapMemory(jvmHeapMemorySize, totalFlinkMemorySize, offHeapMemorySize);
		}
	}

	return createJobManagerFlinkMemory(jvmHeapMemorySize, offHeapMemorySize);
}
 

private void sanityCheckJvmOverhead(
		Configuration config,
		MemorySize derivedJvmOverheadSize,
		MemorySize totalProcessMemorySize) {
	RangeFraction jvmOverheadRangeFraction = getJvmOverheadRangeFraction(config);
	if (derivedJvmOverheadSize.getBytes() > jvmOverheadRangeFraction.getMaxSize().getBytes() ||
		derivedJvmOverheadSize.getBytes() < jvmOverheadRangeFraction.getMinSize().getBytes()) {
		throw new IllegalConfigurationException("Derived JVM Overhead size ("
			+ derivedJvmOverheadSize.toHumanReadableString() + ") is not in configured JVM Overhead range ["
			+ jvmOverheadRangeFraction.getMinSize().toHumanReadableString() + ", "
			+ jvmOverheadRangeFraction.getMaxSize().toHumanReadableString() + "].");
	}
	if (config.contains(options.getJvmOptions().getJvmOverheadFraction()) &&
		!derivedJvmOverheadSize.equals(totalProcessMemorySize.multiply(jvmOverheadRangeFraction.getFraction()))) {
		LOG.info(
			"The derived JVM Overhead size ({}) does not match " +
				"the configured or default JVM Overhead fraction ({}) from the configured Total Process Memory size ({}). " +
				"The derived JVM OVerhead size will be used.",
			derivedJvmOverheadSize.toHumanReadableString(),
			jvmOverheadRangeFraction.getFraction(),
			totalProcessMemorySize.toHumanReadableString());
	}
}
 

/**
 * Factory method to startWith a new instance, providing a
 * new instance of {@link MiniCluster}
 *
 * @param parallelism global setting for parallel execution.
 * @return new instance of {@link DataSetTestEnvironment}
 * @throws Exception
 */
public static DataSetTestEnvironment createTestEnvironment(int parallelism) {

    int taskSlots = Runtime.getRuntime().availableProcessors();

    Configuration configuration = new Configuration();
    configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0");

    if (!configuration.contains(RestOptions.BIND_PORT)) {
        configuration.setString(RestOptions.BIND_PORT, "0");
    }

    int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, taskSlots);

    MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder()
            .setConfiguration(configuration)
            .setNumSlotsPerTaskManager(numSlotsPerTaskManager)
            .build();

    MiniCluster miniCluster = new MiniCluster(cfg);

    return new DataSetTestEnvironment(miniCluster, parallelism);
}
 

/**
 * Parses the hosts / ports for communication and data exchange from configuration.
 *
 * @param configuration configuration object
 * @return the data port
 */
private static int getDataBindPort(Configuration configuration) {
	final int dataBindPort;
	if (configuration.contains(NettyShuffleEnvironmentOptions.DATA_BIND_PORT)) {
		dataBindPort = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_BIND_PORT);
		ConfigurationParserUtils.checkConfigParameter(
			dataBindPort >= 0, dataBindPort, NettyShuffleEnvironmentOptions.DATA_BIND_PORT.key(),
			"Leave config parameter empty to fallback to '" +
				NettyShuffleEnvironmentOptions.DATA_PORT.key() + "' automatically.");
	} else {
		dataBindPort = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT);
		ConfigurationParserUtils.checkConfigParameter(
			dataBindPort >= 0, dataBindPort, NettyShuffleEnvironmentOptions.DATA_PORT.key(),
			"Leave config parameter empty or use 0 to let the system choose a port automatically.");
	}
	return dataBindPort;
}
 

/**
 * Creates a {@link LocalEnvironment} for local program execution that also starts the
 * web monitoring UI.
 *
 * <p>The local execution environment will run the program in a multi-threaded fashion in
 * the same JVM as the environment was created in. It will use the parallelism specified in the
 * parameter.
 *
 * <p>If the configuration key 'rest.port' was set in the configuration, that particular
 * port will be used for the web UI. Otherwise, the default port (8081) will be used.
 */
@PublicEvolving
public static ExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) {
	checkNotNull(conf, "conf");

	conf.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);

	if (!conf.contains(RestOptions.PORT)) {
		// explicitly set this option so that it's not set to 0 later
		conf.setInteger(RestOptions.PORT, RestOptions.PORT.defaultValue());
	}

	return createLocalEnvironment(conf, -1);
}
 

private void startMiniCluster() throws Exception {
	final Configuration configuration = new Configuration(miniClusterResourceConfiguration.getConfiguration());
	configuration.setString(CoreOptions.TMP_DIRS, temporaryFolder.newFolder().getAbsolutePath());

	// we need to set this since a lot of test expect this because TestBaseUtils.startCluster()
	// enabled this by default
	if (!configuration.contains(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE)) {
		configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
	}

	if (!configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) {
		configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, DEFAULT_MANAGED_MEMORY_SIZE);
	}

	// set rest and rpc port to 0 to avoid clashes with concurrent MiniClusters
	configuration.setInteger(JobManagerOptions.PORT, 0);
	configuration.setString(RestOptions.BIND_PORT, "0");

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(miniClusterResourceConfiguration.getNumberTaskManagers())
		.setNumSlotsPerTaskManager(miniClusterResourceConfiguration.getNumberSlotsPerTaskManager())
		.build();

	miniCluster = new MiniCluster(miniClusterConfiguration);

	miniCluster.start();

	final URI restAddress = miniCluster.getRestAddress().get();
	createClientConfiguration(restAddress);
}
 

/**
 * Returns whether the new network buffer memory configuration is present in the configuration
 * object, i.e. at least one new parameter is given or the old one is not present.
 *
 * @param config configuration object
 * @return <tt>true</tt> if the new configuration method is used, <tt>false</tt> otherwise
 */
@SuppressWarnings("deprecation")
@VisibleForTesting
public static boolean hasNewNetworkConfig(final Configuration config) {
	return config.contains(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION) ||
		config.contains(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN) ||
		config.contains(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX) ||
		!config.contains(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS);
}
 

/**
 * Set temporary configuration directories if necessary.
 *
 * @param configuration flink config to patch
 * @param defaultDirs in case no tmp directories is set, next directories will be applied
 */
public static void updateTmpDirectoriesInConfiguration(
		Configuration configuration,
		@Nullable String defaultDirs) {
	if (configuration.contains(CoreOptions.TMP_DIRS)) {
		LOG.info("Overriding Fink's temporary file directories with those " +
			"specified in the Flink config: {}", configuration.getValue(CoreOptions.TMP_DIRS));
	} else if (defaultDirs != null) {
		LOG.info("Setting directories for temporary files to: {}", defaultDirs);
		configuration.setString(CoreOptions.TMP_DIRS, defaultDirs);
		configuration.setBoolean(USE_LOCAL_DEFAULT_TMP_DIRS, true);
	}
}
 

/**
 * Creates a {@link LocalEnvironment} for local program execution that also starts the
 * web monitoring UI.
 *
 * <p>The local execution environment will run the program in a multi-threaded fashion in
 * the same JVM as the environment was created in. It will use the parallelism specified in the
 * parameter.
 *
 * <p>If the configuration key 'rest.port' was set in the configuration, that particular
 * port will be used for the web UI. Otherwise, the default port (8081) will be used.
 */
@PublicEvolving
public static ExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) {
	checkNotNull(conf, "conf");

	conf.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);

	if (!conf.contains(RestOptions.PORT)) {
		// explicitly set this option so that it's not set to 0 later
		conf.setInteger(RestOptions.PORT, RestOptions.PORT.defaultValue());
	}

	return createLocalEnvironment(conf, -1);
}
 

/**
 * Creates a {@link LocalStreamEnvironment} for local program execution that also starts the
 * web monitoring UI.
 *
 * <p>The local execution environment will run the program in a multi-threaded fashion in
 * the same JVM as the environment was created in. It will use the parallelism specified in the
 * parameter.
 *
 * <p>If the configuration key 'rest.port' was set in the configuration, that particular
 * port will be used for the web UI. Otherwise, the default port (8081) will be used.
 */
@PublicEvolving
public static StreamExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) {
	checkNotNull(conf, "conf");

	conf.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);

	if (!conf.contains(RestOptions.PORT)) {
		// explicitly set this option so that it's not set to 0 later
		conf.setInteger(RestOptions.PORT, RestOptions.PORT.defaultValue());
	}

	return createLocalEnvironment(defaultLocalParallelism, conf);
}
 

private static <T> void setConfigOptionToDefaultIfNotSet(
		Configuration config,
		ConfigOption<T> option,
		T defaultValue) {
	if (!config.contains(option)) {
		LOG.info(
			"The configuration option {} required for local execution is not set, setting it to its default value {}",
			option,
			defaultValue);
		config.set(option, defaultValue);
	}
}
 

/**
 * Creates a {@link LocalEnvironment} for local program execution that also starts the
 * web monitoring UI.
 *
 * <p>The local execution environment will run the program in a multi-threaded fashion in
 * the same JVM as the environment was created in. It will use the parallelism specified in the
 * parameter.
 *
 * <p>If the configuration key 'rest.port' was set in the configuration, that particular
 * port will be used for the web UI. Otherwise, the default port (8081) will be used.
 */
@PublicEvolving
public static ExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) {
	checkNotNull(conf, "conf");

	if (!conf.contains(RestOptions.PORT)) {
		// explicitly set this option so that it's not set to 0 later
		conf.setInteger(RestOptions.PORT, RestOptions.PORT.defaultValue());
	}

	return createLocalEnvironment(conf, -1);
}
 

private void startMiniCluster() throws Exception {
	final Configuration configuration = new Configuration(miniClusterResourceConfiguration.getConfiguration());
	configuration.setString(CoreOptions.TMP_DIRS, temporaryFolder.newFolder().getAbsolutePath());

	// we need to set this since a lot of test expect this because TestBaseUtils.startCluster()
	// enabled this by default
	if (!configuration.contains(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE)) {
		configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
	}

	if (!configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) {
		configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, DEFAULT_MANAGED_MEMORY_SIZE);
	}

	// set rest and rpc port to 0 to avoid clashes with concurrent MiniClusters
	configuration.setInteger(JobManagerOptions.PORT, 0);
	configuration.setString(RestOptions.BIND_PORT, "0");

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(miniClusterResourceConfiguration.getNumberTaskManagers())
		.setNumSlotsPerTaskManager(miniClusterResourceConfiguration.getNumberSlotsPerTaskManager())
		.build();

	miniCluster = new MiniCluster(miniClusterConfiguration);

	miniCluster.start();

	final URI restAddress = miniCluster.getRestAddress().get();
	createClientConfiguration(restAddress);
}
 

/**
 * Set temporary configuration directories if necessary.
 *
 * @param configuration flink config to patch
 * @param defaultDirs in case no tmp directories is set, next directories will be applied
 */
public static void updateTmpDirectoriesInConfiguration(
		Configuration configuration,
		@Nullable String defaultDirs) {
	if (configuration.contains(CoreOptions.TMP_DIRS)) {
		LOG.info("Overriding Fink's temporary file directories with those " +
			"specified in the Flink config: {}", configuration.getValue(CoreOptions.TMP_DIRS));
	} else if (defaultDirs != null) {
		LOG.info("Setting directories for temporary files to: {}", defaultDirs);
		configuration.setString(CoreOptions.TMP_DIRS, defaultDirs);
		configuration.setBoolean(USE_LOCAL_DEFAULT_TMP_DIRS, true);
	}
}
 

@SuppressWarnings("deprecation")
private static void logIfIgnoringOldConfigs(Configuration configuration) {
	if (configuration.contains(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS)) {
		LOG.info("Ignoring old (but still present) network buffer configuration via {}.",
			NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS.key());
	}
}
 

private static Time getSlotRequestTimeout(final Configuration configuration) {
	final long slotRequestTimeoutMs;
	if (configuration.contains(ResourceManagerOptions.SLOT_REQUEST_TIMEOUT)) {
		LOGGER.warn("Config key {} is deprecated; use {} instead.",
			ResourceManagerOptions.SLOT_REQUEST_TIMEOUT,
			JobManagerOptions.SLOT_REQUEST_TIMEOUT);
		slotRequestTimeoutMs = configuration.getLong(ResourceManagerOptions.SLOT_REQUEST_TIMEOUT);
	} else {
		slotRequestTimeoutMs = configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT);
	}
	return Time.milliseconds(slotRequestTimeoutMs);
}
 

public static Configuration loadConfiguration(String workingDirectory, Map<String, String> env, Logger log) {
		Configuration configuration = GlobalConfiguration.loadConfiguration(workingDirectory);

		final String remoteKeytabPrincipal = env.get(YarnConfigKeys.KEYTAB_PRINCIPAL);

		final String zooKeeperNamespace = env.get(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE);

		final Map<String, String> dynamicProperties = FlinkYarnSessionCli.getDynamicProperties(
			env.get(YarnConfigKeys.ENV_DYNAMIC_PROPERTIES));

		final String hostname = env.get(ApplicationConstants.Environment.NM_HOST.key());
		Preconditions.checkState(
			hostname != null,
			"ApplicationMaster hostname variable %s not set",
			ApplicationConstants.Environment.NM_HOST.key());

		configuration.setString(JobManagerOptions.ADDRESS, hostname);
		configuration.setString(RestOptions.ADDRESS, hostname);

		// TODO: Support port ranges for the AM
//		final String portRange = configuration.getString(
//			ConfigConstants.YARN_APPLICATION_MASTER_PORT,
//			ConfigConstants.DEFAULT_YARN_JOB_MANAGER_PORT);

		for (Map.Entry<String, String> property : dynamicProperties.entrySet()) {
			configuration.setString(property.getKey(), property.getValue());
		}

		if (zooKeeperNamespace != null) {
			configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zooKeeperNamespace);
		}

		// if a web monitor shall be started, set the port to random binding
		if (configuration.getInteger(WebOptions.PORT, 0) >= 0) {
			configuration.setInteger(WebOptions.PORT, 0);
		}

		if (!configuration.contains(RestOptions.BIND_PORT)) {
			// set the REST port to 0 to select it randomly
			configuration.setString(RestOptions.BIND_PORT, "0");
		}

		// if the user has set the deprecated YARN-specific config keys, we add the
		// corresponding generic config keys instead. that way, later code needs not
		// deal with deprecated config keys

		BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
			ConfigConstants.YARN_APPLICATION_MASTER_ENV_PREFIX,
			ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX);

		BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
			ConfigConstants.YARN_TASK_MANAGER_ENV_PREFIX,
			ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX);

		final String keytabPath;

		if (env.get(YarnConfigKeys.KEYTAB_PATH) == null) {
			keytabPath = null;
		} else {
			File f = new File(workingDirectory, Utils.KEYTAB_FILE_NAME);
			keytabPath = f.getAbsolutePath();
		}

		if (keytabPath != null && remoteKeytabPrincipal != null) {
			configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
			configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
		}

		final String localDirs = env.get(ApplicationConstants.Environment.LOCAL_DIRS.key());
		BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, localDirs);

		return configuration;
	}
 

private static boolean isTaskHeapMemorySizeExplicitlyConfigured(final Configuration config) {
	return config.contains(TaskManagerOptions.TASK_HEAP_MEMORY);
}
 

private static boolean isTotalFlinkMemorySizeExplicitlyConfigured(final Configuration config) {
	return config.contains(TaskManagerOptions.TOTAL_FLINK_MEMORY);
}