Java Code Examples for org.apache.flink.configuration.Configuration#addAll()

/**
* Sets the value of a new config key to the value of a deprecated config key. Taking into
* account the changed prefix.
* @param config Config to write
* @param deprecatedPrefix Old prefix of key
* @param designatedPrefix New prefix of key
*/
public static void substituteDeprecatedConfigPrefix(
		Configuration config,
		String deprecatedPrefix,
		String designatedPrefix) {

	// set the designated key only if it is not set already
	final int prefixLen = deprecatedPrefix.length();

	Configuration replacement = new Configuration();

	for (String key : config.keySet()) {
		if (key.startsWith(deprecatedPrefix)) {
			String newKey = designatedPrefix + key.substring(prefixLen);
			if (!config.containsKey(newKey)) {
				replacement.setString(newKey, config.getString(key, null));
			}
		}
	}

	config.addAll(replacement);
}
 

private void validateInConfigWithExplicitTotalFlinkMem(
	final Configuration customConfig, Consumer<TaskExecutorProcessSpec> validateFunc) {
	log.info("Validating in configuration with explicit total flink memory size.");
	final Configuration config = configWithExplicitTotalFlinkMem();
	config.addAll(customConfig);
	TaskExecutorProcessSpec taskExecutorProcessSpec = TaskExecutorProcessUtils.processSpecFromConfig(config);
	assertThat(taskExecutorProcessSpec.getTotalFlinkMemorySize(), is(TOTAL_FLINK_MEM_SIZE));
	validateFunc.accept(taskExecutorProcessSpec);
}
 

private void validateInConfigWithExplicitTotalProcessMem(
	final Configuration customConfig, Consumer<TaskExecutorProcessSpec> validateFunc) {
	log.info("Validating in configuration with explicit total process memory size.");
	final Configuration config = configWithExplicitTotalProcessMem();
	config.addAll(customConfig);
	TaskExecutorProcessSpec taskExecutorProcessSpec = TaskExecutorProcessUtils.processSpecFromConfig(config);
	assertThat(taskExecutorProcessSpec.getTotalProcessMemorySize(), is(TOTAL_PROCESS_MEM_SIZE));
	validateFunc.accept(taskExecutorProcessSpec);
}
 

/**
 * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user specified name.
 *
 * @param jobName
 *            name of the job
 * @return The result of the job execution, containing elapsed time and accumulators.
 */
@Override
public JobExecutionResult execute(String jobName) throws Exception {
    // transform the streaming program into a JobGraph
    StreamGraph streamGraph = getStreamGraph();
    streamGraph.setJobName(jobName);

    JobGraph jobGraph = streamGraph.getJobGraph();

    Configuration configuration = new Configuration();
    configuration.addAll(jobGraph.getJobConfiguration());

    configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS,
            jobGraph.getMaximumParallelism());

    // add (and override) the settings with what the user defined
    configuration.addAll(_conf);

    _exec = new LocalFlinkMiniCluster(configuration, true);

    try {
        _exec.start();
        return _exec.submitJobAndWait(jobGraph, getConfig().isSysoutLoggingEnabled());
    } finally {
        transformations.clear();
        _exec.stop();
        _exec = null;
    }
}
 

@Before
public void setUp() throws Exception {
	final Configuration clientConfig = new Configuration();
	clientConfig.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);
	clientConfig.setLong(RestOptions.RETRY_DELAY, 0);
	clientConfig.addAll(CLUSTER.getClientConfiguration());

	client = new RestClusterClient<>(
		clientConfig,
		StandaloneClusterId.getInstance()
	);
}
 

public void appendConfiguration(Configuration config) throws IOException {
	final Configuration mergedConfig = new Configuration();
	mergedConfig.addAll(defaultConfig);
	mergedConfig.addAll(config);

	final List<String> configurationLines = mergedConfig.toMap().entrySet().stream()
		.map(entry -> entry.getKey() + ": " + entry.getValue())
		.collect(Collectors.toList());

	Files.write(conf.resolve("flink-conf.yaml"), configurationLines);
}
 

private void validateInConfigWithExplicitTotalFlinkAndTaskHeapMem(
	final Configuration customConfig, Consumer<TaskExecutorProcessSpec> validateFunc) {
	log.info("Validating in configuration with explicit total flink and task heap memory size.");
	final Configuration config = configWithExplicitTotalFlinkAndTaskHeapMem();
	config.addAll(customConfig);
	TaskExecutorProcessSpec taskExecutorProcessSpec = TaskExecutorProcessUtils.processSpecFromConfig(config);
	assertThat(taskExecutorProcessSpec.getTotalFlinkMemorySize(), is(TOTAL_FLINK_MEM_SIZE));
	assertThat(taskExecutorProcessSpec.getTaskHeapSize(), is(TASK_HEAP_SIZE));
	validateFunc.accept(taskExecutorProcessSpec);
}
 

private Configuration combineWithGlobalJobConfiguration(Configuration parameters) {
  Configuration combined = new Configuration();
  combined.addAll(parameters);

  GlobalJobParameters globalJobParameters =
      getRuntimeContext().getExecutionConfig().getGlobalJobParameters();

  Preconditions.checkState(globalJobParameters instanceof Configuration);
  Configuration configuration = (Configuration) globalJobParameters;

  combined.addAll(configuration);
  return combined;
}
 

@Before
public void setUp() throws Exception {
	final Configuration configuration = new Configuration();
	configuration.addAll(createBackPressureSamplingConfiguration());
	configuration.addAll(createNetworkBufferConfiguration());

	final TestingMiniClusterConfiguration testingMiniClusterConfiguration = new TestingMiniClusterConfiguration.Builder()
		.setNumSlotsPerTaskManager(NUM_TASKS)
		.setConfiguration(configuration)
		.build();

	testingMiniCluster = new TestingMiniCluster(testingMiniClusterConfiguration);
	testingMiniCluster.start();
	dispatcherGateway = testingMiniCluster.getDispatcherGatewayFuture().get();
}
 

private void validateInConfigWithExplicitTotalProcessMem(
	Configuration customConfig, Consumer<JobManagerProcessSpec> validateFunc) {
	log.info("Validating in configuration with explicit total process memory size.");
	Configuration config = configWithExplicitTotalProcessMem();
	config.addAll(customConfig);
	JobManagerProcessSpec jobManagerProcessSpec = JobManagerProcessUtils.processSpecFromConfig(config);
	assertThat(jobManagerProcessSpec.getTotalProcessMemorySize(), is(TOTAL_PROCESS_MEM_SIZE));
	validateFunc.accept(jobManagerProcessSpec);
}
 

private void validateInConfigWithExplicitTotalFlinkAndJvmHeapMem(
		Configuration customConfig,
		Consumer<JobManagerProcessSpec> validateFunc) {
	log.info("Validating in configuration with explicit total flink and jvm heap memory size.");
	Configuration config = configWithExplicitTotalFlinkAndJvmHeapMem();
	config.addAll(customConfig);
	JobManagerProcessSpec jobManagerProcessSpec = JobManagerProcessUtils.processSpecFromConfig(config);
	assertThat(jobManagerProcessSpec.getTotalFlinkMemorySize(), is(TOTAL_FLINK_MEM_SIZE));
	assertThat(jobManagerProcessSpec.getJvmHeapMemorySize(), is(JVM_HEAP_SIZE));
	validateFunc.accept(jobManagerProcessSpec);
}
 

public void appendConfiguration(Configuration config) throws IOException {
	final Configuration mergedConfig = new Configuration();
	mergedConfig.addAll(defaultConfig);
	mergedConfig.addAll(config);

	final List<String> configurationLines = mergedConfig.toMap().entrySet().stream()
		.map(entry -> entry.getKey() + ": " + entry.getValue())
		.collect(Collectors.toList());

	Files.write(conf.resolve("flink-conf.yaml"), configurationLines);
}
 

private void validateInConfigWithExplicitTaskHeapAndManagedMem(
	final Configuration customConfig, Consumer<TaskExecutorProcessSpec> validateFunc) {
	log.info("Validating in configuration with explicit task heap and managed memory size.");
	final Configuration config = configWithExplicitTaskHeapAndManageMem();
	config.addAll(customConfig);
	TaskExecutorProcessSpec taskExecutorProcessSpec = TaskExecutorProcessUtils.processSpecFromConfig(config);
	assertThat(taskExecutorProcessSpec.getTaskHeapSize(), is(TASK_HEAP_SIZE));
	assertThat(taskExecutorProcessSpec.getManagedMemorySize(), is(MANAGED_MEM_SIZE));
	validateFunc.accept(taskExecutorProcessSpec);
}
 

private void validateFailInConfigWithExplicitJvmHeap(Configuration customConfig) {
	log.info("Validating failing in configuration with explicit jvm heap.");
	Configuration config = configWithExplicitJvmHeap();
	config.addAll(customConfig);
	validateFail(config);
}
 

private void validateFailInConfigWithExplicitTotalFlinkAndTaskHeapMem(final Configuration customConfig) {
	log.info("Validating failing in configuration with explicit total flink and task heap memory size.");
	final Configuration config = configWithExplicitTotalFlinkAndTaskHeapMem();
	config.addAll(customConfig);
	validateFail(config);
}
 

private void validateFailInConfigWithExplicitTotalProcessMem(Configuration customConfig) {
	log.info("Validating failing in configuration with explicit total process memory size.");
	Configuration config = configWithExplicitTotalProcessMem();
	config.addAll(customConfig);
	validateFail(config);
}
 

/**
 * Executes the JobGraph of the on a mini cluster of ClusterUtil with a user
 * specified name.
 *
 * @return The result of the job execution, containing elapsed time and accumulators.
 */
@Override
public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
	JobGraph jobGraph = streamGraph.getJobGraph();
	jobGraph.setAllowQueuedScheduling(true);

	Configuration configuration = new Configuration();
	configuration.addAll(jobGraph.getJobConfiguration());
	configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0");

	// add (and override) the settings with what the user defined
	configuration.addAll(this.configuration);

	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism());

	MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumSlotsPerTaskManager(numSlotsPerTaskManager)
		.build();

	if (LOG.isInfoEnabled()) {
		LOG.info("Running job on local embedded Flink mini cluster");
	}

	MiniCluster miniCluster = new MiniCluster(cfg);

	try {
		miniCluster.start();
		configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

		return miniCluster.executeJobBlocking(jobGraph);
	}
	finally {
		transformations.clear();
		miniCluster.close();
	}
}
 

private void validateFailInConfigWithExplicitTotalProcessMem(final Configuration customConfig) {
	log.info("Validating failing in configuration with explicit total process memory size.");
	final Configuration config = configWithExplicitTotalProcessMem();
	config.addAll(customConfig);
	validateFail(config);
}
 

private void validateFailInConfigWithExplicitTotalFlinkAndJvmHeapMem(Configuration customConfig) {
	log.info("Validating failing in configuration with explicit total flink and jvm heap memory size.");
	Configuration config = configWithExplicitTotalFlinkAndJvmHeapMem();
	config.addAll(customConfig);
	validateFail(config);
}
 

private void validateFailInConfigWithExplicitTotalFlinkMem(Configuration customConfig) {
	log.info("Validating failing in configuration with explicit total flink memory size.");
	Configuration config = configWithExplicitTotalFlinkMem();
	config.addAll(customConfig);
	validateFail(config);
}