Java Code Examples for org.apache.flink.configuration.Configuration#clone()

/**
 * Verifies that {@link TaskManagerServicesConfiguration#hasNewNetworkBufConf(Configuration)}
 * returns the correct result for mixed old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void hasNewNetworkBufConfMixed() throws Exception {
	Configuration config = new Configuration();
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));

	config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 1);
	assertFalse(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config));

	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));

	config1 = config.clone();
	config1.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, "1024");
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));

	config1 = config.clone();
	config1.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, "1024");
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));
}
 

/**
 * Verifies that {@link NettyShuffleEnvironmentConfiguration#hasNewNetworkConfig(Configuration)}
 * returns the correct result for mixed old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void hasNewNetworkBufConfMixed() throws Exception {
	Configuration config = new Configuration();
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config));

	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 1);
	assertFalse(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config));

	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));

	config1 = config.clone();
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, "1024");
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));

	config1 = config.clone();
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, "1024");
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));
}
 

/**
 * Test for {@link TaskManagerServices#calculateNetworkBufferMemory(long, Configuration)} using mixed
 * old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void calculateNetworkBufMixed() throws Exception {
	Configuration config = new Configuration();
	config.setInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS, 1);

	final Float defaultFrac = TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.defaultValue();
	final Long defaultMin = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes();
	final Long defaultMax = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.defaultValue()).getBytes();


	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertEquals(enforceBounds((long) (0.1f * (10L << 20)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((64L << 20 + 1), config1));
	assertEquals(enforceBounds((long) (0.1f * (10L << 30)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config1));

	config1 = config.clone();
	long newMin = MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(); // smallest value possible
	config1.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(newMin));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), newMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 20), config1));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), newMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config1));

	config1 = config.clone();
	long newMax = Math.max(64L << 20 + 1, MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes());
	config1.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(newMax));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), defaultMin, newMax),
		TaskManagerServices.calculateNetworkBufferMemory((64L << 20 + 1), config1));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), defaultMin, newMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config1));
	assertTrue(TaskManagerServicesConfiguration.hasNewNetworkBufConf(config1));

	// old + any new parameter = new:
	calculateNetworkBufNew(config);
}
 

/**
 * Test for {@link NettyShuffleEnvironmentConfiguration#calculateNetworkBufferMemory(long, Configuration)} using mixed
 * old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void calculateNetworkBufMixed() throws Exception {
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 1);

	final Float defaultFrac = NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION.defaultValue();
	final Long defaultMin = MemorySize.parse(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes();
	final Long defaultMax = MemorySize.parse(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX.defaultValue()).getBytes();


	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertEquals(enforceBounds((long) (0.1f * (10L << 20)), defaultMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((64L << 20 + 1), config1));
	assertEquals(enforceBounds((long) (0.1f * (10L << 30)), defaultMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 30), config1));

	config1 = config.clone();
	long newMin = MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(); // smallest value possible
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(newMin));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), newMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 20), config1));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), newMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 30), config1));

	config1 = config.clone();
	long newMax = Math.max(64L << 20 + 1, MemorySize.parse(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes());
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(newMax));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), defaultMin, newMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((64L << 20 + 1), config1));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), defaultMin, newMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 30), config1));
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));

	// old + any new parameter = new:
	calculateNetworkBufNew(config);
}
 

/**
 * Get properties map for the cluster-side after removal of some keys.
 */
private Map<String, String> getClusterSidePropertiesMap(Configuration flinkConfig) {
	final Configuration clusterSideConfig = flinkConfig.clone();
	// Remove some configuration options that should not be taken to cluster side.
	clusterSideConfig.removeConfig(KubernetesConfigOptions.KUBE_CONFIG_FILE);
	clusterSideConfig.removeConfig(DeploymentOptionsInternal.CONF_DIR);
	return clusterSideConfig.toMap();
}