org.apache.flink.configuration.MemorySize Java Examples

public static TaskExecutorProcessSpec processSpecFromWorkerResourceSpec(
	final Configuration config, final WorkerResourceSpec workerResourceSpec) {

	final MemorySize frameworkHeapMemorySize = TaskExecutorFlinkMemoryUtils.getFrameworkHeapMemorySize(config);
	final MemorySize frameworkOffHeapMemorySize = TaskExecutorFlinkMemoryUtils.getFrameworkOffHeapMemorySize(config);

	final TaskExecutorFlinkMemory flinkMemory = new TaskExecutorFlinkMemory(
		frameworkHeapMemorySize,
		frameworkOffHeapMemorySize,
		workerResourceSpec.getTaskHeapSize(),
		workerResourceSpec.getTaskOffHeapSize(),
		workerResourceSpec.getNetworkMemSize(),
		workerResourceSpec.getManagedMemSize());

	final JvmMetaspaceAndOverhead jvmMetaspaceAndOverhead =
		PROCESS_MEMORY_UTILS.deriveJvmMetaspaceAndOverheadFromTotalFlinkMemory(
			config, flinkMemory.getTotalFlinkMemorySize());

	return new TaskExecutorProcessSpec(workerResourceSpec.getCpuCores(), flinkMemory, jvmMetaspaceAndOverhead);
}
 

private static Configuration getClusterConfiguration() {
	Configuration config = new Configuration();
	try {
		File logDir = File.createTempFile("TestBaseUtils-logdir", null);
		assertTrue("Unable to delete temp file", logDir.delete());
		assertTrue("Unable to create temp directory", logDir.mkdir());
		File logFile = new File(logDir, "jobmanager.log");
		File outFile = new File(logDir, "jobmanager.out");

		Files.createFile(logFile.toPath());
		Files.createFile(outFile.toPath());

		config.setString(WebOptions.LOG_PATH, logFile.getAbsolutePath());
		config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, logFile.getAbsolutePath());
	} catch (Exception e) {
		throw new AssertionError("Could not setup test.", e);
	}

	// !!DO NOT REMOVE!! next line is required for tests
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("12m"));

	return config;
}
 

public ShuffleEnvironmentContext(
		Configuration configuration,
		ResourceID taskExecutorResourceId,
		MemorySize networkMemorySize,
		boolean localCommunicationOnly,
		InetAddress hostAddress,
		TaskEventPublisher eventPublisher,
		MetricGroup parentMetricGroup,
		Executor ioExecutor) {
	this.configuration = checkNotNull(configuration);
	this.taskExecutorResourceId = checkNotNull(taskExecutorResourceId);
	this.networkMemorySize = networkMemorySize;
	this.localCommunicationOnly = localCommunicationOnly;
	this.hostAddress = checkNotNull(hostAddress);
	this.eventPublisher = checkNotNull(eventPublisher);
	this.parentMetricGroup = checkNotNull(parentMetricGroup);
	this.ioExecutor = ioExecutor;
}
 

/**
 * Creates a new ResourceProfile.
 *
 * @param cpuCores The number of CPU cores (possibly fractional, i.e., 0.2 cores)
 * @param taskHeapMemory The size of the task heap memory.
 * @param taskOffHeapMemory The size of the task off-heap memory.
 * @param managedMemory The size of the managed memory.
 * @param networkMemory The size of the network memory.
 * @param extendedResources The extended resources such as GPU and FPGA
 */
private ResourceProfile(
		final Resource cpuCores,
		final MemorySize taskHeapMemory,
		final MemorySize taskOffHeapMemory,
		final MemorySize managedMemory,
		final MemorySize networkMemory,
		final Map<String, Resource> extendedResources) {

	checkNotNull(cpuCores);
	checkArgument(cpuCores instanceof CPUResource, "cpuCores must be CPUResource");

	this.cpuCores = cpuCores;
	this.taskHeapMemory = checkNotNull(taskHeapMemory);
	this.taskOffHeapMemory = checkNotNull(taskOffHeapMemory);
	this.managedMemory = checkNotNull(managedMemory);
	this.networkMemory = checkNotNull(networkMemory);
	if (extendedResources != null) {
		this.extendedResources.putAll(extendedResources);
	}
}
 

private Configuration getConfiguration() throws Exception {
	// Flink configuration
	final Configuration config = new Configuration();

	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, DEFAULT_PARALLELISM);

	UUID id = UUID.randomUUID();
	final File checkpointDir = TEMP_FOLDER.newFolder("checkpoints_" + id).getAbsoluteFile();
	final File savepointDir = TEMP_FOLDER.newFolder("savepoints_" + id).getAbsoluteFile();

	if (!checkpointDir.exists() || !savepointDir.exists()) {
		throw new Exception("Test setup failed: failed to create (temporary) directories.");
	}

	LOG.info("Created temporary checkpoint directory: " + checkpointDir + ".");
	LOG.info("Created savepoint directory: " + savepointDir + ".");

	config.setString(CheckpointingOptions.STATE_BACKEND, "memory");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
	config.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, MemorySize.ZERO);
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());
	config.setLong(HeartbeatManagerOptions.HEARTBEAT_INTERVAL, 300L);

	return config;
}
 

private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}
 

@BeforeClass
public static void setUp() throws Exception {

	Configuration config = new Configuration();

	// we need to use the "filesystem" state backend to ensure FLINK-2543 is not happening again.
	config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// Savepoint path
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// required as we otherwise run out of memory
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("80m"));

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(2)
			.setNumberSlotsPerTaskManager(2)
			.setConfiguration(config)
			.build());

	miniClusterResource.before();
}
 

@Test
public void testLoadDynamicalProperties() throws IOException, FlinkParseException {
	final File tmpDir = temporaryFolder.newFolder();
	final File confFile = new File(tmpDir, GlobalConfiguration.FLINK_CONF_FILENAME);
	final PrintWriter pw1 = new PrintWriter(confFile);
	final long managedMemory = 1024 * 1024 * 256;
	pw1.println(JobManagerOptions.ADDRESS.key() + ": localhost");
	pw1.println(TaskManagerOptions.MANAGED_MEMORY_SIZE.key() + ": " + managedMemory + "b");
	pw1.close();

	final String jmHost = "host1";
	final int jmPort = 12345;
	String[] args = new String[] {
		"--configDir", tmpDir.toString(),
		"-D" + JobManagerOptions.ADDRESS.key() + "=" + jmHost,
		"-D" + JobManagerOptions.PORT.key() + "=" + jmPort
	};
	Configuration configuration = TaskManagerRunner.loadConfiguration(args);
	assertEquals(MemorySize.parse(managedMemory + "b"), configuration.get(TaskManagerOptions.MANAGED_MEMORY_SIZE));
	assertEquals(jmHost, configuration.get(JobManagerOptions.ADDRESS));
	assertEquals(jmPort, configuration.getInteger(JobManagerOptions.PORT));
}
 

@Test
public void testOffHeapMemoryDerivedFromJvmHeapAndTotalFlinkMemory() {
	MemorySize jvmHeap = MemorySize.ofMebiBytes(150);
	MemorySize defaultOffHeap = JobManagerOptions.OFF_HEAP_MEMORY.defaultValue();
	MemorySize expectedOffHeap = MemorySize.ofMebiBytes(100).add(defaultOffHeap);
	MemorySize totalFlinkMemory = jvmHeap.add(expectedOffHeap);

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.TOTAL_FLINK_MEMORY, totalFlinkMemory);
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeap);

	JobManagerProcessSpec jobManagerProcessSpec = JobManagerProcessUtils.processSpecFromConfig(conf);
	assertThat(jobManagerProcessSpec.getJvmDirectMemorySize(), is(expectedOffHeap));
	MatcherAssert.assertThat(
		testLoggerResource.getMessages(),
		hasItem(containsString(String.format(
			"The Off-Heap Memory size (%s) is derived the configured Total Flink Memory size (%s) minus " +
				"the configured JVM Heap Memory size (%s). The default Off-Heap Memory size (%s) is ignored.",
			expectedOffHeap.toHumanReadableString(),
			totalFlinkMemory.toHumanReadableString(),
			jvmHeap.toHumanReadableString(),
			defaultOffHeap.toHumanReadableString()))));
}
 

/**
 * Test for {@link NettyShuffleEnvironmentConfiguration#calculateNetworkBufferMemory(long, Configuration)} using old
 * configurations via {@link NettyShuffleEnvironmentOptions#NETWORK_NUM_BUFFERS}.
 */
@SuppressWarnings("deprecation")
@Test
public void calculateNetworkBufOld() {
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 1);

	// note: actual network buffer memory size is independent of the totalJavaMemorySize
	assertEquals(MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(10L << 20, config));
	assertEquals(MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(64L << 20, config));

	// test integer overflow in the memory size
	int numBuffers = (int) ((2L << 32) / MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes()); // 2^33
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, numBuffers);
	assertEquals(2L << 32, NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(2L << 33, config));
}
 

/**
 * Calculates the amount of memory used for network buffers based on the total memory to use and
 * the according configuration parameters.
 *
 * <p>The following configuration parameters are involved:
 * <ul>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX}</li>
 * </ul>.
 *
 * @param config configuration object
 * @param networkMemoryByFraction memory of network buffers based on JVM memory size and network fraction
 * @param maxAllowedMemory maximum memory used for checking the results of network memory
 *
 * @return memory to use for network buffers (in bytes)
 */
private static long calculateNewNetworkBufferMemory(Configuration config, long networkMemoryByFraction, long maxAllowedMemory) {
	float networkBufFraction = config.getFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION);
	long networkBufMin = MemorySize.parse(config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN)).getBytes();
	long networkBufMax = MemorySize.parse(config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX)).getBytes();

	int pageSize = ConfigurationParserUtils.getPageSize(config);

	checkNewNetworkConfig(pageSize, networkBufFraction, networkBufMin, networkBufMax);

	long networkBufBytes = Math.min(networkBufMax, Math.max(networkBufMin, networkMemoryByFraction));

	ConfigurationParserUtils.checkConfigParameter(networkBufBytes < maxAllowedMemory,
		"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
		"(" + NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
			NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
			NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
		"Network buffer memory size too large: " + networkBufBytes + " >= " +
			maxAllowedMemory + " (maximum JVM memory size)");

	return networkBufBytes;
}
 

/**
 * Tests that the command line arguments override the configuration settings
 * when the {@link ClusterSpecification} is created.
 */
@Test
public void testCommandLineClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	final int taskManagerMemory = 7331;
	final int slotsPerTaskManager = 30;

	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(jobManagerMemory));
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(taskManagerMemory));
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {"-yjm", String.valueOf(jobManagerMemory) + "m", "-ytm", String.valueOf(taskManagerMemory) + "m", "-ys", String.valueOf(slotsPerTaskManager)};
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 

/**
 * Test for {@link TaskManagerServices#calculateNetworkBufferMemory(long, Configuration)} using new
 * configurations via {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_FRACTION},
 * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MIN} and
 * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MAX}.
 */
@Test
public void calculateNetworkBufNew() throws Exception {
	Configuration config = new Configuration();

	// (1) defaults
	final Float defaultFrac = TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.defaultValue();
	final Long defaultMin = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes();
	final Long defaultMax = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.defaultValue()).getBytes();
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((64L << 20 + 1), config));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config));

	calculateNetworkBufNew(config);
}
 

@Test
public void testConfigManagedMemoryLegacySize() {
	final MemorySize managedMemSize = MemorySize.parse("100m");

	@SuppressWarnings("deprecation")
	final ConfigOption<MemorySize> legacyOption = TaskManagerOptions.MANAGED_MEMORY_SIZE;

	Configuration conf = new Configuration();
	conf.set(legacyOption, managedMemSize);

	// validate in configurations without explicit managed memory size,
	// to avoid checking against overwritten managed memory size
	validateInConfigurationsWithoutExplicitManagedMem(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getManagedMemorySize(), is(managedMemSize)));
}
 

private static Configuration getConfig() {
	Configuration config = new Configuration();
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
	config.setBoolean(WebOptions.SUBMIT_ENABLE, false);
	return config;
}
 

@Test(expected = IllegalConfigurationException.class)
public void testConsistencyCheckOfDerivedNetworkMemoryLessThanMinFails() {
	final Configuration configuration = setupConfigWithFlinkAndTaskHeapToDeriveGivenNetworkMem(500);
	configuration.set(TaskManagerOptions.NETWORK_MEMORY_MIN, MemorySize.parse("900m"));
	configuration.set(TaskManagerOptions.NETWORK_MEMORY_MAX, MemorySize.parse("1000m"));
	// internal validation should fail
	TaskExecutorProcessUtils.processSpecFromConfig(configuration);
}
 

@Test
public void testConfigTaskOffheapMemory() {
	final MemorySize taskOffHeapSize = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.TASK_OFF_HEAP_MEMORY, taskOffHeapSize);

	validateInAllConfigurations(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getTaskOffHeapSize(), is(taskOffHeapSize)));
}
 

public static JvmArgTestingProcessMemorySpec generate() {
	return new JvmArgTestingProcessMemorySpec(
		MemorySize.ofMebiBytes(1),
		MemorySize.ofMebiBytes(2),
		MemorySize.ofMebiBytes(3)
	);
}
 

public JvmMetaspaceAndOverheadOptions(
		ConfigOption<MemorySize> jvmMetaspaceOption,
		ConfigOption<MemorySize> jvmOverheadMin,
		ConfigOption<MemorySize> jvmOverheadMax,
		ConfigOption<Float> jvmOverheadFraction) {
	this.jvmMetaspaceOption = jvmMetaspaceOption;
	this.jvmOverheadMin = jvmOverheadMin;
	this.jvmOverheadMax = jvmOverheadMax;
	this.jvmOverheadFraction = jvmOverheadFraction;
}
 

/**
 * Returns a Flink {@link MemorySize} under the given key if it exists.
 */
public Optional<MemorySize> getOptionalMemorySize(String key) {
	return optionalGet(key).map((value) -> {
		try {
			return MemorySize.parse(value, MemorySize.MemoryUnit.BYTES);
		} catch (Exception e) {
			throw new ValidationException("Invalid memory size value for key '" + key + "'.", e);
		}
	});
}
 

public DefaultConfigurableOptionsFactory setWriteBufferSize(String writeBufferSize) {
	Preconditions.checkArgument(MemorySize.parseBytes(writeBufferSize) > 0,
		"Invalid configuration " + writeBufferSize + " for write-buffer size.");

	setInternal(WRITE_BUFFER_SIZE.key(), writeBufferSize);
	return this;
}
 

@Test
public void includesCPUAndMemoryInToStringIfTheyAreBelowThreshold() {
	ResourceProfile resourceProfile = createResourceProfile(1.0, MemorySize.ofMebiBytes(4));
	assertThat(
		resourceProfile.toString(),
		allOf(
			containsCPUCores(),
			containsTaskHeapMemory()));
}
 

public DefaultConfigurableOptionsFactory setBlockCacheSize(String blockCacheSize) {
	Preconditions.checkArgument(MemorySize.parseBytes(blockCacheSize) > 0,
		"Invalid configuration " + blockCacheSize + " for block cache size.");
	setInternal(BLOCK_CACHE_SIZE.key(), blockCacheSize);

	return this;
}
 

@Test
public void testJvmHeapExceedsTotalFlinkMemoryFailure() {
	MemorySize totalFlinkMemory = MemorySize.ofMebiBytes(100);
	MemorySize jvmHeap = MemorySize.ofMebiBytes(150);

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.TOTAL_FLINK_MEMORY, totalFlinkMemory);
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeap);

	validateFail(conf);
}
 

/**
 * Validates taking the application-defined file system state backend and adding with additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testLoadFileSystemStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final Path expectedCheckpointsPath = new Path(new URI(appCheckpointDir));
	final Path expectedSavepointsPath = new Path(savepointDir);

	final int threshold = 1000000;
	final int writeBufferSize = 4000000;

	final FsStateBackend backend = new FsStateBackend(new URI(appCheckpointDir), null, threshold, writeBufferSize, TernaryBoolean.TRUE);

	final Configuration config = new Configuration();
	config.setString(backendKey, "jobmanager"); // this should not be picked up 
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, MemorySize.parse("20")); // this should not be picked up
	config.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, 3000000); // this should not be picked up

	final StateBackend loadedBackend =
			StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof FsStateBackend);

	final FsStateBackend fs = (FsStateBackend) loadedBackend;
	assertEquals(expectedCheckpointsPath, fs.getCheckpointPath());
	assertEquals(expectedSavepointsPath, fs.getSavepointPath());
	assertEquals(threshold, fs.getMinFileSizeThreshold());
	assertEquals(writeBufferSize, fs.getWriteBufferSize());
}
 

@Test
public void testConfigFrameworkHeapMemory() {
	final MemorySize frameworkHeapSize = MemorySize.parse("100m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY, frameworkHeapSize);

	validateInAllConfigurations(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getFrameworkHeapSize(), is(frameworkHeapSize)));
}
 

@Test
public void testLogFailureOfJvmHeapSizeMinSizeVerification() {
	MemorySize jvmHeapMemory = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeapMemory);

	JobManagerProcessUtils.processSpecFromConfig(conf);
	MatcherAssert.assertThat(
		testLoggerResource.getMessages(),
		hasItem(containsString(String.format(
			"The configured or derived JVM heap memory size (%s) is less than its recommended minimum value (%s)",
			jvmHeapMemory.toHumanReadableString(),
			JobManagerOptions.MIN_JVM_HEAP_SIZE.toHumanReadableString()))));
}
 

/**
 * Tests that the command line arguments override the configuration settings
 * when the {@link ClusterSpecification} is created.
 */
@Test
public void testCommandLineClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	final int taskManagerMemory = 7331;
	final int slotsPerTaskManager = 30;

	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(jobManagerMemory));
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(taskManagerMemory));
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {
			"-e", KubernetesSessionClusterExecutor.NAME,
			"-D" + JobManagerOptions.TOTAL_PROCESS_MEMORY.key() + "=" + jobManagerMemory + "m",
			"-D" + TaskManagerOptions.TOTAL_PROCESS_MEMORY.key() + "=" + taskManagerMemory + "m",
			"-D" + TaskManagerOptions.NUM_TASK_SLOTS.key() + "=" + slotsPerTaskManager
	};

	final KubernetesSessionCli cli = new KubernetesSessionCli(
			configuration,
			tmp.getRoot().getAbsolutePath());

	Configuration executorConfig = cli.getEffectiveConfiguration(args);
	ClusterClientFactory<String> clientFactory = getClusterClientFactory(executorConfig);
	ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 

private long getTargetFileSizeBase() {
	return MemorySize.parseBytes(getInternal(TARGET_FILE_SIZE_BASE.key()));
}
 

private static String addMemorySizeString(String resourceStr, String name, MemorySize size) {
	String comma = resourceStr.isEmpty() ? "" : ", ";
	String memorySizeStr = size == null || size.compareTo(MAX_MEMORY_SIZE_TO_LOG) > 0 ?
		"" : comma + name + '=' + size.toHumanReadableString();
	return resourceStr + memorySizeStr;
}