Java Code Examples for org.apache.flink.configuration.MemorySize#parse()

@Test
public void testFlinkInternalMemorySizeAddUpFailure() {
	final MemorySize totalFlinkMemory = MemorySize.parse("499m");
	final MemorySize frameworkHeap = MemorySize.parse("100m");
	final MemorySize taskHeap = MemorySize.parse("100m");
	final MemorySize taskOffHeap = MemorySize.parse("100m");
	final MemorySize network = MemorySize.parse("100m");
	final MemorySize managed = MemorySize.parse("100m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.TOTAL_FLINK_MEMORY, totalFlinkMemory);
	conf.set(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY, frameworkHeap);
	conf.set(TaskManagerOptions.TASK_HEAP_MEMORY, taskHeap);
	conf.set(TaskManagerOptions.TASK_OFF_HEAP_MEMORY, taskOffHeap);
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MIN, network);
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MAX, network);
	conf.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, managed);

	validateFail(conf);
}
 

@Test
public void testConfigTotalProcessMemoryAddUpFailure() {
	MemorySize totalProcessMemory = MemorySize.parse("699m");
	MemorySize totalFlinkMemory = MemorySize.parse("500m");
	MemorySize jvmMetaspace = MemorySize.parse("100m");
	MemorySize jvmOverhead = MemorySize.parse("100m");

	Configuration conf = new Configuration();
	conf.set(options.getTotalProcessMemoryOption(), totalProcessMemory);
	conf.set(options.getTotalFlinkMemoryOption(), totalFlinkMemory);
	conf.set(options.getJvmOptions().getJvmMetaspaceOption(), jvmMetaspace);
	conf.set(options.getJvmOptions().getJvmOverheadMin(), jvmOverhead);
	conf.set(options.getJvmOptions().getJvmOverheadMax(), jvmOverhead);

	validateFail(conf);
}
 

@Test
public void testConfigNetworkMemoryLegacyNumOfBuffers() {
	final MemorySize pageSize = MemorySize.parse("32k");
	final int numOfBuffers = 1024;
	final MemorySize networkSize = pageSize.multiply(numOfBuffers);

	@SuppressWarnings("deprecation")
	final ConfigOption<Integer> legacyOption = NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS;

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.MEMORY_SEGMENT_SIZE, pageSize);
	conf.setInteger(legacyOption, numOfBuffers);

	// validate in configurations without explicit total flink/process memory, otherwise explicit configured
	// network memory size might conflict with total flink/process memory minus other memory sizes
	validateInConfigWithExplicitTaskHeapAndManagedMem(conf, taskExecutorProcessSpec ->
		assertThat(taskExecutorProcessSpec.getNetworkMemSize(), is(networkSize)));
	validateInConfigurationsWithoutExplicitTaskHeapMem(conf, taskExecutorProcessSpec ->
		assertThat(taskExecutorProcessSpec.getNetworkMemSize(), is(networkSize)));
}
 

@Test
public void testConfigNetworkMemoryFraction() {
	final MemorySize networkMin = MemorySize.ZERO;
	final MemorySize networkMax = MemorySize.parse("1t");
	final float fraction = 0.2f;

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MAX, networkMax);
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MIN, networkMin);
	conf.setFloat(TaskManagerOptions.NETWORK_MEMORY_FRACTION, fraction);

	// validate in configurations without explicit total flink/process memory, otherwise explicit configured
	// network memory fraction might conflict with total flink/process memory minus other memory sizes
	validateInConfigWithExplicitTaskHeapAndManagedMem(conf, taskExecutorProcessSpec ->
		assertThat(taskExecutorProcessSpec.getNetworkMemSize(), is(taskExecutorProcessSpec.getTotalFlinkMemorySize().multiply(fraction))));
}
 

@Test
public void testConfigOffHeapMemory() {
	MemorySize offHeapMemory = MemorySize.parse("100m");

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.OFF_HEAP_MEMORY, offHeapMemory);

	validateInAllConfigurationsWithoutExplicitTotalFlinkAndJvmHeapMem(
		conf,
		jobManagerProcessSpec -> assertThat(jobManagerProcessSpec.getJvmDirectMemorySize(), is(offHeapMemory)));
}
 

@Test
public void testConfigJvmMetaspaceSize() {
	MemorySize jvmMetaspaceSize = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(options.getJvmOptions().getJvmMetaspaceOption(), jvmMetaspaceSize);

	validateInAllConfigurations(
		conf,
		processSpec -> assertThat(processSpec.getJvmMetaspaceSize(), is(jvmMetaspaceSize)));
}
 

@Test
public void testConfigTotalProcessMemorySize() {
	MemorySize totalProcessMemorySize = MemorySize.parse("2g");

	Configuration conf = new Configuration();
	conf.set(options.getTotalProcessMemoryOption(), totalProcessMemorySize);

	T processSpec = processSpecFromConfig(conf);
	assertThat(processSpec.getTotalProcessMemorySize(), is(totalProcessMemorySize));
}
 

@Test
public void testConfigTotalFlinkMemory() {
	MemorySize totalFlinkMemorySize = MemorySize.parse("1g");

	Configuration conf = new Configuration();
	conf.set(options.getTotalFlinkMemoryOption(), totalFlinkMemorySize);

	T processSpec = processSpecFromConfig(conf);
	assertThat(processSpec.getTotalFlinkMemorySize(), is(totalFlinkMemorySize));
}
 

@Test
public void testFlinkInternalMemorySizeAddUpFailure() {
	MemorySize totalFlinkMemory = MemorySize.parse("199m");
	MemorySize jvmHeap = MemorySize.parse("100m");
	MemorySize offHeapMemory = MemorySize.parse("100m");

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.TOTAL_FLINK_MEMORY, totalFlinkMemory);
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeap);
	conf.set(JobManagerOptions.OFF_HEAP_MEMORY, offHeapMemory);

	validateFail(conf);
}
 

@Test
public void testConfigManagedMemorySize() {
	final MemorySize managedMemSize = MemorySize.parse("100m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemSize);

	// validate in configurations without explicit managed memory size,
	// to avoid checking against overwritten managed memory size
	validateInConfigurationsWithoutExplicitManagedMem(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getManagedMemorySize(), is(managedMemSize)));
}
 

@Test
public void testLogFailureOfJvmHeapSizeMinSizeVerification() {
	MemorySize jvmHeapMemory = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeapMemory);

	JobManagerProcessUtils.processSpecFromConfig(conf);
	MatcherAssert.assertThat(
		testLoggerResource.getMessages(),
		hasItem(containsString(String.format(
			"The configured or derived JVM heap memory size (%s) is less than its recommended minimum value (%s)",
			jvmHeapMemory.toHumanReadableString(),
			JobManagerOptions.MIN_JVM_HEAP_SIZE.toHumanReadableString()))));
}
 

@Test
public void testConfigNetworkMemoryRangeFailure() {
	final MemorySize networkMin = MemorySize.parse("200m");
	final MemorySize networkMax = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MAX, networkMax);
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MIN, networkMin);

	validateFailInAllConfigurations(conf);
}
 

@Test
public void testConfigNetworkMemoryRange() {
	final MemorySize networkMin = MemorySize.parse("200m");
	final MemorySize networkMax = MemorySize.parse("500m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MAX, networkMax);
	conf.set(TaskManagerOptions.NETWORK_MEMORY_MIN, networkMin);

	validateInAllConfigurations(conf, taskExecutorProcessSpec -> {
		assertThat(taskExecutorProcessSpec.getNetworkMemSize().getBytes(), greaterThanOrEqualTo(networkMin.getBytes()));
		assertThat(taskExecutorProcessSpec.getNetworkMemSize().getBytes(), lessThanOrEqualTo(networkMax.getBytes()));
	});
}
 

@Test
public void testConfigJvmOverheadRange() {
	MemorySize minSize = MemorySize.parse("50m");
	MemorySize maxSize = MemorySize.parse("200m");

	Configuration conf = new Configuration();
	conf.set(options.getJvmOptions().getJvmOverheadMax(), maxSize);
	conf.set(options.getJvmOptions().getJvmOverheadMin(), minSize);

	validateInAllConfigurations(conf, JobManagerProcessSpec -> {
		assertThat(JobManagerProcessSpec.getJvmOverheadSize().getBytes(),
			greaterThanOrEqualTo(minSize.getBytes()));
		assertThat(JobManagerProcessSpec.getJvmOverheadSize().getBytes(), lessThanOrEqualTo(maxSize.getBytes()));
	});
}
 

@Test
public void testConfigJvmHeapMemory() {
	MemorySize jvmHeapSize = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeapSize);

	JobManagerProcessSpec jobManagerProcessSpec = JobManagerProcessUtils.processSpecFromConfig(conf);
	assertThat(jobManagerProcessSpec.getJvmHeapMemorySize(), is(jvmHeapSize));
}
 

@Test
public void testConfigJvmOverheadRangeFailure() {
	MemorySize minSize = MemorySize.parse("200m");
	MemorySize maxSize = MemorySize.parse("50m");

	Configuration conf = new Configuration();
	conf.set(options.getJvmOptions().getJvmOverheadMax(), maxSize);
	conf.set(options.getJvmOptions().getJvmOverheadMin(), minSize);

	validateFailInAllConfigurations(conf);
}
 

@Test
public void testConfigFrameworkHeapMemory() {
	final MemorySize frameworkHeapSize = MemorySize.parse("100m");

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY, frameworkHeapSize);

	validateInAllConfigurations(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getFrameworkHeapSize(), is(frameworkHeapSize)));
}
 

@Test
public void testConfigBothNewOptionAndLegacyHeapSize() {
	MemorySize newOptionValue = MemorySize.parse("1g");
	MemorySize legacyHeapSize = MemorySize.parse("2g");

	Configuration conf = new Configuration();
	conf.set(getNewOptionForLegacyHeapOption(), newOptionValue);
	conf.set(legacyMemoryOptions.getHeap(), legacyHeapSize);

	testConfigLegacyHeapMemory(conf, newOptionValue);
}
 

/**
 * Validates loading a file system state backend with additional parameters from the cluster configuration.
 */
@Test
public void testLoadFileSystemStateBackend() throws Exception {
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();
	final Path expectedCheckpointsPath = new Path(checkpointDir);
	final Path expectedSavepointsPath = new Path(savepointDir);
	final MemorySize threshold = MemorySize.parse("900kb");
	final int minWriteBufferSize = 1024;
	final boolean async = !CheckpointingOptions.ASYNC_SNAPSHOTS.defaultValue();

	// we configure with the explicit string (rather than AbstractStateBackend#X_STATE_BACKEND_NAME)
	// to guard against config-breaking changes of the name 
	final Configuration config1 = new Configuration();
	config1.setString(backendKey, "filesystem");
	config1.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
	config1.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config1.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, threshold);
	config1.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, minWriteBufferSize);
	config1.setBoolean(CheckpointingOptions.ASYNC_SNAPSHOTS, async);

	final Configuration config2 = new Configuration();
	config2.setString(backendKey, FsStateBackendFactory.class.getName());
	config2.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
	config2.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config2.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, threshold);
	config1.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, minWriteBufferSize);
	config2.setBoolean(CheckpointingOptions.ASYNC_SNAPSHOTS, async);

	StateBackend backend1 = StateBackendLoader.loadStateBackendFromConfig(config1, cl, null);
	StateBackend backend2 = StateBackendLoader.loadStateBackendFromConfig(config2, cl, null);

	assertTrue(backend1 instanceof FsStateBackend);
	assertTrue(backend2 instanceof FsStateBackend);

	FsStateBackend fs1 = (FsStateBackend) backend1;
	FsStateBackend fs2 = (FsStateBackend) backend2;

	assertEquals(expectedCheckpointsPath, fs1.getCheckpointPath());
	assertEquals(expectedCheckpointsPath, fs2.getCheckpointPath());
	assertEquals(expectedSavepointsPath, fs1.getSavepointPath());
	assertEquals(expectedSavepointsPath, fs2.getSavepointPath());
	assertEquals(threshold.getBytes(), fs1.getMinFileSizeThreshold());
	assertEquals(threshold.getBytes(), fs2.getMinFileSizeThreshold());
	assertEquals(Math.max(threshold.getBytes(), minWriteBufferSize), fs1.getWriteBufferSize());
	assertEquals(Math.max(threshold.getBytes(), minWriteBufferSize), fs2.getWriteBufferSize());
	assertEquals(async, fs1.isUsingAsynchronousSnapshots());
	assertEquals(async, fs2.isUsingAsynchronousSnapshots());
}
 

/**
 * Create mock RM dependencies.
 */
Context() throws Exception {
	rpcService = new TestingRpcService();
	fatalErrorHandler = new TestingFatalErrorHandler();
	rmServices = new MockMesosResourceManagerRuntimeServices();
	mesosServices = new MockMesosServices();

	// TaskExecutor templating
	ContainerSpecification containerSpecification = new ContainerSpecification();

	MemorySize totalProcessMemory = MemorySize.parse("2g");
	TaskExecutorProcessSpec spec = TaskExecutorProcessUtils
		.newProcessSpecBuilder(flinkConfig)
		.withCpuCores(1.0)
		.withTotalProcessMemory(totalProcessMemory)
		.build();
	ContaineredTaskManagerParameters containeredParams =
		new ContaineredTaskManagerParameters(spec, new HashMap<String, String>());
	MesosTaskManagerParameters tmParams = new MesosTaskManagerParameters(
		1, 0, 0, MesosTaskManagerParameters.ContainerType.MESOS, Option.<String>empty(), containeredParams,
		Collections.<Protos.Volume>emptyList(), Collections.<Protos.Parameter>emptyList(), false,
		Collections.<ConstraintEvaluator>emptyList(), "", Option.<String>empty(),
		Option.<String>empty(), Collections.<String>emptyList());

	// resource manager
	rmResourceID = ResourceID.generate();
	resourceManager =
		new TestingMesosResourceManager(
			rpcService,
			RM_ADDRESS,
			rmResourceID,
			rmServices.highAvailabilityServices,
			rmServices.heartbeatServices,
			rmServices.slotManager,
			rmServices.jobLeaderIdService,
			fatalErrorHandler,
			// Mesos specifics
			flinkConfig,
			mesosServices,
			rmServices.mesosConfig,
			tmParams,
			containerSpecification,
			UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup());
	workerResourceSpec = WorkerResourceSpec.fromTaskExecutorProcessSpec(spec);

	// TaskExecutors
	task1Executor = mockTaskExecutor(task1);
	task2Executor = mockTaskExecutor(task2);
	task3Executor = mockTaskExecutor(task3);

	// JobMaster
	jobMaster1 = mockJobMaster(rmServices, new JobID(1, 0));
}