Java Code Examples for org.apache.flink.configuration.MemorySize#ofMebiBytes()

@Test
public void testOffHeapMemoryDerivedFromJvmHeapAndTotalFlinkMemory() {
	MemorySize jvmHeap = MemorySize.ofMebiBytes(150);
	MemorySize defaultOffHeap = JobManagerOptions.OFF_HEAP_MEMORY.defaultValue();
	MemorySize expectedOffHeap = MemorySize.ofMebiBytes(100).add(defaultOffHeap);
	MemorySize totalFlinkMemory = jvmHeap.add(expectedOffHeap);

	Configuration conf = new Configuration();
	conf.set(JobManagerOptions.TOTAL_FLINK_MEMORY, totalFlinkMemory);
	conf.set(JobManagerOptions.JVM_HEAP_MEMORY, jvmHeap);

	JobManagerProcessSpec jobManagerProcessSpec = JobManagerProcessUtils.processSpecFromConfig(conf);
	assertThat(jobManagerProcessSpec.getJvmDirectMemorySize(), is(expectedOffHeap));
	MatcherAssert.assertThat(
		testLoggerResource.getMessages(),
		hasItem(containsString(String.format(
			"The Off-Heap Memory size (%s) is derived the configured Total Flink Memory size (%s) minus " +
				"the configured JVM Heap Memory size (%s). The default Off-Heap Memory size (%s) is ignored.",
			expectedOffHeap.toHumanReadableString(),
			totalFlinkMemory.toHumanReadableString(),
			jvmHeap.toHumanReadableString(),
			defaultOffHeap.toHumanReadableString()))));
}
 

private static Configuration setAllRequiredOptionsExceptOne(ConfigOption<?> optionToNotSet) {
	Configuration configuration = new Configuration();
	if (!TaskManagerOptions.CPU_CORES.equals(optionToNotSet)) {
		configuration.set(TaskManagerOptions.CPU_CORES, 1.0);
	}

	// skip network to exclude min/max mismatch config failure
	MemorySize network = MemorySize.ofMebiBytes(3);
	configuration.set(TaskManagerOptions.NETWORK_MEMORY_MIN, network);
	configuration.set(TaskManagerOptions.NETWORK_MEMORY_MAX, network);

	//noinspection unchecked
	TaskExecutorResourceUtils.CONFIG_OPTIONS
		.stream()
		.filter(option -> !option.equals(TaskManagerOptions.CPU_CORES))
		.filter(option -> !option.equals(optionToNotSet))
		.forEach(option -> configuration.set((ConfigOption<MemorySize>) option, MemorySize.ofMebiBytes(1)));

	return configuration;
}
 

private Configuration getConfigurationWithJvmMetaspaceAndTotalFlinkMemory(
		long jvmMetaspaceSizeMb,
		long totalProcessMemorySizeMb) {
	MemorySize jvmMetaspaceSize = MemorySize.ofMebiBytes(jvmMetaspaceSizeMb);
	MemorySize totalProcessMemorySize = MemorySize.ofMebiBytes(totalProcessMemorySizeMb);
	Configuration conf = new Configuration();
	conf.set(options.getJvmOptions().getJvmMetaspaceOption(), jvmMetaspaceSize);
	conf.set(options.getTotalProcessMemoryOption(), totalProcessMemorySize);
	return conf;
}
 

@Test
public void testDerivedTotalProcessMemoryGreaterThanConfiguredFailureWithTotalFlinkMemory() {
	Configuration conf = getConfigurationWithJvmMetaspaceAndTotalFlinkMemory(100, 200);
	// Total Flink memory + JVM Metaspace > Total Process Memory (no space for JVM overhead)
	MemorySize totalFlinkMemorySize = MemorySize.ofMebiBytes(150);
	conf.set(options.getTotalFlinkMemoryOption(), totalFlinkMemorySize);
	validateFail(conf);
}
 

@Test
public void testDerivedTotalProcessMemoryGreaterThanConfiguredFailureWithFineGrainedOptions() {
	Configuration conf = getConfigurationWithJvmMetaspaceAndTotalFlinkMemory(100, 200);
	// Total Flink memory + JVM Metaspace > Total Process Memory (no space for JVM overhead)
	MemorySize totalFlinkMemorySize = MemorySize.ofMebiBytes(150);
	configWithFineGrainedOptions(conf, totalFlinkMemorySize);
	validateFail(conf);
}
 

@Test
public void testNetworkMaxAdjustForLocalExecutionIfMinSet() {
	MemorySize networkMemorySize = MemorySize.ofMebiBytes(1);
	Configuration configuration = new Configuration();
	configuration.set(TaskManagerOptions.NETWORK_MEMORY_MIN, networkMemorySize);
	TaskExecutorResourceUtils.adjustForLocalExecution(configuration);

	assertThat(configuration.get(TaskManagerOptions.NETWORK_MEMORY_MIN), is(networkMemorySize));
	assertThat(configuration.get(TaskManagerOptions.NETWORK_MEMORY_MAX), is(networkMemorySize));
}
 

@Test
public void testNetworkMinAdjustForLocalExecutionIfMaxSet() {
	MemorySize networkMemorySize = MemorySize.ofMebiBytes(1);
	Configuration configuration = new Configuration();
	configuration.set(TaskManagerOptions.NETWORK_MEMORY_MAX, networkMemorySize);
	TaskExecutorResourceUtils.adjustForLocalExecution(configuration);

	assertThat(configuration.get(TaskManagerOptions.NETWORK_MEMORY_MIN), is(networkMemorySize));
	assertThat(configuration.get(TaskManagerOptions.NETWORK_MEMORY_MAX), is(networkMemorySize));
}
 

public static JvmArgTestingProcessMemorySpec generate() {
	return new JvmArgTestingProcessMemorySpec(
		MemorySize.ofMebiBytes(1),
		MemorySize.ofMebiBytes(2),
		MemorySize.ofMebiBytes(3)
	);
}
 

public Builder setNetworkMemoryMB(int networkMemoryMB) {
	this.networkMemSize = MemorySize.ofMebiBytes(networkMemoryMB);
	return this;
}
 

@Test
public void testEquals() {
	ResourceSpec rs1 = ResourceSpec.newBuilder(1.0, 100).build();
	ResourceSpec rs2 = ResourceSpec.newBuilder(1.0, 100).build();
	assertEquals(ResourceProfile.fromResourceSpec(rs1), ResourceProfile.fromResourceSpec(rs2));

	ResourceSpec rs3 = ResourceSpec.newBuilder(1.0, 100).
			setGPUResource(2.2).
			build();
	ResourceSpec rs4 = ResourceSpec.newBuilder(1.0, 100).
			setGPUResource(1.1).
			build();
	assertNotEquals(ResourceProfile.fromResourceSpec(rs3), ResourceProfile.fromResourceSpec(rs4));

	ResourceSpec rs5 = ResourceSpec.newBuilder(1.0, 100).
			setGPUResource(2.2).
			build();
	MemorySize networkMemory = MemorySize.ofMebiBytes(100);
	assertEquals(ResourceProfile.fromResourceSpec(rs3, networkMemory), ResourceProfile.fromResourceSpec(rs5, networkMemory));

	final ResourceProfile rp1 = ResourceProfile.newBuilder()
		.setCpuCores(1.0)
		.setTaskHeapMemoryMB(100)
		.setTaskOffHeapMemoryMB(100)
		.setManagedMemoryMB(100)
		.setNetworkMemoryMB(100)
		.build();
	final ResourceProfile rp2 = ResourceProfile.newBuilder()
		.setCpuCores(1.1)
		.setTaskHeapMemoryMB(100)
		.setTaskOffHeapMemoryMB(100)
		.setManagedMemoryMB(100)
		.setNetworkMemoryMB(100)
		.build();
	final ResourceProfile rp3 = ResourceProfile.newBuilder()
		.setCpuCores(1.0)
		.setTaskHeapMemoryMB(110)
		.setTaskOffHeapMemoryMB(100)
		.setManagedMemoryMB(100)
		.setNetworkMemoryMB(100)
		.build();
	final ResourceProfile rp4 = ResourceProfile.newBuilder()
		.setCpuCores(1.0)
		.setTaskHeapMemoryMB(100)
		.setTaskOffHeapMemoryMB(110)
		.setManagedMemoryMB(100)
		.setNetworkMemoryMB(100)
		.build();
	final ResourceProfile rp5 = ResourceProfile.newBuilder()
		.setCpuCores(1.0)
		.setTaskHeapMemoryMB(100)
		.setTaskOffHeapMemoryMB(100)
		.setManagedMemoryMB(110)
		.setNetworkMemoryMB(100)
		.build();
	final ResourceProfile rp6 = ResourceProfile.newBuilder()
		.setCpuCores(1.0)
		.setTaskHeapMemoryMB(100)
		.setTaskOffHeapMemoryMB(100)
		.setManagedMemoryMB(110)
		.setNetworkMemoryMB(100)
		.build();
	final ResourceProfile rp7 = ResourceProfile.newBuilder()
		.setCpuCores(1.0)
		.setTaskHeapMemoryMB(100)
		.setTaskOffHeapMemoryMB(100)
		.setManagedMemoryMB(100)
		.setNetworkMemoryMB(110)
		.build();
	final ResourceProfile rp8 = ResourceProfile.newBuilder()
		.setCpuCores(1.0)
		.setTaskHeapMemoryMB(100)
		.setTaskOffHeapMemoryMB(100)
		.setManagedMemoryMB(100)
		.setNetworkMemoryMB(100)
		.build();

	assertNotEquals(rp1, rp2);
	assertNotEquals(rp1, rp3);
	assertNotEquals(rp1, rp4);
	assertNotEquals(rp1, rp5);
	assertNotEquals(rp1, rp6);
	assertNotEquals(rp1, rp7);
	assertEquals(rp1, rp8);
}
 

public Builder setManagedMemoryMB(int managedMemoryMB) {
	this.managedMemSize = MemorySize.ofMebiBytes(managedMemoryMB);
	return this;
}
 

public static Builder newBuilder(double cpuCores, int taskHeapMemoryMB) {
	return new Builder(new CPUResource(cpuCores), MemorySize.ofMebiBytes(taskHeapMemoryMB));
}
 

public Builder setTaskHeapMemoryMB(int taskHeapMemoryMB) {
	this.taskHeapSize = MemorySize.ofMebiBytes(taskHeapMemoryMB);
	return this;
}
 

public Builder setNetworkMemoryMB(int networkMemoryMB) {
	this.networkMemory = MemorySize.ofMebiBytes(networkMemoryMB);
	return this;
}
 

public Builder setManagedMemoryMB(int managedMemoryMB) {
	this.managedMemory = MemorySize.ofMebiBytes(managedMemoryMB);
	return this;
}
 

public Builder setTaskOffHeapMemoryMB(int taskOffHeapMemoryMB) {
	this.taskOffHeapMemory = MemorySize.ofMebiBytes(taskOffHeapMemoryMB);
	return this;
}
 

public Builder setTaskHeapMemoryMB(int taskHeapMemoryMB) {
	this.taskHeapMemory = MemorySize.ofMebiBytes(taskHeapMemoryMB);
	return this;
}
 

public Builder setManagedMemoryMB(int managedMemoryMB) {
	this.managedMemory = MemorySize.ofMebiBytes(managedMemoryMB);
	return this;
}
 

public Builder setOffTaskHeapMemoryMB(int taskOffHeapMemoryMB) {
	this.taskOffHeapMemory = MemorySize.ofMebiBytes(taskOffHeapMemoryMB);
	return this;
}
 

public Builder setTaskHeapMemoryMB(int taskHeapMemoryMB) {
	this.taskHeapMemory = MemorySize.ofMebiBytes(taskHeapMemoryMB);
	return this;
}