Java Code Examples for org.apache.flink.configuration.MemorySize#getBytes()

/**
 * Calculates the number of network buffers based on configuration and jvm heap size.
 *
 * @param configuration configuration object
 * @param networkMemorySize the size of memory reserved for shuffle environment
 * @param pageSize size of memory segment
 * @return the number of network buffers
 */
private static int calculateNumberOfNetworkBuffers(
	Configuration configuration,
	MemorySize networkMemorySize,
	int pageSize) {

	logIfIgnoringOldConfigs(configuration);

	// tolerate offcuts between intended and allocated memory due to segmentation (will be available to the user-space memory)
	long numberOfNetworkBuffersLong = networkMemorySize.getBytes() / pageSize;
	if (numberOfNetworkBuffersLong > Integer.MAX_VALUE) {
		throw new IllegalArgumentException("The given number of memory bytes (" + networkMemorySize.getBytes()
			+ ") corresponds to more than MAX_INT pages.");
	}

	return (int) numberOfNetworkBuffersLong;
}
 

private JvmMetaspaceAndOverhead deriveJvmMetaspaceAndOverheadWithTotalProcessMemory(
		Configuration config,
		MemorySize totalProcessMemorySize) {
	MemorySize jvmMetaspaceSize = getMemorySizeFromConfig(config, options.getJvmOptions().getJvmMetaspaceOption());
	MemorySize jvmOverheadSize = deriveWithFraction(
		"jvm overhead memory",
		totalProcessMemorySize,
		getJvmOverheadRangeFraction(config));
	JvmMetaspaceAndOverhead jvmMetaspaceAndOverhead = new JvmMetaspaceAndOverhead(jvmMetaspaceSize, jvmOverheadSize);

	if (jvmMetaspaceAndOverhead.getTotalJvmMetaspaceAndOverheadSize().getBytes() > totalProcessMemorySize.getBytes()) {
		throw new IllegalConfigurationException(
			"Sum of configured JVM Metaspace (" + jvmMetaspaceAndOverhead.getMetaspace().toHumanReadableString()
				+ ") and JVM Overhead (" + jvmMetaspaceAndOverhead.getOverhead().toHumanReadableString()
				+ ") exceed configured Total Process Memory (" + totalProcessMemorySize.toHumanReadableString() + ").");
	}

	return jvmMetaspaceAndOverhead;
}
 

private MemorySize deriveJvmOverheadFromTotalFlinkMemoryAndOtherComponents(
		Configuration config,
		MemorySize totalFlinkMemorySize) {
	MemorySize totalProcessMemorySize = getMemorySizeFromConfig(config, options.getTotalProcessMemoryOption());
	MemorySize jvmMetaspaceSize = getMemorySizeFromConfig(config, options.getJvmOptions().getJvmMetaspaceOption());
	MemorySize totalFlinkAndJvmMetaspaceSize = totalFlinkMemorySize.add(jvmMetaspaceSize);
	if (totalProcessMemorySize.getBytes() < totalFlinkAndJvmMetaspaceSize.getBytes()) {
		throw new IllegalConfigurationException(
			"The configured Total Process Memory size (%s) is less than the sum of the derived " +
				"Total Flink Memory size (%s) and the configured or default JVM Metaspace size  (%s).",
			totalProcessMemorySize.toHumanReadableString(),
			totalFlinkMemorySize.toHumanReadableString(),
			jvmMetaspaceSize.toHumanReadableString());
	}
	MemorySize jvmOverheadSize = totalProcessMemorySize.subtract(totalFlinkAndJvmMetaspaceSize);
	sanityCheckJvmOverhead(config, jvmOverheadSize, totalProcessMemorySize);
	return jvmOverheadSize;
}
 

private void sanityCheckJvmOverhead(
		Configuration config,
		MemorySize derivedJvmOverheadSize,
		MemorySize totalProcessMemorySize) {
	RangeFraction jvmOverheadRangeFraction = getJvmOverheadRangeFraction(config);
	if (derivedJvmOverheadSize.getBytes() > jvmOverheadRangeFraction.getMaxSize().getBytes() ||
		derivedJvmOverheadSize.getBytes() < jvmOverheadRangeFraction.getMinSize().getBytes()) {
		throw new IllegalConfigurationException("Derived JVM Overhead size ("
			+ derivedJvmOverheadSize.toHumanReadableString() + ") is not in configured JVM Overhead range ["
			+ jvmOverheadRangeFraction.getMinSize().toHumanReadableString() + ", "
			+ jvmOverheadRangeFraction.getMaxSize().toHumanReadableString() + "].");
	}
	if (config.contains(options.getJvmOptions().getJvmOverheadFraction()) &&
		!derivedJvmOverheadSize.equals(totalProcessMemorySize.multiply(jvmOverheadRangeFraction.getFraction()))) {
		LOG.info(
			"The derived JVM Overhead size ({}) does not match " +
				"the configured or default JVM Overhead fraction ({}) from the configured Total Process Memory size ({}). " +
				"The derived JVM OVerhead size will be used.",
			derivedJvmOverheadSize.toHumanReadableString(),
			jvmOverheadRangeFraction.getFraction(),
			totalProcessMemorySize.toHumanReadableString());
	}
}
 

private static MemorySize capToMinMax(
		String memoryDescription,
		MemorySize relative,
		RangeFraction rangeFraction) {
	long size = relative.getBytes();
	if (size > rangeFraction.getMaxSize().getBytes()) {
		LOG.info(
			"The derived from fraction {} ({}) is greater than its max value {}, max value will be used instead",
			memoryDescription,
			relative.toHumanReadableString(),
			rangeFraction.getMaxSize().toHumanReadableString());
		size = rangeFraction.getMaxSize().getBytes();
	} else if (size < rangeFraction.getMinSize().getBytes()) {
		LOG.info(
			"The derived from fraction {} ({}) is less than its min value {}, min value will be used instead",
			memoryDescription,
			relative.toHumanReadableString(),
			rangeFraction.getMinSize().toHumanReadableString());
		size = rangeFraction.getMinSize().getBytes();
	}
	return new MemorySize(size);
}
 

private static MemorySize deriveOffHeapMemory(
		MemorySize jvmHeapMemorySize,
		MemorySize totalFlinkMemorySize,
		MemorySize defaultOffHeapMemorySize) {
	if (totalFlinkMemorySize.getBytes() < jvmHeapMemorySize.getBytes()) {
		throw new IllegalConfigurationException(String.format(
			"The configured JVM Heap Memory (%s) exceeds the configured Total Flink Memory (%s). " +
				"Please, make the configuration consistent or configure only one option: either JVM Heap " +
				"or Total Flink Memory.",
			jvmHeapMemorySize.toHumanReadableString(),
			totalFlinkMemorySize.toHumanReadableString()));
	}
	MemorySize offHeapMemorySize = totalFlinkMemorySize.subtract(jvmHeapMemorySize);
	if (offHeapMemorySize.getBytes() != defaultOffHeapMemorySize.getBytes()) {
		LOG.info(
			"The Off-Heap Memory size ({}) is derived the configured Total Flink Memory size ({}) minus " +
				"the configured JVM Heap Memory size ({}). The default Off-Heap Memory size ({}) is ignored.",
			offHeapMemorySize.toHumanReadableString(),
			totalFlinkMemorySize.toHumanReadableString(),
			jvmHeapMemorySize.toHumanReadableString(),
			defaultOffHeapMemorySize.toHumanReadableString());
	}
	return offHeapMemorySize;
}
 

private static void sanityCheckNetworkMemory(
	final Configuration config,
	final MemorySize derivedNetworkMemorySize,
	final MemorySize totalFlinkMemorySize) {
	if (isUsingLegacyNetworkConfigs(config)) {
		final MemorySize configuredNetworkMemorySize = getNetworkMemorySizeWithLegacyConfig(config);
		if (!configuredNetworkMemorySize.equals(derivedNetworkMemorySize)) {
			throw new IllegalConfigurationException(
				"Derived Network Memory size (" + derivedNetworkMemorySize.toHumanReadableString()
					+ ") does not match configured Network Memory size (" + configuredNetworkMemorySize.toHumanReadableString() + ").");
		}
	} else {
		final RangeFraction networkRangeFraction = getNetworkMemoryRangeFraction(config);
		if (derivedNetworkMemorySize.getBytes() > networkRangeFraction.getMaxSize().getBytes() ||
			derivedNetworkMemorySize.getBytes() < networkRangeFraction.getMinSize().getBytes()) {
			throw new IllegalConfigurationException("Derived Network Memory size ("
				+ derivedNetworkMemorySize.toHumanReadableString() + ") is not in configured Network Memory range ["
				+ networkRangeFraction.getMinSize().toHumanReadableString() + ", "
				+ networkRangeFraction.getMaxSize().toHumanReadableString() + "].");
		}
		if (isNetworkMemoryFractionExplicitlyConfigured(config) &&
			!derivedNetworkMemorySize.equals(totalFlinkMemorySize.multiply(networkRangeFraction.getFraction()))) {
			LOG.info(
				"The derived Network Memory size ({}) does not match " +
					"the configured Network Memory fraction ({}) from the configured Total Flink Memory size ({}). " +
					"The derived Network Memory size will be used.",
				derivedNetworkMemorySize.toHumanReadableString(),
				networkRangeFraction.getFraction(),
				totalFlinkMemorySize.toHumanReadableString());
		}
	}
}
 

private static void sanityCheckTotalFlinkMemory(
		MemorySize totalFlinkMemorySize,
		MemorySize jvmHeapMemorySize,
		MemorySize offHeapMemorySize) {
	MemorySize derivedTotalFlinkMemorySize = jvmHeapMemorySize.add(offHeapMemorySize);
	if (derivedTotalFlinkMemorySize.getBytes() != totalFlinkMemorySize.getBytes()) {
		throw new IllegalConfigurationException(String.format(
			"Sum of the configured JVM Heap Memory (%s) and the configured Off-heap Memory (%s) " +
				"does not match the configured Total Flink Memory (%s). Please, make the configuration consistent " +
				"or configure only one option: either JVM Heap or Total Flink Memory.",
			jvmHeapMemorySize.toHumanReadableString(),
			offHeapMemorySize.toHumanReadableString(),
			totalFlinkMemorySize.toHumanReadableString()));
	}
}
 

@Override
protected void configWithFineGrainedOptions(Configuration configuration, MemorySize totalFlinkMemorySize) {
	MemorySize componentSize = new MemorySize(totalFlinkMemorySize.getBytes() / 6);
	configuration.set(TaskManagerOptions.TASK_HEAP_MEMORY, componentSize);
	configuration.set(TaskManagerOptions.TASK_OFF_HEAP_MEMORY, componentSize);
	configuration.set(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY, componentSize);
	configuration.set(TaskManagerOptions.FRAMEWORK_OFF_HEAP_MEMORY, componentSize);
	configuration.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, componentSize);
	// network is the 6th component, fixed implicitly
}
 

@Override
protected void configWithFineGrainedOptions(Configuration configuration, MemorySize totalFlinkMemorySize) {
	MemorySize heapSize = new MemorySize(totalFlinkMemorySize.getBytes() / 2);
	MemorySize offHeapSize = totalFlinkMemorySize.subtract(heapSize);
	configuration.set(JobManagerOptions.JVM_HEAP_MEMORY, heapSize);
	configuration.set(JobManagerOptions.OFF_HEAP_MEMORY, offHeapSize);
}
 

@Override
public TaskExecutorFlinkMemory deriveFromRequiredFineGrainedOptions(Configuration config) {
	final MemorySize taskHeapMemorySize = getTaskHeapMemorySize(config);
	final MemorySize managedMemorySize = getManagedMemorySize(config);

	final MemorySize frameworkHeapMemorySize = getFrameworkHeapMemorySize(config);
	final MemorySize frameworkOffHeapMemorySize = getFrameworkOffHeapMemorySize(config);
	final MemorySize taskOffHeapMemorySize = getTaskOffHeapMemorySize(config);

	final MemorySize networkMemorySize;
	final MemorySize totalFlinkExcludeNetworkMemorySize =
		frameworkHeapMemorySize.add(frameworkOffHeapMemorySize).add(taskHeapMemorySize).add(taskOffHeapMemorySize).add(managedMemorySize);

	if (isTotalFlinkMemorySizeExplicitlyConfigured(config)) {
		// derive network memory from total flink memory, and check against network min/max
		final MemorySize totalFlinkMemorySize = getTotalFlinkMemorySize(config);
		if (totalFlinkExcludeNetworkMemorySize.getBytes() > totalFlinkMemorySize.getBytes()) {
			throw new IllegalConfigurationException(
				"Sum of configured Framework Heap Memory (" + frameworkHeapMemorySize.toHumanReadableString()
					+ "), Framework Off-Heap Memory (" + frameworkOffHeapMemorySize.toHumanReadableString()
					+ "), Task Heap Memory (" + taskHeapMemorySize.toHumanReadableString()
					+ "), Task Off-Heap Memory (" + taskOffHeapMemorySize.toHumanReadableString()
					+ ") and Managed Memory (" + managedMemorySize.toHumanReadableString()
					+ ") exceed configured Total Flink Memory (" + totalFlinkMemorySize.toHumanReadableString() + ").");
		}
		networkMemorySize = totalFlinkMemorySize.subtract(totalFlinkExcludeNetworkMemorySize);
		sanityCheckNetworkMemoryWithExplicitlySetTotalFlinkAndHeapMemory(config, networkMemorySize, totalFlinkMemorySize);
	} else {
		// derive network memory from network configs
		networkMemorySize = isUsingLegacyNetworkConfigs(config) ? getNetworkMemorySizeWithLegacyConfig(config) :
			deriveNetworkMemoryWithInverseFraction(config, totalFlinkExcludeNetworkMemorySize);
	}

	final TaskExecutorFlinkMemory flinkInternalMemory = new TaskExecutorFlinkMemory(
		frameworkHeapMemorySize,
		frameworkOffHeapMemorySize,
		taskHeapMemorySize,
		taskOffHeapMemorySize,
		networkMemorySize,
		managedMemorySize);
	sanityCheckTotalFlinkMemory(config, flinkInternalMemory);

	return flinkInternalMemory;
}
 

@Override
public TaskExecutorFlinkMemory deriveFromTotalFlinkMemory(
	final Configuration config,
	final MemorySize totalFlinkMemorySize) {
	final MemorySize frameworkHeapMemorySize = getFrameworkHeapMemorySize(config);
	final MemorySize frameworkOffHeapMemorySize = getFrameworkOffHeapMemorySize(config);
	final MemorySize taskOffHeapMemorySize = getTaskOffHeapMemorySize(config);

	final MemorySize taskHeapMemorySize;
	final MemorySize networkMemorySize;
	final MemorySize managedMemorySize;

	if (isTaskHeapMemorySizeExplicitlyConfigured(config)) {
		// task heap memory is configured,
		// derive managed memory first, leave the remaining to network memory and check against network min/max
		taskHeapMemorySize = getTaskHeapMemorySize(config);
		managedMemorySize = deriveManagedMemoryAbsoluteOrWithFraction(config, totalFlinkMemorySize);
		final MemorySize totalFlinkExcludeNetworkMemorySize =
			frameworkHeapMemorySize.add(frameworkOffHeapMemorySize).add(taskHeapMemorySize).add(taskOffHeapMemorySize).add(managedMemorySize);
		if (totalFlinkExcludeNetworkMemorySize.getBytes() > totalFlinkMemorySize.getBytes()) {
			throw new IllegalConfigurationException(
				"Sum of configured Framework Heap Memory (" + frameworkHeapMemorySize.toHumanReadableString()
					+ "), Framework Off-Heap Memory (" + frameworkOffHeapMemorySize.toHumanReadableString()
					+ "), Task Heap Memory (" + taskHeapMemorySize.toHumanReadableString()
					+ "), Task Off-Heap Memory (" + taskOffHeapMemorySize.toHumanReadableString()
					+ ") and Managed Memory (" + managedMemorySize.toHumanReadableString()
					+ ") exceed configured Total Flink Memory (" + totalFlinkMemorySize.toHumanReadableString() + ").");
		}
		networkMemorySize = totalFlinkMemorySize.subtract(totalFlinkExcludeNetworkMemorySize);
		sanityCheckNetworkMemoryWithExplicitlySetTotalFlinkAndHeapMemory(config, networkMemorySize, totalFlinkMemorySize);
	} else {
		// task heap memory is not configured
		// derive managed memory and network memory, leave the remaining to task heap memory
		managedMemorySize = deriveManagedMemoryAbsoluteOrWithFraction(config, totalFlinkMemorySize);

		networkMemorySize = isUsingLegacyNetworkConfigs(config) ? getNetworkMemorySizeWithLegacyConfig(config) :
			deriveNetworkMemoryWithFraction(config, totalFlinkMemorySize);
		final MemorySize totalFlinkExcludeTaskHeapMemorySize =
			frameworkHeapMemorySize.add(frameworkOffHeapMemorySize).add(taskOffHeapMemorySize).add(managedMemorySize).add(networkMemorySize);
		if (totalFlinkExcludeTaskHeapMemorySize.getBytes() > totalFlinkMemorySize.getBytes()) {
			throw new IllegalConfigurationException(
				"Sum of configured Framework Heap Memory (" + frameworkHeapMemorySize.toHumanReadableString()
					+ "), Framework Off-Heap Memory (" + frameworkOffHeapMemorySize.toHumanReadableString()
					+ "), Task Off-Heap Memory (" + taskOffHeapMemorySize.toHumanReadableString()
					+ "), Managed Memory (" + managedMemorySize.toHumanReadableString()
					+ ") and Network Memory (" + networkMemorySize.toHumanReadableString()
					+ ") exceed configured Total Flink Memory (" + totalFlinkMemorySize.toHumanReadableString() + ").");
		}
		taskHeapMemorySize = totalFlinkMemorySize.subtract(totalFlinkExcludeTaskHeapMemorySize);
	}

	final TaskExecutorFlinkMemory flinkInternalMemory = new TaskExecutorFlinkMemory(
		frameworkHeapMemorySize,
		frameworkOffHeapMemorySize,
		taskHeapMemorySize,
		taskOffHeapMemorySize,
		networkMemorySize,
		managedMemorySize);
	sanityCheckTotalFlinkMemory(config, flinkInternalMemory);

	return flinkInternalMemory;
}