Java Code Examples for org.apache.flink.configuration.MemorySize#toHumanReadableString()

private static void sanityCheckTotalFlinkMemory(final Configuration config, final TaskExecutorFlinkMemory flinkInternalMemory) {
	if (isTotalFlinkMemorySizeExplicitlyConfigured(config)) {
		final MemorySize configuredTotalFlinkMemorySize = getTotalFlinkMemorySize(config);
		if (!configuredTotalFlinkMemorySize.equals(flinkInternalMemory.getTotalFlinkMemorySize())) {
			throw new IllegalConfigurationException(
				"Configured and Derived Flink internal memory sizes (total " + flinkInternalMemory.getTotalFlinkMemorySize().toHumanReadableString()
					+ ") do not add up to the configured Total Flink Memory size (" + configuredTotalFlinkMemorySize.toHumanReadableString()
					+ "). Configured and Derived Flink internal memory sizes are: "
					+ "Framework Heap Memory (" + flinkInternalMemory.getFrameworkHeap().toHumanReadableString()
					+ "), Framework Off-Heap Memory (" + flinkInternalMemory.getFrameworkOffHeap().toHumanReadableString()
					+ "), Task Heap Memory (" + flinkInternalMemory.getTaskHeap().toHumanReadableString()
					+ "), Task Off-Heap Memory (" + flinkInternalMemory.getTaskOffHeap().toHumanReadableString()
					+ "), Network Memory (" + flinkInternalMemory.getNetwork().toHumanReadableString()
					+ "), Managed Memory (" + flinkInternalMemory.getManaged().toHumanReadableString() + ").");
		}
	}
}
 

private JvmMetaspaceAndOverhead deriveJvmMetaspaceAndOverheadWithTotalProcessMemory(
		Configuration config,
		MemorySize totalProcessMemorySize) {
	MemorySize jvmMetaspaceSize = getMemorySizeFromConfig(config, options.getJvmOptions().getJvmMetaspaceOption());
	MemorySize jvmOverheadSize = deriveWithFraction(
		"jvm overhead memory",
		totalProcessMemorySize,
		getJvmOverheadRangeFraction(config));
	JvmMetaspaceAndOverhead jvmMetaspaceAndOverhead = new JvmMetaspaceAndOverhead(jvmMetaspaceSize, jvmOverheadSize);

	if (jvmMetaspaceAndOverhead.getTotalJvmMetaspaceAndOverheadSize().getBytes() > totalProcessMemorySize.getBytes()) {
		throw new IllegalConfigurationException(
			"Sum of configured JVM Metaspace (" + jvmMetaspaceAndOverhead.getMetaspace().toHumanReadableString()
				+ ") and JVM Overhead (" + jvmMetaspaceAndOverhead.getOverhead().toHumanReadableString()
				+ ") exceed configured Total Process Memory (" + totalProcessMemorySize.toHumanReadableString() + ").");
	}

	return jvmMetaspaceAndOverhead;
}
 

private MemorySize deriveJvmOverheadFromTotalFlinkMemoryAndOtherComponents(
		Configuration config,
		MemorySize totalFlinkMemorySize) {
	MemorySize totalProcessMemorySize = getMemorySizeFromConfig(config, options.getTotalProcessMemoryOption());
	MemorySize jvmMetaspaceSize = getMemorySizeFromConfig(config, options.getJvmOptions().getJvmMetaspaceOption());
	MemorySize totalFlinkAndJvmMetaspaceSize = totalFlinkMemorySize.add(jvmMetaspaceSize);
	if (totalProcessMemorySize.getBytes() < totalFlinkAndJvmMetaspaceSize.getBytes()) {
		throw new IllegalConfigurationException(
			"The configured Total Process Memory size (%s) is less than the sum of the derived " +
				"Total Flink Memory size (%s) and the configured or default JVM Metaspace size  (%s).",
			totalProcessMemorySize.toHumanReadableString(),
			totalFlinkMemorySize.toHumanReadableString(),
			jvmMetaspaceSize.toHumanReadableString());
	}
	MemorySize jvmOverheadSize = totalProcessMemorySize.subtract(totalFlinkAndJvmMetaspaceSize);
	sanityCheckJvmOverhead(config, jvmOverheadSize, totalProcessMemorySize);
	return jvmOverheadSize;
}
 

private void sanityCheckJvmOverhead(
		Configuration config,
		MemorySize derivedJvmOverheadSize,
		MemorySize totalProcessMemorySize) {
	RangeFraction jvmOverheadRangeFraction = getJvmOverheadRangeFraction(config);
	if (derivedJvmOverheadSize.getBytes() > jvmOverheadRangeFraction.getMaxSize().getBytes() ||
		derivedJvmOverheadSize.getBytes() < jvmOverheadRangeFraction.getMinSize().getBytes()) {
		throw new IllegalConfigurationException("Derived JVM Overhead size ("
			+ derivedJvmOverheadSize.toHumanReadableString() + ") is not in configured JVM Overhead range ["
			+ jvmOverheadRangeFraction.getMinSize().toHumanReadableString() + ", "
			+ jvmOverheadRangeFraction.getMaxSize().toHumanReadableString() + "].");
	}
	if (config.contains(options.getJvmOptions().getJvmOverheadFraction()) &&
		!derivedJvmOverheadSize.equals(totalProcessMemorySize.multiply(jvmOverheadRangeFraction.getFraction()))) {
		LOG.info(
			"The derived JVM Overhead size ({}) does not match " +
				"the configured or default JVM Overhead fraction ({}) from the configured Total Process Memory size ({}). " +
				"The derived JVM OVerhead size will be used.",
			derivedJvmOverheadSize.toHumanReadableString(),
			jvmOverheadRangeFraction.getFraction(),
			totalProcessMemorySize.toHumanReadableString());
	}
}
 

private static void sanityCheckNetworkMemory(
	final Configuration config,
	final MemorySize derivedNetworkMemorySize,
	final MemorySize totalFlinkMemorySize) {
	if (isUsingLegacyNetworkConfigs(config)) {
		final MemorySize configuredNetworkMemorySize = getNetworkMemorySizeWithLegacyConfig(config);
		if (!configuredNetworkMemorySize.equals(derivedNetworkMemorySize)) {
			throw new IllegalConfigurationException(
				"Derived Network Memory size (" + derivedNetworkMemorySize.toHumanReadableString()
					+ ") does not match configured Network Memory size (" + configuredNetworkMemorySize.toHumanReadableString() + ").");
		}
	} else {
		final RangeFraction networkRangeFraction = getNetworkMemoryRangeFraction(config);
		if (derivedNetworkMemorySize.getBytes() > networkRangeFraction.getMaxSize().getBytes() ||
			derivedNetworkMemorySize.getBytes() < networkRangeFraction.getMinSize().getBytes()) {
			throw new IllegalConfigurationException("Derived Network Memory size ("
				+ derivedNetworkMemorySize.toHumanReadableString() + ") is not in configured Network Memory range ["
				+ networkRangeFraction.getMinSize().toHumanReadableString() + ", "
				+ networkRangeFraction.getMaxSize().toHumanReadableString() + "].");
		}
		if (isNetworkMemoryFractionExplicitlyConfigured(config) &&
			!derivedNetworkMemorySize.equals(totalFlinkMemorySize.multiply(networkRangeFraction.getFraction()))) {
			LOG.info(
				"The derived Network Memory size ({}) does not match " +
					"the configured Network Memory fraction ({}) from the configured Total Flink Memory size ({}). " +
					"The derived Network Memory size will be used.",
				derivedNetworkMemorySize.toHumanReadableString(),
				networkRangeFraction.getFraction(),
				totalFlinkMemorySize.toHumanReadableString());
		}
	}
}
 

@Override
public JobManagerFlinkMemory deriveFromTotalFlinkMemory(Configuration config, MemorySize totalFlinkMemorySize) {
	MemorySize offHeapMemorySize = ProcessMemoryUtils.getMemorySizeFromConfig(config, JobManagerOptions.OFF_HEAP_MEMORY);
	if (totalFlinkMemorySize.compareTo(offHeapMemorySize) < 1) {
		throw new IllegalConfigurationException(
			"The configured Total Flink Memory (%s) is less than the configured Off-heap Memory (%s).",
			totalFlinkMemorySize.toHumanReadableString(),
			offHeapMemorySize.toHumanReadableString());
	}
	MemorySize derivedJvmHeapMemorySize = totalFlinkMemorySize.subtract(offHeapMemorySize);
	return createJobManagerFlinkMemory(derivedJvmHeapMemorySize, offHeapMemorySize);
}
 

private static String addMemorySizeString(String resourceStr, String name, MemorySize size) {
	String comma = resourceStr.isEmpty() ? "" : ", ";
	String memorySizeStr = size == null || size.compareTo(MAX_MEMORY_SIZE_TO_LOG) > 0 ?
		"" : comma + name + '=' + size.toHumanReadableString();
	return resourceStr + memorySizeStr;
}
 

@Override
public TaskExecutorFlinkMemory deriveFromRequiredFineGrainedOptions(Configuration config) {
	final MemorySize taskHeapMemorySize = getTaskHeapMemorySize(config);
	final MemorySize managedMemorySize = getManagedMemorySize(config);

	final MemorySize frameworkHeapMemorySize = getFrameworkHeapMemorySize(config);
	final MemorySize frameworkOffHeapMemorySize = getFrameworkOffHeapMemorySize(config);
	final MemorySize taskOffHeapMemorySize = getTaskOffHeapMemorySize(config);

	final MemorySize networkMemorySize;
	final MemorySize totalFlinkExcludeNetworkMemorySize =
		frameworkHeapMemorySize.add(frameworkOffHeapMemorySize).add(taskHeapMemorySize).add(taskOffHeapMemorySize).add(managedMemorySize);

	if (isTotalFlinkMemorySizeExplicitlyConfigured(config)) {
		// derive network memory from total flink memory, and check against network min/max
		final MemorySize totalFlinkMemorySize = getTotalFlinkMemorySize(config);
		if (totalFlinkExcludeNetworkMemorySize.getBytes() > totalFlinkMemorySize.getBytes()) {
			throw new IllegalConfigurationException(
				"Sum of configured Framework Heap Memory (" + frameworkHeapMemorySize.toHumanReadableString()
					+ "), Framework Off-Heap Memory (" + frameworkOffHeapMemorySize.toHumanReadableString()
					+ "), Task Heap Memory (" + taskHeapMemorySize.toHumanReadableString()
					+ "), Task Off-Heap Memory (" + taskOffHeapMemorySize.toHumanReadableString()
					+ ") and Managed Memory (" + managedMemorySize.toHumanReadableString()
					+ ") exceed configured Total Flink Memory (" + totalFlinkMemorySize.toHumanReadableString() + ").");
		}
		networkMemorySize = totalFlinkMemorySize.subtract(totalFlinkExcludeNetworkMemorySize);
		sanityCheckNetworkMemoryWithExplicitlySetTotalFlinkAndHeapMemory(config, networkMemorySize, totalFlinkMemorySize);
	} else {
		// derive network memory from network configs
		networkMemorySize = isUsingLegacyNetworkConfigs(config) ? getNetworkMemorySizeWithLegacyConfig(config) :
			deriveNetworkMemoryWithInverseFraction(config, totalFlinkExcludeNetworkMemorySize);
	}

	final TaskExecutorFlinkMemory flinkInternalMemory = new TaskExecutorFlinkMemory(
		frameworkHeapMemorySize,
		frameworkOffHeapMemorySize,
		taskHeapMemorySize,
		taskOffHeapMemorySize,
		networkMemorySize,
		managedMemorySize);
	sanityCheckTotalFlinkMemory(config, flinkInternalMemory);

	return flinkInternalMemory;
}
 

@Override
public TaskExecutorFlinkMemory deriveFromTotalFlinkMemory(
	final Configuration config,
	final MemorySize totalFlinkMemorySize) {
	final MemorySize frameworkHeapMemorySize = getFrameworkHeapMemorySize(config);
	final MemorySize frameworkOffHeapMemorySize = getFrameworkOffHeapMemorySize(config);
	final MemorySize taskOffHeapMemorySize = getTaskOffHeapMemorySize(config);

	final MemorySize taskHeapMemorySize;
	final MemorySize networkMemorySize;
	final MemorySize managedMemorySize;

	if (isTaskHeapMemorySizeExplicitlyConfigured(config)) {
		// task heap memory is configured,
		// derive managed memory first, leave the remaining to network memory and check against network min/max
		taskHeapMemorySize = getTaskHeapMemorySize(config);
		managedMemorySize = deriveManagedMemoryAbsoluteOrWithFraction(config, totalFlinkMemorySize);
		final MemorySize totalFlinkExcludeNetworkMemorySize =
			frameworkHeapMemorySize.add(frameworkOffHeapMemorySize).add(taskHeapMemorySize).add(taskOffHeapMemorySize).add(managedMemorySize);
		if (totalFlinkExcludeNetworkMemorySize.getBytes() > totalFlinkMemorySize.getBytes()) {
			throw new IllegalConfigurationException(
				"Sum of configured Framework Heap Memory (" + frameworkHeapMemorySize.toHumanReadableString()
					+ "), Framework Off-Heap Memory (" + frameworkOffHeapMemorySize.toHumanReadableString()
					+ "), Task Heap Memory (" + taskHeapMemorySize.toHumanReadableString()
					+ "), Task Off-Heap Memory (" + taskOffHeapMemorySize.toHumanReadableString()
					+ ") and Managed Memory (" + managedMemorySize.toHumanReadableString()
					+ ") exceed configured Total Flink Memory (" + totalFlinkMemorySize.toHumanReadableString() + ").");
		}
		networkMemorySize = totalFlinkMemorySize.subtract(totalFlinkExcludeNetworkMemorySize);
		sanityCheckNetworkMemoryWithExplicitlySetTotalFlinkAndHeapMemory(config, networkMemorySize, totalFlinkMemorySize);
	} else {
		// task heap memory is not configured
		// derive managed memory and network memory, leave the remaining to task heap memory
		managedMemorySize = deriveManagedMemoryAbsoluteOrWithFraction(config, totalFlinkMemorySize);

		networkMemorySize = isUsingLegacyNetworkConfigs(config) ? getNetworkMemorySizeWithLegacyConfig(config) :
			deriveNetworkMemoryWithFraction(config, totalFlinkMemorySize);
		final MemorySize totalFlinkExcludeTaskHeapMemorySize =
			frameworkHeapMemorySize.add(frameworkOffHeapMemorySize).add(taskOffHeapMemorySize).add(managedMemorySize).add(networkMemorySize);
		if (totalFlinkExcludeTaskHeapMemorySize.getBytes() > totalFlinkMemorySize.getBytes()) {
			throw new IllegalConfigurationException(
				"Sum of configured Framework Heap Memory (" + frameworkHeapMemorySize.toHumanReadableString()
					+ "), Framework Off-Heap Memory (" + frameworkOffHeapMemorySize.toHumanReadableString()
					+ "), Task Off-Heap Memory (" + taskOffHeapMemorySize.toHumanReadableString()
					+ "), Managed Memory (" + managedMemorySize.toHumanReadableString()
					+ ") and Network Memory (" + networkMemorySize.toHumanReadableString()
					+ ") exceed configured Total Flink Memory (" + totalFlinkMemorySize.toHumanReadableString() + ").");
		}
		taskHeapMemorySize = totalFlinkMemorySize.subtract(totalFlinkExcludeTaskHeapMemorySize);
	}

	final TaskExecutorFlinkMemory flinkInternalMemory = new TaskExecutorFlinkMemory(
		frameworkHeapMemorySize,
		frameworkOffHeapMemorySize,
		taskHeapMemorySize,
		taskOffHeapMemorySize,
		networkMemorySize,
		managedMemorySize);
	sanityCheckTotalFlinkMemory(config, flinkInternalMemory);

	return flinkInternalMemory;
}