Java Code Examples for org.apache.flink.configuration.Configuration#getFloat()

public static RangeFraction getRangeFraction(
		MemorySize minSize,
		MemorySize maxSize,
		ConfigOption<Float> fractionOption,
		Configuration config) {
	double fraction = config.getFloat(fractionOption);
	try {
		return new RangeFraction(minSize, maxSize, fraction);
	} catch (IllegalArgumentException e) {
		throw new IllegalConfigurationException(
			String.format(
				"Inconsistently configured %s (%s) and its min (%s), max (%s) value",
				fractionOption,
				fraction,
				minSize.toHumanReadableString(),
				maxSize.toHumanReadableString()),
			e);
	}
}
 

/**
 * This tests that per default the off heap memory is set to what the network buffers require.
 */
@Test
public void testOffHeapMemoryWithDefaultConfiguration() {
	Configuration conf = new Configuration();

	ContaineredTaskManagerParameters params =
		ContaineredTaskManagerParameters.create(conf, CONTAINER_MEMORY, 1);

	final float memoryCutoffRatio = conf.getFloat(
		ConfigConstants.CONTAINERIZED_HEAP_CUTOFF_RATIO,
		ConfigConstants.DEFAULT_YARN_HEAP_CUTOFF_RATIO);
	final int minCutoff = conf.getInteger(
		ConfigConstants.CONTAINERIZED_HEAP_CUTOFF_MIN,
		ConfigConstants.DEFAULT_YARN_HEAP_CUTOFF);

	long cutoff = Math.max((long) (CONTAINER_MEMORY * memoryCutoffRatio), minCutoff);
	final long networkBufMB =
		calculateNetworkBufferMemory(
			(CONTAINER_MEMORY - cutoff) << 20, // megabytes to bytes
			conf) >> 20; // bytes to megabytes
	assertEquals(networkBufMB + cutoff, params.taskManagerDirectMemoryLimitMB());
}
 

/**
 * Calculates the amount of memory used for network buffers based on the total memory to use and
 * the according configuration parameters.
 *
 * <p>The following configuration parameters are involved:
 * <ul>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX}, and</li>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_NUM_BUFFERS} (fallback if the ones above do not exist)</li>
 * </ul>.
 *
 * @param totalProcessMemory overall available memory to use (in bytes)
 * @param config configuration object
 *
 * @return memory to use for network buffers (in bytes)
 */
@SuppressWarnings("deprecation")
public static long calculateNetworkBufferMemory(long totalProcessMemory, Configuration config) {
	final int segmentSize = ConfigurationParserUtils.getPageSize(config);

	final long networkReservedMemory;
	if (hasNewNetworkConfig(config)) {
		float networkBufFraction = config.getFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION);
		long networkMemoryByFraction = (long) (totalProcessMemory * networkBufFraction);
		networkReservedMemory = calculateNewNetworkBufferMemory(config, networkMemoryByFraction, totalProcessMemory);
	} else {
		// use old (deprecated) network buffers parameter
		int numNetworkBuffers = config.getInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS);
		networkReservedMemory = (long) numNetworkBuffers * (long) segmentSize;

		checkOldNetworkConfig(numNetworkBuffers);

		ConfigurationParserUtils.checkConfigParameter(networkReservedMemory < totalProcessMemory,
			networkReservedMemory, NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS.key(),
			"Network buffer memory size too large: " + networkReservedMemory + " >= " +
				totalProcessMemory + " (total JVM memory size)");
	}

	return networkReservedMemory;
}
 

/**
 * Calculates the amount of memory used for network buffers based on the total memory to use and
 * the according configuration parameters.
 *
 * <p>The following configuration parameters are involved:
 * <ul>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX}</li>
 * </ul>.
 *
 * @param config configuration object
 * @param networkMemoryByFraction memory of network buffers based on JVM memory size and network fraction
 * @param maxAllowedMemory maximum memory used for checking the results of network memory
 *
 * @return memory to use for network buffers (in bytes)
 */
private static long calculateNewNetworkBufferMemory(Configuration config, long networkMemoryByFraction, long maxAllowedMemory) {
	float networkBufFraction = config.getFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION);
	long networkBufMin = MemorySize.parse(config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN)).getBytes();
	long networkBufMax = MemorySize.parse(config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX)).getBytes();

	int pageSize = ConfigurationParserUtils.getPageSize(config);

	checkNewNetworkConfig(pageSize, networkBufFraction, networkBufMin, networkBufMax);

	long networkBufBytes = Math.min(networkBufMax, Math.max(networkBufMin, networkMemoryByFraction));

	ConfigurationParserUtils.checkConfigParameter(networkBufBytes < maxAllowedMemory,
		"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
		"(" + NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
			NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
			NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
		"Network buffer memory size too large: " + networkBufBytes + " >= " +
			maxAllowedMemory + " (maximum JVM memory size)");

	return networkBufBytes;
}
 

/**
 * Gets the size of managed memory from the heap size after subtracting network buffer memory.
 * All values are in bytes.
 */
public static long getManagedMemoryFromHeapAndManaged(Configuration config, long heapAndManagedMemory) {
	if (config.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) {
		// take the configured absolute value
		final String sizeValue = config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE);
		try {
			return MemorySize.parse(sizeValue, MEGA_BYTES).getBytes();
		}
		catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException(
				"Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e);
		}
	}
	else {
		// calculate managed memory size via fraction
		final float fraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
		return (long) (fraction * heapAndManagedMemory);
	}
}
 

public JobGraphGenerator(Configuration config) {
	this.defaultMaxFan = config.getInteger(AlgorithmOptions.SPILLING_MAX_FAN);
	this.defaultSortSpillingThreshold = config.getFloat(AlgorithmOptions.SORT_SPILLING_THRESHOLD);
	this.useLargeRecordHandler = config.getBoolean(
			ConfigConstants.USE_LARGE_RECORD_HANDLER_KEY,
			ConfigConstants.DEFAULT_USE_LARGE_RECORD_HANDLER);
}
 

/**
 * Calcuate cutoff memory size used by container, it will throw an {@link IllegalArgumentException}
 * if the config is invalid or return the cutoff value if valid.
 *
 * @param config The Flink configuration.
 * @param containerMemoryMB The size of the complete container, in megabytes.
 *
 * @return cutoff memory size used by container.
 */
public static long calculateCutoffMB(Configuration config, long containerMemoryMB) {
	Preconditions.checkArgument(containerMemoryMB > 0);

	// (1) check cutoff ratio
	final float memoryCutoffRatio = config.getFloat(
		ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO);

	if (memoryCutoffRatio >= 1 || memoryCutoffRatio <= 0) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO.key() + "' must be between 0 and 1. Value given="
			+ memoryCutoffRatio);
	}

	// (2) check min cutoff value
	final int minCutoff = config.getInteger(
		ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN);

	if (minCutoff >= containerMemoryMB) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.key() + "'='" + minCutoff
			+ "' is larger than the total container memory " + containerMemoryMB);
	}

	// (3) check between heap and off-heap
	long cutoff = (long) (containerMemoryMB * memoryCutoffRatio);
	if (cutoff < minCutoff) {
		cutoff = minCutoff;
	}
	return cutoff;
}
 

private static void setManagedMemoryByFraction(final Configuration config) {
  if (!config.containsKey("taskmanager.memory.managed.size")) {
    float managedMemoryFraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
    long freeHeapMemory = EnvironmentInformation.getSizeOfFreeHeapMemoryWithDefrag();
    long managedMemorySize = (long) (freeHeapMemory * managedMemoryFraction);
    config.setString("taskmanager.memory.managed.size", String.valueOf(managedMemorySize));
  }
}
 

public JobGraphGenerator(Configuration config) {
	this.defaultMaxFan = config.getInteger(AlgorithmOptions.SPILLING_MAX_FAN);
	this.defaultSortSpillingThreshold = config.getFloat(AlgorithmOptions.SORT_SPILLING_THRESHOLD);
	this.useLargeRecordHandler = config.getBoolean(
			ConfigConstants.USE_LARGE_RECORD_HANDLER_KEY,
			ConfigConstants.DEFAULT_USE_LARGE_RECORD_HANDLER);
}
 

/**
 * This tests that per default the off heap memory is set to what the network buffers require.
 */
@Test
public void testOffHeapMemoryWithDefaultConfiguration() {
	Configuration conf = new Configuration();

	ContaineredTaskManagerParameters params =
		ContaineredTaskManagerParameters.create(conf, CONTAINER_MEMORY, 1);

	final float memoryCutoffRatio = conf.getFloat(
		ConfigConstants.CONTAINERIZED_HEAP_CUTOFF_RATIO,
		ConfigConstants.DEFAULT_YARN_HEAP_CUTOFF_RATIO);
	final int minCutoff = conf.getInteger(
		ConfigConstants.CONTAINERIZED_HEAP_CUTOFF_MIN,
		ConfigConstants.DEFAULT_YARN_HEAP_CUTOFF);

	long cutoff = Math.max((long) (CONTAINER_MEMORY * memoryCutoffRatio), minCutoff);
	final long networkBufMB = TaskManagerServices.getReservedNetworkMemory(
		conf,
		(CONTAINER_MEMORY - cutoff) << 20 // megabytes to bytes
	) >> 20; // bytes to megabytes

	// this is unfortunately necessary due to rounding errors that happen due to back and
	// forth conversion between bytes and megabytes
	// ideally all logic calculates precisely with bytes and we use coarser units only un
	// user-facing configuration and parametrization classes
	assertThat(networkBufMB + cutoff,
		anyOf(
			equalTo(params.taskManagerDirectMemoryLimitMB()),
			equalTo(params.taskManagerDirectMemoryLimitMB() - 1)));
}
 

/**
 * Parses the configuration to get the fraction of managed memory and validates the value.
 *
 * @param configuration configuration object
 * @return fraction of managed memory
 */
public static float getManagedMemoryFraction(Configuration configuration) {
	float managedMemoryFraction = configuration.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);

	checkConfigParameter(managedMemoryFraction > 0.0f && managedMemoryFraction < 1.0f, managedMemoryFraction,
		TaskManagerOptions.MANAGED_MEMORY_FRACTION.key(),
		"MemoryManager fraction of the free memory must be between 0.0 and 1.0");

	return managedMemoryFraction;
}
 

/**
 * Calculates the amount of memory used for network buffers inside the current JVM instance
 * based on the available heap or the max heap size and the according configuration parameters.
 *
 * <p>For containers or when started via scripts, if started with a memory limit and set to use
 * off-heap memory, the maximum heap size for the JVM is adjusted accordingly and we are able
 * to extract the intended values from this.
 *
 * <p>The following configuration parameters are involved:
 * <ul>
 *  <li>{@link TaskManagerOptions#MANAGED_MEMORY_SIZE},</li>
 *  <li>{@link TaskManagerOptions#MANAGED_MEMORY_FRACTION},</li>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX}, and</li>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_NUM_BUFFERS} (fallback if the ones above do not exist)</li>
 * </ul>.
 *
 * @param config configuration object
 * @param maxJvmHeapMemory the maximum JVM heap size (in bytes)
 *
 * @return memory to use for network buffers (in bytes)
 */
@VisibleForTesting
public static long calculateNewNetworkBufferMemory(Configuration config, long maxJvmHeapMemory) {
	// The maximum heap memory has been adjusted as in TaskManagerServices#calculateHeapSizeMB
	// and we need to invert these calculations.
	final long heapAndManagedMemory;
	final MemoryType memoryType = ConfigurationParserUtils.getMemoryType(config);
	if (memoryType == MemoryType.HEAP) {
		heapAndManagedMemory = maxJvmHeapMemory;
	} else if (memoryType == MemoryType.OFF_HEAP) {
		long configuredMemory = ConfigurationParserUtils.getManagedMemorySize(config) << 20; // megabytes to bytes
		if (configuredMemory > 0) {
			// The maximum heap memory has been adjusted according to configuredMemory, i.e.
			// maxJvmHeap = jvmHeapNoNet - configuredMemory
			heapAndManagedMemory = maxJvmHeapMemory + configuredMemory;
		} else {
			// The maximum heap memory has been adjusted according to the fraction, i.e.
			// maxJvmHeap = jvmHeapNoNet - jvmHeapNoNet * managedFraction = jvmHeapNoNet * (1 - managedFraction)
			heapAndManagedMemory = (long) (maxJvmHeapMemory / (1.0 - ConfigurationParserUtils.getManagedMemoryFraction(config)));
		}
	} else {
		throw new RuntimeException("No supported memory type detected.");
	}

	// finally extract the network buffer memory size again from:
	// heapAndManagedMemory = totalProcessMemory - networkReservedMemory
	//                      = totalProcessMemory - Math.min(networkBufMax, Math.max(networkBufMin, totalProcessMemory * networkBufFraction)
	// totalProcessMemory = heapAndManagedMemory / (1.0 - networkBufFraction)
	float networkBufFraction = config.getFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION);

	// Converts to double for higher precision. Converting via string achieve higher precision for those
	// numbers can not be represented preciously by float, like 0.4f.
	double heapAndManagedFraction = 1.0 - Double.valueOf(Float.toString(networkBufFraction));
	long totalProcessMemory = (long) (heapAndManagedMemory / heapAndManagedFraction);
	long networkMemoryByFraction = (long) (totalProcessMemory * networkBufFraction);

	// Do not need to check the maximum allowed memory since the computed total memory should always
	// be larger than the computed network buffer memory as long as the fraction is less than 1.
	return calculateNewNetworkBufferMemory(config, networkMemoryByFraction, Long.MAX_VALUE);
}
 

/**
 * Calcuate cutoff memory size used by container, it will throw an {@link IllegalArgumentException}
 * if the config is invalid or return the cutoff value if valid.
 *
 * @param config The Flink configuration.
 * @param containerMemoryMB The size of the complete container, in megabytes.
 *
 * @return cutoff memory size used by container.
 */
public static long calculateCutoffMB(Configuration config, long containerMemoryMB) {
	Preconditions.checkArgument(containerMemoryMB > 0);

	// (1) check cutoff ratio
	final float memoryCutoffRatio = config.getFloat(
		ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO);

	if (memoryCutoffRatio >= 1 || memoryCutoffRatio <= 0) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO.key() + "' must be between 0 and 1. Value given="
			+ memoryCutoffRatio);
	}

	// (2) check min cutoff value
	final int minCutoff = config.getInteger(
		ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN);

	if (minCutoff >= containerMemoryMB) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.key() + "'='" + minCutoff
			+ "' is larger than the total container memory " + containerMemoryMB);
	}

	// (3) check between heap and off-heap
	long cutoff = (long) (containerMemoryMB * memoryCutoffRatio);
	if (cutoff < minCutoff) {
		cutoff = minCutoff;
	}
	return cutoff;
}
 

public JobGraphGenerator(Configuration config) {
	this.defaultMaxFan = config.getInteger(AlgorithmOptions.SPILLING_MAX_FAN);
	this.defaultSortSpillingThreshold = config.getFloat(AlgorithmOptions.SORT_SPILLING_THRESHOLD);
	this.useLargeRecordHandler = config.getBoolean(
			ConfigConstants.USE_LARGE_RECORD_HANDLER_KEY,
			ConfigConstants.DEFAULT_USE_LARGE_RECORD_HANDLER);
}
 

/**
 * Utility method to extract TaskManager config parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration The configuration.
 * @param remoteAddress identifying the IP address under which the TaskManager will be accessible
 * @param localCommunication True, to skip initializing the network stack.
 *                                      Use only in cases where only one task manager runs.
 * @return TaskExecutorConfiguration that wrappers InstanceConnectionInfo, NetworkEnvironmentConfiguration, etc.
 */
public static TaskManagerServicesConfiguration fromConfiguration(
		Configuration configuration,
		InetAddress remoteAddress,
		boolean localCommunication) throws Exception {

	// we need this because many configs have been written with a "-1" entry
	int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1);
	if (slots == -1) {
		slots = 1;
	}

	final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);
	String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration);

	if (localStateRootDir.length == 0) {
		// default to temp dirs.
		localStateRootDir = tmpDirs;
	}

	boolean localRecoveryMode = configuration.getBoolean(
		CheckpointingOptions.LOCAL_RECOVERY.key(),
		CheckpointingOptions.LOCAL_RECOVERY.defaultValue());

	final NetworkEnvironmentConfiguration networkConfig = parseNetworkEnvironmentConfiguration(
		configuration,
		localCommunication,
		remoteAddress,
		slots);

	final QueryableStateConfiguration queryableStateConfig =
			parseQueryableStateConfiguration(configuration);

	// extract memory settings
	long configuredMemory;
	String managedMemorySizeDefaultVal = TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue();
	if (!configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(managedMemorySizeDefaultVal)) {
		try {
			configuredMemory = MemorySize.parse(configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE), MEGA_BYTES).getMebiBytes();
		} catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException(
				"Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e);
		}
	} else {
		configuredMemory = Long.valueOf(managedMemorySizeDefaultVal);
	}

	checkConfigParameter(
		configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()) ||
			configuredMemory > 0, configuredMemory,
		TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
		"MemoryManager needs at least one MB of memory. " +
			"If you leave this config parameter empty, the system automatically " +
			"pick a fraction of the available memory.");

	// check whether we use heap or off-heap memory
	final MemoryType memType;
	if (configuration.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP)) {
		memType = MemoryType.OFF_HEAP;
	} else {
		memType = MemoryType.HEAP;
	}

	boolean preAllocateMemory = configuration.getBoolean(TaskManagerOptions.MANAGED_MEMORY_PRE_ALLOCATE);

	float memoryFraction = configuration.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
	checkConfigParameter(memoryFraction > 0.0f && memoryFraction < 1.0f, memoryFraction,
		TaskManagerOptions.MANAGED_MEMORY_FRACTION.key(),
		"MemoryManager fraction of the free memory must be between 0.0 and 1.0");

	long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	return new TaskManagerServicesConfiguration(
		remoteAddress,
		tmpDirs,
		localStateRootDir,
		localRecoveryMode,
		networkConfig,
		queryableStateConfig,
		slots,
		configuredMemory,
		memType,
		preAllocateMemory,
		memoryFraction,
		timerServiceShutdownTimeout,
		retryingRegistrationConfiguration,
		ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));
}
 

/**
 * Calculates the amount of heap memory to use (to set via <tt>-Xmx</tt> and <tt>-Xms</tt>)
 * based on the total memory to use and the given configuration parameters.
 *
 * @param totalJavaMemorySizeMB
 * 		overall available memory to use (heap and off-heap)
 * @param config
 * 		configuration object
 *
 * @return heap memory to use (in megabytes)
 */
public static long calculateHeapSizeMB(long totalJavaMemorySizeMB, Configuration config) {
	Preconditions.checkArgument(totalJavaMemorySizeMB > 0);

	// subtract the Java memory used for network buffers (always off-heap)
	final long networkBufMB =
		calculateNetworkBufferMemory(
			totalJavaMemorySizeMB << 20, // megabytes to bytes
			config) >> 20; // bytes to megabytes
	final long remainingJavaMemorySizeMB = totalJavaMemorySizeMB - networkBufMB;

	// split the available Java memory between heap and off-heap

	final boolean useOffHeap = config.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP);

	final long heapSizeMB;
	if (useOffHeap) {

		long offHeapSize;
		String managedMemorySizeDefaultVal = TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue();
		if (!config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(managedMemorySizeDefaultVal)) {
			try {
				offHeapSize = MemorySize.parse(config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE), MEGA_BYTES).getMebiBytes();
			} catch (IllegalArgumentException e) {
				throw new IllegalConfigurationException(
					"Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e);
			}
		} else {
			offHeapSize = Long.valueOf(managedMemorySizeDefaultVal);
		}

		if (offHeapSize <= 0) {
			// calculate off-heap section via fraction
			double fraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
			offHeapSize = (long) (fraction * remainingJavaMemorySizeMB);
		}

		TaskManagerServicesConfiguration
			.checkConfigParameter(offHeapSize < remainingJavaMemorySizeMB, offHeapSize,
				TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
				"Managed memory size too large for " + networkBufMB +
					" MB network buffer memory and a total of " + totalJavaMemorySizeMB +
					" MB JVM memory");

		heapSizeMB = remainingJavaMemorySizeMB - offHeapSize;
	} else {
		heapSizeMB = remainingJavaMemorySizeMB;
	}

	return heapSizeMB;
}
 

/**
 * Calculates the amount of memory used for network buffers based on the total memory to use and
 * the according configuration parameters.
 *
 * <p>The following configuration parameters are involved:
 * <ul>
 *  <li>{@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
 * 	<li>{@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
 * 	<li>{@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MAX}, and</li>
 *  <li>{@link TaskManagerOptions#NETWORK_NUM_BUFFERS} (fallback if the ones above do not exist)</li>
 * </ul>.
 *
 * @param totalJavaMemorySize
 * 		overall available memory to use (heap and off-heap, in bytes)
 * @param config
 * 		configuration object
 *
 * @return memory to use for network buffers (in bytes); at least one memory segment
 */
@SuppressWarnings("deprecation")
public static long calculateNetworkBufferMemory(long totalJavaMemorySize, Configuration config) {
	Preconditions.checkArgument(totalJavaMemorySize > 0);

	int segmentSize =
		checkedDownCast(MemorySize.parse(config.getString(TaskManagerOptions.MEMORY_SEGMENT_SIZE)).getBytes());

	final long networkBufBytes;
	if (TaskManagerServicesConfiguration.hasNewNetworkBufConf(config)) {
		// new configuration based on fractions of available memory with selectable min and max
		float networkBufFraction = config.getFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION);
		long networkBufMin = MemorySize.parse(config.getString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN)).getBytes();
		long networkBufMax = MemorySize.parse(config.getString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX)).getBytes();

		TaskManagerServicesConfiguration
			.checkNetworkBufferConfig(segmentSize, networkBufFraction, networkBufMin, networkBufMax);

		networkBufBytes = Math.min(networkBufMax, Math.max(networkBufMin,
			(long) (networkBufFraction * totalJavaMemorySize)));

		TaskManagerServicesConfiguration
			.checkConfigParameter(networkBufBytes < totalJavaMemorySize,
				"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
				"(" + TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
					TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
					TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
				"Network buffer memory size too large: " + networkBufBytes + " >= " +
					totalJavaMemorySize + " (total JVM memory size)");
		TaskManagerServicesConfiguration
			.checkConfigParameter(networkBufBytes >= segmentSize,
				"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
				"(" + TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
					TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
					TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
				"Network buffer memory size too small: " + networkBufBytes + " < " +
					segmentSize + " (" + TaskManagerOptions.MEMORY_SEGMENT_SIZE.key() + ")");
	} else {
		// use old (deprecated) network buffers parameter
		int numNetworkBuffers = config.getInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS);
		networkBufBytes = (long) numNetworkBuffers * (long) segmentSize;

		TaskManagerServicesConfiguration.checkNetworkConfigOld(numNetworkBuffers);

		TaskManagerServicesConfiguration
			.checkConfigParameter(networkBufBytes < totalJavaMemorySize,
				networkBufBytes, TaskManagerOptions.NETWORK_NUM_BUFFERS.key(),
				"Network buffer memory size too large: " + networkBufBytes + " >= " +
					totalJavaMemorySize + " (total JVM memory size)");
		TaskManagerServicesConfiguration
			.checkConfigParameter(networkBufBytes >= segmentSize,
				networkBufBytes, TaskManagerOptions.NETWORK_NUM_BUFFERS.key(),
				"Network buffer memory size too small: " + networkBufBytes + " < " +
					segmentSize + " (" + TaskManagerOptions.MEMORY_SEGMENT_SIZE.key() + ")");
	}

	return networkBufBytes;
}