org.apache.flink.configuration.TaskManagerOptions Java Examples

/**
 * Validates the (new) network buffer configuration.
 *
 * @param pageSize 				size of memory buffers
 * @param networkBufFraction	fraction of JVM memory to use for network buffers
 * @param networkBufMin 		minimum memory size for network buffers (in bytes)
 * @param networkBufMax 		maximum memory size for network buffers (in bytes)
 *
 * @throws IllegalConfigurationException if the condition does not hold
 */
protected static void checkNetworkBufferConfig(
		final int pageSize, final float networkBufFraction, final long networkBufMin,
		final long networkBufMax) throws IllegalConfigurationException {

	checkConfigParameter(networkBufFraction > 0.0f && networkBufFraction < 1.0f, networkBufFraction,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key(),
		"Network buffer memory fraction of the free memory must be between 0.0 and 1.0");

	checkConfigParameter(networkBufMin >= pageSize, networkBufMin,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key(),
		"Minimum memory for network buffers must allow at least one network " +
			"buffer with respect to the memory segment size");

	checkConfigParameter(networkBufMax >= pageSize, networkBufMax,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key(),
		"Maximum memory for network buffers must allow at least one network " +
			"buffer with respect to the memory segment size");

	checkConfigParameter(networkBufMax >= networkBufMin, networkBufMax,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key(),
		"Maximum memory for network buffers must not be smaller than minimum memory (" +
			TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ": " + networkBufMin + ")");
}
 

@Test
public void testFlinkOnYarnTMMemOption() {
    String flinkonYarnTMMemOption = "-ytm";
    Map<String, String> map = FlinkOnYarnConfigMapping.flinkOnYarnConfigMap;
    for (Map.Entry<String, String> entry : map.entrySet()) {
        if (entry.getValue().equals(flinkonYarnTMMemOption)) {
            String flinkConfigOption = entry.getKey();

            boolean matchedAnyOne;
            matchedAnyOne = flinkConfigOption.equals(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY.key());
            if (!matchedAnyOne) {
                if (TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY.hasFallbackKeys()) {
                    Iterator<FallbackKey> deprecatedKeyIterator = TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY
                            .fallbackKeys().iterator();
                    while (deprecatedKeyIterator.hasNext()) {
                        matchedAnyOne = matchedAnyOne && flinkConfigOption.equals(deprecatedKeyIterator.next().getKey());
                    }
                }
            }

            Assert.assertTrue(matchedAnyOne);
        }
    }
}
 

/**
 * Tests that the TaskManagerRunner startup fails if the network stack cannot be initialized.
 */
@Test
public void testStartupWhenNetworkStackFailsToInitialize() throws Exception {
	final ServerSocket blocker = new ServerSocket(0, 50, InetAddress.getByName(LOCAL_HOST));

	try {
		final Configuration cfg = new Configuration();
		cfg.setInteger(TaskManagerOptions.DATA_PORT, blocker.getLocalPort());

		startTaskManager(
			cfg,
			rpcService,
			highAvailabilityServices);

		fail("Should throw IOException when the network stack cannot be initialized.");
	} catch (IOException e) {
		// ignored
	} finally {
		IOUtils.closeQuietly(blocker);
	}
}
 

@Test
public void createResourceManager_WithLessMemoryThanContainerizedHeapCutoffMin_ShouldSucceed() throws Exception {
	final StandaloneResourceManagerFactory resourceManagerFactory = StandaloneResourceManagerFactory.INSTANCE;

	final TestingRpcService rpcService = new TestingRpcService();
	try {
		final Configuration configuration = new Configuration();
		configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, new MemorySize(128 * 1024 * 1024).toString());
		configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 600);

		final ResourceManager<ResourceID> ignored = resourceManagerFactory.createResourceManager(
			configuration,
			ResourceID.generate(),
			rpcService,
			new TestingHighAvailabilityServices(),
			new TestingHeartbeatServices(),
			NoOpMetricRegistry.INSTANCE,
			new TestingFatalErrorHandler(),
			new ClusterInformation("foobar", 1234),
			null,
			UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());
	} finally {
		RpcUtils.terminateRpcService(rpcService, Time.seconds(10L));
	}
}
 

/**
 * Tests that RM and TM create the same slot resource profiles.
 */
@Test
public void testCreateWorkerSlotProfiles() {
	final Configuration config = new Configuration();
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "100m");
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 5);

	final ResourceProfile rmCalculatedResourceProfile =
		ResourceManager.createWorkerSlotProfiles(config).iterator().next();

	final ResourceProfile tmCalculatedResourceProfile =
		TaskManagerServices.computeSlotResourceProfile(
			config.getInteger(TaskManagerOptions.NUM_TASK_SLOTS),
			MemorySize.parse(config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE)).getBytes());

	assertEquals(rmCalculatedResourceProfile, tmCalculatedResourceProfile);
}
 

private void tryShutdownTimerService() {

		if (timerService != null && !timerService.isTerminated()) {

			try {
				final long timeoutMs = getEnvironment().getTaskManagerInfo().getConfiguration().
					getLong(TaskManagerOptions.TASK_CANCELLATION_TIMEOUT_TIMERS);

				if (!timerService.shutdownServiceUninterruptible(timeoutMs)) {
					LOG.warn("Timer service shutdown exceeded time limit of {} ms while waiting for pending " +
						"timers. Will continue with shutdown procedure.", timeoutMs);
				}
			} catch (Throwable t) {
				// catch and log the exception to not replace the original exception
				LOG.error("Could not shut down timer service", t);
			}
		}
	}
 

/**
 * Returns a flink configuration object with the given values.
 *
 * @param javaMemMB
 * 		total JVM memory to use (in megabytes)
 * @param useOffHeap
 * 		whether to use off-heap memory (<tt>true</tt>) or not (<tt>false</tt>)
 * @param netBufMemFrac
 * 		fraction of JVM memory to use for network buffers
 * @param netBufMemMin
 * 		minimum memory size for network buffers (in bytes)
 * @param netBufMemMax
 * 		maximum memory size for network buffers (in bytes)
 * @param managedMemSizeMB
 * 		amount of managed memory (in megabytes)
 * @param managedMemFrac
 * 		fraction of free memory to use for managed memory (if <tt>managedMemSizeMB</tt> is
 * 		<tt>-1</tt>)
 *
 * @return flink configuration
 */
private static Configuration getConfig(
		final int javaMemMB, final boolean useOffHeap, final float netBufMemFrac,
		final long netBufMemMin, final long netBufMemMax, final int managedMemSizeMB,
		final float managedMemFrac) {

	Configuration config = new Configuration();

	config.setLong(KEY_TASKM_MEM_SIZE, javaMemMB);
	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, useOffHeap);

	config.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, netBufMemFrac);
	config.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(netBufMemMin));
	config.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(netBufMemMax));

	if (managedMemSizeMB == 0) {
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0");
	} else {
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemSizeMB + "m");
	}
	config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, managedMemFrac);

	return config;
}
 

/**
 * Test for {@link NettyShuffleEnvironmentConfiguration#calculateNetworkBufferMemory(long, Configuration)} using old
 * configurations via {@link NettyShuffleEnvironmentOptions#NETWORK_NUM_BUFFERS}.
 */
@SuppressWarnings("deprecation")
@Test
public void calculateNetworkBufOld() {
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 1);

	// note: actual network buffer memory size is independent of the totalJavaMemorySize
	assertEquals(MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(10L << 20, config));
	assertEquals(MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(64L << 20, config));

	// test integer overflow in the memory size
	int numBuffers = (int) ((2L << 32) / MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes()); // 2^33
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, numBuffers);
	assertEquals(2L << 32, NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(2L << 33, config));
}
 

@Test
public void testCreatingTaskManagerRpcServiceShouldFailIfRpcPortRangeIsInvalid() throws Exception {
	final Configuration config = new Configuration(createFlinkConfigWithPredefinedTaskManagerHostname("example.org"));
	config.setString(TaskManagerOptions.RPC_PORT, "-1");

	final HighAvailabilityServices highAvailabilityServices = createHighAvailabilityServices(config);

	try {
		TaskManagerRunner.createRpcService(config, highAvailabilityServices);
		fail("Should fail because -1 is not a valid port range");
	} catch (final IllegalArgumentException e) {
		assertThat(e.getMessage(),  containsString("Invalid port range definition: -1"));
	} finally {
		highAvailabilityServices.closeAndCleanupAllData();
	}
}
 

@Test
public void testMaximumRegistrationDuration() throws Exception {
	configuration.setString(TaskManagerOptions.REGISTRATION_TIMEOUT, "10 ms");

	final TaskExecutor taskExecutor = createTaskExecutor(new TaskManagerServicesBuilder().build());

	taskExecutor.start();

	try {
		final Throwable error = testingFatalErrorHandler.getErrorFuture().get();
		assertThat(error, is(notNullValue()));
		assertThat(ExceptionUtils.stripExecutionException(error), instanceOf(RegistrationTimeoutException.class));

		testingFatalErrorHandler.clearError();
	} finally {
		RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
	}
}
 

@Test
public void testCreatingTaskManagerRpcServiceShouldFailIfRpcPortRangeIsInvalid() throws Exception {
	final Configuration config = new Configuration(createFlinkConfigWithPredefinedTaskManagerHostname("example.org"));
	config.setString(TaskManagerOptions.RPC_PORT, "-1");

	final HighAvailabilityServices highAvailabilityServices = createHighAvailabilityServices(config);

	try {
		TaskManagerRunner.createRpcService(config, highAvailabilityServices);
		fail("Should fail because -1 is not a valid port range");
	} catch (final IllegalArgumentException e) {
		assertThat(e.getMessage(),  containsString("Invalid port range definition: -1"));
	} finally {
		highAvailabilityServices.closeAndCleanupAllData();
	}
}
 

/**
 * Parses the configuration to get the page size and validates the value.
 *
 * @param configuration configuration object
 * @return size of memory segment
 */
public static int getPageSize(Configuration configuration) {
	final int pageSize = checkedDownCast(MemorySize.parse(
		configuration.getString(TaskManagerOptions.MEMORY_SEGMENT_SIZE)).getBytes());

	// check page size of for minimum size
	checkConfigParameter(
		pageSize >= MemoryManager.MIN_PAGE_SIZE,
		pageSize,
		TaskManagerOptions.MEMORY_SEGMENT_SIZE.key(),
		"Minimum memory segment size is " + MemoryManager.MIN_PAGE_SIZE);
	// check page size for power of two
	checkConfigParameter(
		MathUtils.isPowerOf2(pageSize),
		pageSize,
		TaskManagerOptions.MEMORY_SEGMENT_SIZE.key(),
		"Memory segment size must be a power of 2.");

	return pageSize;
}
 

/**
 * Calculates the amount of heap memory to use (to set via <tt>-Xmx</tt> and <tt>-Xms</tt>)
 * based on the total memory to use and the given configuration parameters.
 *
 * @param totalJavaMemorySizeMB
 * 		overall available memory to use (heap and off-heap)
 * @param config
 * 		configuration object
 *
 * @return heap memory to use (in megabytes)
 */
public static long calculateHeapSizeMB(long totalJavaMemorySizeMB, Configuration config) {
	Preconditions.checkArgument(totalJavaMemorySizeMB > 0);

	// all values below here are in bytes

	final long totalProcessMemory = megabytesToBytes(totalJavaMemorySizeMB);
	final long networkReservedMemory = getReservedNetworkMemory(config, totalProcessMemory);
	final long heapAndManagedMemory = totalProcessMemory - networkReservedMemory;

	if (config.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP)) {
		final long managedMemorySize = getManagedMemoryFromHeapAndManaged(config, heapAndManagedMemory);

		ConfigurationParserUtils.checkConfigParameter(managedMemorySize < heapAndManagedMemory, managedMemorySize,
			TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
				"Managed memory size too large for " + (networkReservedMemory >> 20) +
					" MB network buffer memory and a total of " + totalJavaMemorySizeMB +
					" MB JVM memory");

		return bytesToMegabytes(heapAndManagedMemory - managedMemorySize);
	}
	else {
		return bytesToMegabytes(heapAndManagedMemory);
	}
}
 

/**
 * Test for {@link TaskManagerServices#calculateNetworkBufferMemory(long, Configuration)} using new
 * configurations via {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_FRACTION},
 * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MIN} and
 * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MAX}.
 */
@Test
public void calculateNetworkBufNew() throws Exception {
	Configuration config = new Configuration();

	// (1) defaults
	final Float defaultFrac = TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.defaultValue();
	final Long defaultMin = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes();
	final Long defaultMax = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.defaultValue()).getBytes();
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((64L << 20 + 1), config));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config));

	calculateNetworkBufNew(config);
}
 

@BeforeClass
public static void setUp() throws Exception {

	Configuration config = new Configuration();

	// we need to use the "filesystem" state backend to ensure FLINK-2543 is not happening again.
	config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// Savepoint path
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// required as we otherwise run out of memory
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "80m");

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(2)
			.setNumberSlotsPerTaskManager(2)
			.setConfiguration(config)
			.build());

	miniClusterResource.before();
}
 

private static String determineTaskManagerBindAddressByConnectingToResourceManager(
		final Configuration configuration,
		final HighAvailabilityServices haServices) throws LeaderRetrievalException {

	final Time lookupTimeout = Time.milliseconds(AkkaUtils.getLookupTimeout(configuration).toMillis());

	final InetAddress taskManagerAddress = LeaderRetrievalUtils.findConnectingAddress(
		haServices.getResourceManagerLeaderRetriever(),
		lookupTimeout);

	LOG.info("TaskManager will use hostname/address '{}' ({}) for communication.",
		taskManagerAddress.getHostName(), taskManagerAddress.getHostAddress());

	HostBindPolicy bindPolicy = HostBindPolicy.fromString(configuration.getString(TaskManagerOptions.HOST_BIND_POLICY));
	return bindPolicy == HostBindPolicy.IP ? taskManagerAddress.getHostAddress() : taskManagerAddress.getHostName();
}
 

/**
 * The invoke() method holds a lock (trigger awaitLatch after acquisition)
 * and cancel cannot complete because it also tries to acquire the same lock.
 * This is resolved by the watch dog, no fatal error.
 */
@Test
public void testInterruptibleSharedLockInInvokeAndCancel() throws Exception {
	final TaskManagerActions taskManagerActions = new ProhibitFatalErrorTaskManagerActions();

	final Configuration config = new Configuration();
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_INTERVAL, 5);
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_TIMEOUT, 50);

	final Task task = new TaskBuilder()
		.setInvokable(InvokableInterruptibleSharedLockInInvokeAndCancel.class)
		.setTaskManagerConfig(config)
		.setTaskManagerActions(taskManagerActions)
		.build();

	task.startTaskThread();

	awaitLatch.await();

	task.cancelExecution();
	task.getExecutingThread().join();
}
 

private JobExecutorService createJobExecutorService(Configuration configuration) throws Exception {
	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(
			configuration.getInteger(
				ConfigConstants.LOCAL_NUMBER_TASK_MANAGER,
				ConfigConstants.DEFAULT_LOCAL_NUMBER_TASK_MANAGER))
		.setRpcServiceSharing(RpcServiceSharing.SHARED)
		.setNumSlotsPerTaskManager(
			configuration.getInteger(
				TaskManagerOptions.NUM_TASK_SLOTS, 1))
		.build();

	final MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration);
	miniCluster.start();

	configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

	return miniCluster;
}
 

/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	final int taskManagerMemory = 7331;
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 

/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	final int taskManagerMemory = 7331;
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 

/**
 * Generate a task manager configuration.
 * @param baseConfig Config to start from.
 * @param jobManagerHostname Job manager host name.
 * @param jobManagerPort Port of the job manager.
 * @param numSlots Number of slots to configure.
 * @param registrationTimeout Timeout for registration
 * @return TaskManager configuration
 */
public static Configuration generateTaskManagerConfiguration(
			Configuration baseConfig,
			String jobManagerHostname,
			int jobManagerPort,
			int numSlots,
			FiniteDuration registrationTimeout) {

	Configuration cfg = cloneConfiguration(baseConfig);

	if (jobManagerHostname != null && !jobManagerHostname.isEmpty()) {
		cfg.setString(JobManagerOptions.ADDRESS, jobManagerHostname);
	}

	if (jobManagerPort > 0) {
		cfg.setInteger(JobManagerOptions.PORT, jobManagerPort);
	}

	cfg.setString(TaskManagerOptions.REGISTRATION_TIMEOUT, registrationTimeout.toString());
	if (numSlots != -1){
		cfg.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numSlots);
	}

	return cfg;
}
 

public TestingMiniClusterConfiguration build() {
	final Configuration modifiedConfiguration = new Configuration(configuration);
	modifiedConfiguration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numSlotsPerTaskManager);
	modifiedConfiguration.setString(
		RestOptions.ADDRESS,
		modifiedConfiguration.getString(RestOptions.ADDRESS, "localhost"));
	modifiedConfiguration.setInteger(
		RestOptions.PORT,
		modifiedConfiguration.getInteger(RestOptions.PORT, 0));

	return new TestingMiniClusterConfiguration(
		modifiedConfiguration,
		numTaskManagers,
		rpcServiceSharing,
		commonBindAddress,
		numberDispatcherResourceManagerComponents,
		localCommunication);
}
 

/**
 * Generate a task manager configuration.
 * @param baseConfig Config to start from.
 * @param jobManagerHostname Job manager host name.
 * @param jobManagerPort Port of the job manager.
 * @param numSlots Number of slots to configure.
 * @param registrationTimeout Timeout for registration
 * @return TaskManager configuration
 */
public static Configuration generateTaskManagerConfiguration(
			Configuration baseConfig,
			String jobManagerHostname,
			int jobManagerPort,
			int numSlots,
			FiniteDuration registrationTimeout) {

	Configuration cfg = cloneConfiguration(baseConfig);

	if (jobManagerHostname != null && !jobManagerHostname.isEmpty()) {
		cfg.setString(JobManagerOptions.ADDRESS, jobManagerHostname);
	}

	if (jobManagerPort > 0) {
		cfg.setInteger(JobManagerOptions.PORT, jobManagerPort);
	}

	cfg.setString(TaskManagerOptions.REGISTRATION_TIMEOUT, registrationTimeout.toString());
	if (numSlots != -1){
		cfg.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numSlots);
	}

	return cfg;
}
 

/**
 * Parses the configuration to get the managed memory size and validates the value.
 *
 * @param configuration configuration object
 * @return managed memory size (in megabytes)
 */
public static long getManagedMemorySize(Configuration configuration) {
	long managedMemorySize;
	String managedMemorySizeDefaultVal = TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue();
	if (!configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(managedMemorySizeDefaultVal)) {
		try {
			managedMemorySize = MemorySize.parse(
				configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE), MEGA_BYTES).getMebiBytes();
		} catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException("Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e);
		}
	} else {
		managedMemorySize = Long.valueOf(managedMemorySizeDefaultVal);
	}

	checkConfigParameter(configuration.getString(
		TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()) || managedMemorySize > 0,
		managedMemorySize, TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
		"MemoryManager needs at least one MB of memory. " +
			"If you leave this config parameter empty, the system automatically pick a fraction of the available memory.");

	return managedMemorySize;
}
 

private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}
 

protected Configuration createClusterConfig() throws IOException {
	TemporaryFolder temporaryFolder = new TemporaryFolder();
	temporaryFolder.create();
	final File haDir = temporaryFolder.newFolder();

	Configuration config = new Configuration();
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "48m");
	// the default network buffers size (10% of heap max =~ 150MB) seems to much for this test case
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(80L << 20)); // 80 MB
	config.setString(AkkaOptions.FRAMESIZE, String.valueOf(MAX_MEM_STATE_SIZE) + "b");

	if (zkServer != null) {
		config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, haDir.toURI().toString());
	}
	return config;
}
 

/**
 * Returns a flink configuration object with the given values.
 *
 * @param javaMemMB
 * 		total JVM memory to use (in megabytes)
 * @param useOffHeap
 * 		whether to use off-heap memory (<tt>true</tt>) or not (<tt>false</tt>)
 * @param netBufMemFrac
 * 		fraction of JVM memory to use for network buffers
 * @param netBufMemMin
 * 		minimum memory size for network buffers (in bytes)
 * @param netBufMemMax
 * 		maximum memory size for network buffers (in bytes)
 * @param managedMemSizeMB
 * 		amount of managed memory (in megabytes)
 * @param managedMemFrac
 * 		fraction of free memory to use for managed memory (if <tt>managedMemSizeMB</tt> is
 * 		<tt>-1</tt>)
 *
 * @return flink configuration
 */
private static Configuration getConfig(
		final int javaMemMB, final boolean useOffHeap, final float netBufMemFrac,
		final long netBufMemMin, final long netBufMemMax, final int managedMemSizeMB,
		final float managedMemFrac) {

	Configuration config = new Configuration();

	config.setLong(KEY_TASKM_MEM_SIZE, javaMemMB);
	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, useOffHeap);

	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, netBufMemFrac);
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(netBufMemMin));
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(netBufMemMax));

	if (managedMemSizeMB == 0) {
		config.removeConfig(TaskManagerOptions.MANAGED_MEMORY_SIZE);
	} else {
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemSizeMB + "m");
	}
	config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, managedMemFrac);

	return config;
}
 

protected Configuration createClusterConfig() throws IOException {
	TemporaryFolder temporaryFolder = new TemporaryFolder();
	temporaryFolder.create();
	final File haDir = temporaryFolder.newFolder();

	Configuration config = new Configuration();
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "48m");
	// the default network buffers size (10% of heap max =~ 150MB) seems to much for this test case
	config.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(80L << 20)); // 80 MB
	config.setString(AkkaOptions.FRAMESIZE, String.valueOf(MAX_MEM_STATE_SIZE) + "b");

	if (zkServer != null) {
		config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, haDir.toURI().toString());
	}
	return config;
}
 

@Test
public void remoteModeInsufficientBuffersSender() throws Exception {
	StreamNetworkThroughputBenchmark env = new StreamNetworkThroughputBenchmark();
	int writers = 2;
	int channels = 2;

	expectedException.expect(IOException.class);
	expectedException.expectMessage("Insufficient number of network buffers");

	env.setUp(writers, channels, 100, false, writers * channels - 1, writers * channels * TaskManagerOptions.NETWORK_BUFFERS_PER_CHANNEL.defaultValue());
}
 

/**
 * Test for {@link NettyShuffleEnvironmentConfiguration#calculateNetworkBufferMemory(long, Configuration)} using mixed
 * old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void calculateNetworkBufMixed() throws Exception {
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 1);

	final Float defaultFrac = NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION.defaultValue();
	final Long defaultMin = MemorySize.parse(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes();
	final Long defaultMax = MemorySize.parse(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX.defaultValue()).getBytes();


	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertEquals(enforceBounds((long) (0.1f * (10L << 20)), defaultMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((64L << 20 + 1), config1));
	assertEquals(enforceBounds((long) (0.1f * (10L << 30)), defaultMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 30), config1));

	config1 = config.clone();
	long newMin = MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(); // smallest value possible
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(newMin));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), newMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 20), config1));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), newMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 30), config1));

	config1 = config.clone();
	long newMax = Math.max(64L << 20 + 1, MemorySize.parse(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes());
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(newMax));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), defaultMin, newMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((64L << 20 + 1), config1));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), defaultMin, newMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 30), config1));
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));

	// old + any new parameter = new:
	calculateNetworkBufNew(config);
}