org.apache.flink.configuration.NettyShuffleEnvironmentOptions Java Examples

@Test(timeout = 10000L)
public void testLogNotFoundHandling() throws Throwable {
	final int dataPort = NetUtils.getAvailablePort();
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.DATA_PORT, dataPort);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);
	config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, "/i/dont/exist");

	try (TaskSubmissionTestEnvironment env =
		new Builder(jobId)
			.setConfiguration(config)
			.setLocalCommunication(false)
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		try {
			CompletableFuture<TransientBlobKey> logFuture =
				tmGateway.requestFileUploadByType(FileType.LOG, timeout);
			logFuture.get();
		} catch (Exception e) {
			assertThat(e.getMessage(), containsString("The file LOG does not exist on the TaskExecutor."));
		}
	}
}
 

private MiniClusterWithClientResource trySetUpCluster() throws Exception {
	try {
		Configuration config = new Configuration();
		config.setString(NettyShuffleEnvironmentOptions.TRANSPORT_TYPE, "epoll");
		MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setConfiguration(config)
				.setNumberTaskManagers(NUM_TASK_MANAGERS)
				.setNumberSlotsPerTaskManager(1)
				.build());
		cluster.before();
		return cluster;
	}
	catch (UnsatisfiedLinkError ex) {
		// If we failed to init netty because we are not on Linux platform, abort the test.
		if (findThrowableWithMessage(ex, "Only supported on Linux").isPresent()) {
			throw new AssumptionViolatedException("This test is only supported on linux");
		}
		throw ex;
	}
}
 

/**
 * Calculates the amount of memory used for network buffers based on the total memory to use and
 * the according configuration parameters.
 *
 * <p>The following configuration parameters are involved:
 * <ul>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX}</li>
 * </ul>.
 *
 * @param config configuration object
 * @param networkMemoryByFraction memory of network buffers based on JVM memory size and network fraction
 * @param maxAllowedMemory maximum memory used for checking the results of network memory
 *
 * @return memory to use for network buffers (in bytes)
 */
private static long calculateNewNetworkBufferMemory(Configuration config, long networkMemoryByFraction, long maxAllowedMemory) {
	float networkBufFraction = config.getFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION);
	long networkBufMin = MemorySize.parse(config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN)).getBytes();
	long networkBufMax = MemorySize.parse(config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX)).getBytes();

	int pageSize = ConfigurationParserUtils.getPageSize(config);

	checkNewNetworkConfig(pageSize, networkBufFraction, networkBufMin, networkBufMax);

	long networkBufBytes = Math.min(networkBufMax, Math.max(networkBufMin, networkMemoryByFraction));

	ConfigurationParserUtils.checkConfigParameter(networkBufBytes < maxAllowedMemory,
		"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
		"(" + NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
			NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
			NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
		"Network buffer memory size too large: " + networkBufBytes + " >= " +
			maxAllowedMemory + " (maximum JVM memory size)");

	return networkBufBytes;
}
 

/**
 * Calculates the amount of memory used for network buffers based on the total memory to use and
 * the according configuration parameters.
 *
 * <p>The following configuration parameters are involved:
 * <ul>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
 * 	<li>{@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX}, and</li>
 *  <li>{@link NettyShuffleEnvironmentOptions#NETWORK_NUM_BUFFERS} (fallback if the ones above do not exist)</li>
 * </ul>.
 *
 * @param totalProcessMemory overall available memory to use (in bytes)
 * @param config configuration object
 *
 * @return memory to use for network buffers (in bytes)
 */
@SuppressWarnings("deprecation")
public static long calculateNetworkBufferMemory(long totalProcessMemory, Configuration config) {
	final int segmentSize = ConfigurationParserUtils.getPageSize(config);

	final long networkReservedMemory;
	if (hasNewNetworkConfig(config)) {
		float networkBufFraction = config.getFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION);
		long networkMemoryByFraction = (long) (totalProcessMemory * networkBufFraction);
		networkReservedMemory = calculateNewNetworkBufferMemory(config, networkMemoryByFraction, totalProcessMemory);
	} else {
		// use old (deprecated) network buffers parameter
		int numNetworkBuffers = config.getInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS);
		networkReservedMemory = (long) numNetworkBuffers * (long) segmentSize;

		checkOldNetworkConfig(numNetworkBuffers);

		ConfigurationParserUtils.checkConfigParameter(networkReservedMemory < totalProcessMemory,
			networkReservedMemory, NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS.key(),
			"Network buffer memory size too large: " + networkReservedMemory + " >= " +
				totalProcessMemory + " (total JVM memory size)");
	}

	return networkReservedMemory;
}
 

protected Configuration createClusterConfig() throws IOException {
	TemporaryFolder temporaryFolder = new TemporaryFolder();
	temporaryFolder.create();
	final File haDir = temporaryFolder.newFolder();

	Configuration config = new Configuration();
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "48m");
	// the default network buffers size (10% of heap max =~ 150MB) seems to much for this test case
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(80L << 20)); // 80 MB
	config.setString(AkkaOptions.FRAMESIZE, String.valueOf(MAX_MEM_STATE_SIZE) + "b");

	if (zkServer != null) {
		config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, haDir.toURI().toString());
	}
	return config;
}
 

/**
 * Returns a configuration for the tests.
 *
 * @param managedMemory         see {@link TaskManagerOptions#MANAGED_MEMORY_SIZE}
 * @param managedMemoryFraction see {@link TaskManagerOptions#MANAGED_MEMORY_FRACTION}
 * @param networkBufFraction	see {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION}
 * @param networkBufMin			see {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN}
 * @param networkBufMax			see {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX}
 * @param memoryType			on-heap or off-heap
 *
 * @return configuration object
 */
private static Configuration getConfig(
	final long managedMemory,
	final float managedMemoryFraction,
	float networkBufFraction,
	long networkBufMin,
	long networkBufMax,
	MemoryType memoryType) {

	final Configuration configuration = new Configuration();

	configuration.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), managedMemory);
	configuration.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION.key(), managedMemoryFraction);
	configuration.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key(), networkBufFraction);
	configuration.setLong(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.key(), networkBufMin);
	configuration.setLong(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX.key(), networkBufMax);
	configuration.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP.key(), memoryType == MemoryType.OFF_HEAP);

	return configuration;
}
 

/**
 * Verifies that {@link  NettyShuffleEnvironmentConfiguration#fromConfiguration(Configuration, long, boolean, InetAddress)}
 * returns the correct result for new configurations via
 * {@link NettyShuffleEnvironmentOptions#NETWORK_REQUEST_BACKOFF_INITIAL},
 * {@link NettyShuffleEnvironmentOptions#NETWORK_REQUEST_BACKOFF_MAX},
 * {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_PER_CHANNEL} and
 * {@link NettyShuffleEnvironmentOptions#NETWORK_EXTRA_BUFFERS_PER_GATE}
 */
@Test
public void testNetworkRequestBackoffAndBuffers() {

	// set some non-default values
	final Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_PER_CHANNEL, 10);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_EXTRA_BUFFERS_PER_GATE, 100);

	final  NettyShuffleEnvironmentConfiguration networkConfig =  NettyShuffleEnvironmentConfiguration.fromConfiguration(
		config,
		MEM_SIZE_PARAM,
		true,
		InetAddress.getLoopbackAddress());

	assertEquals(networkConfig.partitionRequestInitialBackoff(), 100);
	assertEquals(networkConfig.partitionRequestMaxBackoff(), 200);
	assertEquals(networkConfig.networkBuffersPerChannel(), 10);
	assertEquals(networkConfig.floatingNetworkBuffersPerGate(), 100);
}
 

/**
 * Test for {@link NettyShuffleEnvironmentConfiguration#calculateNetworkBufferMemory(long, Configuration)} using old
 * configurations via {@link NettyShuffleEnvironmentOptions#NETWORK_NUM_BUFFERS}.
 */
@SuppressWarnings("deprecation")
@Test
public void calculateNetworkBufOld() {
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 1);

	// note: actual network buffer memory size is independent of the totalJavaMemorySize
	assertEquals(MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(10L << 20, config));
	assertEquals(MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(64L << 20, config));

	// test integer overflow in the memory size
	int numBuffers = (int) ((2L << 32) / MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes()); // 2^33
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, numBuffers);
	assertEquals(2L << 32, NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(2L << 33, config));
}
 

/**
 * Test for {@link NettyShuffleEnvironmentConfiguration#calculateNetworkBufferMemory(long, Configuration)} using new
 * configurations via {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION},
 * {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN} and
 * {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX}.
 */
@Test
public void calculateNetworkBufNew() throws Exception {
	Configuration config = new Configuration();

	// (1) defaults
	final Float defaultFrac = NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION.defaultValue();
	final Long defaultMin = MemorySize.parse(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes();
	final Long defaultMax = MemorySize.parse(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX.defaultValue()).getBytes();
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), defaultMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((64L << 20 + 1), config));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), defaultMin, defaultMax),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory((10L << 30), config));

	calculateNetworkBufNew(config);
}
 

/**
 * Verifies that {@link NettyShuffleEnvironmentConfiguration#hasNewNetworkConfig(Configuration)}
 * returns the correct result for mixed old/new configurations.
 */
@SuppressWarnings("deprecation")
@Test
public void hasNewNetworkBufConfMixed() throws Exception {
	Configuration config = new Configuration();
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config));

	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 1);
	assertFalse(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config));

	// old + 1 new parameter = new:
	Configuration config1 = config.clone();
	config1.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f);
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));

	config1 = config.clone();
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, "1024");
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));

	config1 = config.clone();
	config1.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, "1024");
	assertTrue(NettyShuffleEnvironmentConfiguration.hasNewNetworkConfig(config1));
}
 

@Test(timeout = 10000L)
public void testLogNotFoundHandling() throws Throwable {
	final int dataPort = NetUtils.getAvailablePort();
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.DATA_PORT, dataPort);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);
	config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, "/i/dont/exist");

	try (TaskSubmissionTestEnvironment env =
		new TaskSubmissionTestEnvironment.Builder(jobId)
			.setConfiguration(config)
			.setLocalCommunication(false)
			.build()) {
		TaskExecutorGateway tmGateway = env.getTaskExecutorGateway();
		try {
			CompletableFuture<TransientBlobKey> logFuture =
				tmGateway.requestFileUpload(FileType.LOG, timeout);
			logFuture.get();
		} catch (Exception e) {
			assertThat(e.getMessage(), containsString("The file LOG does not exist on the TaskExecutor."));
		}
	}
}
 

/**
 * Verifies that {@link Task#setupPartitionsAndGates(ResultPartitionWriter[], InputGate[])}} fails if the bare minimum of
 * required buffers is not available (we are one buffer short).
 */
@Test
public void testRegisterTaskWithInsufficientBuffers() throws Exception {
	final int bufferCount;
	// outgoing: 1 buffer per channel (always)
	if (!enableCreditBasedFlowControl) {
		// incoming: 1 buffer per channel
		bufferCount = 19;
	} else {
		// incoming: 2 exclusive buffers per channel
		bufferCount = 10 + 10 * NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_PER_CHANNEL.defaultValue() - 1;
	}

	expectedException.expect(IOException.class);
	expectedException.expectMessage("Insufficient number of network buffers");
	testRegisterTaskWithLimitedBuffers(bufferCount);
}
 

private void executeTest(JobGraph jobGraph) throws Exception {
	Configuration configuration = new Configuration();
	configuration.set(TaskManagerOptions.TOTAL_FLINK_MEMORY, MemorySize.parse("1g"));
	configuration.setBoolean(NettyShuffleEnvironmentOptions.BLOCKING_SHUFFLE_COMPRESSION_ENABLED, true);

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(NUM_TASKMANAGERS)
		.setNumSlotsPerTaskManager(NUM_SLOTS)
		.build();

	try (MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration)) {
		miniCluster.start();

		MiniClusterClient miniClusterClient = new MiniClusterClient(configuration, miniCluster);
		// wait for the submission to succeed
		JobID jobID = miniClusterClient.submitJob(jobGraph).get();

		CompletableFuture<JobResult> resultFuture = miniClusterClient.requestJobResult(jobID);
		assertFalse(resultFuture.get().getSerializedThrowable().isPresent());
	}
}
 

private MiniClusterWithClientResource trySetUpCluster() throws Exception {
	try {
		Configuration config = new Configuration();
		config.setString(NettyShuffleEnvironmentOptions.TRANSPORT_TYPE, "epoll");
		MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setConfiguration(config)
				.setNumberTaskManagers(NUM_TASK_MANAGERS)
				.setNumberSlotsPerTaskManager(1)
				.build());
		cluster.before();
		return cluster;
	}
	catch (UnsatisfiedLinkError ex) {
		// If we failed to init netty because we are not on Linux platform, abort the test.
		if (findThrowableWithMessage(ex, "Only supported on Linux").isPresent()) {
			throw new AssumptionViolatedException("This test is only supported on linux");
		}
		throw ex;
	}
}
 

/**
 * Parses the hosts / ports for communication and data exchange from configuration.
 *
 * @param configuration configuration object
 * @return the data port
 */
private static int getDataBindPort(Configuration configuration) {
	final int dataBindPort;
	if (configuration.contains(NettyShuffleEnvironmentOptions.DATA_BIND_PORT)) {
		dataBindPort = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_BIND_PORT);
		ConfigurationParserUtils.checkConfigParameter(
			dataBindPort >= 0, dataBindPort, NettyShuffleEnvironmentOptions.DATA_BIND_PORT.key(),
			"Leave config parameter empty to fallback to '" +
				NettyShuffleEnvironmentOptions.DATA_PORT.key() + "' automatically.");
	} else {
		dataBindPort = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT);
		ConfigurationParserUtils.checkConfigParameter(
			dataBindPort >= 0, dataBindPort, NettyShuffleEnvironmentOptions.DATA_PORT.key(),
			"Leave config parameter empty or use 0 to let the system choose a port automatically.");
	}
	return dataBindPort;
}
 

/**
 * Returns a flink configuration object with the given values.
 *
 * @param javaMemMB
 * 		total JVM memory to use (in megabytes)
 * @param useOffHeap
 * 		whether to use off-heap memory (<tt>true</tt>) or not (<tt>false</tt>)
 * @param netBufMemFrac
 * 		fraction of JVM memory to use for network buffers
 * @param netBufMemMin
 * 		minimum memory size for network buffers (in bytes)
 * @param netBufMemMax
 * 		maximum memory size for network buffers (in bytes)
 * @param managedMemSizeMB
 * 		amount of managed memory (in megabytes)
 * @param managedMemFrac
 * 		fraction of free memory to use for managed memory (if <tt>managedMemSizeMB</tt> is
 * 		<tt>-1</tt>)
 *
 * @return flink configuration
 */
private static Configuration getConfig(
		final int javaMemMB, final boolean useOffHeap, final float netBufMemFrac,
		final long netBufMemMin, final long netBufMemMax, final int managedMemSizeMB,
		final float managedMemFrac) {

	Configuration config = new Configuration();

	config.setLong(KEY_TASKM_MEM_SIZE, javaMemMB);
	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, useOffHeap);

	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, netBufMemFrac);
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(netBufMemMin));
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(netBufMemMax));

	if (managedMemSizeMB == 0) {
		config.removeConfig(TaskManagerOptions.MANAGED_MEMORY_SIZE);
	} else {
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemSizeMB + "m");
	}
	config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, managedMemFrac);

	return config;
}
 

@Test
public void testConfigNetworkMemoryLegacyNumOfBuffers() {
	final MemorySize pageSize = MemorySize.parse("32k");
	final int numOfBuffers = 1024;
	final MemorySize networkSize = pageSize.multiply(numOfBuffers);

	@SuppressWarnings("deprecation")
	final ConfigOption<Integer> legacyOption = NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS;

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.MEMORY_SEGMENT_SIZE, pageSize);
	conf.setInteger(legacyOption, numOfBuffers);

	// validate in configurations without explicit total flink/process memory, otherwise explicit configured
	// network memory size might conflict with total flink/process memory minus other memory sizes
	validateInConfigWithExplicitTaskHeapAndManagedMem(conf, taskExecutorProcessSpec ->
		assertThat(taskExecutorProcessSpec.getNetworkMemSize(), is(networkSize)));
	validateInConfigurationsWithoutExplicitTaskHeapMem(conf, taskExecutorProcessSpec ->
		assertThat(taskExecutorProcessSpec.getNetworkMemSize(), is(networkSize)));
}
 

/**
 * Verifies that {@link  NettyShuffleEnvironmentConfiguration#fromConfiguration(Configuration, MemorySize, boolean, InetAddress)}
 * returns the correct result for new configurations via
 * {@link NettyShuffleEnvironmentOptions#NETWORK_REQUEST_BACKOFF_INITIAL},
 * {@link NettyShuffleEnvironmentOptions#NETWORK_REQUEST_BACKOFF_MAX},
 * {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_PER_CHANNEL} and
 * {@link NettyShuffleEnvironmentOptions#NETWORK_EXTRA_BUFFERS_PER_GATE}
 */
@Test
public void testNetworkRequestBackoffAndBuffers() {

	// set some non-default values
	final Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_PER_CHANNEL, 10);
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_EXTRA_BUFFERS_PER_GATE, 100);

	final  NettyShuffleEnvironmentConfiguration networkConfig =  NettyShuffleEnvironmentConfiguration.fromConfiguration(
		config,
		MEM_SIZE_PARAM,
		true,
		InetAddress.getLoopbackAddress());

	assertEquals(networkConfig.partitionRequestInitialBackoff(), 100);
	assertEquals(networkConfig.partitionRequestMaxBackoff(), 200);
	assertEquals(networkConfig.networkBuffersPerChannel(), 10);
	assertEquals(networkConfig.floatingNetworkBuffersPerGate(), 100);
}
 

/**
 * Tests that the TaskManagerRunner startup fails if the network stack cannot be initialized.
 */
@Test
public void testStartupWhenNetworkStackFailsToInitialize() throws Exception {
	final ServerSocket blocker = new ServerSocket(0, 50, InetAddress.getByName(LOCAL_HOST));

	try {
		final Configuration cfg = createFlinkConfiguration();
		cfg.setInteger(NettyShuffleEnvironmentOptions.DATA_PORT, blocker.getLocalPort());
		cfg.setString(TaskManagerOptions.BIND_HOST, LOCAL_HOST);

		startTaskManager(
			cfg,
			rpcService,
			highAvailabilityServices);

		fail("Should throw IOException when the network stack cannot be initialized.");
	} catch (IOException e) {
		// ignored
	} finally {
		IOUtils.closeQuietly(blocker);
	}
}
 

/**
 * Tests that the TaskManagerRunner startup fails if the network stack cannot be initialized.
 */
@Test
public void testStartupWhenNetworkStackFailsToInitialize() throws Exception {
	final ServerSocket blocker = new ServerSocket(0, 50, InetAddress.getByName(LOCAL_HOST));

	try {
		final Configuration cfg = new Configuration();
		cfg.setInteger(NettyShuffleEnvironmentOptions.DATA_PORT, blocker.getLocalPort());

		startTaskManager(
			cfg,
			rpcService,
			highAvailabilityServices);

		fail("Should throw IOException when the network stack cannot be initialized.");
	} catch (IOException e) {
		// ignored
	} finally {
		IOUtils.closeQuietly(blocker);
	}
}
 

protected Configuration createConfiguration(boolean compressionEnabled, String subpartitionType) {
	Configuration configuration = super.createConfiguration();

	configuration.setBoolean(NettyShuffleEnvironmentOptions.BLOCKING_SHUFFLE_COMPRESSION_ENABLED, compressionEnabled);
	configuration.setString(NettyShuffleEnvironmentOptions.NETWORK_BLOCKING_SHUFFLE_TYPE, subpartitionType);
	configuration.setString(CoreOptions.TMP_DIRS, FileUtils.getCurrentWorkingDirectory().toAbsolutePath().toUri().toString());
	return configuration;
}
 

/**
 * Parses the hosts / ports for communication and data exchange from configuration.
 *
 * @param configuration configuration object
 * @return the data port
 */
private static int getDataport(Configuration configuration) {
	final int dataport = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT);
	ConfigurationParserUtils.checkConfigParameter(dataport >= 0, dataport, NettyShuffleEnvironmentOptions.DATA_PORT.key(),
		"Leave config parameter empty or use 0 to let the system choose a port automatically.");

	return dataport;
}
 

private static BoundedBlockingSubpartitionType getBlockingSubpartitionType(Configuration config) {
	String transport = config.getString(NettyShuffleEnvironmentOptions.NETWORK_BLOCKING_SHUFFLE_TYPE);

	switch (transport) {
		case "mmap":
			return BoundedBlockingSubpartitionType.FILE_MMAP;
		case "file":
			return BoundedBlockingSubpartitionType.FILE;
		default:
			return BoundedBlockingSubpartitionType.AUTO;
	}
}
 

private static BufferStorage createBufferStorage(
		CheckpointingMode checkpointMode,
		IOManager ioManager,
		int pageSize,
		Configuration taskManagerConfig,
		String taskName) throws IOException {
	switch (checkpointMode) {
		case EXACTLY_ONCE: {
			long maxAlign = taskManagerConfig.getLong(TaskManagerOptions.TASK_CHECKPOINT_ALIGNMENT_BYTES_LIMIT);
			if (!(maxAlign == -1 || maxAlign > 0)) {
				throw new IllegalConfigurationException(
					TaskManagerOptions.TASK_CHECKPOINT_ALIGNMENT_BYTES_LIMIT.key()
						+ " must be positive or -1 (infinite)");
			}

			if (taskManagerConfig.getBoolean(NettyShuffleEnvironmentOptions.NETWORK_CREDIT_MODEL)) {
				return new CachedBufferStorage(pageSize, maxAlign, taskName);
			} else {
				return new BufferSpiller(ioManager, pageSize, maxAlign, taskName);
			}
		}
		case AT_LEAST_ONCE:
			return new EmptyBufferStorage();
		default:
			throw new UnsupportedOperationException("Unrecognized Checkpointing Mode: " + checkpointMode);
	}
}
 

private static Configuration getFlinkConfiguration() {
	final Configuration config = new Configuration();
	config.setString(AkkaOptions.ASK_TIMEOUT, TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT());
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, NUMBER_OF_NETWORK_BUFFERS);

	return config;
}
 

@Test
public void remoteModeInsufficientBuffersSender() throws Exception {
	StreamNetworkThroughputBenchmark env = new StreamNetworkThroughputBenchmark();
	int writers = 2;
	int channels = 2;

	expectedException.expect(IOException.class);
	expectedException.expectMessage("Insufficient number of network buffers");

	env.setUp(writers, channels, 100, false, writers * channels - 1, writers * channels * NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_PER_CHANNEL.defaultValue());
}
 

/**
 * Verifies that {@link Task#setupPartitionsAndGates(ResultPartitionWriter[], InputGate[])}} sets up (un)bounded buffer pool
 * instances for various types of input and output channels working with the bare minimum of
 * required buffers.
 */
@Test
public void testRegisterTaskWithLimitedBuffers() throws Exception {
	// outgoing: 1 buffer per channel + 1 extra buffer per ResultPartition
	// incoming: 2 exclusive buffers per channel + 1 floating buffer per single gate
	final int bufferCount = 18 + 10 * NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_PER_CHANNEL.defaultValue();

	testRegisterTaskWithLimitedBuffers(bufferCount);
}
 

@Test
public void testSequentialReading() throws Exception {
	// setup
	final Configuration configuration = new Configuration();
	configuration.setString(RestOptions.BIND_PORT, "0");
	configuration.setString(NettyShuffleEnvironmentOptions.NETWORK_BOUNDED_BLOCKING_SUBPARTITION_TYPE, "file");

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(parallelism)
		.setNumSlotsPerTaskManager(1)
		.build();

	try (final MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration)) {
		miniCluster.start();

		final MiniClusterClient client = new MiniClusterClient(configuration, miniCluster);
		final JobGraph jobGraph = createJobGraph();
		final CompletableFuture<JobSubmissionResult> submitFuture = client.submitJob(jobGraph);
		// wait for the submission to succeed
		final JobSubmissionResult result = submitFuture.get();

		final CompletableFuture<JobResult> resultFuture = client.requestJobResult(result.getJobID());
		final JobResult jobResult = resultFuture.get();

		assertThat(jobResult.getSerializedThrowable().isPresent(), is(false));
	}
}
 

@SuppressWarnings("deprecation")
private static void logIfIgnoringOldConfigs(Configuration configuration) {
	if (configuration.contains(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS)) {
		LOG.info("Ignoring old (but still present) network buffer configuration via {}.",
			NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS.key());
	}
}
 

private static Configuration getConfiguration() {
	verifyJvmOptions();
	Configuration config = new Configuration();
	config.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
	config.setString(AkkaOptions.ASK_TIMEOUT, TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT());
	config.set(TaskManagerOptions.MEMORY_SEGMENT_SIZE, MemorySize.parse("4096"));
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 2048);

	return config;
}