org.apache.flink.configuration.IllegalConfigurationException Java Examples

/**
 * Creates the high-availability services for the TaskManagers participating in
 * a Flink YARN application.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forYarnTaskManager(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnPreConfiguredMasterNonHaServices(
				flinkConfig,
				hadoopConfig,
				HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 

@Override
public FsStateBackend createFromConfig(Configuration config, ClassLoader classLoader) throws IllegalConfigurationException {
	// we need to explicitly read the checkpoint directory here, because that
	// is a required constructor parameter
	final String checkpointDir = config.getString(CheckpointingOptions.CHECKPOINTS_DIRECTORY);
	if (checkpointDir == null) {
		throw new IllegalConfigurationException(
				"Cannot create the file system state backend: The configuration does not specify the " +
						"checkpoint directory '" + CheckpointingOptions.CHECKPOINTS_DIRECTORY.key() + '\'');
	}

	try {
		return new FsStateBackend(checkpointDir).configure(config, classLoader);
	}
	catch (IllegalArgumentException e) {
		throw new IllegalConfigurationException("Invalid configuration for the state backend", e);
	}
}
 

/**
 * Creates the high-availability services for the TaskManagers participating in
 * a Flink YARN application.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forYarnTaskManager(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnPreConfiguredMasterNonHaServices(
				flinkConfig,
				hadoopConfig,
				HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 

/**
 * Write 'myid' file to the 'dataDir' in the given ZooKeeper configuration.
 *
 * <blockquote>
 * Every machine that is part of the ZooKeeper ensemble should know about every other machine in
 * the ensemble. You accomplish this with the series of lines of the form
 * server.id=host:port:port. The parameters host and port are straightforward. You attribute the
 * server id to each machine by creating a file named myid, one for each server, which resides
 * in that server's data directory, as specified by the configuration file parameter dataDir.
 * </blockquote>
 *
 * @param zkProps ZooKeeper configuration.
 * @param id      The ID of this {@link QuorumPeer}.
 * @throws IllegalConfigurationException Thrown, if 'dataDir' property not set in given
 *                                       ZooKeeper properties.
 * @throws IOException                   Thrown, if 'dataDir' does not exist and cannot be
 *                                       created.
 * @see <a href="http://zookeeper.apache.org/doc/r3.4.6/zookeeperAdmin.html">
 * ZooKeeper Administrator's Guide</a>
 */

private static void writeMyIdToDataDir(Properties zkProps, int id) throws IOException {

	// Check dataDir and create if necessary
	if (zkProps.getProperty("dataDir") == null) {
		throw new IllegalConfigurationException("No dataDir configured.");
	}

	File dataDir = new File(zkProps.getProperty("dataDir"));

	if (!dataDir.isDirectory() && !dataDir.mkdirs()) {
		throw new IOException("Cannot create dataDir '" + dataDir + "'.");
	}

	dataDir.deleteOnExit();

	LOG.info("Writing {} to myid file in 'dataDir'.", id);
	
	// Write myid to file. We use a File Writer, because that properly propagates errors,
	// while the PrintWriter swallows errors
	try (FileWriter writer = new FileWriter(new File(dataDir, "myid"))) {
		writer.write(String.valueOf(id));
	}
}
 

/**
 * Validates the (new) network buffer configuration.
 *
 * @param pageSize 				size of memory buffers
 * @param networkBufFraction	fraction of JVM memory to use for network buffers
 * @param networkBufMin 		minimum memory size for network buffers (in bytes)
 * @param networkBufMax 		maximum memory size for network buffers (in bytes)
 *
 * @throws IllegalConfigurationException if the condition does not hold
 */
protected static void checkNetworkBufferConfig(
		final int pageSize, final float networkBufFraction, final long networkBufMin,
		final long networkBufMax) throws IllegalConfigurationException {

	checkConfigParameter(networkBufFraction > 0.0f && networkBufFraction < 1.0f, networkBufFraction,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key(),
		"Network buffer memory fraction of the free memory must be between 0.0 and 1.0");

	checkConfigParameter(networkBufMin >= pageSize, networkBufMin,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key(),
		"Minimum memory for network buffers must allow at least one network " +
			"buffer with respect to the memory segment size");

	checkConfigParameter(networkBufMax >= pageSize, networkBufMax,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key(),
		"Maximum memory for network buffers must allow at least one network " +
			"buffer with respect to the memory segment size");

	checkConfigParameter(networkBufMax >= networkBufMin, networkBufMax,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key(),
		"Maximum memory for network buffers must not be smaller than minimum memory (" +
			TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ": " + networkBufMin + ")");
}
 

/**
 * Gets the size of managed memory from the heap size after subtracting network buffer memory.
 * All values are in bytes.
 */
public static long getManagedMemoryFromHeapAndManaged(Configuration config, long heapAndManagedMemory) {
	if (config.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) {
		// take the configured absolute value
		final String sizeValue = config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE);
		try {
			return MemorySize.parse(sizeValue, MEGA_BYTES).getBytes();
		}
		catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException(
				"Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e);
		}
	}
	else {
		// calculate managed memory size via fraction
		final float fraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
		return (long) (fraction * heapAndManagedMemory);
	}
}
 

@Nullable
private static Path parameterOrConfigured(@Nullable Path path, Configuration config, ConfigOption<String> option) {
	if (path != null) {
		return path;
	}
	else {
		String configValue = config.getString(option);
		try {
			return configValue == null ? null : new Path(configValue);
		}
		catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException("Cannot parse value for " + option.key() +
					" : " + configValue + " . Not a valid path.");
		}
	}
}
 

@Override
public FsStateBackend createFromConfig(Configuration config, ClassLoader classLoader) throws IllegalConfigurationException {
	// we need to explicitly read the checkpoint directory here, because that
	// is a required constructor parameter
	final String checkpointDir = config.getString(CheckpointingOptions.CHECKPOINTS_DIRECTORY);
	if (checkpointDir == null) {
		throw new IllegalConfigurationException(
				"Cannot create the file system state backend: The configuration does not specify the " +
						"checkpoint directory '" + CheckpointingOptions.CHECKPOINTS_DIRECTORY.key() + '\'');
	}

	try {
		return new FsStateBackend(checkpointDir).configure(config, classLoader);
	}
	catch (IllegalArgumentException e) {
		throw new IllegalConfigurationException("Invalid configuration for the state backend", e);
	}
}
 

/**
 * Write 'myid' file to the 'dataDir' in the given ZooKeeper configuration.
 *
 * <blockquote>
 * Every machine that is part of the ZooKeeper ensemble should know about every other machine in
 * the ensemble. You accomplish this with the series of lines of the form
 * server.id=host:port:port. The parameters host and port are straightforward. You attribute the
 * server id to each machine by creating a file named myid, one for each server, which resides
 * in that server's data directory, as specified by the configuration file parameter dataDir.
 * </blockquote>
 *
 * @param zkProps ZooKeeper configuration.
 * @param id      The ID of this {@link QuorumPeer}.
 * @throws IllegalConfigurationException Thrown, if 'dataDir' property not set in given
 *                                       ZooKeeper properties.
 * @throws IOException                   Thrown, if 'dataDir' does not exist and cannot be
 *                                       created.
 * @see <a href="http://zookeeper.apache.org/doc/r3.4.6/zookeeperAdmin.html">
 * ZooKeeper Administrator's Guide</a>
 */

private static void writeMyIdToDataDir(Properties zkProps, int id) throws IOException {

	// Check dataDir and create if necessary
	if (zkProps.getProperty("dataDir") == null) {
		throw new IllegalConfigurationException("No dataDir configured.");
	}

	File dataDir = new File(zkProps.getProperty("dataDir"));

	if (!dataDir.isDirectory() && !dataDir.mkdirs()) {
		throw new IOException("Cannot create dataDir '" + dataDir + "'.");
	}

	dataDir.deleteOnExit();

	LOG.info("Writing {} to myid file in 'dataDir'.", id);
	
	// Write myid to file. We use a File Writer, because that properly propagates errors,
	// while the PrintWriter swallows errors
	try (FileWriter writer = new FileWriter(new File(dataDir, "myid"))) {
		writer.write(String.valueOf(id));
	}
}
 

/**
 * Creates the high-availability services for a single-job Flink YARN application, to be
 * used in the Application Master that runs both ResourceManager and JobManager.
 *
 * @param flinkConfig  The Flink configuration.
 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
 *
 * @return The created high-availability services.
 *
 * @throws IOException Thrown, if the high-availability services could not be initialized.
 */
public static YarnHighAvailabilityServices forSingleJobAppMaster(
		Configuration flinkConfig,
		org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {

	checkNotNull(flinkConfig, "flinkConfig");
	checkNotNull(hadoopConfig, "hadoopConfig");

	final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
	switch (mode) {
		case NONE:
			return new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig);

		case ZOOKEEPER:
			throw  new UnsupportedOperationException("to be implemented");

		default:
			throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
	}
}
 

/**
 * Creates a SSLEngineFactory to be used by internal communication server endpoints.
 */
public static SSLHandlerFactory createInternalServerSSLEngineFactory(final Configuration config) throws Exception {
	SSLContext sslContext = createInternalSSLContext(config);
	if (sslContext == null) {
		throw new IllegalConfigurationException("SSL is not enabled for internal communication.");
	}

	return new SSLHandlerFactory(
			sslContext,
			getEnabledProtocols(config),
			getEnabledCipherSuites(config),
			false,
			true,
			config.getInteger(SecurityOptions.SSL_INTERNAL_HANDSHAKE_TIMEOUT),
			config.getInteger(SecurityOptions.SSL_INTERNAL_CLOSE_NOTIFY_FLUSH_TIMEOUT));
}
 

/**
 * Creates a SSLEngineFactory to be used by internal communication client endpoints.
 */
public static SSLHandlerFactory createInternalClientSSLEngineFactory(final Configuration config) throws Exception {
	SSLContext sslContext = createInternalSSLContext(config);
	if (sslContext == null) {
		throw new IllegalConfigurationException("SSL is not enabled for internal communication.");
	}

	return new SSLHandlerFactory(
			sslContext,
			getEnabledProtocols(config),
			getEnabledCipherSuites(config),
			true,
			true,
			config.getInteger(SecurityOptions.SSL_INTERNAL_HANDSHAKE_TIMEOUT),
			config.getInteger(SecurityOptions.SSL_INTERNAL_CLOSE_NOTIFY_FLUSH_TIMEOUT));
}
 

/**
 * Parses the configuration to get the managed memory size and validates the value.
 *
 * @param configuration configuration object
 * @return managed memory size (in megabytes)
 */
public static long getManagedMemorySize(Configuration configuration) {
	long managedMemorySize;
	String managedMemorySizeDefaultVal = TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue();
	if (!configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(managedMemorySizeDefaultVal)) {
		try {
			managedMemorySize = MemorySize.parse(
				configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE), MEGA_BYTES).getMebiBytes();
		} catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException("Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e);
		}
	} else {
		managedMemorySize = Long.valueOf(managedMemorySizeDefaultVal);
	}

	checkConfigParameter(configuration.getString(
		TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()) || managedMemorySize > 0,
		managedMemorySize, TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
		"MemoryManager needs at least one MB of memory. " +
			"If you leave this config parameter empty, the system automatically pick a fraction of the available memory.");

	return managedMemorySize;
}
 

/**
 * Creates a {@link SSLHandlerFactory} to be used by the REST Servers.
 *
 * @param config The application configuration.
 */
public static SSLHandlerFactory createRestServerSSLEngineFactory(final Configuration config) throws Exception {
	SSLContext sslContext = createRestServerSSLContext(config);
	if (sslContext == null) {
		throw new IllegalConfigurationException("SSL is not enabled for REST endpoints.");
	}

	return new SSLHandlerFactory(
			sslContext,
			getEnabledProtocols(config),
			getEnabledCipherSuites(config),
			false,
			isRestSSLAuthenticationEnabled(config),
			-1,
			-1);
}
 

/**
 * Creates a {@link SSLHandlerFactory} to be used by the REST Clients.
 *
 * @param config The application configuration.
 */
public static SSLHandlerFactory createRestClientSSLEngineFactory(final Configuration config) throws Exception {
	SSLContext sslContext = createRestClientSSLContext(config);
	if (sslContext == null) {
		throw new IllegalConfigurationException("SSL is not enabled for REST endpoints.");
	}

	return new SSLHandlerFactory(
			sslContext,
			getEnabledProtocols(config),
			getEnabledCipherSuites(config),
			true,
			isRestSSLAuthenticationEnabled(config),
			-1,
			-1);
}
 

/**
 * Return the configured {@link ZkClientACLMode}.
 *
 * @param config The config to parse
 * @return Configured ACL mode or the default defined by {@link HighAvailabilityOptions#ZOOKEEPER_CLIENT_ACL} if not
 * configured.
 */
public static ZkClientACLMode fromConfig(Configuration config) {
	String aclMode = config.getString(HighAvailabilityOptions.ZOOKEEPER_CLIENT_ACL);
	if (aclMode == null || aclMode.equalsIgnoreCase(ZkClientACLMode.OPEN.name())) {
		return ZkClientACLMode.OPEN;
	} else if (aclMode.equalsIgnoreCase(ZkClientACLMode.CREATOR.name())) {
		return ZkClientACLMode.CREATOR;
	} else {
		String message = "Unsupported ACL option: [" + aclMode + "] provided";
		LOG.error(message);
		throw new IllegalConfigurationException(message);
	}
}
 

/**
 * Creates a SSLEngineFactory to be used by internal communication server endpoints.
 */
public static SSLHandlerFactory createInternalServerSSLEngineFactory(final Configuration config) throws Exception {
	SslContext sslContext = createInternalNettySSLContext(config, false);
	if (sslContext == null) {
		throw new IllegalConfigurationException("SSL is not enabled for internal communication.");
	}

	return new SSLHandlerFactory(
			sslContext,
			config.getInteger(SecurityOptions.SSL_INTERNAL_HANDSHAKE_TIMEOUT),
			config.getInteger(SecurityOptions.SSL_INTERNAL_CLOSE_NOTIFY_FLUSH_TIMEOUT));
}
 

/**
 * Creates a factory for SSL Server Sockets from the given configuration.
 * SSL Server Sockets are always part of internal communication.
 */
public static ServerSocketFactory createSSLServerSocketFactory(Configuration config) throws Exception {
	SSLContext sslContext = createInternalSSLContext(config, false);
	if (sslContext == null) {
		throw new IllegalConfigurationException("SSL is not enabled");
	}

	String[] protocols = getEnabledProtocols(config);
	String[] cipherSuites = getEnabledCipherSuites(config);

	SSLServerSocketFactory factory = sslContext.getServerSocketFactory();
	return new ConfiguringSSLServerSocketFactory(factory, protocols, cipherSuites);
}
 

/**
 * Creates a {@link FileSystemStateStorageHelper} instance.
 *
 * @param configuration {@link Configuration} object
 * @param prefix Prefix for the created files
 * @param <T> Type of the state objects
 * @return {@link FileSystemStateStorageHelper} instance
 * @throws IOException if file system state storage cannot be created
 */
public static <T extends Serializable> FileSystemStateStorageHelper<T> createFileSystemStateStorage(
		Configuration configuration,
		String prefix) throws IOException {

	String rootPath = configuration.getValue(HighAvailabilityOptions.HA_STORAGE_PATH);

	if (rootPath == null || StringUtils.isBlank(rootPath)) {
		throw new IllegalConfigurationException("Missing high-availability storage path for metadata." +
				" Specify via configuration key '" + HighAvailabilityOptions.HA_STORAGE_PATH + "'.");
	} else {
		return new FileSystemStateStorageHelper<T>(rootPath, prefix);
	}
}
 

/**
 * Creates a factory for SSL Client Sockets from the given configuration.
 * SSL Client Sockets are always part of internal communication.
 */
public static SocketFactory createSSLClientSocketFactory(Configuration config) throws Exception {
	SSLContext sslContext = createInternalSSLContext(config, true);
	if (sslContext == null) {
		throw new IllegalConfigurationException("SSL is not enabled");
	}

	return sslContext.getSocketFactory();
}
 

/**
 * Returns the configured ZooKeeper quorum (and removes whitespace, because ZooKeeper does not
 * tolerate it).
 */
public static String getZooKeeperEnsemble(Configuration flinkConf)
		throws IllegalConfigurationException {

	String zkQuorum = flinkConf.getValue(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM);

	if (zkQuorum == null || StringUtils.isBlank(zkQuorum)) {
		throw new IllegalConfigurationException("No ZooKeeper quorum specified in config.");
	}

	// Remove all whitespace
	zkQuorum = zkQuorum.replaceAll("\\s+", "");

	return zkQuorum;
}
 

@Test
public void testConfigOverwrite() throws ClusterDeploymentException {
	Configuration configuration = new Configuration();
	// overwrite vcores in config
	configuration.setInteger(YarnConfigOptions.VCORES, Integer.MAX_VALUE);
	configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0);

	YarnClusterDescriptor clusterDescriptor = createYarnClusterDescriptor(configuration);

	clusterDescriptor.setLocalJarPath(new Path(flinkJar.getPath()));

	// configure slots
	ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
		.setMasterMemoryMB(1)
		.setTaskManagerMemoryMB(1)
		.setNumberTaskManagers(1)
		.setSlotsPerTaskManager(1)
		.createClusterSpecification();

	try {
		clusterDescriptor.deploySessionCluster(clusterSpecification);

		fail("The deploy call should have failed.");
	} catch (ClusterDeploymentException e) {
		// we expect the cause to be an IllegalConfigurationException
		if (!(e.getCause() instanceof IllegalConfigurationException)) {
			throw e;
		}
	} finally {
		clusterDescriptor.close();
	}
}
 

/**
 * Tests that REST Server SSL Engine is not created if SSL is disabled.
 */
@Test
public void testRESTServerSSLDisabled() throws Exception {
	Configuration serverConfig = createRestSslConfigWithKeyStore();
	serverConfig.setBoolean(SecurityOptions.SSL_REST_ENABLED, false);

	try {
		SSLUtils.createRestServerSSLEngineFactory(serverConfig);
		fail("exception expected");
	} catch (IllegalConfigurationException ignored) {}
}
 

private JobGraph createJobGraph() {

		// make sure that all vertices start immediately
		jobGraph.setScheduleMode(streamGraph.getScheduleMode());

		// Generate deterministic hashes for the nodes in order to identify them across
		// submission iff they didn't change.
		Map<Integer, byte[]> hashes = defaultStreamGraphHasher.traverseStreamGraphAndGenerateHashes(streamGraph);

		// Generate legacy version hashes for backwards compatibility
		List<Map<Integer, byte[]>> legacyHashes = new ArrayList<>(legacyStreamGraphHashers.size());
		for (StreamGraphHasher hasher : legacyStreamGraphHashers) {
			legacyHashes.add(hasher.traverseStreamGraphAndGenerateHashes(streamGraph));
		}

		Map<Integer, List<Tuple2<byte[], byte[]>>> chainedOperatorHashes = new HashMap<>();

		setChaining(hashes, legacyHashes, chainedOperatorHashes);

		setPhysicalEdges();

		setSlotSharingAndCoLocation();

		configureCheckpointing();

		JobGraphGenerator.addUserArtifactEntries(streamGraph.getUserArtifacts(), jobGraph);

		// set the ExecutionConfig last when it has been finalized
		try {
			jobGraph.setExecutionConfig(streamGraph.getExecutionConfig());
		}
		catch (IOException e) {
			throw new IllegalConfigurationException("Could not serialize the ExecutionConfig." +
					"This indicates that non-serializable types (like custom serializers) were registered");
		}

		return jobGraph;
	}
 

/**
 * Returns an address in a normalized format for Akka.
 * When an IPv6 address is specified, it normalizes the IPv6 address to avoid
 * complications with the exact URL match policy of Akka.
 * @param host The hostname, IPv4 or IPv6 address
 * @return host which will be normalized if it is an IPv6 address
 */
public static String unresolvedHostToNormalizedString(String host) {
	// Return loopback interface address if host is null
	// This represents the behavior of {@code InetAddress.getByName } and RFC 3330
	if (host == null) {
		host = InetAddress.getLoopbackAddress().getHostAddress();
	} else {
		host = host.trim().toLowerCase();
		if (host.startsWith("[") && host.endsWith("]")) {
			String address = host.substring(1, host.length() - 1);
			if (IPAddressUtil.isIPv6LiteralAddress(address)) {
				host = address;
			}
		}
	}

	// normalize and valid address
	if (IPAddressUtil.isIPv6LiteralAddress(host)) {
		byte[] ipV6Address = IPAddressUtil.textToNumericFormatV6(host);
		host = getIPv6UrlRepresentation(ipV6Address);
	} else if (!IPAddressUtil.isIPv4LiteralAddress(host)) {
		try {
			// We don't allow these in hostnames
			Preconditions.checkArgument(!host.startsWith("."));
			Preconditions.checkArgument(!host.endsWith("."));
			Preconditions.checkArgument(!host.contains(":"));
		} catch (Exception e) {
			throw new IllegalConfigurationException("The configured hostname is not valid", e);
		}
	}

	return host;
}
 

public static CheckpointBarrierHandler createCheckpointBarrierHandler(
		StreamTask<?, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		IOManager ioManager,
		InputGate inputGate,
		Configuration taskManagerConfig) throws IOException {

	CheckpointBarrierHandler barrierHandler;
	if (checkpointMode == CheckpointingMode.EXACTLY_ONCE) {
		long maxAlign = taskManagerConfig.getLong(TaskManagerOptions.TASK_CHECKPOINT_ALIGNMENT_BYTES_LIMIT);
		if (!(maxAlign == -1 || maxAlign > 0)) {
			throw new IllegalConfigurationException(
				TaskManagerOptions.TASK_CHECKPOINT_ALIGNMENT_BYTES_LIMIT.key()
				+ " must be positive or -1 (infinite)");
		}

		if (taskManagerConfig.getBoolean(TaskManagerOptions.NETWORK_CREDIT_MODEL)) {
			barrierHandler = new BarrierBuffer(inputGate, new CachedBufferBlocker(inputGate.getPageSize()), maxAlign);
		} else {
			barrierHandler = new BarrierBuffer(inputGate, new BufferSpiller(ioManager, inputGate.getPageSize()), maxAlign);
		}
	} else if (checkpointMode == CheckpointingMode.AT_LEAST_ONCE) {
		barrierHandler = new BarrierTracker(inputGate);
	} else {
		throw new IllegalArgumentException("Unrecognized Checkpointing Mode: " + checkpointMode);
	}

	if (checkpointedTask != null) {
		barrierHandler.registerCheckpointEventHandler(checkpointedTask);
	}

	return barrierHandler;
}
 

private JobGraph createJobGraph() {

		// make sure that all vertices start immediately
		jobGraph.setScheduleMode(ScheduleMode.EAGER);

		// Generate deterministic hashes for the nodes in order to identify them across
		// submission iff they didn't change.
		Map<Integer, byte[]> hashes = defaultStreamGraphHasher.traverseStreamGraphAndGenerateHashes(streamGraph);

		// Generate legacy version hashes for backwards compatibility
		List<Map<Integer, byte[]>> legacyHashes = new ArrayList<>(legacyStreamGraphHashers.size());
		for (StreamGraphHasher hasher : legacyStreamGraphHashers) {
			legacyHashes.add(hasher.traverseStreamGraphAndGenerateHashes(streamGraph));
		}

		Map<Integer, List<Tuple2<byte[], byte[]>>> chainedOperatorHashes = new HashMap<>();

		setChaining(hashes, legacyHashes, chainedOperatorHashes);

		setPhysicalEdges();

		setSlotSharingAndCoLocation();

		configureCheckpointing();

		JobGraphGenerator.addUserArtifactEntries(streamGraph.getEnvironment().getCachedFiles(), jobGraph);

		// set the ExecutionConfig last when it has been finalized
		try {
			jobGraph.setExecutionConfig(streamGraph.getExecutionConfig());
		}
		catch (IOException e) {
			throw new IllegalConfigurationException("Could not serialize the ExecutionConfig." +
					"This indicates that non-serializable types (like custom serializers) were registered");
		}

		return jobGraph;
	}
 

@SuppressWarnings("unchecked")
@Override
public void readParametersFromConfig(Configuration config, ClassLoader cl) throws ClassNotFoundException {
	// figure out how many key fields there are
	final int numKeyFields = config.getInteger(NUM_KEYS, -1);
	if (numKeyFields < 0) {
		throw new IllegalConfigurationException("The number of keys for the comparator is invalid: " + numKeyFields);
	}
	
	final int[] positions = new int[numKeyFields];
	final Class<? extends Value>[] types = new Class[numKeyFields];
	final boolean[] direction = new boolean[numKeyFields];
	
	// read the individual key positions and types
	for (int i = 0; i < numKeyFields; i++) {
		// next key position
		final int p = config.getInteger(KEY_POS_PREFIX + i, -1);
		if (p >= 0) {
			positions[i] = p;
		} else {
			throw new IllegalConfigurationException("Contained invalid position for key no positions for keys.");
		}
		
		// next key type
		final String name = config.getString(KEY_CLASS_PREFIX + i, null);
		if (name != null) {
			types[i] = (Class<? extends Value>) Class.forName(name, true, cl).asSubclass(Value.class);
		} else {
			throw new IllegalConfigurationException("The key type (" + i +
				") for the comparator is null"); 
		}
		
		// next key sort direction
		direction[i] = config.getBoolean(KEY_SORT_DIRECTION_PREFIX + i, true);
	}
	
	this.positions = positions;
	this.types = types;
	this.sortDirections = direction;
}
 

/**
 * Tests that REST Client SSL creation fails with bad SSL configuration.
 */
@Test
public void testRESTClientSSLMissingPassword() throws Exception {
	Configuration config = new Configuration();
	config.setBoolean(SecurityOptions.SSL_REST_ENABLED, true);
	config.setString(SecurityOptions.SSL_REST_TRUSTSTORE, TRUST_STORE_PATH);

	try {
		SSLUtils.createRestClientSSLEngineFactory(config);
		fail("exception expected");
	} catch (IllegalConfigurationException ignored) {}
}
 

/**
 * Tests that REST Client SSL creation fails with bad SSL configuration.
 */
@Test
public void testRESTClientSSLMissingTrustStore() throws Exception {
	Configuration config = new Configuration();
	config.setBoolean(SecurityOptions.SSL_REST_ENABLED, true);
	config.setString(SecurityOptions.SSL_REST_TRUSTSTORE_PASSWORD, "some password");

	try {
		SSLUtils.createRestClientSSLEngineFactory(config);
		fail("exception expected");
	} catch (IllegalConfigurationException ignored) {}
}