Java Code Examples for org.apache.flink.util.NetUtils#getPortRangeFromString()

/**
 * Creates the {@link QueryableStateConfiguration} from the given Configuration.
 */
private static QueryableStateConfiguration parseQueryableStateConfiguration(Configuration config) {
	if (!config.getBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER)) {
		return null;
	}

	final Iterator<Integer> proxyPorts = NetUtils.getPortRangeFromString(
			config.getString(QueryableStateOptions.PROXY_PORT_RANGE));
	final Iterator<Integer> serverPorts = NetUtils.getPortRangeFromString(
			config.getString(QueryableStateOptions.SERVER_PORT_RANGE));

	final int numProxyServerNetworkThreads = config.getInteger(QueryableStateOptions.PROXY_NETWORK_THREADS);
	final int numProxyServerQueryThreads = config.getInteger(QueryableStateOptions.PROXY_ASYNC_QUERY_THREADS);

	final int numStateServerNetworkThreads = config.getInteger(QueryableStateOptions.SERVER_NETWORK_THREADS);
	final int numStateServerQueryThreads = config.getInteger(QueryableStateOptions.SERVER_ASYNC_QUERY_THREADS);

	return new QueryableStateConfiguration(
			proxyPorts,
			serverPorts,
			numProxyServerNetworkThreads,
			numProxyServerQueryThreads,
			numStateServerNetworkThreads,
			numStateServerQueryThreads);
}
 

@Override
public void open(MetricConfig config) {
	super.open(config);

	String portsConfig = config.getString(ARG_PORT, DEFAULT_PORT);
	Iterator<Integer> ports = NetUtils.getPortRangeFromString(portsConfig);

	while (ports.hasNext()) {
		int port = ports.next();
		try {
			// internally accesses CollectorRegistry.defaultRegistry
			httpServer = new HTTPServer(port);
			this.port = port;
			log.info("Started PrometheusReporter HTTP server on port {}.", port);
			break;
		} catch (IOException ioe) { //assume port conflict
			log.debug("Could not start PrometheusReporter HTTP server on port {}.", port, ioe);
		}
	}
	if (httpServer == null) {
		throw new RuntimeException("Could not start PrometheusReporter HTTP server on any configured port. Ports: " + portsConfig);
	}
}
 

/**
 * Creates the {@link QueryableStateConfiguration} from the given Configuration.
 */
public static QueryableStateConfiguration fromConfiguration(Configuration config) {
	if (!config.getBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER)) {
		return null;
	}

	final Iterator<Integer> proxyPorts = NetUtils.getPortRangeFromString(
		config.getString(QueryableStateOptions.PROXY_PORT_RANGE));
	final Iterator<Integer> serverPorts = NetUtils.getPortRangeFromString(
		config.getString(QueryableStateOptions.SERVER_PORT_RANGE));

	final int numProxyServerNetworkThreads = config.getInteger(QueryableStateOptions.PROXY_NETWORK_THREADS);
	final int numProxyServerQueryThreads = config.getInteger(QueryableStateOptions.PROXY_ASYNC_QUERY_THREADS);

	final int numStateServerNetworkThreads = config.getInteger(QueryableStateOptions.SERVER_NETWORK_THREADS);
	final int numStateServerQueryThreads = config.getInteger(QueryableStateOptions.SERVER_ASYNC_QUERY_THREADS);

	return new QueryableStateConfiguration(
		proxyPorts,
		serverPorts,
		numProxyServerNetworkThreads,
		numProxyServerQueryThreads,
		numStateServerNetworkThreads,
		numStateServerQueryThreads);
}
 

@Override
public void open(MetricConfig config) {
	super.open(config);

	String portsConfig = config.getString(ARG_PORT, DEFAULT_PORT);
	Iterator<Integer> ports = NetUtils.getPortRangeFromString(portsConfig);

	while (ports.hasNext()) {
		int port = ports.next();
		try {
			// internally accesses CollectorRegistry.defaultRegistry
			httpServer = new HTTPServer(port);
			this.port = port;
			log.info("Started PrometheusReporter HTTP server on port {}.", port);
			break;
		} catch (IOException ioe) { //assume port conflict
			log.debug("Could not start PrometheusReporter HTTP server on port {}.", port, ioe);
		}
	}
	if (httpServer == null) {
		throw new RuntimeException("Could not start PrometheusReporter HTTP server on any configured port. Ports: " + portsConfig);
	}
}
 

/**
 * Creates the {@link QueryableStateConfiguration} from the given Configuration.
 */
public static QueryableStateConfiguration fromConfiguration(Configuration config) {
	if (!config.getBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER)) {
		return null;
	}

	final Iterator<Integer> proxyPorts = NetUtils.getPortRangeFromString(
		config.getString(QueryableStateOptions.PROXY_PORT_RANGE));
	final Iterator<Integer> serverPorts = NetUtils.getPortRangeFromString(
		config.getString(QueryableStateOptions.SERVER_PORT_RANGE));

	final int numProxyServerNetworkThreads = config.getInteger(QueryableStateOptions.PROXY_NETWORK_THREADS);
	final int numProxyServerQueryThreads = config.getInteger(QueryableStateOptions.PROXY_ASYNC_QUERY_THREADS);

	final int numStateServerNetworkThreads = config.getInteger(QueryableStateOptions.SERVER_NETWORK_THREADS);
	final int numStateServerQueryThreads = config.getInteger(QueryableStateOptions.SERVER_ASYNC_QUERY_THREADS);

	return new QueryableStateConfiguration(
		proxyPorts,
		serverPorts,
		numProxyServerNetworkThreads,
		numProxyServerQueryThreads,
		numStateServerNetworkThreads,
		numStateServerQueryThreads);
}
 

@Override
public void open(MetricConfig config) {
	super.open(config);

	String portsConfig = config.getString(ARG_PORT, DEFAULT_PORT);
	Iterator<Integer> ports = NetUtils.getPortRangeFromString(portsConfig);

	while (ports.hasNext()) {
		int port = ports.next();
		try {
			// internally accesses CollectorRegistry.defaultRegistry
			httpServer = new HTTPServer(port);
			this.port = port;
			log.info("Started PrometheusReporter HTTP server on port {}.", port);
			break;
		} catch (IOException ioe) { //assume port conflict
			log.debug("Could not start PrometheusReporter HTTP server on port {}.", port, ioe);
		}
	}
	if (httpServer == null) {
		throw new RuntimeException("Could not start PrometheusReporter HTTP server on any configured port. Ports: " + portsConfig);
	}
}
 

@Override
public void open(MetricConfig config) {
	String portsConfig = config.getString(ARG_PORT, null);

	if (portsConfig != null) {
		Iterator<Integer> ports = NetUtils.getPortRangeFromString(portsConfig);

		JMXServer server = new JMXServer();
		while (ports.hasNext()) {
			int port = ports.next();
			try {
				server.start(port);
				LOG.info("Started JMX server on port " + port + ".");
				// only set our field if the server was actually started
				jmxServer = server;
				break;
			} catch (IOException ioe) { //assume port conflict
				LOG.debug("Could not start JMX server on port " + port + ".", ioe);
				try {
					server.stop();
				} catch (Exception e) {
					LOG.debug("Could not stop JMX server.", e);
				}
			}
		}
		if (jmxServer == null) {
			throw new RuntimeException("Could not start JMX server on any configured port. Ports: " + portsConfig);
		}
	}
	LOG.info("Configured JMXReporter with {port:{}}", portsConfig);
}
 

JMXReporter(@Nullable final String portsConfig) {
	this.mBeanServer = ManagementFactory.getPlatformMBeanServer();
	this.registeredMetrics = new HashMap<>();

	if (portsConfig != null) {
		Iterator<Integer> ports = NetUtils.getPortRangeFromString(portsConfig);

		JMXServer successfullyStartedServer = null;
		while (ports.hasNext() && successfullyStartedServer == null) {
			JMXServer server = new JMXServer();
			int port = ports.next();
			try {
				server.start(port);
				LOG.info("Started JMX server on port " + port + ".");
				successfullyStartedServer = server;
			} catch (IOException ioe) { //assume port conflict
				LOG.debug("Could not start JMX server on port " + port + ".", ioe);
				try {
					server.stop();
				} catch (Exception e) {
					LOG.debug("Could not stop JMX server.", e);
				}
			}
		}
		if (successfullyStartedServer == null) {
			throw new RuntimeException("Could not start JMX server on any configured port. Ports: " + portsConfig);
		}
		this.jmxServer = successfullyStartedServer;
	} else {
		this.jmxServer = null;
	}
	LOG.info("Configured JMXReporter with {port:{}}", portsConfig);
}
 

JMXReporter(@Nullable final String portsConfig) {
	this.mBeanServer = ManagementFactory.getPlatformMBeanServer();
	this.registeredMetrics = new HashMap<>();

	if (portsConfig != null) {
		Iterator<Integer> ports = NetUtils.getPortRangeFromString(portsConfig);

		JMXServer successfullyStartedServer = null;
		while (ports.hasNext() && successfullyStartedServer == null) {
			JMXServer server = new JMXServer();
			int port = ports.next();
			try {
				server.start(port);
				LOG.info("Started JMX server on port " + port + ".");
				successfullyStartedServer = server;
			} catch (IOException ioe) { //assume port conflict
				LOG.debug("Could not start JMX server on port " + port + ".", ioe);
				try {
					server.stop();
				} catch (Exception e) {
					LOG.debug("Could not stop JMX server.", e);
				}
			}
		}
		if (successfullyStartedServer == null) {
			throw new RuntimeException("Could not start JMX server on any configured port. Ports: " + portsConfig);
		}
		this.jmxServer = successfullyStartedServer;
	} else {
		this.jmxServer = null;
	}
	LOG.info("Configured JMXReporter with {port:{}}", portsConfig);
}
 

/**
 * Gets the configuration describing the queryable state as deactivated.
 */
public static QueryableStateConfiguration disabled() {
	final Iterator<Integer> proxyPorts = NetUtils.getPortRangeFromString(QueryableStateOptions.PROXY_PORT_RANGE.defaultValue());
	final Iterator<Integer> serverPorts = NetUtils.getPortRangeFromString(QueryableStateOptions.SERVER_PORT_RANGE.defaultValue());
	return new QueryableStateConfiguration(proxyPorts, serverPorts, 0, 0, 0, 0);
}
 

/**
 * Instantiates a new BLOB server and binds it to a free network port.
 *
 * @param config Configuration to be used to instantiate the BlobServer
 * @param blobStore BlobStore to store blobs persistently
 *
 * @throws IOException
 * 		thrown if the BLOB server cannot bind to a free network port or if the
 * 		(local or distributed) file storage cannot be created or is not usable
 */
public BlobServer(Configuration config, BlobStore blobStore) throws IOException {
	this.blobServiceConfiguration = checkNotNull(config);
	this.blobStore = checkNotNull(blobStore);
	this.readWriteLock = new ReentrantReadWriteLock();

	// configure and create the storage directory
	this.storageDir = BlobUtils.initLocalStorageDirectory(config);
	LOG.info("Created BLOB server storage directory {}", storageDir);

	// configure the maximum number of concurrent connections
	final int maxConnections = config.getInteger(BlobServerOptions.FETCH_CONCURRENT);
	if (maxConnections >= 1) {
		this.maxConnections = maxConnections;
	}
	else {
		LOG.warn("Invalid value for maximum connections in BLOB server: {}. Using default value of {}",
				maxConnections, BlobServerOptions.FETCH_CONCURRENT.defaultValue());
		this.maxConnections = BlobServerOptions.FETCH_CONCURRENT.defaultValue();
	}

	// configure the backlog of connections
	int backlog = config.getInteger(BlobServerOptions.FETCH_BACKLOG);
	if (backlog < 1) {
		LOG.warn("Invalid value for BLOB connection backlog: {}. Using default value of {}",
				backlog, BlobServerOptions.FETCH_BACKLOG.defaultValue());
		backlog = BlobServerOptions.FETCH_BACKLOG.defaultValue();
	}

	// Initializing the clean up task
	this.cleanupTimer = new Timer(true);

	this.cleanupInterval = config.getLong(BlobServerOptions.CLEANUP_INTERVAL) * 1000;
	this.cleanupTimer
		.schedule(new TransientBlobCleanupTask(blobExpiryTimes, readWriteLock.writeLock(),
			storageDir, LOG), cleanupInterval, cleanupInterval);

	this.shutdownHook = ShutdownHookUtil.addShutdownHook(this, getClass().getSimpleName(), LOG);

	//  ----------------------- start the server -------------------

	final String serverPortRange = config.getString(BlobServerOptions.PORT);
	final Iterator<Integer> ports = NetUtils.getPortRangeFromString(serverPortRange);

	final ServerSocketFactory socketFactory;
	if (SSLUtils.isInternalSSLEnabled(config) && config.getBoolean(BlobServerOptions.SSL_ENABLED)) {
		try {
			socketFactory = SSLUtils.createSSLServerSocketFactory(config);
		}
		catch (Exception e) {
			throw new IOException("Failed to initialize SSL for the blob server", e);
		}
	}
	else {
		socketFactory = ServerSocketFactory.getDefault();
	}

	final int finalBacklog = backlog;
	this.serverSocket = NetUtils.createSocketFromPorts(ports,
			(port) -> socketFactory.createServerSocket(port, finalBacklog));

	if (serverSocket == null) {
		throw new IOException("Unable to open BLOB Server in specified port range: " + serverPortRange);
	}

	// start the server thread
	setName("BLOB Server listener at " + getPort());
	setDaemon(true);

	if (LOG.isInfoEnabled()) {
		LOG.info("Started BLOB server at {}:{} - max concurrent requests: {} - max backlog: {}",
				serverSocket.getInetAddress().getHostAddress(), getPort(), maxConnections, backlog);
	}
}
 

/**
 * Gets the configuration describing the queryable state as deactivated.
 */
public static QueryableStateConfiguration disabled() {
	final Iterator<Integer> proxyPorts = NetUtils.getPortRangeFromString(QueryableStateOptions.PROXY_PORT_RANGE.defaultValue());
	final Iterator<Integer> serverPorts = NetUtils.getPortRangeFromString(QueryableStateOptions.SERVER_PORT_RANGE.defaultValue());
	return new QueryableStateConfiguration(proxyPorts, serverPorts, 0, 0, 0, 0);
}
 

/**
 * Instantiates a new BLOB server and binds it to a free network port.
 *
 * @param config Configuration to be used to instantiate the BlobServer
 * @param blobStore BlobStore to store blobs persistently
 *
 * @throws IOException
 * 		thrown if the BLOB server cannot bind to a free network port or if the
 * 		(local or distributed) file storage cannot be created or is not usable
 */
public BlobServer(Configuration config, BlobStore blobStore) throws IOException {
	this.blobServiceConfiguration = checkNotNull(config);
	this.blobStore = checkNotNull(blobStore);
	this.readWriteLock = new ReentrantReadWriteLock();

	// configure and create the storage directory
	this.storageDir = BlobUtils.initLocalStorageDirectory(config);
	LOG.info("Created BLOB server storage directory {}", storageDir);

	// configure the maximum number of concurrent connections
	final int maxConnections = config.getInteger(BlobServerOptions.FETCH_CONCURRENT);
	if (maxConnections >= 1) {
		this.maxConnections = maxConnections;
	}
	else {
		LOG.warn("Invalid value for maximum connections in BLOB server: {}. Using default value of {}",
				maxConnections, BlobServerOptions.FETCH_CONCURRENT.defaultValue());
		this.maxConnections = BlobServerOptions.FETCH_CONCURRENT.defaultValue();
	}

	// configure the backlog of connections
	int backlog = config.getInteger(BlobServerOptions.FETCH_BACKLOG);
	if (backlog < 1) {
		LOG.warn("Invalid value for BLOB connection backlog: {}. Using default value of {}",
				backlog, BlobServerOptions.FETCH_BACKLOG.defaultValue());
		backlog = BlobServerOptions.FETCH_BACKLOG.defaultValue();
	}

	// Initializing the clean up task
	this.cleanupTimer = new Timer(true);

	this.cleanupInterval = config.getLong(BlobServerOptions.CLEANUP_INTERVAL) * 1000;
	this.cleanupTimer
		.schedule(new TransientBlobCleanupTask(blobExpiryTimes, readWriteLock.writeLock(),
			storageDir, LOG), cleanupInterval, cleanupInterval);

	this.shutdownHook = ShutdownHookUtil.addShutdownHook(this, getClass().getSimpleName(), LOG);

	//  ----------------------- start the server -------------------

	final String serverPortRange = config.getString(BlobServerOptions.PORT);
	final Iterator<Integer> ports = NetUtils.getPortRangeFromString(serverPortRange);

	final ServerSocketFactory socketFactory;
	if (SSLUtils.isInternalSSLEnabled(config) && config.getBoolean(BlobServerOptions.SSL_ENABLED)) {
		try {
			socketFactory = SSLUtils.createSSLServerSocketFactory(config);
		}
		catch (Exception e) {
			throw new IOException("Failed to initialize SSL for the blob server", e);
		}
	}
	else {
		socketFactory = ServerSocketFactory.getDefault();
	}

	final int finalBacklog = backlog;
	this.serverSocket = NetUtils.createSocketFromPorts(ports,
			(port) -> socketFactory.createServerSocket(port, finalBacklog));

	if (serverSocket == null) {
		throw new IOException("Unable to open BLOB Server in specified port range: " + serverPortRange);
	}

	// start the server thread
	setName("BLOB Server listener at " + getPort());
	setDaemon(true);

	if (LOG.isInfoEnabled()) {
		LOG.info("Started BLOB server at {}:{} - max concurrent requests: {} - max backlog: {}",
				serverSocket.getInetAddress().getHostAddress(), getPort(), maxConnections, backlog);
	}
}
 

/**
 * Gets the configuration describing the queryable state as deactivated.
 */
public static QueryableStateConfiguration disabled() {
	final Iterator<Integer> proxyPorts = NetUtils.getPortRangeFromString(QueryableStateOptions.PROXY_PORT_RANGE.defaultValue());
	final Iterator<Integer> serverPorts = NetUtils.getPortRangeFromString(QueryableStateOptions.SERVER_PORT_RANGE.defaultValue());
	return new QueryableStateConfiguration(proxyPorts, serverPorts, 0, 0, 0, 0);
}
 

/**
 * Instantiates a new BLOB server and binds it to a free network port.
 *
 * @param config Configuration to be used to instantiate the BlobServer
 * @param blobStore BlobStore to store blobs persistently
 *
 * @throws IOException
 * 		thrown if the BLOB server cannot bind to a free network port or if the
 * 		(local or distributed) file storage cannot be created or is not usable
 */
public BlobServer(Configuration config, BlobStore blobStore) throws IOException {
	this.blobServiceConfiguration = checkNotNull(config);
	this.blobStore = checkNotNull(blobStore);
	this.readWriteLock = new ReentrantReadWriteLock();

	// configure and create the storage directory
	this.storageDir = BlobUtils.initLocalStorageDirectory(config);
	LOG.info("Created BLOB server storage directory {}", storageDir);

	// configure the maximum number of concurrent connections
	final int maxConnections = config.getInteger(BlobServerOptions.FETCH_CONCURRENT);
	if (maxConnections >= 1) {
		this.maxConnections = maxConnections;
	}
	else {
		LOG.warn("Invalid value for maximum connections in BLOB server: {}. Using default value of {}",
				maxConnections, BlobServerOptions.FETCH_CONCURRENT.defaultValue());
		this.maxConnections = BlobServerOptions.FETCH_CONCURRENT.defaultValue();
	}

	// configure the backlog of connections
	int backlog = config.getInteger(BlobServerOptions.FETCH_BACKLOG);
	if (backlog < 1) {
		LOG.warn("Invalid value for BLOB connection backlog: {}. Using default value of {}",
				backlog, BlobServerOptions.FETCH_BACKLOG.defaultValue());
		backlog = BlobServerOptions.FETCH_BACKLOG.defaultValue();
	}

	// Initializing the clean up task
	this.cleanupTimer = new Timer(true);

	this.cleanupInterval = config.getLong(BlobServerOptions.CLEANUP_INTERVAL) * 1000;
	this.cleanupTimer
		.schedule(new TransientBlobCleanupTask(blobExpiryTimes, readWriteLock.writeLock(),
			storageDir, LOG), cleanupInterval, cleanupInterval);

	this.shutdownHook = ShutdownHookUtil.addShutdownHook(this, getClass().getSimpleName(), LOG);

	//  ----------------------- start the server -------------------

	final String serverPortRange = config.getString(BlobServerOptions.PORT);
	final Iterator<Integer> ports = NetUtils.getPortRangeFromString(serverPortRange);

	final ServerSocketFactory socketFactory;
	if (SSLUtils.isInternalSSLEnabled(config) && config.getBoolean(BlobServerOptions.SSL_ENABLED)) {
		try {
			socketFactory = SSLUtils.createSSLServerSocketFactory(config);
		}
		catch (Exception e) {
			throw new IOException("Failed to initialize SSL for the blob server", e);
		}
	}
	else {
		socketFactory = ServerSocketFactory.getDefault();
	}

	final int finalBacklog = backlog;
	final String bindHost = config.getOptional(JobManagerOptions.BIND_HOST).orElseGet(NetUtils::getWildcardIPAddress);

	this.serverSocket = NetUtils.createSocketFromPorts(ports,
			(port) -> socketFactory.createServerSocket(port, finalBacklog, InetAddress.getByName(bindHost)));

	if (serverSocket == null) {
		throw new IOException("Unable to open BLOB Server in specified port range: " + serverPortRange);
	}

	// start the server thread
	setName("BLOB Server listener at " + getPort());
	setDaemon(true);

	if (LOG.isInfoEnabled()) {
		LOG.info("Started BLOB server at {}:{} - max concurrent requests: {} - max backlog: {}",
				serverSocket.getInetAddress().getHostAddress(), getPort(), maxConnections, backlog);
	}
}