Java Code Examples for org.apache.flink.configuration.Configuration#getInteger()

public AbstractClientProvider(Configuration properties) {
	this.stsRoleArn = properties.getString(BlinkOptions.STS.STS_ROLE_ARN);
	this.stsAccessId = properties.getString(BlinkOptions.STS.STS_ACCESS_ID);
	this.stsAccessKey = properties.getString(BlinkOptions.STS.STS_ACCESS_KEY);
	this.stsAssumeRoleFor = properties.getString(BlinkOptions.STS.STS_UID);
	this.stsSessionName = String.valueOf(System.currentTimeMillis());
	this.stsExpireSeconds = properties.getInteger(BlinkOptions.STS.STS_ROLEARN_UPDATE_SECONDS);
	useSts = true;
	synchronized (mutex) {
		if (this.cacheArnResponse == null) {
			this.cacheArnResponse = CacheBuilder.newBuilder().concurrencyLevel(5).initialCapacity(1).maximumSize(3)
												.expireAfterWrite(stsExpireSeconds, TimeUnit.SECONDS).build(
							new StsIdentityLoader(this, properties));
		}
	}
}
 

public CliFrontend(
		Configuration configuration,
		ClusterClientServiceLoader clusterClientServiceLoader,
		List<CustomCommandLine> customCommandLines) {
	this.configuration = checkNotNull(configuration);
	this.customCommandLines = checkNotNull(customCommandLines);
	this.clusterClientServiceLoader = checkNotNull(clusterClientServiceLoader);

	FileSystem.initialize(configuration, PluginUtils.createPluginManagerFromRootFolder(configuration));

	this.customCommandLineOptions = new Options();

	for (CustomCommandLine customCommandLine : customCommandLines) {
		customCommandLine.addGeneralOptions(customCommandLineOptions);
		customCommandLine.addRunOptions(customCommandLineOptions);
	}

	this.clientTimeout = configuration.get(ClientOptions.CLIENT_TIMEOUT);
	this.defaultParallelism = configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM);
}
 

/**
 * Factory method to startWith a new instance, providing a new instance of {@link MiniCluster}
 *
 * @param parallelism global setting for parallel execution.
 * @return new instance of {@link DataStreamTestEnvironment}
 * @throws Exception
 */
public static DataStreamTestEnvironment createTestEnvironment(int parallelism) {
	int taskSlots = Runtime.getRuntime().availableProcessors();

	Configuration configuration = new Configuration();
	configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0");

	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, taskSlots);

	MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder()
			.setConfiguration(configuration)
			.setNumSlotsPerTaskManager(numSlotsPerTaskManager)
			.build();

	MiniCluster miniCluster = new MiniCluster(cfg);

	return new DataStreamTestEnvironment(
			miniCluster,
			parallelism);
}
 

/**
 * Creates and returns a new {@link RestClientConfiguration} from the given {@link Configuration}.
 *
 * @param config configuration from which the REST client endpoint configuration should be created from
 * @return REST client endpoint configuration
 * @throws ConfigurationException if SSL was configured incorrectly
 */

public static RestClientConfiguration fromConfiguration(Configuration config) throws ConfigurationException {
	Preconditions.checkNotNull(config);

	final SSLHandlerFactory sslHandlerFactory;
	if (SSLUtils.isRestSSLEnabled(config)) {
		try {
			sslHandlerFactory = SSLUtils.createRestClientSSLEngineFactory(config);
		} catch (Exception e) {
			throw new ConfigurationException("Failed to initialize SSLContext for the REST client", e);
		}
	} else {
		sslHandlerFactory = null;
	}

	final long connectionTimeout = config.getLong(RestOptions.CONNECTION_TIMEOUT);

	final long idlenessTimeout = config.getLong(RestOptions.IDLENESS_TIMEOUT);

	int maxContentLength = config.getInteger(RestOptions.CLIENT_MAX_CONTENT_LENGTH);

	return new RestClientConfiguration(sslHandlerFactory, connectionTimeout, idlenessTimeout, maxContentLength);
}
 

/**
 * Gets and verify the io-executor pool size based on configuration.
 *
 * @param config The configuration to read.
 * @return The legal io-executor pool size.
 */
public static int getPoolSize(Configuration config) {
	final int poolSize = config.getInteger(ClusterOptions.CLUSTER_IO_EXECUTOR_POOL_SIZE, 4 * Hardware.getNumberCPUCores());
	Preconditions.checkArgument(poolSize > 0,
		String.format("Illegal pool size (%s) of io-executor, please re-configure '%s'.",
			poolSize, ClusterOptions.CLUSTER_IO_EXECUTOR_POOL_SIZE.key()));
	return poolSize;
}
 

public JobGraphGenerator(Configuration config) {
	this.defaultMaxFan = config.getInteger(AlgorithmOptions.SPILLING_MAX_FAN);
	this.defaultSortSpillingThreshold = config.getFloat(AlgorithmOptions.SORT_SPILLING_THRESHOLD);
	this.useLargeRecordHandler = config.getBoolean(
			ConfigConstants.USE_LARGE_RECORD_HANDLER_KEY,
			ConfigConstants.DEFAULT_USE_LARGE_RECORD_HANDLER);
}
 

public static FailureRateRestartStrategyFactory createFactory(Configuration configuration) throws Exception {
	int maxFailuresPerInterval = configuration.getInteger(ConfigConstants.RESTART_STRATEGY_FAILURE_RATE_MAX_FAILURES_PER_INTERVAL, 1);
	String failuresIntervalString = configuration.getString(
			ConfigConstants.RESTART_STRATEGY_FAILURE_RATE_FAILURE_RATE_INTERVAL, Duration.apply(1, TimeUnit.MINUTES).toString()
	);
	String timeoutString = configuration.getString(AkkaOptions.WATCH_HEARTBEAT_INTERVAL);
	String delayString = configuration.getString(ConfigConstants.RESTART_STRATEGY_FAILURE_RATE_DELAY, timeoutString);

	Duration failuresInterval = Duration.apply(failuresIntervalString);
	Duration delay = Duration.apply(delayString);


	return new FailureRateRestartStrategyFactory(maxFailuresPerInterval, Time.milliseconds(failuresInterval.toMillis()), Time.milliseconds(delay.toMillis()));
}
 

/**
 * Calcuate cutoff memory size used by container, it will throw an {@link IllegalArgumentException}
 * if the config is invalid or return the cutoff value if valid.
 *
 * @param config The Flink configuration.
 * @param containerMemoryMB The size of the complete container, in megabytes.
 *
 * @return cutoff memory size used by container.
 */
public static long calculateCutoffMB(Configuration config, long containerMemoryMB) {
	Preconditions.checkArgument(containerMemoryMB > 0);

	// (1) check cutoff ratio
	final float memoryCutoffRatio = config.getFloat(
		ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO);

	if (memoryCutoffRatio >= 1 || memoryCutoffRatio <= 0) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO.key() + "' must be between 0 and 1. Value given="
			+ memoryCutoffRatio);
	}

	// (2) check min cutoff value
	final int minCutoff = config.getInteger(
		ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN);

	if (minCutoff >= containerMemoryMB) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.key() + "'='" + minCutoff
			+ "' is larger than the total container memory " + containerMemoryMB);
	}

	// (3) check between heap and off-heap
	long cutoff = (long) (containerMemoryMB * memoryCutoffRatio);
	if (cutoff < minCutoff) {
		cutoff = minCutoff;
	}
	return cutoff;
}
 

public static void writeFileInfoToConfig(String name, DistributedCacheEntry e, Configuration conf) {
	int num = conf.getInteger(CACHE_FILE_NUM, 0) + 1;
	conf.setInteger(CACHE_FILE_NUM, num);
	conf.setString(CACHE_FILE_NAME + num, name);
	conf.setString(CACHE_FILE_PATH + num, e.filePath);
	conf.setBoolean(CACHE_FILE_EXE + num, e.isExecutable || new File(e.filePath).canExecute());
	conf.setBoolean(CACHE_FILE_DIR + num, e.isZipped || new File(e.filePath).isDirectory());
	if (e.blobKey != null) {
		conf.setBytes(CACHE_FILE_BLOB_KEY + num, e.blobKey);
	}
}
 

protected static void loadConfigParameters(Configuration parameters) {
	int maxSamples = parameters.getInteger(OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES);
	int minSamples = parameters.getInteger(OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES);
	
	if (maxSamples < 0) {
		LOG.error("Invalid default maximum number of line samples: " + maxSamples + ". Using default value of " +
			OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES.key());
		maxSamples = OptimizerOptions.DELIMITED_FORMAT_MAX_LINE_SAMPLES.defaultValue();
	}
	if (minSamples < 0) {
		LOG.error("Invalid default minimum number of line samples: " + minSamples + ". Using default value of " +
			OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES.key());
		minSamples = OptimizerOptions.DELIMITED_FORMAT_MIN_LINE_SAMPLES.defaultValue();
	}
	
	DEFAULT_MAX_NUM_SAMPLES = maxSamples;
	
	if (minSamples > maxSamples) {
		LOG.error("Default minimum number of line samples cannot be greater the default maximum number " +
				"of line samples: min=" + minSamples + ", max=" + maxSamples + ". Defaulting minimum to maximum.");
		DEFAULT_MIN_NUM_SAMPLES = maxSamples;
	} else {
		DEFAULT_MIN_NUM_SAMPLES = minSamples;
	}
	
	int maxLen = parameters.getInteger(OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN);
	if (maxLen <= 0) {
		maxLen = OptimizerOptions.DELIMITED_FORMAT_MAX_SAMPLE_LEN.defaultValue();
		LOG.error("Invalid value for the maximum sample record length. Using default value of " + maxLen + '.');
	} else if (maxLen < DEFAULT_READ_BUFFER_SIZE) {
		maxLen = DEFAULT_READ_BUFFER_SIZE;
		LOG.warn("Increasing maximum sample record length to size of the read buffer (" + maxLen + ").");
	}
	MAX_SAMPLE_LEN = maxLen;
}
 

/**
 * Parses the hosts / ports for communication and data exchange from configuration.
 *
 * @param configuration configuration object
 * @return the data port
 */
private static int getDataport(Configuration configuration) {
	final int dataport = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT);
	ConfigurationParserUtils.checkConfigParameter(dataport >= 0, dataport, NettyShuffleEnvironmentOptions.DATA_PORT.key(),
		"Leave config parameter empty or use 0 to let the system choose a port automatically.");

	return dataport;
}
 

private static MesosArtifactServer createArtifactServer(Configuration configuration, String hostname) throws Exception {
	final int artifactServerPort = configuration.getInteger(MesosOptions.ARTIFACT_SERVER_PORT, 0);

	// a random prefix is affixed to artifact URLs to ensure uniqueness in the Mesos fetcher cache
	final String artifactServerPrefix = UUID.randomUUID().toString();

	return new MesosArtifactServer(artifactServerPrefix, hostname, artifactServerPort, configuration);
}
 

public static ClusterSpecification fromConfiguration(Configuration configuration) {
	int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1);

	int jobManagerMemoryMb = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes();
	int taskManagerMemoryMb = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes();

	return new ClusterSpecificationBuilder()
		.setMasterMemoryMB(jobManagerMemoryMb)
		.setTaskManagerMemoryMB(taskManagerMemoryMb)
		.setNumberTaskManagers(1)
		.setSlotsPerTaskManager(slots)
		.createClusterSpecification();
}
 

@Override
public void open(Configuration parameters) throws Exception {
    super.open(parameters);
    batchSize = parameters.getInteger(BATCH_SIZE, 1000);
    attributes = Serialization.deserializeObject(parameters.getBytes(ATTRIBUTES, null));
}
 

public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				JobMaster.JOB_MANAGER_NAME,
				addressResolution,
				configuration);
			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				ResourceManager.RESOURCE_MANAGER_NAME,
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				Dispatcher.DISPATCHER_NAME,
				addressResolution,
				configuration);

			final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
				"%s must be set",
				RestOptions.ADDRESS.key());
			final int port = configuration.getInteger(RestOptions.PORT);
			final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
			final String protocol = enableSSL ? "https://" : "http://";

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				jobManagerRpcUrl,
				String.format("%s%s:%s", protocol, address, port));
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}
 

private void testValidSslConnection(Configuration sslConfig) throws Exception {
	OneShotLatch serverChannelInitComplete = new OneShotLatch();
	final SslHandler[] serverSslHandler = new SslHandler[1];

	NettyProtocol protocol = new NoOpProtocol();

	NettyConfig nettyConfig = createNettyConfig(sslConfig);

	final NettyBufferPool bufferPool = new NettyBufferPool(1);
	final NettyServer server = NettyTestUtil.initServer(
		nettyConfig,
		bufferPool,
		sslHandlerFactory ->
			new TestingServerChannelInitializer(
				protocol,
				sslHandlerFactory,
				serverChannelInitComplete,
				serverSslHandler));
	final NettyClient client = NettyTestUtil.initClient(nettyConfig, protocol, bufferPool);
	final NettyServerAndClient serverAndClient = new NettyServerAndClient(server, client);

	Channel ch = NettyTestUtil.connect(serverAndClient);

	SslHandler clientSslHandler = (SslHandler) ch.pipeline().get("ssl");
	assertEqualsOrDefault(sslConfig, SSL_INTERNAL_HANDSHAKE_TIMEOUT, clientSslHandler.getHandshakeTimeoutMillis());
	assertEqualsOrDefault(sslConfig, SSL_INTERNAL_CLOSE_NOTIFY_FLUSH_TIMEOUT, clientSslHandler.getCloseNotifyFlushTimeoutMillis());

	// should be able to send text data
	ch.pipeline().addLast(new StringDecoder()).addLast(new StringEncoder());
	ch.writeAndFlush("test").sync();

	// session context is only be available after a session was setup -> this should be true after data was sent
	serverChannelInitComplete.await();
	assertNotNull(serverSslHandler[0]);

	// verify server parameters
	assertEqualsOrDefault(sslConfig, SSL_INTERNAL_HANDSHAKE_TIMEOUT, serverSslHandler[0].getHandshakeTimeoutMillis());
	assertEqualsOrDefault(sslConfig, SSL_INTERNAL_CLOSE_NOTIFY_FLUSH_TIMEOUT, serverSslHandler[0].getCloseNotifyFlushTimeoutMillis());
	SSLSessionContext sessionContext = serverSslHandler[0].engine().getSession().getSessionContext();
	assertNotNull("bug in unit test setup: session context not available", sessionContext);
	// note: can't verify session cache setting at the client - delegate to server instead (with our own channel initializer)
	assertEqualsOrDefault(sslConfig, SSL_INTERNAL_SESSION_CACHE_SIZE, sessionContext.getSessionCacheSize());
	int sessionTimeout = sslConfig.getInteger(SSL_INTERNAL_SESSION_TIMEOUT);
	if (sessionTimeout != -1) {
		// session timeout config is in milliseconds but the context returns it in seconds
		assertEquals(sessionTimeout / 1000, sessionContext.getSessionTimeout());
	} else {
		assertTrue("default value (-1) should not be propagated", sessionContext.getSessionTimeout() >= 0);
	}

	NettyTestUtil.shutdown(serverAndClient);
}
 

/**
 * Creates and returns a new {@link RestServerEndpointConfiguration} from the given {@link Configuration}.
 *
 * @param config configuration from which the REST server endpoint configuration should be created from
 * @return REST server endpoint configuration
 * @throws ConfigurationException if SSL was configured incorrectly
 */
public static RestServerEndpointConfiguration fromConfiguration(Configuration config) throws ConfigurationException {
	Preconditions.checkNotNull(config);

	final String restAddress = Preconditions.checkNotNull(config.getString(RestOptions.ADDRESS),
		"%s must be set",
		RestOptions.ADDRESS.key());

	final String restBindAddress = config.getString(RestOptions.BIND_ADDRESS);
	final String portRangeDefinition = config.getString(RestOptions.BIND_PORT);

	final SSLHandlerFactory sslHandlerFactory;
	if (SSLUtils.isRestSSLEnabled(config)) {
		try {
			sslHandlerFactory = SSLUtils.createRestServerSSLEngineFactory(config);
		} catch (Exception e) {
			throw new ConfigurationException("Failed to initialize SSLEngineFactory for REST server endpoint.", e);
		}
	} else {
		sslHandlerFactory = null;
	}

	final Path uploadDir = Paths.get(
		config.getString(WebOptions.UPLOAD_DIR,	config.getString(WebOptions.TMP_DIR)),
		"flink-web-upload");

	final int maxContentLength = config.getInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH);

	final Map<String, String> responseHeaders = Collections.singletonMap(
		HttpHeaders.Names.ACCESS_CONTROL_ALLOW_ORIGIN,
		config.getString(WebOptions.ACCESS_CONTROL_ALLOW_ORIGIN));

	return new RestServerEndpointConfiguration(
		restAddress,
		restBindAddress,
		portRangeDefinition,
		sslHandlerFactory,
		uploadDir,
		maxContentLength,
		responseHeaders);
}
 

@Override
public void open(Configuration config) {
	int val = config.getInteger(TEST_KEY, -1);
	Assert.assertEquals(TEST_VALUE, val);
}
 

@Override
public OptionsFactory configure(Configuration configuration) {
	this.backgroundJobs = configuration.getInteger(BACKGROUND_JOBS_OPTION, DEFAULT_BACKGROUND_JOBS);
	return this;
}
 

/**
 * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user
 * specified name.
 *
 * @param jobName
 *            name of the job
 * @return The result of the job execution, containing elapsed time and accumulators.
 */
@Override
public JobExecutionResult execute(String jobName) throws Exception {
	// transform the streaming program into a JobGraph
	StreamGraph streamGraph = getStreamGraph();
	streamGraph.setJobName(jobName);

	JobGraph jobGraph = streamGraph.getJobGraph();
	jobGraph.setAllowQueuedScheduling(true);

	Configuration configuration = new Configuration();
	configuration.addAll(jobGraph.getJobConfiguration());
	configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0");

	// add (and override) the settings with what the user defined
	configuration.addAll(this.configuration);

	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism());

	MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumSlotsPerTaskManager(numSlotsPerTaskManager)
		.build();

	if (LOG.isInfoEnabled()) {
		LOG.info("Running job on local embedded Flink mini cluster");
	}

	MiniCluster miniCluster = new MiniCluster(cfg);

	try {
		miniCluster.start();
		configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

		return miniCluster.executeJobBlocking(jobGraph);
	}
	finally {
		transformations.clear();
		miniCluster.close();
	}
}