Java Code Examples for org.apache.flink.configuration.ConfigurationUtils#parseTempDirectories()

public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}
 

@Override
protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
		Configuration configuration,
		ScheduledExecutor scheduledExecutor) throws IOException {
	final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]);

	final Time expirationTime =  Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME));
	final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE);

	return new FileArchivedExecutionGraphStore(
		tmpDir,
		expirationTime,
		maximumCacheSizeBytes,
		scheduledExecutor,
		Ticker.systemTicker());
}
 

public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}
 

@Override
protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
		Configuration configuration,
		ScheduledExecutor scheduledExecutor) throws IOException {
	final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]);

	final Time expirationTime =  Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME));
	final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE);

	return new FileArchivedExecutionGraphStore(
		tmpDir,
		expirationTime,
		maximumCacheSizeBytes,
		scheduledExecutor,
		Ticker.systemTicker());
}
 

private SavepointEnvironment(RuntimeContext ctx, Configuration configuration, int maxParallelism, int indexOfSubtask, PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskState) {
	this.jobID = new JobID();
	this.vertexID = new JobVertexID();
	this.attemptID = new ExecutionAttemptID();
	this.ctx = Preconditions.checkNotNull(ctx);
	this.configuration = Preconditions.checkNotNull(configuration);

	Preconditions.checkArgument(maxParallelism > 0 && indexOfSubtask < maxParallelism);
	this.maxParallelism = maxParallelism;
	this.indexOfSubtask = indexOfSubtask;

	this.registry = new KvStateRegistry().createTaskRegistry(jobID, vertexID);
	this.taskStateManager = new SavepointTaskStateManager(prioritizedOperatorSubtaskState);
	this.ioManager = new IOManagerAsync(ConfigurationUtils.parseTempDirectories(configuration));
	this.memoryManager = MemoryManager.forDefaultPageSize(64 * 1024 * 1024);
	this.accumulatorRegistry = new AccumulatorRegistry(jobID, attemptID);
}
 

public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}
 

@Override
protected ArchivedExecutionGraphStore createSerializableExecutionGraphStore(
		Configuration configuration,
		ScheduledExecutor scheduledExecutor) throws IOException {
	final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]);

	final Time expirationTime =  Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME));
	final int maximumCapacity = configuration.getInteger(JobManagerOptions.JOB_STORE_MAX_CAPACITY);
	final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE);

	return new FileArchivedExecutionGraphStore(
		tmpDir,
		expirationTime,
		maximumCapacity,
		maximumCacheSizeBytes,
		scheduledExecutor,
		Ticker.systemTicker());
}
 

/**
 * Utility method to extract TaskManager config parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration The configuration.
 * @param remoteAddress identifying the IP address under which the TaskManager will be accessible
 * @param localCommunication True, to skip initializing the network stack.
 *                                      Use only in cases where only one task manager runs.
 * @return TaskExecutorConfiguration that wrappers InstanceConnectionInfo, NetworkEnvironmentConfiguration, etc.
 */
public static TaskManagerServicesConfiguration fromConfiguration(
		Configuration configuration,
		InetAddress remoteAddress,
		boolean localCommunication) throws Exception {

	// we need this because many configs have been written with a "-1" entry
	int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1);
	if (slots == -1) {
		slots = 1;
	}

	final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);
	String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration);

	if (localStateRootDir.length == 0) {
		// default to temp dirs.
		localStateRootDir = tmpDirs;
	}

	boolean localRecoveryMode = configuration.getBoolean(
		CheckpointingOptions.LOCAL_RECOVERY.key(),
		CheckpointingOptions.LOCAL_RECOVERY.defaultValue());

	final NetworkEnvironmentConfiguration networkConfig = parseNetworkEnvironmentConfiguration(
		configuration,
		localCommunication,
		remoteAddress,
		slots);

	final QueryableStateConfiguration queryableStateConfig =
			parseQueryableStateConfiguration(configuration);

	// extract memory settings
	long configuredMemory;
	String managedMemorySizeDefaultVal = TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue();
	if (!configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(managedMemorySizeDefaultVal)) {
		try {
			configuredMemory = MemorySize.parse(configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE), MEGA_BYTES).getMebiBytes();
		} catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException(
				"Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e);
		}
	} else {
		configuredMemory = Long.valueOf(managedMemorySizeDefaultVal);
	}

	checkConfigParameter(
		configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()) ||
			configuredMemory > 0, configuredMemory,
		TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
		"MemoryManager needs at least one MB of memory. " +
			"If you leave this config parameter empty, the system automatically " +
			"pick a fraction of the available memory.");

	// check whether we use heap or off-heap memory
	final MemoryType memType;
	if (configuration.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP)) {
		memType = MemoryType.OFF_HEAP;
	} else {
		memType = MemoryType.HEAP;
	}

	boolean preAllocateMemory = configuration.getBoolean(TaskManagerOptions.MANAGED_MEMORY_PRE_ALLOCATE);

	float memoryFraction = configuration.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
	checkConfigParameter(memoryFraction > 0.0f && memoryFraction < 1.0f, memoryFraction,
		TaskManagerOptions.MANAGED_MEMORY_FRACTION.key(),
		"MemoryManager fraction of the free memory must be between 0.0 and 1.0");

	long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	return new TaskManagerServicesConfiguration(
		remoteAddress,
		tmpDirs,
		localStateRootDir,
		localRecoveryMode,
		networkConfig,
		queryableStateConfig,
		slots,
		configuredMemory,
		memType,
		preAllocateMemory,
		memoryFraction,
		timerServiceShutdownTimeout,
		retryingRegistrationConfiguration,
		ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));
}
 

/**
 * Utility method to extract network related parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration configuration object
 * @param maxJvmHeapMemory the maximum JVM heap size (in bytes)
 * @param localTaskManagerCommunication true, to skip initializing the network stack
 * @param taskManagerAddress identifying the IP address under which the TaskManager will be accessible
 * @return NettyShuffleEnvironmentConfiguration
 */
public static NettyShuffleEnvironmentConfiguration fromConfiguration(
	Configuration configuration,
	long maxJvmHeapMemory,
	boolean localTaskManagerCommunication,
	InetAddress taskManagerAddress) {

	final int dataport = getDataport(configuration);

	final int pageSize = ConfigurationParserUtils.getPageSize(configuration);

	final int numberOfNetworkBuffers = calculateNumberOfNetworkBuffers(configuration, maxJvmHeapMemory);

	final NettyConfig nettyConfig = createNettyConfig(configuration, localTaskManagerCommunication, taskManagerAddress, dataport);

	int initialRequestBackoff = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_INITIAL);
	int maxRequestBackoff = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_MAX);

	int buffersPerChannel = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_PER_CHANNEL);
	int extraBuffersPerGate = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_EXTRA_BUFFERS_PER_GATE);

	boolean isCreditBased = nettyConfig != null && configuration.getBoolean(NettyShuffleEnvironmentOptions.NETWORK_CREDIT_MODEL);

	boolean isNetworkDetailedMetrics = configuration.getBoolean(NettyShuffleEnvironmentOptions.NETWORK_DETAILED_METRICS);

	String[] tempDirs = ConfigurationUtils.parseTempDirectories(configuration);

	Duration requestSegmentsTimeout = Duration.ofMillis(configuration.getLong(
			NettyShuffleEnvironmentOptions.NETWORK_EXCLUSIVE_BUFFERS_REQUEST_TIMEOUT_MILLISECONDS));

	BoundedBlockingSubpartitionType blockingSubpartitionType = getBlockingSubpartitionType(configuration);

	boolean forcePartitionReleaseOnConsumption =
		configuration.getBoolean(NettyShuffleEnvironmentOptions.FORCE_PARTITION_RELEASE_ON_CONSUMPTION);

	return new NettyShuffleEnvironmentConfiguration(
		numberOfNetworkBuffers,
		pageSize,
		initialRequestBackoff,
		maxRequestBackoff,
		buffersPerChannel,
		extraBuffersPerGate,
		requestSegmentsTimeout,
		isCreditBased,
		isNetworkDetailedMetrics,
		nettyConfig,
		tempDirs,
		blockingSubpartitionType,
		forcePartitionReleaseOnConsumption);
}
 

/**
 * Utility method to extract TaskManager config parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration The configuration.
 * @param resourceID resource ID of the task manager
 * @param remoteAddress identifying the IP address under which the TaskManager will be accessible
 * @param freeHeapMemoryWithDefrag an estimate of the size of the free heap memory
 * @param maxJvmHeapMemory the maximum JVM heap size
 * @param localCommunicationOnly True if only local communication is possible.
 *                               Use only in cases where only one task manager runs.
 *
 * @return configuration of task manager services used to create them
 */
public static TaskManagerServicesConfiguration fromConfiguration(
		Configuration configuration,
		ResourceID resourceID,
		InetAddress remoteAddress,
		long freeHeapMemoryWithDefrag,
		long maxJvmHeapMemory,
		boolean localCommunicationOnly) {
	final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);
	String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration);
	if (localStateRootDir.length == 0) {
		// default to temp dirs.
		localStateRootDir = tmpDirs;
	}

	boolean localRecoveryMode = configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY);

	final QueryableStateConfiguration queryableStateConfig = QueryableStateConfiguration.fromConfiguration(configuration);

	boolean preAllocateMemory = configuration.getBoolean(TaskManagerOptions.MANAGED_MEMORY_PRE_ALLOCATE);

	long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	return new TaskManagerServicesConfiguration(
		configuration,
		resourceID,
		remoteAddress,
		localCommunicationOnly,
		tmpDirs,
		localStateRootDir,
		freeHeapMemoryWithDefrag,
		maxJvmHeapMemory,
		localRecoveryMode,
		queryableStateConfig,
		ConfigurationParserUtils.getSlot(configuration),
		ConfigurationParserUtils.getManagedMemorySize(configuration),
		ConfigurationParserUtils.getMemoryType(configuration),
		preAllocateMemory,
		ConfigurationParserUtils.getManagedMemoryFraction(configuration),
		ConfigurationParserUtils.getPageSize(configuration),
		timerServiceShutdownTimeout,
		retryingRegistrationConfiguration,
		ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));
}
 

/**
 * Utility method to extract network related parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration configuration object
 * @param networkMemorySize the size of memory reserved for shuffle environment
 * @param localTaskManagerCommunication true, to skip initializing the network stack
 * @param taskManagerAddress identifying the IP address under which the TaskManager will be accessible
 * @return NettyShuffleEnvironmentConfiguration
 */
public static NettyShuffleEnvironmentConfiguration fromConfiguration(
	Configuration configuration,
	MemorySize networkMemorySize,
	boolean localTaskManagerCommunication,
	InetAddress taskManagerAddress) {

	final int dataBindPort = getDataBindPort(configuration);

	final int pageSize = ConfigurationParserUtils.getPageSize(configuration);

	final NettyConfig nettyConfig = createNettyConfig(configuration, localTaskManagerCommunication, taskManagerAddress, dataBindPort);

	final int numberOfNetworkBuffers = calculateNumberOfNetworkBuffers(
		configuration,
		networkMemorySize,
		pageSize);

	int initialRequestBackoff = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_INITIAL);
	int maxRequestBackoff = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_MAX);

	int buffersPerChannel = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_PER_CHANNEL);
	int extraBuffersPerGate = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_EXTRA_BUFFERS_PER_GATE);

	int maxBuffersPerChannel = configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_MAX_BUFFERS_PER_CHANNEL);

	boolean isNetworkDetailedMetrics = configuration.getBoolean(NettyShuffleEnvironmentOptions.NETWORK_DETAILED_METRICS);

	String[] tempDirs = ConfigurationUtils.parseTempDirectories(configuration);

	Duration requestSegmentsTimeout = Duration.ofMillis(configuration.getLong(
			NettyShuffleEnvironmentOptions.NETWORK_EXCLUSIVE_BUFFERS_REQUEST_TIMEOUT_MILLISECONDS));

	BoundedBlockingSubpartitionType blockingSubpartitionType = getBlockingSubpartitionType(configuration);

	boolean forcePartitionReleaseOnConsumption =
		configuration.getBoolean(NettyShuffleEnvironmentOptions.FORCE_PARTITION_RELEASE_ON_CONSUMPTION);

	boolean blockingShuffleCompressionEnabled =
		configuration.get(NettyShuffleEnvironmentOptions.BLOCKING_SHUFFLE_COMPRESSION_ENABLED);
	String compressionCodec = configuration.getString(NettyShuffleEnvironmentOptions.SHUFFLE_COMPRESSION_CODEC);

	return new NettyShuffleEnvironmentConfiguration(
		numberOfNetworkBuffers,
		pageSize,
		initialRequestBackoff,
		maxRequestBackoff,
		buffersPerChannel,
		extraBuffersPerGate,
		requestSegmentsTimeout,
		isNetworkDetailedMetrics,
		nettyConfig,
		tempDirs,
		blockingSubpartitionType,
		forcePartitionReleaseOnConsumption,
		blockingShuffleCompressionEnabled,
		compressionCodec,
		maxBuffersPerChannel);
}
 

/**
 * Utility method to extract TaskManager config parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration The configuration.
 * @param resourceID resource ID of the task manager
 * @param externalAddress identifying the IP address under which the TaskManager will be accessible
 * @param localCommunicationOnly True if only local communication is possible.
 *                               Use only in cases where only one task manager runs.
 *
 * @return configuration of task manager services used to create them
 */
public static TaskManagerServicesConfiguration fromConfiguration(
		Configuration configuration,
		ResourceID resourceID,
		String externalAddress,
		boolean localCommunicationOnly,
		TaskExecutorResourceSpec taskExecutorResourceSpec) throws Exception {
	final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);
	String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration);
	if (localStateRootDir.length == 0) {
		// default to temp dirs.
		localStateRootDir = tmpDirs;
	}

	boolean localRecoveryMode = configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY);

	final QueryableStateConfiguration queryableStateConfig = QueryableStateConfiguration.fromConfiguration(configuration);

	long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	final int externalDataPort = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT);

	String bindAddr = configuration.getString(TaskManagerOptions.BIND_HOST, NetUtils.getWildcardIPAddress());
	InetAddress bindAddress = InetAddress.getByName(bindAddr);

	final String classLoaderResolveOrder =
		configuration.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);

	final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(configuration);

	final int numIoThreads = ClusterEntrypointUtils.getPoolSize(configuration);

	return new TaskManagerServicesConfiguration(
		configuration,
		resourceID,
		externalAddress,
		bindAddress,
		externalDataPort,
		localCommunicationOnly,
		tmpDirs,
		localStateRootDir,
		localRecoveryMode,
		queryableStateConfig,
		ConfigurationParserUtils.getSlot(configuration),
		ConfigurationParserUtils.getPageSize(configuration),
		taskExecutorResourceSpec,
		timerServiceShutdownTimeout,
		retryingRegistrationConfiguration,
		ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration),
		FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder),
		alwaysParentFirstLoaderPatterns,
		numIoThreads);
}