Java Code Examples for org.apache.flink.configuration.ConfigurationUtils#getSystemResourceMetricsProbingInterval()

/**
 * Utility method to extract TaskManager config parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration The configuration.
 * @param remoteAddress identifying the IP address under which the TaskManager will be accessible
 * @param localCommunication True, to skip initializing the network stack.
 *                                      Use only in cases where only one task manager runs.
 * @return TaskExecutorConfiguration that wrappers InstanceConnectionInfo, NetworkEnvironmentConfiguration, etc.
 */
public static TaskManagerServicesConfiguration fromConfiguration(
		Configuration configuration,
		InetAddress remoteAddress,
		boolean localCommunication) throws Exception {

	// we need this because many configs have been written with a "-1" entry
	int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1);
	if (slots == -1) {
		slots = 1;
	}

	final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);
	String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration);

	if (localStateRootDir.length == 0) {
		// default to temp dirs.
		localStateRootDir = tmpDirs;
	}

	boolean localRecoveryMode = configuration.getBoolean(
		CheckpointingOptions.LOCAL_RECOVERY.key(),
		CheckpointingOptions.LOCAL_RECOVERY.defaultValue());

	final NetworkEnvironmentConfiguration networkConfig = parseNetworkEnvironmentConfiguration(
		configuration,
		localCommunication,
		remoteAddress,
		slots);

	final QueryableStateConfiguration queryableStateConfig =
			parseQueryableStateConfiguration(configuration);

	// extract memory settings
	long configuredMemory;
	String managedMemorySizeDefaultVal = TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue();
	if (!configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(managedMemorySizeDefaultVal)) {
		try {
			configuredMemory = MemorySize.parse(configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE), MEGA_BYTES).getMebiBytes();
		} catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException(
				"Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e);
		}
	} else {
		configuredMemory = Long.valueOf(managedMemorySizeDefaultVal);
	}

	checkConfigParameter(
		configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()) ||
			configuredMemory > 0, configuredMemory,
		TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
		"MemoryManager needs at least one MB of memory. " +
			"If you leave this config parameter empty, the system automatically " +
			"pick a fraction of the available memory.");

	// check whether we use heap or off-heap memory
	final MemoryType memType;
	if (configuration.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP)) {
		memType = MemoryType.OFF_HEAP;
	} else {
		memType = MemoryType.HEAP;
	}

	boolean preAllocateMemory = configuration.getBoolean(TaskManagerOptions.MANAGED_MEMORY_PRE_ALLOCATE);

	float memoryFraction = configuration.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
	checkConfigParameter(memoryFraction > 0.0f && memoryFraction < 1.0f, memoryFraction,
		TaskManagerOptions.MANAGED_MEMORY_FRACTION.key(),
		"MemoryManager fraction of the free memory must be between 0.0 and 1.0");

	long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	return new TaskManagerServicesConfiguration(
		remoteAddress,
		tmpDirs,
		localStateRootDir,
		localRecoveryMode,
		networkConfig,
		queryableStateConfig,
		slots,
		configuredMemory,
		memType,
		preAllocateMemory,
		memoryFraction,
		timerServiceShutdownTimeout,
		retryingRegistrationConfiguration,
		ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));
}
 

/**
 * Utility method to extract TaskManager config parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration The configuration.
 * @param resourceID resource ID of the task manager
 * @param remoteAddress identifying the IP address under which the TaskManager will be accessible
 * @param freeHeapMemoryWithDefrag an estimate of the size of the free heap memory
 * @param maxJvmHeapMemory the maximum JVM heap size
 * @param localCommunicationOnly True if only local communication is possible.
 *                               Use only in cases where only one task manager runs.
 *
 * @return configuration of task manager services used to create them
 */
public static TaskManagerServicesConfiguration fromConfiguration(
		Configuration configuration,
		ResourceID resourceID,
		InetAddress remoteAddress,
		long freeHeapMemoryWithDefrag,
		long maxJvmHeapMemory,
		boolean localCommunicationOnly) {
	final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);
	String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration);
	if (localStateRootDir.length == 0) {
		// default to temp dirs.
		localStateRootDir = tmpDirs;
	}

	boolean localRecoveryMode = configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY);

	final QueryableStateConfiguration queryableStateConfig = QueryableStateConfiguration.fromConfiguration(configuration);

	boolean preAllocateMemory = configuration.getBoolean(TaskManagerOptions.MANAGED_MEMORY_PRE_ALLOCATE);

	long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	return new TaskManagerServicesConfiguration(
		configuration,
		resourceID,
		remoteAddress,
		localCommunicationOnly,
		tmpDirs,
		localStateRootDir,
		freeHeapMemoryWithDefrag,
		maxJvmHeapMemory,
		localRecoveryMode,
		queryableStateConfig,
		ConfigurationParserUtils.getSlot(configuration),
		ConfigurationParserUtils.getManagedMemorySize(configuration),
		ConfigurationParserUtils.getMemoryType(configuration),
		preAllocateMemory,
		ConfigurationParserUtils.getManagedMemoryFraction(configuration),
		ConfigurationParserUtils.getPageSize(configuration),
		timerServiceShutdownTimeout,
		retryingRegistrationConfiguration,
		ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));
}
 

/**
 * Utility method to extract TaskManager config parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration The configuration.
 * @param resourceID resource ID of the task manager
 * @param externalAddress identifying the IP address under which the TaskManager will be accessible
 * @param localCommunicationOnly True if only local communication is possible.
 *                               Use only in cases where only one task manager runs.
 *
 * @return configuration of task manager services used to create them
 */
public static TaskManagerServicesConfiguration fromConfiguration(
		Configuration configuration,
		ResourceID resourceID,
		String externalAddress,
		boolean localCommunicationOnly,
		TaskExecutorResourceSpec taskExecutorResourceSpec) throws Exception {
	final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);
	String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration);
	if (localStateRootDir.length == 0) {
		// default to temp dirs.
		localStateRootDir = tmpDirs;
	}

	boolean localRecoveryMode = configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY);

	final QueryableStateConfiguration queryableStateConfig = QueryableStateConfiguration.fromConfiguration(configuration);

	long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	final int externalDataPort = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT);

	String bindAddr = configuration.getString(TaskManagerOptions.BIND_HOST, NetUtils.getWildcardIPAddress());
	InetAddress bindAddress = InetAddress.getByName(bindAddr);

	final String classLoaderResolveOrder =
		configuration.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);

	final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(configuration);

	final int numIoThreads = ClusterEntrypointUtils.getPoolSize(configuration);

	return new TaskManagerServicesConfiguration(
		configuration,
		resourceID,
		externalAddress,
		bindAddress,
		externalDataPort,
		localCommunicationOnly,
		tmpDirs,
		localStateRootDir,
		localRecoveryMode,
		queryableStateConfig,
		ConfigurationParserUtils.getSlot(configuration),
		ConfigurationParserUtils.getPageSize(configuration),
		taskExecutorResourceSpec,
		timerServiceShutdownTimeout,
		retryingRegistrationConfiguration,
		ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration),
		FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder),
		alwaysParentFirstLoaderPatterns,
		numIoThreads);
}