Java Code Examples for org.apache.flink.configuration.CoreOptions#getParentFirstLoaderPatterns()

The following examples show how to use org.apache.flink.configuration.CoreOptions#getParentFirstLoaderPatterns() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: JobManagerSharedServices.java From Flink-CEPplus with Apache License 2.0

4 votes

public static JobManagerSharedServices fromConfiguration(
		Configuration config,
		BlobServer blobServer) throws Exception {

	checkNotNull(config);
	checkNotNull(blobServer);

	final String classLoaderResolveOrder =
		config.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);

	final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(config);

	final BlobLibraryCacheManager libraryCacheManager =
		new BlobLibraryCacheManager(
			blobServer,
			FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder),
			alwaysParentFirstLoaderPatterns);

	final FiniteDuration timeout;
	try {
		timeout = AkkaUtils.getTimeout(config);
	} catch (NumberFormatException e) {
		throw new IllegalConfigurationException(AkkaUtils.formatDurationParsingErrorMessage());
	}

	final ScheduledExecutorService futureExecutor = Executors.newScheduledThreadPool(
			Hardware.getNumberCPUCores(),
			new ExecutorThreadFactory("jobmanager-future"));

	final StackTraceSampleCoordinator stackTraceSampleCoordinator =
		new StackTraceSampleCoordinator(futureExecutor, timeout.toMillis());
	final int cleanUpInterval = config.getInteger(WebOptions.BACKPRESSURE_CLEANUP_INTERVAL);
	final BackPressureStatsTrackerImpl backPressureStatsTracker = new BackPressureStatsTrackerImpl(
		stackTraceSampleCoordinator,
		cleanUpInterval,
		config.getInteger(WebOptions.BACKPRESSURE_NUM_SAMPLES),
		config.getInteger(WebOptions.BACKPRESSURE_REFRESH_INTERVAL),
		Time.milliseconds(config.getInteger(WebOptions.BACKPRESSURE_DELAY)));

	futureExecutor.scheduleWithFixedDelay(
		backPressureStatsTracker::cleanUpOperatorStatsCache,
		cleanUpInterval,
		cleanUpInterval,
		TimeUnit.MILLISECONDS);

	return new JobManagerSharedServices(
		futureExecutor,
		libraryCacheManager,
		RestartStrategyFactory.createRestartStrategyFactory(config),
		stackTraceSampleCoordinator,
		backPressureStatsTracker,
		blobServer);
}

Example 2

Source File: PluginConfig.java From flink with Apache License 2.0

4 votes

public static PluginConfig fromConfiguration(Configuration configuration) {
	return new PluginConfig(
		getPluginsDirPath(configuration),
		CoreOptions.getParentFirstLoaderPatterns(configuration));
}

Example 3

Source File: JobManagerSharedServices.java From flink with Apache License 2.0

4 votes

public static JobManagerSharedServices fromConfiguration(
		Configuration config,
		BlobServer blobServer) throws Exception {

	checkNotNull(config);
	checkNotNull(blobServer);

	final String classLoaderResolveOrder =
		config.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);

	final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(config);

	final BlobLibraryCacheManager libraryCacheManager =
		new BlobLibraryCacheManager(
			blobServer,
			FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder),
			alwaysParentFirstLoaderPatterns);

	final FiniteDuration timeout;
	try {
		timeout = AkkaUtils.getTimeout(config);
	} catch (NumberFormatException e) {
		throw new IllegalConfigurationException(AkkaUtils.formatDurationParsingErrorMessage());
	}

	final ScheduledExecutorService futureExecutor = Executors.newScheduledThreadPool(
			Hardware.getNumberCPUCores(),
			new ExecutorThreadFactory("jobmanager-future"));

	final StackTraceSampleCoordinator stackTraceSampleCoordinator =
		new StackTraceSampleCoordinator(futureExecutor, timeout.toMillis());
	final int cleanUpInterval = config.getInteger(WebOptions.BACKPRESSURE_CLEANUP_INTERVAL);
	final BackPressureStatsTrackerImpl backPressureStatsTracker = new BackPressureStatsTrackerImpl(
		stackTraceSampleCoordinator,
		cleanUpInterval,
		config.getInteger(WebOptions.BACKPRESSURE_NUM_SAMPLES),
		config.getInteger(WebOptions.BACKPRESSURE_REFRESH_INTERVAL),
		Time.milliseconds(config.getInteger(WebOptions.BACKPRESSURE_DELAY)));

	futureExecutor.scheduleWithFixedDelay(
		backPressureStatsTracker::cleanUpOperatorStatsCache,
		cleanUpInterval,
		cleanUpInterval,
		TimeUnit.MILLISECONDS);

	return new JobManagerSharedServices(
		futureExecutor,
		libraryCacheManager,
		RestartStrategyFactory.createRestartStrategyFactory(config),
		stackTraceSampleCoordinator,
		backPressureStatsTracker,
		blobServer);
}

Example 4

Source File: JobManagerSharedServices.java From flink with Apache License 2.0

4 votes

public static JobManagerSharedServices fromConfiguration(
		Configuration config,
		BlobServer blobServer,
		FatalErrorHandler fatalErrorHandler) {

	checkNotNull(config);
	checkNotNull(blobServer);

	final String classLoaderResolveOrder =
		config.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);

	final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(config);

	final boolean failOnJvmMetaspaceOomError = config.getBoolean(CoreOptions.FAIL_ON_USER_CLASS_LOADING_METASPACE_OOM);
	final BlobLibraryCacheManager libraryCacheManager =
		new BlobLibraryCacheManager(
			blobServer,
			BlobLibraryCacheManager.defaultClassLoaderFactory(
				FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder),
				alwaysParentFirstLoaderPatterns,
				failOnJvmMetaspaceOomError ? fatalErrorHandler : null));

	final Duration akkaTimeout;
	try {
		akkaTimeout = AkkaUtils.getTimeout(config);
	} catch (NumberFormatException e) {
		throw new IllegalConfigurationException(AkkaUtils.formatDurationParsingErrorMessage());
	}

	final ScheduledExecutorService futureExecutor = Executors.newScheduledThreadPool(
			Hardware.getNumberCPUCores(),
			new ExecutorThreadFactory("jobmanager-future"));

	final int numSamples = config.getInteger(WebOptions.BACKPRESSURE_NUM_SAMPLES);
	final long delayBetweenSamples = config.getInteger(WebOptions.BACKPRESSURE_DELAY);
	final BackPressureRequestCoordinator coordinator = new BackPressureRequestCoordinator(
		futureExecutor,
		akkaTimeout.toMillis() + numSamples * delayBetweenSamples);

	final int cleanUpInterval = config.getInteger(WebOptions.BACKPRESSURE_CLEANUP_INTERVAL);
	final BackPressureStatsTrackerImpl backPressureStatsTracker = new BackPressureStatsTrackerImpl(
		coordinator,
		cleanUpInterval,
		config.getInteger(WebOptions.BACKPRESSURE_REFRESH_INTERVAL));

	futureExecutor.scheduleWithFixedDelay(
		backPressureStatsTracker::cleanUpOperatorStatsCache,
		cleanUpInterval,
		cleanUpInterval,
		TimeUnit.MILLISECONDS);

	return new JobManagerSharedServices(
		futureExecutor,
		libraryCacheManager,
		coordinator,
		backPressureStatsTracker,
		blobServer);
}

Example 5

Source File: TaskManagerServicesConfiguration.java From flink with Apache License 2.0

4 votes

/**
 * Utility method to extract TaskManager config parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration The configuration.
 * @param resourceID resource ID of the task manager
 * @param externalAddress identifying the IP address under which the TaskManager will be accessible
 * @param localCommunicationOnly True if only local communication is possible.
 *                               Use only in cases where only one task manager runs.
 *
 * @return configuration of task manager services used to create them
 */
public static TaskManagerServicesConfiguration fromConfiguration(
		Configuration configuration,
		ResourceID resourceID,
		String externalAddress,
		boolean localCommunicationOnly,
		TaskExecutorResourceSpec taskExecutorResourceSpec) throws Exception {
	final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);
	String[] localStateRootDir = ConfigurationUtils.parseLocalStateDirectories(configuration);
	if (localStateRootDir.length == 0) {
		// default to temp dirs.
		localStateRootDir = tmpDirs;
	}

	boolean localRecoveryMode = configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY);

	final QueryableStateConfiguration queryableStateConfig = QueryableStateConfiguration.fromConfiguration(configuration);

	long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	final int externalDataPort = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT);

	String bindAddr = configuration.getString(TaskManagerOptions.BIND_HOST, NetUtils.getWildcardIPAddress());
	InetAddress bindAddress = InetAddress.getByName(bindAddr);

	final String classLoaderResolveOrder =
		configuration.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);

	final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(configuration);

	final int numIoThreads = ClusterEntrypointUtils.getPoolSize(configuration);

	return new TaskManagerServicesConfiguration(
		configuration,
		resourceID,
		externalAddress,
		bindAddress,
		externalDataPort,
		localCommunicationOnly,
		tmpDirs,
		localStateRootDir,
		localRecoveryMode,
		queryableStateConfig,
		ConfigurationParserUtils.getSlot(configuration),
		ConfigurationParserUtils.getPageSize(configuration),
		taskExecutorResourceSpec,
		timerServiceShutdownTimeout,
		retryingRegistrationConfiguration,
		ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration),
		FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder),
		alwaysParentFirstLoaderPatterns,
		numIoThreads);
}