org.apache.flink.runtime.util.ExecutorThreadFactory Java Examples

public static ExecutorService createExecutorService(int numThreads, int threadPriority, String componentName) {
	if (threadPriority < Thread.MIN_PRIORITY || threadPriority > Thread.MAX_PRIORITY) {
		throw new IllegalArgumentException(
			String.format(
				"The thread priority must be within (%s, %s) but it was %s.",
				Thread.MIN_PRIORITY,
				Thread.MAX_PRIORITY,
				threadPriority));
	}

	return Executors.newFixedThreadPool(
		numThreads,
		new ExecutorThreadFactory.Builder()
			.setThreadPriority(threadPriority)
			.setPoolName("Flink-" + componentName)
			.build());
}
 

public static ScheduledExecutorService createExecutorService(int numThreads, int threadPriority, String componentName) {
	if (threadPriority < Thread.MIN_PRIORITY || threadPriority > Thread.MAX_PRIORITY) {
		throw new IllegalArgumentException(
			String.format(
				"The thread priority must be within (%s, %s) but it was %s.",
				Thread.MIN_PRIORITY,
				Thread.MAX_PRIORITY,
				threadPriority));
	}

	return Executors.newScheduledThreadPool(
		numThreads,
		new ExecutorThreadFactory.Builder()
			.setThreadPriority(threadPriority)
			.setPoolName("Flink-" + componentName)
			.build());
}
 

/**
 * Connects to the target database and initializes the prepared statement.
 *
 * @param taskNumber The number of the parallel instance.
 */
@Override
public void open(int taskNumber, int numTasks) throws IOException {
	super.open(taskNumber, numTasks);
	jdbcStatementExecutor = createAndOpenStatementExecutor(statementExecutorFactory);
	if (executionOptions.getBatchIntervalMs() != 0 && executionOptions.getBatchSize() != 1) {
		this.scheduler = Executors.newScheduledThreadPool(1, new ExecutorThreadFactory("jdbc-upsert-output-format"));
		this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(() -> {
			synchronized (JdbcBatchingOutputFormat.this) {
				if (!closed) {
					try {
						flush();
					} catch (Exception e) {
						flushException = e;
					}
				}
			}
		}, executionOptions.getBatchIntervalMs(), executionOptions.getBatchIntervalMs(), TimeUnit.MILLISECONDS);
	}
}
 

public static ExecutorService createExecutorService(int numThreads, int threadPriority, String componentName) {
	if (threadPriority < Thread.MIN_PRIORITY || threadPriority > Thread.MAX_PRIORITY) {
		throw new IllegalArgumentException(
			String.format(
				"The thread priority must be within (%s, %s) but it was %s.",
				Thread.MIN_PRIORITY,
				Thread.MAX_PRIORITY,
				threadPriority));
	}

	return Executors.newFixedThreadPool(
		numThreads,
		new ExecutorThreadFactory.Builder()
			.setThreadPriority(threadPriority)
			.setPoolName("Flink-" + componentName)
			.build());
}
 

private RestClusterClient(
	Configuration configuration,
	@Nullable RestClient restClient,
	T clusterId,
	WaitStrategy waitStrategy,
	ClientHighAvailabilityServices clientHAServices) throws Exception {
	this.configuration = checkNotNull(configuration);

	this.restClusterClientConfiguration = RestClusterClientConfiguration.fromConfiguration(configuration);

	if (restClient != null) {
		this.restClient = restClient;
	} else {
		this.restClient = new RestClient(restClusterClientConfiguration.getRestClientConfiguration(), executorService);
	}

	this.waitStrategy = checkNotNull(waitStrategy);
	this.clusterId = checkNotNull(clusterId);

	this.clientHAServices = checkNotNull(clientHAServices);

	this.webMonitorRetrievalService = clientHAServices.getClusterRestEndpointLeaderRetriever();
	this.retryExecutorService = Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-RestClusterClient-Retry"));
	startLeaderRetrievers();
}
 

@Override
public void start() throws Exception {
	this.executorService =
		Executors.newSingleThreadExecutor(
			new ExecutorThreadFactory(
				"collect-sink-operator-coordinator-executor-thread-pool"));
}
 

@Before
public void setUp() throws Exception {
	restServerEndpointConfiguration = RestServerEndpointConfiguration.fromConfiguration(restConfig);
	mockGatewayRetriever = () -> CompletableFuture.completedFuture(mockRestfulGateway);

	executor = Executors.newSingleThreadExecutor(new ExecutorThreadFactory(RestClusterClientTest.class.getSimpleName()));

	jobGraph = new JobGraph("testjob");
	jobId = jobGraph.getJobID();
}
 

@VisibleForTesting
RestClusterClient(
		Configuration configuration,
		@Nullable RestClient restClient,
		T clusterId,
		WaitStrategy waitStrategy,
		@Nullable LeaderRetrievalService webMonitorRetrievalService) throws Exception {
	super(configuration);
	this.restClusterClientConfiguration = RestClusterClientConfiguration.fromConfiguration(configuration);

	if (restClient != null) {
		this.restClient = restClient;
	} else {
		this.restClient = new RestClient(restClusterClientConfiguration.getRestClientConfiguration(), executorService);
	}

	this.waitStrategy = Preconditions.checkNotNull(waitStrategy);
	this.clusterId = Preconditions.checkNotNull(clusterId);

	if (webMonitorRetrievalService == null) {
		this.webMonitorRetrievalService = highAvailabilityServices.getWebMonitorLeaderRetriever();
	} else {
		this.webMonitorRetrievalService = webMonitorRetrievalService;
	}
	this.retryExecutorService = Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-RestClusterClient-Retry"));
	startLeaderRetrievers();
}
 

protected void initializeServices(Configuration configuration) throws Exception {

		LOG.info("Initializing cluster services.");

		synchronized (lock) {
			final String bindAddress = configuration.getString(JobManagerOptions.ADDRESS);
			final String portRange = getRPCPortRange(configuration);

			commonRpcService = createRpcService(configuration, bindAddress, portRange);

			// update the configuration used to create the high availability services
			configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress());
			configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort());

			ioExecutor = Executors.newFixedThreadPool(
				Hardware.getNumberCPUCores(),
				new ExecutorThreadFactory("cluster-io"));
			haServices = createHaServices(configuration, ioExecutor);
			blobServer = new BlobServer(configuration, haServices.createBlobStore());
			blobServer.start();
			heartbeatServices = createHeartbeatServices(configuration);
			metricRegistry = createMetricRegistry(configuration);

			final RpcService metricQueryServiceRpcService = MetricUtils.startMetricsRpcService(configuration, bindAddress);
			metricRegistry.startQueryService(metricQueryServiceRpcService, null);

			archivedExecutionGraphStore = createSerializableExecutionGraphStore(configuration, commonRpcService.getScheduledExecutor());
		}
	}
 

public RestClient(RestClientConfiguration configuration, Executor executor) {
	Preconditions.checkNotNull(configuration);
	this.executor = Preconditions.checkNotNull(executor);
	this.terminationFuture = new CompletableFuture<>();

	final SSLHandlerFactory sslHandlerFactory = configuration.getSslHandlerFactory();
	ChannelInitializer<SocketChannel> initializer = new ChannelInitializer<SocketChannel>() {
		@Override
		protected void initChannel(SocketChannel socketChannel) {
			try {
				// SSL should be the first handler in the pipeline
				if (sslHandlerFactory != null) {
					socketChannel.pipeline().addLast("ssl", sslHandlerFactory.createNettySSLHandler(socketChannel.alloc()));
				}

				socketChannel.pipeline()
					.addLast(new HttpClientCodec())
					.addLast(new HttpObjectAggregator(configuration.getMaxContentLength()))
					.addLast(new ChunkedWriteHandler()) // required for multipart-requests
					.addLast(new IdleStateHandler(configuration.getIdlenessTimeout(), configuration.getIdlenessTimeout(), configuration.getIdlenessTimeout(), TimeUnit.MILLISECONDS))
					.addLast(new ClientHandler());
			} catch (Throwable t) {
				t.printStackTrace();
				ExceptionUtils.rethrow(t);
			}
		}
	};
	NioEventLoopGroup group = new NioEventLoopGroup(1, new ExecutorThreadFactory("flink-rest-client-netty"));

	bootstrap = new Bootstrap();
	bootstrap
		.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, Math.toIntExact(configuration.getConnectionTimeout()))
		.group(group)
		.channel(NioSocketChannel.class)
		.handler(initializer);

	LOG.info("Rest client endpoint started.");
}
 

public CliTableauResultView(
		final Terminal terminal,
		final Executor sqlExecutor,
		final String sessionId,
		final ResultDescriptor resultDescriptor) {
	this.terminal = terminal;
	this.sqlExecutor = sqlExecutor;
	this.sessionId = sessionId;
	this.resultDescriptor = resultDescriptor;
	this.displayResultExecutorService = Executors.newSingleThreadExecutor(new ExecutorThreadFactory("CliTableauResultView"));
}
 

public RestClient(RestClientConfiguration configuration, Executor executor) {
	Preconditions.checkNotNull(configuration);
	this.executor = Preconditions.checkNotNull(executor);
	this.terminationFuture = new CompletableFuture<>();

	final SSLHandlerFactory sslHandlerFactory = configuration.getSslHandlerFactory();
	ChannelInitializer<SocketChannel> initializer = new ChannelInitializer<SocketChannel>() {
		@Override
		protected void initChannel(SocketChannel socketChannel) {
			try {
				// SSL should be the first handler in the pipeline
				if (sslHandlerFactory != null) {
					socketChannel.pipeline().addLast("ssl", sslHandlerFactory.createNettySSLHandler(socketChannel.alloc()));
				}

				socketChannel.pipeline()
					.addLast(new HttpClientCodec())
					.addLast(new HttpObjectAggregator(configuration.getMaxContentLength()))
					.addLast(new ChunkedWriteHandler()) // required for multipart-requests
					.addLast(new IdleStateHandler(configuration.getIdlenessTimeout(), configuration.getIdlenessTimeout(), configuration.getIdlenessTimeout(), TimeUnit.MILLISECONDS))
					.addLast(new ClientHandler());
			} catch (Throwable t) {
				t.printStackTrace();
				ExceptionUtils.rethrow(t);
			}
		}
	};
	NioEventLoopGroup group = new NioEventLoopGroup(1, new ExecutorThreadFactory("flink-rest-client-netty"));

	bootstrap = new Bootstrap();
	bootstrap
		.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, Math.toIntExact(configuration.getConnectionTimeout()))
		.group(group)
		.channel(NioSocketChannel.class)
		.handler(initializer);

	LOG.debug("Rest client endpoint started.");
}
 

protected void initializeServices(Configuration configuration, PluginManager pluginManager) throws Exception {

		LOG.info("Initializing cluster services.");

		synchronized (lock) {
			commonRpcService = AkkaRpcServiceUtils.createRemoteRpcService(
				configuration,
				configuration.getString(JobManagerOptions.ADDRESS),
				getRPCPortRange(configuration),
				configuration.getString(JobManagerOptions.BIND_HOST),
				configuration.getOptional(JobManagerOptions.RPC_BIND_PORT));

			// update the configuration used to create the high availability services
			configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress());
			configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort());

			ioExecutor = Executors.newFixedThreadPool(
				ClusterEntrypointUtils.getPoolSize(configuration),
				new ExecutorThreadFactory("cluster-io"));
			haServices = createHaServices(configuration, ioExecutor);
			blobServer = new BlobServer(configuration, haServices.createBlobStore());
			blobServer.start();
			heartbeatServices = createHeartbeatServices(configuration);
			metricRegistry = createMetricRegistry(configuration, pluginManager);

			final RpcService metricQueryServiceRpcService = MetricUtils.startRemoteMetricsRpcService(configuration, commonRpcService.getAddress());
			metricRegistry.startQueryService(metricQueryServiceRpcService, null);

			final String hostname = RpcUtils.getHostname(commonRpcService);

			processMetricGroup = MetricUtils.instantiateProcessMetricGroup(
				metricRegistry,
				hostname,
				ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration));

			archivedExecutionGraphStore = createSerializableExecutionGraphStore(configuration, commonRpcService.getScheduledExecutor());
		}
	}
 

/**
 * Connects to the target database and initializes the prepared statement.
 *
 * @param taskNumber The number of the parallel instance.
 * @throws IOException Thrown, if the output could not be opened due to an
 * I/O problem.
 */
@Override
public void open(int taskNumber, int numTasks) throws IOException {
	try {
		establishConnection();
		if (keyFields == null || keyFields.length == 0) {
			String insertSQL = dialect.getInsertIntoStatement(tableName, fieldNames);
			jdbcWriter = new AppendOnlyWriter(insertSQL, fieldTypes);
		} else {
			jdbcWriter = UpsertWriter.create(
				dialect, tableName, fieldNames, fieldTypes, keyFields,
				getRuntimeContext().getExecutionConfig().isObjectReuseEnabled());
		}
		jdbcWriter.open(connection);
	} catch (SQLException sqe) {
		throw new IllegalArgumentException("open() failed.", sqe);
	} catch (ClassNotFoundException cnfe) {
		throw new IllegalArgumentException("JDBC driver class not found.", cnfe);
	}

	if (flushIntervalMills != 0) {
		this.scheduler = Executors.newScheduledThreadPool(
				1, new ExecutorThreadFactory("jdbc-upsert-output-format"));
		this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(() -> {
			synchronized (JDBCUpsertOutputFormat.this) {
				if (closed) {
					return;
				}
				try {
					flush();
				} catch (Exception e) {
					flushException = e;
				}
			}
		}, flushIntervalMills, flushIntervalMills, TimeUnit.MILLISECONDS);
	}
}
 

public RestClient(RestClientConfiguration configuration, Executor executor) {
	Preconditions.checkNotNull(configuration);
	this.executor = Preconditions.checkNotNull(executor);
	this.terminationFuture = new CompletableFuture<>();

	final SSLHandlerFactory sslHandlerFactory = configuration.getSslHandlerFactory();
	ChannelInitializer<SocketChannel> initializer = new ChannelInitializer<SocketChannel>() {
		@Override
		protected void initChannel(SocketChannel socketChannel) {
			try {
				// SSL should be the first handler in the pipeline
				if (sslHandlerFactory != null) {
					socketChannel.pipeline().addLast("ssl", sslHandlerFactory.createNettySSLHandler());
				}

				socketChannel.pipeline()
					.addLast(new HttpClientCodec())
					.addLast(new HttpObjectAggregator(configuration.getMaxContentLength()))
					.addLast(new ChunkedWriteHandler()) // required for multipart-requests
					.addLast(new IdleStateHandler(configuration.getIdlenessTimeout(), configuration.getIdlenessTimeout(), configuration.getIdlenessTimeout(), TimeUnit.MILLISECONDS))
					.addLast(new ClientHandler());
			} catch (Throwable t) {
				t.printStackTrace();
				ExceptionUtils.rethrow(t);
			}
		}
	};
	NioEventLoopGroup group = new NioEventLoopGroup(1, new ExecutorThreadFactory("flink-rest-client-netty"));

	bootstrap = new Bootstrap();
	bootstrap
		.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, Math.toIntExact(configuration.getConnectionTimeout()))
		.group(group)
		.channel(NioSocketChannel.class)
		.handler(initializer);

	LOG.info("Rest client endpoint started.");
}
 

@Before
public void setUp() throws Exception {
	restServerEndpointConfiguration = RestServerEndpointConfiguration.fromConfiguration(restConfig);
	mockGatewayRetriever = () -> CompletableFuture.completedFuture(mockRestfulGateway);

	executor = Executors.newSingleThreadExecutor(new ExecutorThreadFactory(RestClusterClientTest.class.getSimpleName()));

	jobGraph = new JobGraph("testjob");
	jobId = jobGraph.getJobID();
}
 

@Before
public void setUp() throws Exception {
	restServerEndpointConfiguration = RestServerEndpointConfiguration.fromConfiguration(restConfig);
	mockGatewayRetriever = () -> CompletableFuture.completedFuture(mockRestfulGateway);

	executor = Executors.newSingleThreadExecutor(new ExecutorThreadFactory(RestClusterClientTest.class.getSimpleName()));

	jobGraph = new JobGraph("testjob");
	jobId = jobGraph.getJobID();
}
 

@VisibleForTesting
RestClusterClient(
		Configuration configuration,
		@Nullable RestClient restClient,
		T clusterId,
		WaitStrategy waitStrategy,
		@Nullable LeaderRetrievalService webMonitorRetrievalService) throws Exception {
	super(configuration);
	this.restClusterClientConfiguration = RestClusterClientConfiguration.fromConfiguration(configuration);

	if (restClient != null) {
		this.restClient = restClient;
	} else {
		this.restClient = new RestClient(restClusterClientConfiguration.getRestClientConfiguration(), executorService);
	}

	this.waitStrategy = Preconditions.checkNotNull(waitStrategy);
	this.clusterId = Preconditions.checkNotNull(clusterId);

	if (webMonitorRetrievalService == null) {
		this.webMonitorRetrievalService = highAvailabilityServices.getWebMonitorLeaderRetriever();
	} else {
		this.webMonitorRetrievalService = webMonitorRetrievalService;
	}
	this.dispatcherRetrievalService = highAvailabilityServices.getDispatcherLeaderRetriever();
	this.retryExecutorService = Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-RestClusterClient-Retry"));
	startLeaderRetrievers();
}
 

protected void initializeServices(Configuration configuration) throws Exception {

		LOG.info("Initializing cluster services.");

		synchronized (lock) {
			final String bindAddress = configuration.getString(JobManagerOptions.ADDRESS);
			final String portRange = getRPCPortRange(configuration);

			commonRpcService = createRpcService(configuration, bindAddress, portRange);

			// update the configuration used to create the high availability services
			configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress());
			configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort());

			ioExecutor = Executors.newFixedThreadPool(
				Hardware.getNumberCPUCores(),
				new ExecutorThreadFactory("cluster-io"));
			haServices = createHaServices(configuration, ioExecutor);
			blobServer = new BlobServer(configuration, haServices.createBlobStore());
			blobServer.start();
			heartbeatServices = createHeartbeatServices(configuration);
			metricRegistry = createMetricRegistry(configuration);

			// TODO: This is a temporary hack until we have ported the MetricQueryService to the new RpcEndpoint
			// Start actor system for metric query service on any available port
			metricQueryServiceActorSystem = MetricUtils.startMetricsActorSystem(configuration, bindAddress, LOG);
			metricRegistry.startQueryService(metricQueryServiceActorSystem, null);

			archivedExecutionGraphStore = createSerializableExecutionGraphStore(configuration, commonRpcService.getScheduledExecutor());

			transientBlobCache = new TransientBlobCache(
				configuration,
				new InetSocketAddress(
					commonRpcService.getAddress(),
					blobServer.getPort()));
		}
	}
 

/**
 * Creates a new MetricRegistry and starts the configured reporter.
 */
public MetricRegistryImpl(MetricRegistryConfiguration config, Collection<ReporterSetup> reporterConfigurations) {
	this(config, reporterConfigurations, Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-MetricRegistry")));
}
 

private Supervisor startSupervisorActor() {
	final ExecutorService terminationFutureExecutor = Executors.newSingleThreadExecutor(new ExecutorThreadFactory("AkkaRpcService-Supervisor-Termination-Future-Executor"));
	final ActorRef actorRef = SupervisorActor.startSupervisorActor(actorSystem, terminationFutureExecutor);

	return Supervisor.create(actorRef, terminationFutureExecutor);
}
 

protected StreamTask(
		Environment environment,
		@Nullable TimerService timerService,
		Thread.UncaughtExceptionHandler uncaughtExceptionHandler,
		StreamTaskActionExecutor actionExecutor,
		TaskMailbox mailbox) throws Exception {

	super(environment);

	this.configuration = new StreamConfig(getTaskConfiguration());
	this.recordWriter = createRecordWriterDelegate(configuration, environment);
	this.actionExecutor = Preconditions.checkNotNull(actionExecutor);
	this.mailboxProcessor = new MailboxProcessor(this::processInput, mailbox, actionExecutor);
	this.mailboxProcessor.initMetric(environment.getMetricGroup());
	this.mainMailboxExecutor = mailboxProcessor.getMainMailboxExecutor();
	this.asyncExceptionHandler = new StreamTaskAsyncExceptionHandler(environment);
	this.asyncOperationsThreadPool = Executors.newCachedThreadPool(
		new ExecutorThreadFactory("AsyncOperations", uncaughtExceptionHandler));

	this.stateBackend = createStateBackend();

	this.subtaskCheckpointCoordinator = new SubtaskCheckpointCoordinatorImpl(
		stateBackend.createCheckpointStorage(getEnvironment().getJobID()),
		getName(),
		actionExecutor,
		getCancelables(),
		getAsyncOperationsThreadPool(),
		getEnvironment(),
		this,
		configuration.isUnalignedCheckpointsEnabled(),
		this::prepareInputSnapshot);

	// if the clock is not already set, then assign a default TimeServiceProvider
	if (timerService == null) {
		ThreadFactory timerThreadFactory = new DispatcherThreadFactory(TRIGGER_THREAD_GROUP, "Time Trigger for " + getName());
		this.timerService = new SystemProcessingTimeService(this::handleTimerException, timerThreadFactory);
	} else {
		this.timerService = timerService;
	}

	this.channelIOExecutor = Executors.newSingleThreadExecutor(new ExecutorThreadFactory("channel-state-unspilling"));
}
 

/**
 * Creates a new MetricRegistry and starts the configured reporter.
 */
public MetricRegistryImpl(MetricRegistryConfiguration config) {
	this.maximumFramesize = config.getQueryServiceMessageSizeLimit();
	this.scopeFormats = config.getScopeFormats();
	this.globalDelimiter = config.getDelimiter();
	this.delimiters = new ArrayList<>(10);
	this.terminationFuture = new CompletableFuture<>();
	this.isShutdown = false;

	// second, instantiate any custom configured reporters
	this.reporters = new ArrayList<>(4);

	List<Tuple2<String, Configuration>> reporterConfigurations = config.getReporterConfigurations();

	this.executor = Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-MetricRegistry"));

	this.queryService = null;
	this.metricQueryServicePath = null;

	if (reporterConfigurations.isEmpty()) {
		// no reporters defined
		// by default, don't report anything
		LOG.info("No metrics reporter configured, no metrics will be exposed/reported.");
	} else {
		// we have some reporters so
		for (Tuple2<String, Configuration> reporterConfiguration: reporterConfigurations) {
			String namedReporter = reporterConfiguration.f0;
			Configuration reporterConfig = reporterConfiguration.f1;

			final String className = reporterConfig.getString(ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, null);
			if (className == null) {
				LOG.error("No reporter class set for reporter " + namedReporter + ". Metrics might not be exposed/reported.");
				continue;
			}

			try {
				String configuredPeriod = reporterConfig.getString(ConfigConstants.METRICS_REPORTER_INTERVAL_SUFFIX, null);
				TimeUnit timeunit = TimeUnit.SECONDS;
				long period = 10;

				if (configuredPeriod != null) {
					try {
						String[] interval = configuredPeriod.split(" ");
						period = Long.parseLong(interval[0]);
						timeunit = TimeUnit.valueOf(interval[1]);
					}
					catch (Exception e) {
						LOG.error("Cannot parse report interval from config: " + configuredPeriod +
								" - please use values like '10 SECONDS' or '500 MILLISECONDS'. " +
								"Using default reporting interval.");
					}
				}

				Class<?> reporterClass = Class.forName(className);
				MetricReporter reporterInstance = (MetricReporter) reporterClass.newInstance();

				MetricConfig metricConfig = new MetricConfig();
				reporterConfig.addAllToProperties(metricConfig);
				LOG.info("Configuring {} with {}.", namedReporter, metricConfig);
				reporterInstance.open(metricConfig);

				if (reporterInstance instanceof Scheduled) {
					LOG.info("Periodically reporting metrics in intervals of {} {} for reporter {} of type {}.", period, timeunit.name(), namedReporter, className);

					executor.scheduleWithFixedDelay(
							new MetricRegistryImpl.ReporterTask((Scheduled) reporterInstance), period, period, timeunit);
				} else {
					LOG.info("Reporting metrics for reporter {} of type {}.", namedReporter, className);
				}
				reporters.add(reporterInstance);

				String delimiterForReporter = reporterConfig.getString(ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, String.valueOf(globalDelimiter));
				if (delimiterForReporter.length() != 1) {
					LOG.warn("Failed to parse delimiter '{}' for reporter '{}', using global delimiter '{}'.", delimiterForReporter, namedReporter, globalDelimiter);
					delimiterForReporter = String.valueOf(globalDelimiter);
				}
				this.delimiters.add(delimiterForReporter.charAt(0));
			}
			catch (Throwable t) {
				LOG.error("Could not instantiate metrics reporter {}. Metrics might not be exposed/reported.", namedReporter, t);
			}
		}
	}
}
 

public static JobManagerSharedServices fromConfiguration(
		Configuration config,
		BlobServer blobServer,
		FatalErrorHandler fatalErrorHandler) {

	checkNotNull(config);
	checkNotNull(blobServer);

	final String classLoaderResolveOrder =
		config.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);

	final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(config);

	final boolean failOnJvmMetaspaceOomError = config.getBoolean(CoreOptions.FAIL_ON_USER_CLASS_LOADING_METASPACE_OOM);
	final BlobLibraryCacheManager libraryCacheManager =
		new BlobLibraryCacheManager(
			blobServer,
			BlobLibraryCacheManager.defaultClassLoaderFactory(
				FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder),
				alwaysParentFirstLoaderPatterns,
				failOnJvmMetaspaceOomError ? fatalErrorHandler : null));

	final Duration akkaTimeout;
	try {
		akkaTimeout = AkkaUtils.getTimeout(config);
	} catch (NumberFormatException e) {
		throw new IllegalConfigurationException(AkkaUtils.formatDurationParsingErrorMessage());
	}

	final ScheduledExecutorService futureExecutor = Executors.newScheduledThreadPool(
			Hardware.getNumberCPUCores(),
			new ExecutorThreadFactory("jobmanager-future"));

	final int numSamples = config.getInteger(WebOptions.BACKPRESSURE_NUM_SAMPLES);
	final long delayBetweenSamples = config.getInteger(WebOptions.BACKPRESSURE_DELAY);
	final BackPressureRequestCoordinator coordinator = new BackPressureRequestCoordinator(
		futureExecutor,
		akkaTimeout.toMillis() + numSamples * delayBetweenSamples);

	final int cleanUpInterval = config.getInteger(WebOptions.BACKPRESSURE_CLEANUP_INTERVAL);
	final BackPressureStatsTrackerImpl backPressureStatsTracker = new BackPressureStatsTrackerImpl(
		coordinator,
		cleanUpInterval,
		config.getInteger(WebOptions.BACKPRESSURE_REFRESH_INTERVAL));

	futureExecutor.scheduleWithFixedDelay(
		backPressureStatsTracker::cleanUpOperatorStatsCache,
		cleanUpInterval,
		cleanUpInterval,
		TimeUnit.MILLISECONDS);

	return new JobManagerSharedServices(
		futureExecutor,
		libraryCacheManager,
		coordinator,
		backPressureStatsTracker,
		blobServer);
}
 

public TaskManagerRunner(Configuration configuration, PluginManager pluginManager) throws Exception {
	this.configuration = checkNotNull(configuration);

	timeout = AkkaUtils.getTimeoutAsTime(configuration);

	this.executor = java.util.concurrent.Executors.newScheduledThreadPool(
		Hardware.getNumberCPUCores(),
		new ExecutorThreadFactory("taskmanager-future"));

	highAvailabilityServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(
		configuration,
		executor,
		HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);

	rpcService = createRpcService(configuration, highAvailabilityServices);

	this.resourceId = new ResourceID(getTaskManagerResourceID(configuration, rpcService.getAddress(), rpcService.getPort()));

	HeartbeatServices heartbeatServices = HeartbeatServices.fromConfiguration(configuration);

	metricRegistry = new MetricRegistryImpl(
		MetricRegistryConfiguration.fromConfiguration(configuration),
		ReporterSetup.fromConfiguration(configuration, pluginManager));

	final RpcService metricQueryServiceRpcService = MetricUtils.startRemoteMetricsRpcService(configuration, rpcService.getAddress());
	metricRegistry.startQueryService(metricQueryServiceRpcService, resourceId);

	blobCacheService = new BlobCacheService(
		configuration, highAvailabilityServices.createBlobStore(), null
	);

	final ExternalResourceInfoProvider externalResourceInfoProvider =
		ExternalResourceUtils.createStaticExternalResourceInfoProvider(
			ExternalResourceUtils.getExternalResourceAmountMap(configuration),
			ExternalResourceUtils.externalResourceDriversFromConfig(configuration, pluginManager));

	taskManager = startTaskManager(
		this.configuration,
		this.resourceId,
		rpcService,
		highAvailabilityServices,
		heartbeatServices,
		metricRegistry,
		blobCacheService,
		false,
		externalResourceInfoProvider,
		this);

	this.terminationFuture = new CompletableFuture<>();
	this.shutdown = false;

	MemoryLogger.startIfConfigured(LOG, configuration, terminationFuture);
}
 

public static TaskExecutor startTaskManager(
		Configuration configuration,
		ResourceID resourceID,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		BlobCacheService blobCacheService,
		boolean localCommunicationOnly,
		ExternalResourceInfoProvider externalResourceInfoProvider,
		FatalErrorHandler fatalErrorHandler) throws Exception {

	checkNotNull(configuration);
	checkNotNull(resourceID);
	checkNotNull(rpcService);
	checkNotNull(highAvailabilityServices);

	LOG.info("Starting TaskManager with ResourceID: {}", resourceID);

	String externalAddress = rpcService.getAddress();

	final TaskExecutorResourceSpec taskExecutorResourceSpec = TaskExecutorResourceUtils.resourceSpecFromConfig(configuration);

	TaskManagerServicesConfiguration taskManagerServicesConfiguration =
		TaskManagerServicesConfiguration.fromConfiguration(
			configuration,
			resourceID,
			externalAddress,
			localCommunicationOnly,
			taskExecutorResourceSpec);

	Tuple2<TaskManagerMetricGroup, MetricGroup> taskManagerMetricGroup = MetricUtils.instantiateTaskManagerMetricGroup(
		metricRegistry,
		externalAddress,
		resourceID,
		taskManagerServicesConfiguration.getSystemResourceMetricsProbingInterval());

	final ExecutorService ioExecutor = Executors.newFixedThreadPool(
		taskManagerServicesConfiguration.getNumIoThreads(),
		new ExecutorThreadFactory("flink-taskexecutor-io"));

	TaskManagerServices taskManagerServices = TaskManagerServices.fromConfiguration(
		taskManagerServicesConfiguration,
		blobCacheService.getPermanentBlobService(),
		taskManagerMetricGroup.f1,
		ioExecutor,
		fatalErrorHandler);

	TaskManagerConfiguration taskManagerConfiguration =
		TaskManagerConfiguration.fromConfiguration(configuration, taskExecutorResourceSpec, externalAddress);

	String metricQueryServiceAddress = metricRegistry.getMetricQueryServiceGatewayRpcAddress();

	return new TaskExecutor(
		rpcService,
		taskManagerConfiguration,
		highAvailabilityServices,
		taskManagerServices,
		externalResourceInfoProvider,
		heartbeatServices,
		taskManagerMetricGroup.f0,
		metricQueryServiceAddress,
		blobCacheService,
		fatalErrorHandler,
		new TaskExecutorPartitionTrackerImpl(taskManagerServices.getShuffleEnvironment()),
		createBackPressureSampleService(configuration, rpcService.getScheduledExecutor()));
}
 

public FileCache(String[] tempDirectories, PermanentBlobService blobService) throws IOException {
	this (tempDirectories, blobService, Executors.newScheduledThreadPool(10,
		new ExecutorThreadFactory("flink-file-cache")), 5000);
}
 

@BeforeClass
public static void setUp() throws ConfigurationException {
	restServerEndpointConfiguration = RestServerEndpointConfiguration.fromConfiguration(REST_CONFIG);
	executor = Executors.newSingleThreadExecutor(new ExecutorThreadFactory(RestClusterClientSavepointTriggerTest.class.getSimpleName()));
}
 

private static ExecutorService createThreadPoolForAsyncIO() {
	return Executors.newFixedThreadPool(2, new ExecutorThreadFactory("FlinkKubeClient-IO"));
}
 

private MiniClusterClient(TestingMiniCluster miniCluster) throws ConfigurationException {
	restAddress = miniCluster.getRestAddress().join();
	executorService = Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-RestClient-IO"));
	restClient = createRestClient();
}