Java Code Examples for org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils#createRandomName()

public Dispatcher(
		RpcService rpcService,
		DispatcherId fencingToken,
		DispatcherBootstrap dispatcherBootstrap,
		DispatcherServices dispatcherServices) throws Exception {
	super(rpcService, AkkaRpcServiceUtils.createRandomName(DISPATCHER_NAME), fencingToken);
	checkNotNull(dispatcherServices);

	this.configuration = dispatcherServices.getConfiguration();
	this.highAvailabilityServices = dispatcherServices.getHighAvailabilityServices();
	this.resourceManagerGatewayRetriever = dispatcherServices.getResourceManagerGatewayRetriever();
	this.heartbeatServices = dispatcherServices.getHeartbeatServices();
	this.blobServer = dispatcherServices.getBlobServer();
	this.fatalErrorHandler = dispatcherServices.getFatalErrorHandler();
	this.jobGraphWriter = dispatcherServices.getJobGraphWriter();
	this.jobManagerMetricGroup = dispatcherServices.getJobManagerMetricGroup();
	this.metricServiceQueryAddress = dispatcherServices.getMetricQueryServiceAddress();

	this.jobManagerSharedServices = JobManagerSharedServices.fromConfiguration(
		configuration,
		blobServer,
		fatalErrorHandler);

	this.runningJobsRegistry = highAvailabilityServices.getRunningJobsRegistry();

	jobManagerRunnerFutures = new HashMap<>(16);

	this.historyServerArchivist = dispatcherServices.getHistoryServerArchivist();

	this.archivedExecutionGraphStore = dispatcherServices.getArchivedExecutionGraphStore();

	this.jobManagerRunnerFactory = dispatcherServices.getJobManagerRunnerFactory();

	this.jobManagerTerminationFutures = new HashMap<>(2);

	this.shutDownFuture = new CompletableFuture<>();

	this.dispatcherBootstrap = checkNotNull(dispatcherBootstrap);
}
 

public JobMaster(
		RpcService rpcService,
		JobMasterConfiguration jobMasterConfiguration,
		ResourceID resourceId,
		JobGraph jobGraph,
		HighAvailabilityServices highAvailabilityService,
		SlotPoolFactory slotPoolFactory,
		SchedulerFactory schedulerFactory,
		JobManagerSharedServices jobManagerSharedServices,
		HeartbeatServices heartbeatServices,
		JobManagerJobMetricGroupFactory jobMetricGroupFactory,
		OnCompletionActions jobCompletionActions,
		FatalErrorHandler fatalErrorHandler,
		ClassLoader userCodeLoader) throws Exception {

	super(rpcService, AkkaRpcServiceUtils.createRandomName(JOB_MANAGER_NAME));

	this.jobMasterConfiguration = checkNotNull(jobMasterConfiguration);
	this.resourceId = checkNotNull(resourceId);
	this.jobGraph = checkNotNull(jobGraph);
	this.rpcTimeout = jobMasterConfiguration.getRpcTimeout();
	this.highAvailabilityServices = checkNotNull(highAvailabilityService);
	this.blobWriter = jobManagerSharedServices.getBlobWriter();
	this.scheduledExecutorService = jobManagerSharedServices.getScheduledExecutorService();
	this.jobCompletionActions = checkNotNull(jobCompletionActions);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.userCodeLoader = checkNotNull(userCodeLoader);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.jobMetricGroupFactory = checkNotNull(jobMetricGroupFactory);

	final String jobName = jobGraph.getName();
	final JobID jid = jobGraph.getJobID();

	log.info("Initializing job {} ({}).", jobName, jid);

	final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration =
			jobGraph.getSerializedExecutionConfig()
					.deserializeValue(userCodeLoader)
					.getRestartStrategy();

	this.restartStrategy = RestartStrategyResolving.resolve(restartStrategyConfiguration,
		jobManagerSharedServices.getRestartStrategyFactory(),
		jobGraph.isCheckpointingEnabled());

	log.info("Using restart strategy {} for {} ({}).", this.restartStrategy, jobName, jid);

	resourceManagerLeaderRetriever = highAvailabilityServices.getResourceManagerLeaderRetriever();

	this.slotPool = checkNotNull(slotPoolFactory).createSlotPool(jobGraph.getJobID());

	this.scheduler = checkNotNull(schedulerFactory).createScheduler(slotPool);

	this.registeredTaskManagers = new HashMap<>(4);

	this.backPressureStatsTracker = checkNotNull(jobManagerSharedServices.getBackPressureStatsTracker());
	this.lastInternalSavepoint = null;

	this.jobManagerJobMetricGroup = jobMetricGroupFactory.create(jobGraph);
	this.executionGraph = createAndRestoreExecutionGraph(jobManagerJobMetricGroup);
	this.jobStatusListener = null;

	this.resourceManagerConnection = null;
	this.establishedResourceManagerConnection = null;

	this.accumulators = new HashMap<>();
}
 

public TaskExecutor(
		RpcService rpcService,
		TaskManagerConfiguration taskManagerConfiguration,
		HighAvailabilityServices haServices,
		TaskManagerServices taskExecutorServices,
		HeartbeatServices heartbeatServices,
		TaskManagerMetricGroup taskManagerMetricGroup,
		@Nullable String metricQueryServicePath,
		BlobCacheService blobCacheService,
		FatalErrorHandler fatalErrorHandler) {

	super(rpcService, AkkaRpcServiceUtils.createRandomName(TASK_MANAGER_NAME));

	checkArgument(taskManagerConfiguration.getNumberSlots() > 0, "The number of slots has to be larger than 0.");

	this.taskManagerConfiguration = checkNotNull(taskManagerConfiguration);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.taskExecutorServices = checkNotNull(taskExecutorServices);
	this.haServices = checkNotNull(haServices);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.taskManagerMetricGroup = checkNotNull(taskManagerMetricGroup);
	this.blobCacheService = checkNotNull(blobCacheService);
	this.metricQueryServicePath = metricQueryServicePath;

	this.taskSlotTable = taskExecutorServices.getTaskSlotTable();
	this.jobManagerTable = taskExecutorServices.getJobManagerTable();
	this.jobLeaderService = taskExecutorServices.getJobLeaderService();
	this.taskManagerLocation = taskExecutorServices.getTaskManagerLocation();
	this.localStateStoresManager = taskExecutorServices.getTaskManagerStateStore();
	this.networkEnvironment = taskExecutorServices.getNetworkEnvironment();
	this.resourceManagerLeaderRetriever = haServices.getResourceManagerLeaderRetriever();

	this.jobManagerConnections = new HashMap<>(4);

	this.hardwareDescription = HardwareDescription.extractFromSystem(
		taskExecutorServices.getMemoryManager().getMemorySize());

	this.resourceManagerAddress = null;
	this.resourceManagerConnection = null;
	this.currentRegistrationTimeoutId = null;

	this.stackTraceSampleService = new StackTraceSampleService(rpcService.getScheduledExecutor());
}
 

public JobMaster(
		RpcService rpcService,
		JobMasterConfiguration jobMasterConfiguration,
		ResourceID resourceId,
		JobGraph jobGraph,
		HighAvailabilityServices highAvailabilityService,
		SlotPoolFactory slotPoolFactory,
		SchedulerFactory schedulerFactory,
		JobManagerSharedServices jobManagerSharedServices,
		HeartbeatServices heartbeatServices,
		JobManagerJobMetricGroupFactory jobMetricGroupFactory,
		OnCompletionActions jobCompletionActions,
		FatalErrorHandler fatalErrorHandler,
		ClassLoader userCodeLoader,
		SchedulerNGFactory schedulerNGFactory,
		ShuffleMaster<?> shuffleMaster,
		PartitionTrackerFactory partitionTrackerFactory) throws Exception {

	super(rpcService, AkkaRpcServiceUtils.createRandomName(JOB_MANAGER_NAME));

	this.jobMasterConfiguration = checkNotNull(jobMasterConfiguration);
	this.resourceId = checkNotNull(resourceId);
	this.jobGraph = checkNotNull(jobGraph);
	this.rpcTimeout = jobMasterConfiguration.getRpcTimeout();
	this.highAvailabilityServices = checkNotNull(highAvailabilityService);
	this.blobWriter = jobManagerSharedServices.getBlobWriter();
	this.scheduledExecutorService = jobManagerSharedServices.getScheduledExecutorService();
	this.jobCompletionActions = checkNotNull(jobCompletionActions);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.userCodeLoader = checkNotNull(userCodeLoader);
	this.schedulerNGFactory = checkNotNull(schedulerNGFactory);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.jobMetricGroupFactory = checkNotNull(jobMetricGroupFactory);

	final String jobName = jobGraph.getName();
	final JobID jid = jobGraph.getJobID();

	log.info("Initializing job {} ({}).", jobName, jid);

	resourceManagerLeaderRetriever = highAvailabilityServices.getResourceManagerLeaderRetriever();

	this.slotPool = checkNotNull(slotPoolFactory).createSlotPool(jobGraph.getJobID());

	this.scheduler = checkNotNull(schedulerFactory).createScheduler(slotPool);

	this.registeredTaskManagers = new HashMap<>(4);
	this.partitionTracker = checkNotNull(partitionTrackerFactory)
		.create(resourceID -> {
			Tuple2<TaskManagerLocation, TaskExecutorGateway> taskManagerInfo = registeredTaskManagers.get(resourceID);
			if (taskManagerInfo == null) {
				return Optional.empty();
			}

			return Optional.of(taskManagerInfo.f1);
		});

	this.backPressureStatsTracker = checkNotNull(jobManagerSharedServices.getBackPressureStatsTracker());

	this.shuffleMaster = checkNotNull(shuffleMaster);

	this.jobManagerJobMetricGroup = jobMetricGroupFactory.create(jobGraph);
	this.schedulerNG = createScheduler(jobManagerJobMetricGroup);
	this.jobStatusListener = null;

	this.resourceManagerConnection = null;
	this.establishedResourceManagerConnection = null;

	this.accumulators = new HashMap<>();
}
 

public TaskExecutor(
		RpcService rpcService,
		TaskManagerConfiguration taskManagerConfiguration,
		HighAvailabilityServices haServices,
		TaskManagerServices taskExecutorServices,
		HeartbeatServices heartbeatServices,
		TaskManagerMetricGroup taskManagerMetricGroup,
		@Nullable String metricQueryServiceAddress,
		BlobCacheService blobCacheService,
		FatalErrorHandler fatalErrorHandler,
		PartitionTable<JobID> partitionTable) {

	super(rpcService, AkkaRpcServiceUtils.createRandomName(TASK_MANAGER_NAME));

	checkArgument(taskManagerConfiguration.getNumberSlots() > 0, "The number of slots has to be larger than 0.");

	this.taskManagerConfiguration = checkNotNull(taskManagerConfiguration);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.taskExecutorServices = checkNotNull(taskExecutorServices);
	this.haServices = checkNotNull(haServices);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.partitionTable = partitionTable;
	this.taskManagerMetricGroup = checkNotNull(taskManagerMetricGroup);
	this.blobCacheService = checkNotNull(blobCacheService);
	this.metricQueryServiceAddress = metricQueryServiceAddress;

	this.taskSlotTable = taskExecutorServices.getTaskSlotTable();
	this.jobManagerTable = taskExecutorServices.getJobManagerTable();
	this.jobLeaderService = taskExecutorServices.getJobLeaderService();
	this.taskManagerLocation = taskExecutorServices.getTaskManagerLocation();
	this.localStateStoresManager = taskExecutorServices.getTaskManagerStateStore();
	this.shuffleEnvironment = taskExecutorServices.getShuffleEnvironment();
	this.kvStateService = taskExecutorServices.getKvStateService();
	this.resourceManagerLeaderRetriever = haServices.getResourceManagerLeaderRetriever();

	this.jobManagerConnections = new HashMap<>(4);

	this.hardwareDescription = HardwareDescription.extractFromSystem(
		taskExecutorServices.getMemoryManager().getMemorySize());

	this.resourceManagerAddress = null;
	this.resourceManagerConnection = null;
	this.currentRegistrationTimeoutId = null;

	this.stackTraceSampleService = new StackTraceSampleService(rpcService.getScheduledExecutor());
}
 

public JobMaster(
		RpcService rpcService,
		JobMasterConfiguration jobMasterConfiguration,
		ResourceID resourceId,
		JobGraph jobGraph,
		HighAvailabilityServices highAvailabilityService,
		SlotPoolFactory slotPoolFactory,
		SchedulerFactory schedulerFactory,
		JobManagerSharedServices jobManagerSharedServices,
		HeartbeatServices heartbeatServices,
		JobManagerJobMetricGroupFactory jobMetricGroupFactory,
		OnCompletionActions jobCompletionActions,
		FatalErrorHandler fatalErrorHandler,
		ClassLoader userCodeLoader,
		SchedulerNGFactory schedulerNGFactory,
		ShuffleMaster<?> shuffleMaster,
		PartitionTrackerFactory partitionTrackerFactory) throws Exception {

	super(rpcService, AkkaRpcServiceUtils.createRandomName(JOB_MANAGER_NAME), null);

	this.jobMasterConfiguration = checkNotNull(jobMasterConfiguration);
	this.resourceId = checkNotNull(resourceId);
	this.jobGraph = checkNotNull(jobGraph);
	this.rpcTimeout = jobMasterConfiguration.getRpcTimeout();
	this.highAvailabilityServices = checkNotNull(highAvailabilityService);
	this.blobWriter = jobManagerSharedServices.getBlobWriter();
	this.scheduledExecutorService = jobManagerSharedServices.getScheduledExecutorService();
	this.jobCompletionActions = checkNotNull(jobCompletionActions);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.userCodeLoader = checkNotNull(userCodeLoader);
	this.schedulerNGFactory = checkNotNull(schedulerNGFactory);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.jobMetricGroupFactory = checkNotNull(jobMetricGroupFactory);

	final String jobName = jobGraph.getName();
	final JobID jid = jobGraph.getJobID();

	log.info("Initializing job {} ({}).", jobName, jid);

	resourceManagerLeaderRetriever = highAvailabilityServices.getResourceManagerLeaderRetriever();

	this.slotPool = checkNotNull(slotPoolFactory).createSlotPool(jobGraph.getJobID());

	this.scheduler = checkNotNull(schedulerFactory).createScheduler(slotPool);

	this.registeredTaskManagers = new HashMap<>(4);
	this.partitionTracker = checkNotNull(partitionTrackerFactory)
		.create(resourceID -> {
			Tuple2<TaskManagerLocation, TaskExecutorGateway> taskManagerInfo = registeredTaskManagers.get(resourceID);
			if (taskManagerInfo == null) {
				return Optional.empty();
			}

			return Optional.of(taskManagerInfo.f1);
		});

	this.backPressureStatsTracker = checkNotNull(jobManagerSharedServices.getBackPressureStatsTracker());

	this.shuffleMaster = checkNotNull(shuffleMaster);

	this.jobManagerJobMetricGroup = jobMetricGroupFactory.create(jobGraph);
	this.schedulerNG = createScheduler(jobManagerJobMetricGroup);
	this.jobStatusListener = null;

	this.resourceManagerConnection = null;
	this.establishedResourceManagerConnection = null;

	this.accumulators = new HashMap<>();
	this.taskManagerHeartbeatManager = NoOpHeartbeatManager.getInstance();
	this.resourceManagerHeartbeatManager = NoOpHeartbeatManager.getInstance();
}
 

public TaskExecutor(
		RpcService rpcService,
		TaskManagerConfiguration taskManagerConfiguration,
		HighAvailabilityServices haServices,
		TaskManagerServices taskExecutorServices,
		ExternalResourceInfoProvider externalResourceInfoProvider,
		HeartbeatServices heartbeatServices,
		TaskManagerMetricGroup taskManagerMetricGroup,
		@Nullable String metricQueryServiceAddress,
		BlobCacheService blobCacheService,
		FatalErrorHandler fatalErrorHandler,
		TaskExecutorPartitionTracker partitionTracker,
		BackPressureSampleService backPressureSampleService) {

	super(rpcService, AkkaRpcServiceUtils.createRandomName(TASK_MANAGER_NAME));

	checkArgument(taskManagerConfiguration.getNumberSlots() > 0, "The number of slots has to be larger than 0.");

	this.taskManagerConfiguration = checkNotNull(taskManagerConfiguration);
	this.taskExecutorServices = checkNotNull(taskExecutorServices);
	this.haServices = checkNotNull(haServices);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.partitionTracker = partitionTracker;
	this.taskManagerMetricGroup = checkNotNull(taskManagerMetricGroup);
	this.blobCacheService = checkNotNull(blobCacheService);
	this.metricQueryServiceAddress = metricQueryServiceAddress;
	this.backPressureSampleService = checkNotNull(backPressureSampleService);
	this.externalResourceInfoProvider = checkNotNull(externalResourceInfoProvider);

	this.libraryCacheManager = taskExecutorServices.getLibraryCacheManager();
	this.taskSlotTable = taskExecutorServices.getTaskSlotTable();
	this.jobTable = taskExecutorServices.getJobTable();
	this.jobLeaderService = taskExecutorServices.getJobLeaderService();
	this.unresolvedTaskManagerLocation = taskExecutorServices.getUnresolvedTaskManagerLocation();
	this.localStateStoresManager = taskExecutorServices.getTaskManagerStateStore();
	this.shuffleEnvironment = taskExecutorServices.getShuffleEnvironment();
	this.kvStateService = taskExecutorServices.getKvStateService();
	this.ioExecutor = taskExecutorServices.getIOExecutor();
	this.resourceManagerLeaderRetriever = haServices.getResourceManagerLeaderRetriever();

	this.hardwareDescription = HardwareDescription.extractFromSystem(taskExecutorServices.getManagedMemorySize());

	this.resourceManagerAddress = null;
	this.resourceManagerConnection = null;
	this.currentRegistrationTimeoutId = null;

	final ResourceID resourceId = taskExecutorServices.getUnresolvedTaskManagerLocation().getResourceID();
	this.jobManagerHeartbeatManager = createJobManagerHeartbeatManager(heartbeatServices, resourceId);
	this.resourceManagerHeartbeatManager = createResourceManagerHeartbeatManager(heartbeatServices, resourceId);
}
 

public ResourceManager(
		RpcService rpcService,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		ResourceManagerPartitionTrackerFactory clusterPartitionTrackerFactory,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		ResourceManagerMetricGroup resourceManagerMetricGroup,
		Time rpcTimeout) {

	super(rpcService, AkkaRpcServiceUtils.createRandomName(RESOURCE_MANAGER_NAME), null);

	this.resourceId = checkNotNull(resourceId);
	this.highAvailabilityServices = checkNotNull(highAvailabilityServices);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.slotManager = checkNotNull(slotManager);
	this.jobLeaderIdService = checkNotNull(jobLeaderIdService);
	this.clusterInformation = checkNotNull(clusterInformation);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.resourceManagerMetricGroup = checkNotNull(resourceManagerMetricGroup);

	this.jobManagerRegistrations = new HashMap<>(4);
	this.jmResourceIdRegistrations = new HashMap<>(4);
	this.taskExecutors = new HashMap<>(8);
	this.taskExecutorGatewayFutures = new HashMap<>(8);

	this.jobManagerHeartbeatManager = NoOpHeartbeatManager.getInstance();
	this.taskManagerHeartbeatManager = NoOpHeartbeatManager.getInstance();

	this.clusterPartitionTracker = checkNotNull(clusterPartitionTrackerFactory).get(
		(taskExecutorResourceId, dataSetIds) -> taskExecutors.get(taskExecutorResourceId).getTaskExecutorGateway()
			.releaseClusterPartitions(dataSetIds, rpcTimeout)
			.exceptionally(throwable -> {
				log.debug("Request for release of cluster partitions belonging to data sets {} was not successful.", dataSetIds, throwable);
				throw new CompletionException(throwable);
			})
	);
}