org.apache.flink.runtime.execution.librarycache.LibraryCacheManager Java Examples

static JobManagerConnection createJobManagerConnection(JobID jobId, JobMasterGateway jobMasterGateway, RpcService testingRpcService, TaskManagerActions taskManagerActions, Time timeout) {
	final LibraryCacheManager libraryCacheManager = mock(LibraryCacheManager.class);
	when(libraryCacheManager.getClassLoader(any(JobID.class))).thenReturn(ClassLoader.getSystemClassLoader());

	final PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	when(partitionProducerStateChecker.requestPartitionProducerState(any(), any(), any()))
		.thenReturn(CompletableFuture.completedFuture(ExecutionState.RUNNING));

	return new JobManagerConnection(
		jobId,
		ResourceID.generate(),
		jobMasterGateway,
		taskManagerActions,
		mock(CheckpointResponder.class),
		new TestGlobalAggregateManager(),
		libraryCacheManager,
		new RpcResultPartitionConsumableNotifier(jobMasterGateway, testingRpcService.getExecutor(), timeout),
		partitionProducerStateChecker);
}
 

public JobManagerConnection(
			JobID jobID,
			ResourceID resourceID,
			JobMasterGateway jobMasterGateway,
			TaskManagerActions taskManagerActions,
			CheckpointResponder checkpointResponder,
			GlobalAggregateManager aggregateManager,
			LibraryCacheManager libraryCacheManager,
			ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
			PartitionProducerStateChecker partitionStateChecker) {
	this.jobID = Preconditions.checkNotNull(jobID);
	this.resourceID = Preconditions.checkNotNull(resourceID);
	this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
	this.taskManagerActions = Preconditions.checkNotNull(taskManagerActions);
	this.checkpointResponder = Preconditions.checkNotNull(checkpointResponder);
	this.aggregateManager = Preconditions.checkNotNull(aggregateManager);
	this.libraryCacheManager = Preconditions.checkNotNull(libraryCacheManager);
	this.resultPartitionConsumableNotifier = Preconditions.checkNotNull(resultPartitionConsumableNotifier);
	this.partitionStateChecker = Preconditions.checkNotNull(partitionStateChecker);
}
 

public JobManagerConnection(
			JobID jobID,
			ResourceID resourceID,
			JobMasterGateway jobMasterGateway,
			TaskManagerActions taskManagerActions,
			CheckpointResponder checkpointResponder,
			GlobalAggregateManager aggregateManager,
			LibraryCacheManager libraryCacheManager,
			ResultPartitionConsumableNotifier resultPartitionConsumableNotifier,
			PartitionProducerStateChecker partitionStateChecker) {
	this.jobID = Preconditions.checkNotNull(jobID);
	this.resourceID = Preconditions.checkNotNull(resourceID);
	this.jobMasterGateway = Preconditions.checkNotNull(jobMasterGateway);
	this.taskManagerActions = Preconditions.checkNotNull(taskManagerActions);
	this.checkpointResponder = Preconditions.checkNotNull(checkpointResponder);
	this.aggregateManager = Preconditions.checkNotNull(aggregateManager);
	this.libraryCacheManager = Preconditions.checkNotNull(libraryCacheManager);
	this.resultPartitionConsumableNotifier = Preconditions.checkNotNull(resultPartitionConsumableNotifier);
	this.partitionStateChecker = Preconditions.checkNotNull(partitionStateChecker);
}
 

public JobManagerSharedServices(
		ScheduledExecutorService scheduledExecutorService,
		LibraryCacheManager libraryCacheManager,
		RestartStrategyFactory restartStrategyFactory,
		StackTraceSampleCoordinator stackTraceSampleCoordinator,
		BackPressureStatsTracker backPressureStatsTracker,
		@Nonnull BlobWriter blobWriter) {

	this.scheduledExecutorService = checkNotNull(scheduledExecutorService);
	this.libraryCacheManager = checkNotNull(libraryCacheManager);
	this.restartStrategyFactory = checkNotNull(restartStrategyFactory);
	this.stackTraceSampleCoordinator = checkNotNull(stackTraceSampleCoordinator);
	this.backPressureStatsTracker = checkNotNull(backPressureStatsTracker);
	this.blobWriter = blobWriter;
}
 

public JobManagerSharedServices(
		ScheduledExecutorService scheduledExecutorService,
		LibraryCacheManager libraryCacheManager,
		BackPressureRequestCoordinator backPressureSampleCoordinator,
		BackPressureStatsTracker backPressureStatsTracker,
		@Nonnull BlobWriter blobWriter) {

	this.scheduledExecutorService = checkNotNull(scheduledExecutorService);
	this.libraryCacheManager = checkNotNull(libraryCacheManager);
	this.backPressureSampleCoordinator = checkNotNull(backPressureSampleCoordinator);
	this.backPressureStatsTracker = checkNotNull(backPressureStatsTracker);
	this.blobWriter = blobWriter;
}
 

/**
 * Exceptions that occur while creating the JobManager or JobManagerRunnerImpl are directly
 * thrown and not reported to the given {@code FatalErrorHandler}.
 *
 * @throws Exception Thrown if the runner cannot be set up, because either one of the
 *                   required services could not be started, or the Job could not be initialized.
 */
public JobManagerRunnerImpl(
		final JobGraph jobGraph,
		final JobMasterServiceFactory jobMasterFactory,
		final HighAvailabilityServices haServices,
		final LibraryCacheManager.ClassLoaderLease classLoaderLease,
		final Executor executor,
		final FatalErrorHandler fatalErrorHandler) throws Exception {

	this.resultFuture = new CompletableFuture<>();
	this.terminationFuture = new CompletableFuture<>();
	this.leadershipOperation = CompletableFuture.completedFuture(null);

	this.jobGraph = checkNotNull(jobGraph);
	this.classLoaderLease = checkNotNull(classLoaderLease);
	this.executor = checkNotNull(executor);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);

	checkArgument(jobGraph.getNumberOfVertices() > 0, "The given job is empty");

	// libraries and class loader first
	final ClassLoader userCodeLoader;
	try {
		userCodeLoader = classLoaderLease.getOrResolveClassLoader(
			jobGraph.getUserJarBlobKeys(),
			jobGraph.getClasspaths());
	} catch (IOException e) {
		throw new Exception("Cannot set up the user code libraries: " + e.getMessage(), e);
	}

	// high availability services next
	this.runningJobsRegistry = haServices.getRunningJobsRegistry();
	this.leaderElectionService = haServices.getJobManagerLeaderElectionService(jobGraph.getJobID());

	this.leaderGatewayFuture = new CompletableFuture<>();

	// now start the JobManager
	this.jobMasterService = jobMasterFactory.createJobMasterService(jobGraph, this, userCodeLoader);
}
 

@Nonnull
private JobManagerRunner createJobManagerRunner(JobMasterServiceFactory jobMasterServiceFactory, LibraryCacheManager libraryCacheManager) throws Exception{
	return new JobManagerRunner(
		jobGraph,
		jobMasterServiceFactory,
		haServices,
		libraryCacheManager,
		TestingUtils.defaultExecutor(),
		fatalErrorHandler);
}
 

@Test
public void testLibraryCacheRegistrationFailed() throws Exception {
	final QueuedNoOpTaskManagerActions taskManagerActions = new QueuedNoOpTaskManagerActions();
	final Task task = createTaskBuilder()
		.setTaskManagerActions(taskManagerActions)
		.setLibraryCacheManager(mock(LibraryCacheManager.class)) // inactive manager
		.build();

	// task should be new and perfect
	assertEquals(ExecutionState.CREATED, task.getExecutionState());
	assertFalse(task.isCanceledOrFailed());
	assertNull(task.getFailureCause());

	// should fail
	task.run();

	// verify final state
	assertEquals(ExecutionState.FAILED, task.getExecutionState());
	assertTrue(task.isCanceledOrFailed());
	assertNotNull(task.getFailureCause());
	assertNotNull(task.getFailureCause().getMessage());
	assertTrue(task.getFailureCause().getMessage().contains("classloader"));

	assertNull(task.getInvokable());

	taskManagerActions.validateListenerMessage(
		ExecutionState.FAILED, task, new Exception("No user code classloader available."));
}
 

@VisibleForTesting
static TaskExecutorJobServices create(
	LibraryCacheManager.ClassLoaderLease classLoaderLease,
	Runnable closeHook) {
	return new TaskExecutorJobServices(
		classLoaderLease,
		closeHook);
}
 

TaskManagerServices(
	UnresolvedTaskManagerLocation unresolvedTaskManagerLocation,
	long managedMemorySize,
	IOManager ioManager,
	ShuffleEnvironment<?, ?> shuffleEnvironment,
	KvStateService kvStateService,
	BroadcastVariableManager broadcastVariableManager,
	TaskSlotTable<Task> taskSlotTable,
	JobTable jobTable,
	JobLeaderService jobLeaderService,
	TaskExecutorLocalStateStoresManager taskManagerStateStore,
	TaskEventDispatcher taskEventDispatcher,
	ExecutorService ioExecutor,
	LibraryCacheManager libraryCacheManager) {

	this.unresolvedTaskManagerLocation = Preconditions.checkNotNull(unresolvedTaskManagerLocation);
	this.managedMemorySize = managedMemorySize;
	this.ioManager = Preconditions.checkNotNull(ioManager);
	this.shuffleEnvironment = Preconditions.checkNotNull(shuffleEnvironment);
	this.kvStateService = Preconditions.checkNotNull(kvStateService);
	this.broadcastVariableManager = Preconditions.checkNotNull(broadcastVariableManager);
	this.taskSlotTable = Preconditions.checkNotNull(taskSlotTable);
	this.jobTable = Preconditions.checkNotNull(jobTable);
	this.jobLeaderService = Preconditions.checkNotNull(jobLeaderService);
	this.taskManagerStateStore = Preconditions.checkNotNull(taskManagerStateStore);
	this.taskEventDispatcher = Preconditions.checkNotNull(taskEventDispatcher);
	this.ioExecutor = Preconditions.checkNotNull(ioExecutor);
	this.libraryCacheManager = Preconditions.checkNotNull(libraryCacheManager);
}
 

private JobManagerConnection associateWithJobManager(
		JobID jobID,
		ResourceID resourceID,
		JobMasterGateway jobMasterGateway) {
	checkNotNull(jobID);
	checkNotNull(resourceID);
	checkNotNull(jobMasterGateway);

	TaskManagerActions taskManagerActions = new TaskManagerActionsImpl(jobMasterGateway);

	CheckpointResponder checkpointResponder = new RpcCheckpointResponder(jobMasterGateway);
	GlobalAggregateManager aggregateManager = new RpcGlobalAggregateManager(jobMasterGateway);

	final LibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(
		blobCacheService.getPermanentBlobService(),
		taskManagerConfiguration.getClassLoaderResolveOrder(),
		taskManagerConfiguration.getAlwaysParentFirstLoaderPatterns());

	ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = new RpcResultPartitionConsumableNotifier(
		jobMasterGateway,
		getRpcService().getExecutor(),
		taskManagerConfiguration.getTimeout());

	PartitionProducerStateChecker partitionStateChecker = new RpcPartitionStateChecker(jobMasterGateway);

	registerQueryableState(jobID, jobMasterGateway);

	return new JobManagerConnection(
		jobID,
		resourceID,
		jobMasterGateway,
		taskManagerActions,
		checkpointResponder,
		aggregateManager,
		libraryCacheManager,
		resultPartitionConsumableNotifier,
		partitionStateChecker);
}
 

public JobManagerSharedServices(
		ScheduledExecutorService scheduledExecutorService,
		LibraryCacheManager libraryCacheManager,
		RestartStrategyFactory restartStrategyFactory,
		StackTraceSampleCoordinator stackTraceSampleCoordinator,
		BackPressureStatsTracker backPressureStatsTracker,
		@Nonnull BlobWriter blobWriter) {

	this.scheduledExecutorService = checkNotNull(scheduledExecutorService);
	this.libraryCacheManager = checkNotNull(libraryCacheManager);
	this.restartStrategyFactory = checkNotNull(restartStrategyFactory);
	this.stackTraceSampleCoordinator = checkNotNull(stackTraceSampleCoordinator);
	this.backPressureStatsTracker = checkNotNull(backPressureStatsTracker);
	this.blobWriter = blobWriter;
}
 

@Nonnull
private JobManagerRunner createJobManagerRunner(JobMasterServiceFactory jobMasterServiceFactory, LibraryCacheManager libraryCacheManager) throws Exception{
	return new JobManagerRunner(
		jobGraph,
		jobMasterServiceFactory,
		haServices,
		libraryCacheManager,
		TestingUtils.defaultExecutor(),
		fatalErrorHandler);
}
 

private JobManagerConnection associateWithJobManager(
		JobID jobID,
		ResourceID resourceID,
		JobMasterGateway jobMasterGateway) {
	checkNotNull(jobID);
	checkNotNull(resourceID);
	checkNotNull(jobMasterGateway);

	TaskManagerActions taskManagerActions = new TaskManagerActionsImpl(jobMasterGateway);

	CheckpointResponder checkpointResponder = new RpcCheckpointResponder(jobMasterGateway);
	GlobalAggregateManager aggregateManager = new RpcGlobalAggregateManager(jobMasterGateway);

	final LibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(
		blobCacheService.getPermanentBlobService(),
		taskManagerConfiguration.getClassLoaderResolveOrder(),
		taskManagerConfiguration.getAlwaysParentFirstLoaderPatterns());

	ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = new RpcResultPartitionConsumableNotifier(
		jobMasterGateway,
		getRpcService().getExecutor(),
		taskManagerConfiguration.getTimeout());

	PartitionProducerStateChecker partitionStateChecker = new RpcPartitionStateChecker(jobMasterGateway);

	registerQueryableState(jobID, jobMasterGateway);

	return new JobManagerConnection(
		jobID,
		resourceID,
		jobMasterGateway,
		taskManagerActions,
		checkpointResponder,
		aggregateManager,
		libraryCacheManager,
		resultPartitionConsumableNotifier,
		partitionStateChecker);
}
 

@Test
public void testLibraryCacheRegistrationFailed() throws Exception {
	final QueuedNoOpTaskManagerActions taskManagerActions = new QueuedNoOpTaskManagerActions();
	final Task task = new TaskBuilder()
		.setTaskManagerActions(taskManagerActions)
		.setLibraryCacheManager(mock(LibraryCacheManager.class)) // inactive manager
		.build();

	// task should be new and perfect
	assertEquals(ExecutionState.CREATED, task.getExecutionState());
	assertFalse(task.isCanceledOrFailed());
	assertNull(task.getFailureCause());

	// should fail
	task.run();

	// verify final state
	assertEquals(ExecutionState.FAILED, task.getExecutionState());
	assertTrue(task.isCanceledOrFailed());
	assertNotNull(task.getFailureCause());
	assertNotNull(task.getFailureCause().getMessage());
	assertTrue(task.getFailureCause().getMessage().contains("classloader"));

	assertNull(task.getInvokable());

	taskManagerActions.validateListenerMessage(
		ExecutionState.FAILED, task, new Exception("No user code classloader available."));
}
 

@Nonnull
private JobManagerRunnerImpl createJobManagerRunner(JobMasterServiceFactory jobMasterServiceFactory, LibraryCacheManager.ClassLoaderLease classLoaderLease) throws Exception{
	return new JobManagerRunnerImpl(
		jobGraph,
		jobMasterServiceFactory,
		haServices,
		classLoaderLease,
		TestingUtils.defaultExecutor(),
		fatalErrorHandler);
}
 

public TestingJobManagerSharedServicesBuilder setLibraryCacheManager(LibraryCacheManager libraryCacheManager) {
	this.libraryCacheManager = libraryCacheManager;
	return this;

}
 

@Nonnull
private JobManagerRunner createJobManagerRunner(LibraryCacheManager.ClassLoaderLease classLoaderLease) throws Exception {
	return createJobManagerRunner(defaultJobMasterServiceFactory, classLoaderLease);
}
 

public LibraryCacheManager getLibraryCacheManager() {
	return libraryCacheManager;
}
 

@Override
public LibraryCacheManager.ClassLoaderHandle getClassLoaderHandle() {
	verifyJobIsNotClosed();
	return jobServices.getClassLoaderHandle();
}
 

/**
 * Creates and returns the task manager services.
 *
 * @param taskManagerServicesConfiguration task manager configuration
 * @param permanentBlobService permanentBlobService used by the services
 * @param taskManagerMetricGroup metric group of the task manager
 * @param ioExecutor executor for async IO operations
 * @param fatalErrorHandler to handle class loading OOMs
 * @return task manager components
 * @throws Exception
 */
public static TaskManagerServices fromConfiguration(
		TaskManagerServicesConfiguration taskManagerServicesConfiguration,
		PermanentBlobService permanentBlobService,
		MetricGroup taskManagerMetricGroup,
		ExecutorService ioExecutor,
		FatalErrorHandler fatalErrorHandler) throws Exception {

	// pre-start checks
	checkTempDirs(taskManagerServicesConfiguration.getTmpDirPaths());

	final TaskEventDispatcher taskEventDispatcher = new TaskEventDispatcher();

	// start the I/O manager, it will create some temp directories.
	final IOManager ioManager = new IOManagerAsync(taskManagerServicesConfiguration.getTmpDirPaths());

	final ShuffleEnvironment<?, ?> shuffleEnvironment = createShuffleEnvironment(
		taskManagerServicesConfiguration,
		taskEventDispatcher,
		taskManagerMetricGroup,
		ioExecutor);
	final int listeningDataPort = shuffleEnvironment.start();

	final KvStateService kvStateService = KvStateService.fromConfiguration(taskManagerServicesConfiguration);
	kvStateService.start();

	final UnresolvedTaskManagerLocation unresolvedTaskManagerLocation = new UnresolvedTaskManagerLocation(
		taskManagerServicesConfiguration.getResourceID(),
		taskManagerServicesConfiguration.getExternalAddress(),
		// we expose the task manager location with the listening port
		// iff the external data port is not explicitly defined
		taskManagerServicesConfiguration.getExternalDataPort() > 0 ?
			taskManagerServicesConfiguration.getExternalDataPort() :
			listeningDataPort);

	final BroadcastVariableManager broadcastVariableManager = new BroadcastVariableManager();

	final TaskSlotTable<Task> taskSlotTable = createTaskSlotTable(
		taskManagerServicesConfiguration.getNumberOfSlots(),
		taskManagerServicesConfiguration.getTaskExecutorResourceSpec(),
		taskManagerServicesConfiguration.getTimerServiceShutdownTimeout(),
		taskManagerServicesConfiguration.getPageSize());

	final JobTable jobTable = DefaultJobTable.create();

	final JobLeaderService jobLeaderService = new DefaultJobLeaderService(unresolvedTaskManagerLocation, taskManagerServicesConfiguration.getRetryingRegistrationConfiguration());

	final String[] stateRootDirectoryStrings = taskManagerServicesConfiguration.getLocalRecoveryStateRootDirectories();

	final File[] stateRootDirectoryFiles = new File[stateRootDirectoryStrings.length];

	for (int i = 0; i < stateRootDirectoryStrings.length; ++i) {
		stateRootDirectoryFiles[i] = new File(stateRootDirectoryStrings[i], LOCAL_STATE_SUB_DIRECTORY_ROOT);
	}

	final TaskExecutorLocalStateStoresManager taskStateManager = new TaskExecutorLocalStateStoresManager(
		taskManagerServicesConfiguration.isLocalRecoveryEnabled(),
		stateRootDirectoryFiles,
		ioExecutor);

	final boolean failOnJvmMetaspaceOomError =
		taskManagerServicesConfiguration.getConfiguration().getBoolean(CoreOptions.FAIL_ON_USER_CLASS_LOADING_METASPACE_OOM);
	final LibraryCacheManager libraryCacheManager = new BlobLibraryCacheManager(
		permanentBlobService,
		BlobLibraryCacheManager.defaultClassLoaderFactory(
			taskManagerServicesConfiguration.getClassLoaderResolveOrder(),
			taskManagerServicesConfiguration.getAlwaysParentFirstLoaderPatterns(),
			failOnJvmMetaspaceOomError ? fatalErrorHandler : null));

	return new TaskManagerServices(
		unresolvedTaskManagerLocation,
		taskManagerServicesConfiguration.getManagedMemorySize().getBytes(),
		ioManager,
		shuffleEnvironment,
		kvStateService,
		broadcastVariableManager,
		taskSlotTable,
		jobTable,
		jobLeaderService,
		taskStateManager,
		taskEventDispatcher,
		ioExecutor,
		libraryCacheManager);
}
 

private TestingJobServices(
		Supplier<LibraryCacheManager.ClassLoaderHandle> classLoaderHandleSupplier,
		Runnable closeRunnable) {
	this.classLoaderHandleSupplier = classLoaderHandleSupplier;
	this.closeRunnable = closeRunnable;
}
 

@Override
public LibraryCacheManager.ClassLoaderHandle getClassLoaderHandle() {
	return classLoaderLease;
}
 

private TaskExecutorJobServices(
	LibraryCacheManager.ClassLoaderLease classLoaderLease,
	Runnable closeHook) {
	this.classLoaderLease = classLoaderLease;
	this.closeHook = closeHook;
}
 

@Override
public LibraryCacheManager.ClassLoaderHandle getClassLoaderHandle() {
	return classLoaderHandleSupplier.get();
}
 

public TestTaskBuilder setClassLoaderHandle(LibraryCacheManager.ClassLoaderHandle classLoaderHandle) {
	this.classLoaderHandle = classLoaderHandle;
	return this;
}
 

public Builder setClassLoaderHandleSupplier(Supplier<LibraryCacheManager.ClassLoaderHandle> classLoaderHandleSupplier) {
	this.classLoaderHandleSupplier = classLoaderHandleSupplier;
	return this;
}
 

public LibraryCacheManager getLibraryCacheManager() {
	return libraryCacheManager;
}
 

public TaskManagerServicesBuilder setLibraryCacheManager(LibraryCacheManager libraryCacheManager) {
	this.libraryCacheManager = libraryCacheManager;
	return this;
}
 

private Task createTask(Class<? extends AbstractInvokable> invokableClass) throws Exception {
	BlobCacheService blobService =
			new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

	LibraryCacheManager libCache = mock(LibraryCacheManager.class);
	when(libCache.getClassLoader(any(JobID.class))).thenReturn(ClassLoader.getSystemClassLoader());

	ResultPartitionConsumableNotifier consumableNotifier = new NoOpResultPartitionConsumableNotifier();
	PartitionProducerStateChecker partitionProducerStateChecker = mock(PartitionProducerStateChecker.class);
	Executor executor = mock(Executor.class);
	ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

	TaskMetricGroup taskMetricGroup = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();

	JobInformation jobInformation = new JobInformation(
			new JobID(),
			"Job Name",
			new SerializedValue<>(new ExecutionConfig()),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList());

	TaskInformation taskInformation = new TaskInformation(
			new JobVertexID(),
			"Test Task",
			1,
			1,
			invokableClass.getName(),
			new Configuration());

	return new Task(
			jobInformation,
			taskInformation,
			new ExecutionAttemptID(),
			new AllocationID(),
			0,
			0,
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			0,
			mock(MemoryManager.class),
			mock(IOManager.class),
			shuffleEnvironment,
			new KvStateService(new KvStateRegistry(), null, null),
			mock(BroadcastVariableManager.class),
			new TaskEventDispatcher(),
			new TestTaskStateManager(),
			mock(TaskManagerActions.class),
			mock(InputSplitProvider.class),
			mock(CheckpointResponder.class),
			new TestGlobalAggregateManager(),
			blobService,
			libCache,
			mock(FileCache.class),
			new TestingTaskManagerRuntimeInfo(),
			taskMetricGroup,
			consumableNotifier,
			partitionProducerStateChecker,
			executor);
}