org.apache.flink.runtime.blob.BlobWriter Java Examples

SchedulerNG createInstance(
Logger log,
JobGraph jobGraph,
BackPressureStatsTracker backPressureStatsTracker,
Executor ioExecutor,
Configuration jobMasterConfiguration,
SlotProvider slotProvider,
ScheduledExecutorService futureExecutor,
ClassLoader userCodeLoader,
CheckpointRecoveryFactory checkpointRecoveryFactory,
Time rpcTimeout,
BlobWriter blobWriter,
JobManagerJobMetricGroup jobManagerJobMetricGroup,
Time slotRequestTimeout,
ShuffleMaster<?> shuffleMaster,
JobMasterPartitionTracker partitionTracker) throws Exception;
 

public Either<SerializedValue<TaskInformation>, PermanentBlobKey> getTaskInformationOrBlobKey() throws IOException {
	// only one thread should offload the task information, so let's also let only one thread
	// serialize the task information!
	synchronized (stateMonitor) {
		if (taskInformationOrBlobKey == null) {
			final BlobWriter blobWriter = graph.getBlobWriter();

			final TaskInformation taskInformation = new TaskInformation(
				jobVertex.getID(),
				jobVertex.getName(),
				parallelism,
				maxParallelism,
				jobVertex.getInvokableClassName(),
				jobVertex.getConfiguration());

			taskInformationOrBlobKey = BlobWriter.serializeAndTryOffload(
				taskInformation,
				getJobId(),
				blobWriter);
		}

		return taskInformationOrBlobKey;
	}
}
 

public Either<SerializedValue<TaskInformation>, PermanentBlobKey> getTaskInformationOrBlobKey() throws IOException {
	// only one thread should offload the task information, so let's also let only one thread
	// serialize the task information!
	synchronized (stateMonitor) {
		if (taskInformationOrBlobKey == null) {
			final BlobWriter blobWriter = graph.getBlobWriter();

			final TaskInformation taskInformation = new TaskInformation(
				jobVertex.getID(),
				jobVertex.getName(),
				parallelism,
				maxParallelism,
				jobVertex.getInvokableClassName(),
				jobVertex.getConfiguration());

			taskInformationOrBlobKey = BlobWriter.serializeAndTryOffload(
				taskInformation,
				getJobId(),
				blobWriter);
		}

		return taskInformationOrBlobKey;
	}
}
 

public Either<SerializedValue<TaskInformation>, PermanentBlobKey> getTaskInformationOrBlobKey() throws IOException {
	// only one thread should offload the task information, so let's also let only one thread
	// serialize the task information!
	synchronized (stateMonitor) {
		if (taskInformationOrBlobKey == null) {
			final BlobWriter blobWriter = graph.getBlobWriter();

			final TaskInformation taskInformation = new TaskInformation(
				jobVertex.getID(),
				jobVertex.getName(),
				parallelism,
				maxParallelism,
				jobVertex.getInvokableClassName(),
				jobVertex.getConfiguration());

			taskInformationOrBlobKey = BlobWriter.serializeAndTryOffload(
				taskInformation,
				getJobId(),
				blobWriter);
		}

		return taskInformationOrBlobKey;
	}
}
 

SchedulerNG createInstance(
Logger log,
JobGraph jobGraph,
BackPressureStatsTracker backPressureStatsTracker,
Executor ioExecutor,
Configuration jobMasterConfiguration,
SlotProvider slotProvider,
ScheduledExecutorService futureExecutor,
ClassLoader userCodeLoader,
CheckpointRecoveryFactory checkpointRecoveryFactory,
Time rpcTimeout,
BlobWriter blobWriter,
JobManagerJobMetricGroup jobManagerJobMetricGroup,
Time slotRequestTimeout,
ShuffleMaster<?> shuffleMaster,
PartitionTracker partitionTracker) throws Exception;
 

public DefaultScheduler(
		final Logger log,
		final JobGraph jobGraph,
		final BackPressureStatsTracker backPressureStatsTracker,
		final Executor ioExecutor,
		final Configuration jobMasterConfiguration,
		final SlotProvider slotProvider,
		final ScheduledExecutorService futureExecutor,
		final ClassLoader userCodeLoader,
		final CheckpointRecoveryFactory checkpointRecoveryFactory,
		final Time rpcTimeout,
		final BlobWriter blobWriter,
		final JobManagerJobMetricGroup jobManagerJobMetricGroup,
		final Time slotRequestTimeout,
		final ShuffleMaster<?> shuffleMaster,
		final PartitionTracker partitionTracker) throws Exception {

	super(
		log,
		jobGraph,
		backPressureStatsTracker,
		ioExecutor,
		jobMasterConfiguration,
		slotProvider,
		futureExecutor,
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		new ThrowingRestartStrategy.ThrowingRestartStrategyFactory(),
		blobWriter,
		jobManagerJobMetricGroup,
		slotRequestTimeout,
		shuffleMaster,
		partitionTracker);
}
 

public JobManagerSharedServices(
		ScheduledExecutorService scheduledExecutorService,
		LibraryCacheManager libraryCacheManager,
		BackPressureRequestCoordinator backPressureSampleCoordinator,
		BackPressureStatsTracker backPressureStatsTracker,
		@Nonnull BlobWriter blobWriter) {

	this.scheduledExecutorService = checkNotNull(scheduledExecutorService);
	this.libraryCacheManager = checkNotNull(libraryCacheManager);
	this.backPressureSampleCoordinator = checkNotNull(backPressureSampleCoordinator);
	this.backPressureStatsTracker = checkNotNull(backPressureStatsTracker);
	this.blobWriter = blobWriter;
}
 

@VisibleForTesting
public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time timeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategy,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout) throws IOException {
	this(
		jobInformation,
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		JobManagerOptions.MAX_ATTEMPTS_HISTORY_SIZE.defaultValue(),
		failoverStrategy,
		slotProvider,
		userClassLoader,
		blobWriter,
		allocationTimeout,
		new NotReleasingPartitionReleaseStrategy.Factory(),
		NettyShuffleMaster.INSTANCE,
		new PartitionTrackerImpl(
			jobInformation.getJobId(),
			NettyShuffleMaster.INSTANCE,
			ignored -> Optional.empty()),
		ScheduleMode.LAZY_FROM_SOURCES,
		false);
}
 

@Override
public SchedulerNG createInstance(
		final Logger log,
		final JobGraph jobGraph,
		final BackPressureStatsTracker backPressureStatsTracker,
		final Executor ioExecutor,
		final Configuration jobMasterConfiguration,
		final SlotProvider slotProvider,
		final ScheduledExecutorService futureExecutor,
		final ClassLoader userCodeLoader,
		final CheckpointRecoveryFactory checkpointRecoveryFactory,
		final Time rpcTimeout,
		final BlobWriter blobWriter,
		final JobManagerJobMetricGroup jobManagerJobMetricGroup,
		final Time slotRequestTimeout,
		final ShuffleMaster<?> shuffleMaster,
		final PartitionTracker partitionTracker) throws Exception {

	return new DefaultScheduler(
		log,
		jobGraph,
		backPressureStatsTracker,
		ioExecutor,
		jobMasterConfiguration,
		slotProvider,
		futureExecutor,
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		blobWriter,
		jobManagerJobMetricGroup,
		slotRequestTimeout,
		shuffleMaster,
		partitionTracker);
}
 

@Override
public SchedulerNG createInstance(
		final Logger log,
		final JobGraph jobGraph,
		final BackPressureStatsTracker backPressureStatsTracker,
		final Executor ioExecutor,
		final Configuration jobMasterConfiguration,
		final SlotProvider slotProvider,
		final ScheduledExecutorService futureExecutor,
		final ClassLoader userCodeLoader,
		final CheckpointRecoveryFactory checkpointRecoveryFactory,
		final Time rpcTimeout,
		final BlobWriter blobWriter,
		final JobManagerJobMetricGroup jobManagerJobMetricGroup,
		final Time slotRequestTimeout,
		final ShuffleMaster<?> shuffleMaster,
		final PartitionTracker partitionTracker) throws Exception {

	return new LegacyScheduler(
		log,
		jobGraph,
		backPressureStatsTracker,
		ioExecutor,
		jobMasterConfiguration,
		slotProvider,
		futureExecutor,
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		restartStrategyFactory,
		blobWriter,
		jobManagerJobMetricGroup,
		slotRequestTimeout,
		shuffleMaster,
		partitionTracker);
}
 

public JobManagerSharedServices(
		ScheduledExecutorService scheduledExecutorService,
		LibraryCacheManager libraryCacheManager,
		RestartStrategyFactory restartStrategyFactory,
		StackTraceSampleCoordinator stackTraceSampleCoordinator,
		BackPressureStatsTracker backPressureStatsTracker,
		@Nonnull BlobWriter blobWriter) {

	this.scheduledExecutorService = checkNotNull(scheduledExecutorService);
	this.libraryCacheManager = checkNotNull(libraryCacheManager);
	this.restartStrategyFactory = checkNotNull(restartStrategyFactory);
	this.stackTraceSampleCoordinator = checkNotNull(stackTraceSampleCoordinator);
	this.backPressureStatsTracker = checkNotNull(backPressureStatsTracker);
	this.blobWriter = blobWriter;
}
 

/**
 * Builds the ExecutionGraph from the JobGraph.
 * If a prior execution graph exists, the JobGraph will be attached. If no prior execution
 * graph exists, then the JobGraph will become attach to a new empty execution graph.
 */
public static ExecutionGraph buildGraph(
		@Nullable ExecutionGraph prior,
		JobGraph jobGraph,
		Configuration jobManagerConfig,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		SlotProvider slotProvider,
		ClassLoader classLoader,
		CheckpointRecoveryFactory recoveryFactory,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		MetricGroup metrics,
		BlobWriter blobWriter,
		Time allocationTimeout,
		Logger log)
	throws JobExecutionException, JobException {

	return buildGraph(
		prior,
		jobGraph,
		jobManagerConfig,
		futureExecutor,
		ioExecutor,
		slotProvider,
		classLoader,
		recoveryFactory,
		rpcTimeout,
		restartStrategy,
		metrics,
		-1,
		blobWriter,
		allocationTimeout,
		log);
}
 

public JobManagerSharedServices(
		ScheduledExecutorService scheduledExecutorService,
		LibraryCacheManager libraryCacheManager,
		RestartStrategyFactory restartStrategyFactory,
		StackTraceSampleCoordinator stackTraceSampleCoordinator,
		BackPressureStatsTracker backPressureStatsTracker,
		@Nonnull BlobWriter blobWriter) {

	this.scheduledExecutorService = checkNotNull(scheduledExecutorService);
	this.libraryCacheManager = checkNotNull(libraryCacheManager);
	this.restartStrategyFactory = checkNotNull(restartStrategyFactory);
	this.stackTraceSampleCoordinator = checkNotNull(stackTraceSampleCoordinator);
	this.backPressureStatsTracker = checkNotNull(backPressureStatsTracker);
	this.blobWriter = blobWriter;
}
 

public TestingExecutionGraphBuilder setBlobWriter(BlobWriter blobWriter) {
	this.blobWriter = blobWriter;
	return this;
}
 

public final BlobWriter getBlobWriter() {
	return blobWriter;
}
 

/**
 * Builds the ExecutionGraph from the JobGraph.
 * If a prior execution graph exists, the JobGraph will be attached. If no prior execution
 * graph exists, then the JobGraph will become attach to a new empty execution graph.
 */
public static ExecutionGraph buildGraph(
		@Nullable ExecutionGraph prior,
		JobGraph jobGraph,
		Configuration jobManagerConfig,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		SlotProvider slotProvider,
		ClassLoader classLoader,
		CheckpointRecoveryFactory recoveryFactory,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		MetricGroup metrics,
		BlobWriter blobWriter,
		Time allocationTimeout,
		Logger log,
		ShuffleMaster<?> shuffleMaster,
		JobMasterPartitionTracker partitionTracker) throws JobExecutionException, JobException {

	final FailoverStrategy.Factory failoverStrategy =
		FailoverStrategyLoader.loadFailoverStrategy(jobManagerConfig, log);

	return buildGraph(
		prior,
		jobGraph,
		jobManagerConfig,
		futureExecutor,
		ioExecutor,
		slotProvider,
		classLoader,
		recoveryFactory,
		rpcTimeout,
		restartStrategy,
		metrics,
		blobWriter,
		allocationTimeout,
		log,
		shuffleMaster,
		partitionTracker,
		failoverStrategy);
}
 

public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		int maxPriorAttemptsHistoryLength,
		FailoverStrategy.Factory failoverStrategyFactory,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout,
		PartitionReleaseStrategy.Factory partitionReleaseStrategyFactory,
		ShuffleMaster<?> shuffleMaster,
		JobMasterPartitionTracker partitionTracker,
		ScheduleMode scheduleMode) throws IOException {

	this.jobInformation = Preconditions.checkNotNull(jobInformation);

	this.blobWriter = Preconditions.checkNotNull(blobWriter);

	this.scheduleMode = checkNotNull(scheduleMode);

	this.jobInformationOrBlobKey = BlobWriter.serializeAndTryOffload(jobInformation, jobInformation.getJobId(), blobWriter);

	this.futureExecutor = Preconditions.checkNotNull(futureExecutor);
	this.ioExecutor = Preconditions.checkNotNull(ioExecutor);

	this.slotProviderStrategy = SlotProviderStrategy.from(
		scheduleMode,
		slotProvider,
		allocationTimeout);
	this.userClassLoader = Preconditions.checkNotNull(userClassLoader, "userClassLoader");

	this.tasks = new HashMap<>(16);
	this.intermediateResults = new HashMap<>(16);
	this.verticesInCreationOrder = new ArrayList<>(16);
	this.currentExecutions = new HashMap<>(16);

	this.jobStatusListeners  = new ArrayList<>();

	this.stateTimestamps = new long[JobStatus.values().length];
	this.stateTimestamps[JobStatus.CREATED.ordinal()] = System.currentTimeMillis();

	this.rpcTimeout = checkNotNull(rpcTimeout);
	this.allocationTimeout = checkNotNull(allocationTimeout);

	this.partitionReleaseStrategyFactory = checkNotNull(partitionReleaseStrategyFactory);

	this.restartStrategy = restartStrategy;
	this.kvStateLocationRegistry = new KvStateLocationRegistry(jobInformation.getJobId(), getAllVertices());

	this.globalModVersion = 1L;

	// the failover strategy must be instantiated last, so that the execution graph
	// is ready by the time the failover strategy sees it
	this.failoverStrategy = checkNotNull(failoverStrategyFactory.create(this), "null failover strategy");

	this.maxPriorAttemptsHistoryLength = maxPriorAttemptsHistoryLength;

	this.schedulingFuture = null;
	this.jobMasterMainThreadExecutor =
		new ComponentMainThreadExecutor.DummyComponentMainThreadExecutor(
			"ExecutionGraph is not initialized with proper main thread executor. " +
				"Call to ExecutionGraph.start(...) required.");

	this.shuffleMaster = checkNotNull(shuffleMaster);

	this.partitionTracker = checkNotNull(partitionTracker);

	this.resultPartitionAvailabilityChecker = new ExecutionGraphResultPartitionAvailabilityChecker(
		this::createResultPartitionId,
		partitionTracker);
}
 

@Override
public SchedulerNG createInstance(
		final Logger log,
		final JobGraph jobGraph,
		final BackPressureStatsTracker backPressureStatsTracker,
		final Executor ioExecutor,
		final Configuration jobMasterConfiguration,
		final SlotProvider slotProvider,
		final ScheduledExecutorService futureExecutor,
		final ClassLoader userCodeLoader,
		final CheckpointRecoveryFactory checkpointRecoveryFactory,
		final Time rpcTimeout,
		final BlobWriter blobWriter,
		final JobManagerJobMetricGroup jobManagerJobMetricGroup,
		final Time slotRequestTimeout,
		final ShuffleMaster<?> shuffleMaster,
		final JobMasterPartitionTracker partitionTracker) throws Exception {

	final SchedulingStrategyFactory schedulingStrategyFactory = createSchedulingStrategyFactory(jobGraph.getScheduleMode());
	final RestartBackoffTimeStrategy restartBackoffTimeStrategy = RestartBackoffTimeStrategyFactoryLoader
		.createRestartBackoffTimeStrategyFactory(
			jobGraph
				.getSerializedExecutionConfig()
				.deserializeValue(userCodeLoader)
				.getRestartStrategy(),
			jobMasterConfiguration,
			jobGraph.isCheckpointingEnabled())
		.create();
	log.info("Using restart back off time strategy {} for {} ({}).", restartBackoffTimeStrategy, jobGraph.getName(), jobGraph.getJobID());

	final ExecutionSlotAllocatorFactory slotAllocatorFactory =
		createExecutionSlotAllocatorFactory(
			jobGraph.getScheduleMode(),
			slotProvider,
			slotRequestTimeout,
			schedulingStrategyFactory);

	return new DefaultScheduler(
		log,
		jobGraph,
		backPressureStatsTracker,
		ioExecutor,
		jobMasterConfiguration,
		futureExecutor,
		new ScheduledExecutorServiceAdapter(futureExecutor),
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		blobWriter,
		jobManagerJobMetricGroup,
		shuffleMaster,
		partitionTracker,
		schedulingStrategyFactory,
		FailoverStrategyFactoryLoader.loadFailoverStrategyFactory(jobMasterConfiguration),
		restartBackoffTimeStrategy,
		new DefaultExecutionVertexOperations(),
		new ExecutionVertexVersioner(),
		slotAllocatorFactory);
}
 

public void setBlobWriter(BlobWriter blobWriter) {
	this.blobWriter = blobWriter;
}
 

public SchedulerBase(
	final Logger log,
	final JobGraph jobGraph,
	final BackPressureStatsTracker backPressureStatsTracker,
	final Executor ioExecutor,
	final Configuration jobMasterConfiguration,
	final SlotProvider slotProvider,
	final ScheduledExecutorService futureExecutor,
	final ClassLoader userCodeLoader,
	final CheckpointRecoveryFactory checkpointRecoveryFactory,
	final Time rpcTimeout,
	final RestartStrategyFactory restartStrategyFactory,
	final BlobWriter blobWriter,
	final JobManagerJobMetricGroup jobManagerJobMetricGroup,
	final Time slotRequestTimeout,
	final ShuffleMaster<?> shuffleMaster,
	final JobMasterPartitionTracker partitionTracker,
	final ExecutionVertexVersioner executionVertexVersioner,
	final boolean legacyScheduling) throws Exception {

	this.log = checkNotNull(log);
	this.jobGraph = checkNotNull(jobGraph);
	this.backPressureStatsTracker = checkNotNull(backPressureStatsTracker);
	this.ioExecutor = checkNotNull(ioExecutor);
	this.jobMasterConfiguration = checkNotNull(jobMasterConfiguration);
	this.slotProvider = checkNotNull(slotProvider);
	this.futureExecutor = checkNotNull(futureExecutor);
	this.userCodeLoader = checkNotNull(userCodeLoader);
	this.checkpointRecoveryFactory = checkNotNull(checkpointRecoveryFactory);
	this.rpcTimeout = checkNotNull(rpcTimeout);

	final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration =
		jobGraph.getSerializedExecutionConfig()
			.deserializeValue(userCodeLoader)
			.getRestartStrategy();

	this.restartStrategy = RestartStrategyResolving.resolve(restartStrategyConfiguration,
		restartStrategyFactory,
		jobGraph.isCheckpointingEnabled());

	if (legacyScheduling) {
		log.info("Using restart strategy {} for {} ({}).", this.restartStrategy, jobGraph.getName(), jobGraph.getJobID());
	}

	this.blobWriter = checkNotNull(blobWriter);
	this.jobManagerJobMetricGroup = checkNotNull(jobManagerJobMetricGroup);
	this.slotRequestTimeout = checkNotNull(slotRequestTimeout);
	this.executionVertexVersioner = checkNotNull(executionVertexVersioner);
	this.legacyScheduling = legacyScheduling;

	this.executionGraph = createAndRestoreExecutionGraph(jobManagerJobMetricGroup, checkNotNull(shuffleMaster), checkNotNull(partitionTracker));
	this.schedulingTopology = executionGraph.getSchedulingTopology();

	final StateLocationRetriever stateLocationRetriever =
		executionVertexId -> getExecutionVertex(executionVertexId).getPreferredLocationBasedOnState();
	final InputsLocationsRetriever inputsLocationsRetriever = new ExecutionGraphToInputsLocationsRetrieverAdapter(executionGraph);
	this.preferredLocationsRetriever = new DefaultPreferredLocationsRetriever(stateLocationRetriever, inputsLocationsRetriever);

	this.coordinatorMap = createCoordinatorMap();
}
 

DefaultScheduler(
	final Logger log,
	final JobGraph jobGraph,
	final BackPressureStatsTracker backPressureStatsTracker,
	final Executor ioExecutor,
	final Configuration jobMasterConfiguration,
	final ScheduledExecutorService futureExecutor,
	final ScheduledExecutor delayExecutor,
	final ClassLoader userCodeLoader,
	final CheckpointRecoveryFactory checkpointRecoveryFactory,
	final Time rpcTimeout,
	final BlobWriter blobWriter,
	final JobManagerJobMetricGroup jobManagerJobMetricGroup,
	final ShuffleMaster<?> shuffleMaster,
	final JobMasterPartitionTracker partitionTracker,
	final SchedulingStrategyFactory schedulingStrategyFactory,
	final FailoverStrategy.Factory failoverStrategyFactory,
	final RestartBackoffTimeStrategy restartBackoffTimeStrategy,
	final ExecutionVertexOperations executionVertexOperations,
	final ExecutionVertexVersioner executionVertexVersioner,
	final ExecutionSlotAllocatorFactory executionSlotAllocatorFactory) throws Exception {

	super(
		log,
		jobGraph,
		backPressureStatsTracker,
		ioExecutor,
		jobMasterConfiguration,
		new ThrowingSlotProvider(), // this is not used any more in the new scheduler
		futureExecutor,
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		new ThrowingRestartStrategy.ThrowingRestartStrategyFactory(),
		blobWriter,
		jobManagerJobMetricGroup,
		Time.seconds(0), // this is not used any more in the new scheduler
		shuffleMaster,
		partitionTracker,
		executionVertexVersioner,
		false);

	this.log = log;

	this.delayExecutor = checkNotNull(delayExecutor);
	this.userCodeLoader = checkNotNull(userCodeLoader);
	this.executionVertexOperations = checkNotNull(executionVertexOperations);

	final FailoverStrategy failoverStrategy = failoverStrategyFactory.create(
		getSchedulingTopology(),
		getResultPartitionAvailabilityChecker());
	log.info("Using failover strategy {} for {} ({}).", failoverStrategy, jobGraph.getName(), jobGraph.getJobID());

	this.executionFailureHandler = new ExecutionFailureHandler(
		getSchedulingTopology(),
		failoverStrategy,
		restartBackoffTimeStrategy);
	this.schedulingStrategy = schedulingStrategyFactory.createInstance(this, getSchedulingTopology());
	this.executionSlotAllocator = checkNotNull(executionSlotAllocatorFactory).createInstance(getPreferredLocationsRetriever());

	this.verticesWaitingForRestart = new HashSet<>();
}
 

public DefaultSchedulerBuilder setBlobWriter(final BlobWriter blobWriter) {
	this.blobWriter = blobWriter;
	return this;
}
 

@Nonnull
public BlobWriter getBlobWriter() {
	return blobWriter;
}
 

public TestingExecutionGraphBuilder setBlobWriter(final BlobWriter blobWriter) {
	this.blobWriter = blobWriter;
	return this;
}
 

public void setBlobWriter(BlobWriter blobWriter) {
	this.blobWriter = blobWriter;
}
 

/**
 * Builds the ExecutionGraph from the JobGraph.
 * If a prior execution graph exists, the JobGraph will be attached. If no prior execution
 * graph exists, then the JobGraph will become attach to a new empty execution graph.
 */
public static ExecutionGraph buildGraph(
		@Nullable ExecutionGraph prior,
		JobGraph jobGraph,
		Configuration jobManagerConfig,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		SlotProvider slotProvider,
		ClassLoader classLoader,
		CheckpointRecoveryFactory recoveryFactory,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		MetricGroup metrics,
		BlobWriter blobWriter,
		Time allocationTimeout,
		Logger log,
		ShuffleMaster<?> shuffleMaster,
		PartitionTracker partitionTracker) throws JobExecutionException, JobException {

	final FailoverStrategy.Factory failoverStrategy =
		FailoverStrategyLoader.loadFailoverStrategy(jobManagerConfig, log);

	return buildGraph(
		prior,
		jobGraph,
		jobManagerConfig,
		futureExecutor,
		ioExecutor,
		slotProvider,
		classLoader,
		recoveryFactory,
		rpcTimeout,
		restartStrategy,
		metrics,
		blobWriter,
		allocationTimeout,
		log,
		shuffleMaster,
		partitionTracker,
		failoverStrategy);
}
 

public final BlobWriter getBlobWriter() {
	return blobWriter;
}
 

public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		int maxPriorAttemptsHistoryLength,
		FailoverStrategy.Factory failoverStrategyFactory,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout,
		PartitionReleaseStrategy.Factory partitionReleaseStrategyFactory,
		ShuffleMaster<?> shuffleMaster,
		PartitionTracker partitionTracker,
		ScheduleMode scheduleMode,
		boolean allowQueuedScheduling) throws IOException {

	checkNotNull(futureExecutor);

	this.jobInformation = Preconditions.checkNotNull(jobInformation);

	this.blobWriter = Preconditions.checkNotNull(blobWriter);

	this.scheduleMode = checkNotNull(scheduleMode);

	this.allowQueuedScheduling = allowQueuedScheduling;

	this.jobInformationOrBlobKey = BlobWriter.serializeAndTryOffload(jobInformation, jobInformation.getJobId(), blobWriter);

	this.futureExecutor = Preconditions.checkNotNull(futureExecutor);
	this.ioExecutor = Preconditions.checkNotNull(ioExecutor);

	this.slotProviderStrategy = SlotProviderStrategy.from(
		scheduleMode,
		slotProvider,
		allocationTimeout,
		allowQueuedScheduling);
	this.userClassLoader = Preconditions.checkNotNull(userClassLoader, "userClassLoader");

	this.tasks = new ConcurrentHashMap<>(16);
	this.intermediateResults = new ConcurrentHashMap<>(16);
	this.verticesInCreationOrder = new ArrayList<>(16);
	this.currentExecutions = new ConcurrentHashMap<>(16);

	this.jobStatusListeners  = new CopyOnWriteArrayList<>();

	this.stateTimestamps = new long[JobStatus.values().length];
	this.stateTimestamps[JobStatus.CREATED.ordinal()] = System.currentTimeMillis();

	this.rpcTimeout = checkNotNull(rpcTimeout);
	this.allocationTimeout = checkNotNull(allocationTimeout);

	this.partitionReleaseStrategyFactory = checkNotNull(partitionReleaseStrategyFactory);

	this.restartStrategy = restartStrategy;
	this.kvStateLocationRegistry = new KvStateLocationRegistry(jobInformation.getJobId(), getAllVertices());

	this.verticesFinished = new AtomicInteger();

	this.globalModVersion = 1L;

	// the failover strategy must be instantiated last, so that the execution graph
	// is ready by the time the failover strategy sees it
	this.failoverStrategy = checkNotNull(failoverStrategyFactory.create(this), "null failover strategy");

	this.maxPriorAttemptsHistoryLength = maxPriorAttemptsHistoryLength;

	this.schedulingFuture = null;
	this.jobMasterMainThreadExecutor =
		new ComponentMainThreadExecutor.DummyComponentMainThreadExecutor(
			"ExecutionGraph is not initialized with proper main thread executor. " +
				"Call to ExecutionGraph.start(...) required.");

	this.shuffleMaster = checkNotNull(shuffleMaster);

	this.partitionTracker = checkNotNull(partitionTracker);

	this.resultPartitionAvailabilityChecker = new ExecutionGraphResultPartitionAvailabilityChecker(
		this::createResultPartitionId,
		partitionTracker);

	LOG.info("Job recovers via failover strategy: {}", failoverStrategy.getStrategyName());
}
 

public LegacyScheduler(
		final Logger log,
		final JobGraph jobGraph,
		final BackPressureStatsTracker backPressureStatsTracker,
		final Executor ioExecutor,
		final Configuration jobMasterConfiguration,
		final SlotProvider slotProvider,
		final ScheduledExecutorService futureExecutor,
		final ClassLoader userCodeLoader,
		final CheckpointRecoveryFactory checkpointRecoveryFactory,
		final Time rpcTimeout,
		final RestartStrategyFactory restartStrategyFactory,
		final BlobWriter blobWriter,
		final JobManagerJobMetricGroup jobManagerJobMetricGroup,
		final Time slotRequestTimeout,
		final ShuffleMaster<?> shuffleMaster,
		final PartitionTracker partitionTracker) throws Exception {

	this.log = checkNotNull(log);
	this.jobGraph = checkNotNull(jobGraph);
	this.backPressureStatsTracker = checkNotNull(backPressureStatsTracker);
	this.ioExecutor = checkNotNull(ioExecutor);
	this.jobMasterConfiguration = checkNotNull(jobMasterConfiguration);
	this.slotProvider = checkNotNull(slotProvider);
	this.futureExecutor = checkNotNull(futureExecutor);
	this.userCodeLoader = checkNotNull(userCodeLoader);
	this.checkpointRecoveryFactory = checkNotNull(checkpointRecoveryFactory);
	this.rpcTimeout = checkNotNull(rpcTimeout);

	final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration =
		jobGraph.getSerializedExecutionConfig()
			.deserializeValue(userCodeLoader)
			.getRestartStrategy();

	this.restartStrategy = RestartStrategyResolving.resolve(restartStrategyConfiguration,
		restartStrategyFactory,
		jobGraph.isCheckpointingEnabled());

	log.info("Using restart strategy {} for {} ({}).", this.restartStrategy, jobGraph.getName(), jobGraph.getJobID());

	this.blobWriter = checkNotNull(blobWriter);
	this.slotRequestTimeout = checkNotNull(slotRequestTimeout);

	this.executionGraph = createAndRestoreExecutionGraph(jobManagerJobMetricGroup, checkNotNull(shuffleMaster), checkNotNull(partitionTracker));
}
 

@Nonnull
public BlobWriter getBlobWriter() {
	return blobWriter;
}