org.apache.flink.runtime.metrics.groups.JobManagerJobMetricGroup Java Examples

The following examples show how to use org.apache.flink.runtime.metrics.groups.JobManagerJobMetricGroup. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JobMaster.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private void resetAndScheduleExecutionGraph() throws Exception {
	validateRunsInMainThread();

	final CompletableFuture<Void> executionGraphAssignedFuture;

	if (executionGraph.getState() == JobStatus.CREATED) {
		executionGraphAssignedFuture = CompletableFuture.completedFuture(null);
		executionGraph.start(getMainThreadExecutor());
	} else {
		suspendAndClearExecutionGraphFields(new FlinkException("ExecutionGraph is being reset in order to be rescheduled."));
		final JobManagerJobMetricGroup newJobManagerJobMetricGroup = jobMetricGroupFactory.create(jobGraph);
		final ExecutionGraph newExecutionGraph = createAndRestoreExecutionGraph(newJobManagerJobMetricGroup);

		executionGraphAssignedFuture = executionGraph.getTerminationFuture().handle(
			(JobStatus ignored, Throwable throwable) -> {
				newExecutionGraph.start(getMainThreadExecutor());
				assignExecutionGraph(newExecutionGraph, newJobManagerJobMetricGroup);
				return null;
			});
	}

	executionGraphAssignedFuture.thenRun(this::scheduleExecutionGraph);
}
 
Example #2
Source File: SchedulerBase.java    From flink with Apache License 2.0 6 votes vote down vote up
private ExecutionGraph createAndRestoreExecutionGraph(
	JobManagerJobMetricGroup currentJobManagerJobMetricGroup,
	ShuffleMaster<?> shuffleMaster,
	JobMasterPartitionTracker partitionTracker) throws Exception {

	ExecutionGraph newExecutionGraph = createExecutionGraph(currentJobManagerJobMetricGroup, shuffleMaster, partitionTracker);

	final CheckpointCoordinator checkpointCoordinator = newExecutionGraph.getCheckpointCoordinator();

	if (checkpointCoordinator != null) {
		// check whether we find a valid checkpoint
		if (!checkpointCoordinator.restoreLatestCheckpointedStateToAll(
			new HashSet<>(newExecutionGraph.getAllVertices().values()),
			false)) {

			// check whether we can restore from a savepoint
			tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, jobGraph.getSavepointRestoreSettings());
		}
	}

	return newExecutionGraph;
}
 
Example #3
Source File: SchedulerNGFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
SchedulerNG createInstance(
Logger log,
JobGraph jobGraph,
BackPressureStatsTracker backPressureStatsTracker,
Executor ioExecutor,
Configuration jobMasterConfiguration,
SlotProvider slotProvider,
ScheduledExecutorService futureExecutor,
ClassLoader userCodeLoader,
CheckpointRecoveryFactory checkpointRecoveryFactory,
Time rpcTimeout,
BlobWriter blobWriter,
JobManagerJobMetricGroup jobManagerJobMetricGroup,
Time slotRequestTimeout,
ShuffleMaster<?> shuffleMaster,
JobMasterPartitionTracker partitionTracker) throws Exception;
 
Example #4
Source File: JobMaster.java    From flink with Apache License 2.0 6 votes vote down vote up
private void resetAndStartScheduler() throws Exception {
	validateRunsInMainThread();

	final CompletableFuture<Void> schedulerAssignedFuture;

	if (schedulerNG.requestJobStatus() == JobStatus.CREATED) {
		schedulerAssignedFuture = CompletableFuture.completedFuture(null);
		schedulerNG.setMainThreadExecutor(getMainThreadExecutor());
	} else {
		suspendAndClearSchedulerFields(new FlinkException("ExecutionGraph is being reset in order to be rescheduled."));
		final JobManagerJobMetricGroup newJobManagerJobMetricGroup = jobMetricGroupFactory.create(jobGraph);
		final SchedulerNG newScheduler = createScheduler(newJobManagerJobMetricGroup);

		schedulerAssignedFuture = schedulerNG.getTerminationFuture().handle(
			(ignored, throwable) -> {
				newScheduler.setMainThreadExecutor(getMainThreadExecutor());
				assignScheduler(newScheduler, newJobManagerJobMetricGroup);
				return null;
			}
		);
	}

	schedulerAssignedFuture.thenRun(this::startScheduling);
}
 
Example #5
Source File: JobMaster.java    From flink with Apache License 2.0 6 votes vote down vote up
private SchedulerNG createScheduler(final JobManagerJobMetricGroup jobManagerJobMetricGroup) throws Exception {
	return schedulerNGFactory.createInstance(
		log,
		jobGraph,
		backPressureStatsTracker,
		scheduledExecutorService,
		jobMasterConfiguration.getConfiguration(),
		scheduler,
		scheduledExecutorService,
		userCodeLoader,
		highAvailabilityServices.getCheckpointRecoveryFactory(),
		rpcTimeout,
		blobWriter,
		jobManagerJobMetricGroup,
		jobMasterConfiguration.getSlotRequestTimeout(),
		shuffleMaster,
		partitionTracker);
}
 
Example #6
Source File: LegacyScheduler.java    From flink with Apache License 2.0 6 votes vote down vote up
private ExecutionGraph createExecutionGraph(
		JobManagerJobMetricGroup currentJobManagerJobMetricGroup,
		ShuffleMaster<?> shuffleMaster,
		final PartitionTracker partitionTracker) throws JobExecutionException, JobException {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobMasterConfiguration,
		futureExecutor,
		ioExecutor,
		slotProvider,
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		restartStrategy,
		currentJobManagerJobMetricGroup,
		blobWriter,
		slotRequestTimeout,
		log,
		shuffleMaster,
		partitionTracker);
}
 
Example #7
Source File: LegacyScheduler.java    From flink with Apache License 2.0 6 votes vote down vote up
private ExecutionGraph createAndRestoreExecutionGraph(
		JobManagerJobMetricGroup currentJobManagerJobMetricGroup,
		ShuffleMaster<?> shuffleMaster,
		PartitionTracker partitionTracker) throws Exception {

	ExecutionGraph newExecutionGraph = createExecutionGraph(currentJobManagerJobMetricGroup, shuffleMaster, partitionTracker);

	final CheckpointCoordinator checkpointCoordinator = newExecutionGraph.getCheckpointCoordinator();

	if (checkpointCoordinator != null) {
		// check whether we find a valid checkpoint
		if (!checkpointCoordinator.restoreLatestCheckpointedState(
			newExecutionGraph.getAllVertices(),
			false,
			false)) {

			// check whether we can restore from a savepoint
			tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, jobGraph.getSavepointRestoreSettings());
		}
	}

	return newExecutionGraph;
}
 
Example #8
Source File: SchedulerNGFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
SchedulerNG createInstance(
Logger log,
JobGraph jobGraph,
BackPressureStatsTracker backPressureStatsTracker,
Executor ioExecutor,
Configuration jobMasterConfiguration,
SlotProvider slotProvider,
ScheduledExecutorService futureExecutor,
ClassLoader userCodeLoader,
CheckpointRecoveryFactory checkpointRecoveryFactory,
Time rpcTimeout,
BlobWriter blobWriter,
JobManagerJobMetricGroup jobManagerJobMetricGroup,
Time slotRequestTimeout,
ShuffleMaster<?> shuffleMaster,
PartitionTracker partitionTracker) throws Exception;
 
Example #9
Source File: JobMaster.java    From flink with Apache License 2.0 6 votes vote down vote up
private void resetAndStartScheduler() throws Exception {
	validateRunsInMainThread();

	final CompletableFuture<Void> schedulerAssignedFuture;

	if (schedulerNG.requestJobStatus() == JobStatus.CREATED) {
		schedulerAssignedFuture = CompletableFuture.completedFuture(null);
		schedulerNG.setMainThreadExecutor(getMainThreadExecutor());
	} else {
		suspendAndClearSchedulerFields(new FlinkException("ExecutionGraph is being reset in order to be rescheduled."));
		final JobManagerJobMetricGroup newJobManagerJobMetricGroup = jobMetricGroupFactory.create(jobGraph);
		final SchedulerNG newScheduler = createScheduler(newJobManagerJobMetricGroup);

		schedulerAssignedFuture = schedulerNG.getTerminationFuture().handle(
			(ignored, throwable) -> {
				newScheduler.setMainThreadExecutor(getMainThreadExecutor());
				assignScheduler(newScheduler, newJobManagerJobMetricGroup);
				return null;
			}
		);
	}

	schedulerAssignedFuture.thenRun(this::startScheduling);
}
 
Example #10
Source File: JobMaster.java    From flink with Apache License 2.0 6 votes vote down vote up
private SchedulerNG createScheduler(final JobManagerJobMetricGroup jobManagerJobMetricGroup) throws Exception {
	return schedulerNGFactory.createInstance(
		log,
		jobGraph,
		backPressureStatsTracker,
		scheduledExecutorService,
		jobMasterConfiguration.getConfiguration(),
		scheduler,
		scheduledExecutorService,
		userCodeLoader,
		highAvailabilityServices.getCheckpointRecoveryFactory(),
		rpcTimeout,
		blobWriter,
		jobManagerJobMetricGroup,
		jobMasterConfiguration.getSlotRequestTimeout(),
		shuffleMaster,
		partitionTracker);
}
 
Example #11
Source File: JobMaster.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private ExecutionGraph createExecutionGraph(JobManagerJobMetricGroup currentJobManagerJobMetricGroup) throws JobExecutionException, JobException {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobMasterConfiguration.getConfiguration(),
		scheduledExecutorService,
		scheduledExecutorService,
		scheduler,
		userCodeLoader,
		highAvailabilityServices.getCheckpointRecoveryFactory(),
		rpcTimeout,
		restartStrategy,
		currentJobManagerJobMetricGroup,
		blobWriter,
		jobMasterConfiguration.getSlotRequestTimeout(),
		log);
}
 
Example #12
Source File: JobMaster.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private ExecutionGraph createAndRestoreExecutionGraph(JobManagerJobMetricGroup currentJobManagerJobMetricGroup) throws Exception {

		ExecutionGraph newExecutionGraph = createExecutionGraph(currentJobManagerJobMetricGroup);

		final CheckpointCoordinator checkpointCoordinator = newExecutionGraph.getCheckpointCoordinator();

		if (checkpointCoordinator != null) {
			// check whether we find a valid checkpoint
			if (!checkpointCoordinator.restoreLatestCheckpointedState(
				newExecutionGraph.getAllVertices(),
				false,
				false)) {

				// check whether we can restore from a savepoint
				tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, jobGraph.getSavepointRestoreSettings());
			}
		}

		return newExecutionGraph;
	}
 
Example #13
Source File: JobMaster.java    From flink with Apache License 2.0 5 votes vote down vote up
private void assignScheduler(
		SchedulerNG newScheduler,
		JobManagerJobMetricGroup newJobManagerJobMetricGroup) {
	validateRunsInMainThread();
	checkState(schedulerNG.requestJobStatus().isTerminalState());
	checkState(jobManagerJobMetricGroup == null);

	schedulerNG = newScheduler;
	jobManagerJobMetricGroup = newJobManagerJobMetricGroup;
}
 
Example #14
Source File: LegacySchedulerFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public SchedulerNG createInstance(
		final Logger log,
		final JobGraph jobGraph,
		final BackPressureStatsTracker backPressureStatsTracker,
		final Executor ioExecutor,
		final Configuration jobMasterConfiguration,
		final SlotProvider slotProvider,
		final ScheduledExecutorService futureExecutor,
		final ClassLoader userCodeLoader,
		final CheckpointRecoveryFactory checkpointRecoveryFactory,
		final Time rpcTimeout,
		final BlobWriter blobWriter,
		final JobManagerJobMetricGroup jobManagerJobMetricGroup,
		final Time slotRequestTimeout,
		final ShuffleMaster<?> shuffleMaster,
		final PartitionTracker partitionTracker) throws Exception {

	return new LegacyScheduler(
		log,
		jobGraph,
		backPressureStatsTracker,
		ioExecutor,
		jobMasterConfiguration,
		slotProvider,
		futureExecutor,
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		restartStrategyFactory,
		blobWriter,
		jobManagerJobMetricGroup,
		slotRequestTimeout,
		shuffleMaster,
		partitionTracker);
}
 
Example #15
Source File: DefaultScheduler.java    From flink with Apache License 2.0 5 votes vote down vote up
public DefaultScheduler(
		final Logger log,
		final JobGraph jobGraph,
		final BackPressureStatsTracker backPressureStatsTracker,
		final Executor ioExecutor,
		final Configuration jobMasterConfiguration,
		final SlotProvider slotProvider,
		final ScheduledExecutorService futureExecutor,
		final ClassLoader userCodeLoader,
		final CheckpointRecoveryFactory checkpointRecoveryFactory,
		final Time rpcTimeout,
		final BlobWriter blobWriter,
		final JobManagerJobMetricGroup jobManagerJobMetricGroup,
		final Time slotRequestTimeout,
		final ShuffleMaster<?> shuffleMaster,
		final PartitionTracker partitionTracker) throws Exception {

	super(
		log,
		jobGraph,
		backPressureStatsTracker,
		ioExecutor,
		jobMasterConfiguration,
		slotProvider,
		futureExecutor,
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		new ThrowingRestartStrategy.ThrowingRestartStrategyFactory(),
		blobWriter,
		jobManagerJobMetricGroup,
		slotRequestTimeout,
		shuffleMaster,
		partitionTracker);
}
 
Example #16
Source File: SchedulerBase.java    From flink with Apache License 2.0 5 votes vote down vote up
private ExecutionGraph createExecutionGraph(
	JobManagerJobMetricGroup currentJobManagerJobMetricGroup,
	ShuffleMaster<?> shuffleMaster,
	final JobMasterPartitionTracker partitionTracker) throws JobExecutionException, JobException {

	final FailoverStrategy.Factory failoverStrategy = legacyScheduling ?
		FailoverStrategyLoader.loadFailoverStrategy(jobMasterConfiguration, log) :
		new NoOpFailoverStrategy.Factory();

	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobMasterConfiguration,
		futureExecutor,
		ioExecutor,
		slotProvider,
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		restartStrategy,
		currentJobManagerJobMetricGroup,
		blobWriter,
		slotRequestTimeout,
		log,
		shuffleMaster,
		partitionTracker,
		failoverStrategy);
}
 
Example #17
Source File: JobMaster.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void assignExecutionGraph(
		ExecutionGraph newExecutionGraph,
		JobManagerJobMetricGroup newJobManagerJobMetricGroup) {
	validateRunsInMainThread();
	checkState(executionGraph.getState().isTerminalState());
	checkState(jobManagerJobMetricGroup == null);

	executionGraph = newExecutionGraph;
	jobManagerJobMetricGroup = newJobManagerJobMetricGroup;
}
 
Example #18
Source File: DefaultSchedulerFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public SchedulerNG createInstance(
		final Logger log,
		final JobGraph jobGraph,
		final BackPressureStatsTracker backPressureStatsTracker,
		final Executor ioExecutor,
		final Configuration jobMasterConfiguration,
		final SlotProvider slotProvider,
		final ScheduledExecutorService futureExecutor,
		final ClassLoader userCodeLoader,
		final CheckpointRecoveryFactory checkpointRecoveryFactory,
		final Time rpcTimeout,
		final BlobWriter blobWriter,
		final JobManagerJobMetricGroup jobManagerJobMetricGroup,
		final Time slotRequestTimeout,
		final ShuffleMaster<?> shuffleMaster,
		final PartitionTracker partitionTracker) throws Exception {

	return new DefaultScheduler(
		log,
		jobGraph,
		backPressureStatsTracker,
		ioExecutor,
		jobMasterConfiguration,
		slotProvider,
		futureExecutor,
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		blobWriter,
		jobManagerJobMetricGroup,
		slotRequestTimeout,
		shuffleMaster,
		partitionTracker);
}
 
Example #19
Source File: JobMaster.java    From flink with Apache License 2.0 5 votes vote down vote up
private void assignScheduler(
		SchedulerNG newScheduler,
		JobManagerJobMetricGroup newJobManagerJobMetricGroup) {
	validateRunsInMainThread();
	checkState(schedulerNG.requestJobStatus().isTerminalState());
	checkState(jobManagerJobMetricGroup == null);

	schedulerNG = newScheduler;
	jobManagerJobMetricGroup = newJobManagerJobMetricGroup;
}
 
Example #20
Source File: SchedulerTestingUtils.java    From flink with Apache License 2.0 4 votes vote down vote up
public DefaultSchedulerBuilder setJobManagerJobMetricGroup(final JobManagerJobMetricGroup jobManagerJobMetricGroup) {
	this.jobManagerJobMetricGroup = jobManagerJobMetricGroup;
	return this;
}
 
Example #21
Source File: DefaultSchedulerFactory.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public SchedulerNG createInstance(
		final Logger log,
		final JobGraph jobGraph,
		final BackPressureStatsTracker backPressureStatsTracker,
		final Executor ioExecutor,
		final Configuration jobMasterConfiguration,
		final SlotProvider slotProvider,
		final ScheduledExecutorService futureExecutor,
		final ClassLoader userCodeLoader,
		final CheckpointRecoveryFactory checkpointRecoveryFactory,
		final Time rpcTimeout,
		final BlobWriter blobWriter,
		final JobManagerJobMetricGroup jobManagerJobMetricGroup,
		final Time slotRequestTimeout,
		final ShuffleMaster<?> shuffleMaster,
		final JobMasterPartitionTracker partitionTracker) throws Exception {

	final SchedulingStrategyFactory schedulingStrategyFactory = createSchedulingStrategyFactory(jobGraph.getScheduleMode());
	final RestartBackoffTimeStrategy restartBackoffTimeStrategy = RestartBackoffTimeStrategyFactoryLoader
		.createRestartBackoffTimeStrategyFactory(
			jobGraph
				.getSerializedExecutionConfig()
				.deserializeValue(userCodeLoader)
				.getRestartStrategy(),
			jobMasterConfiguration,
			jobGraph.isCheckpointingEnabled())
		.create();
	log.info("Using restart back off time strategy {} for {} ({}).", restartBackoffTimeStrategy, jobGraph.getName(), jobGraph.getJobID());

	final ExecutionSlotAllocatorFactory slotAllocatorFactory =
		createExecutionSlotAllocatorFactory(
			jobGraph.getScheduleMode(),
			slotProvider,
			slotRequestTimeout,
			schedulingStrategyFactory);

	return new DefaultScheduler(
		log,
		jobGraph,
		backPressureStatsTracker,
		ioExecutor,
		jobMasterConfiguration,
		futureExecutor,
		new ScheduledExecutorServiceAdapter(futureExecutor),
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		blobWriter,
		jobManagerJobMetricGroup,
		shuffleMaster,
		partitionTracker,
		schedulingStrategyFactory,
		FailoverStrategyFactoryLoader.loadFailoverStrategyFactory(jobMasterConfiguration),
		restartBackoffTimeStrategy,
		new DefaultExecutionVertexOperations(),
		new ExecutionVertexVersioner(),
		slotAllocatorFactory);
}
 
Example #22
Source File: SchedulerBase.java    From flink with Apache License 2.0 4 votes vote down vote up
public SchedulerBase(
	final Logger log,
	final JobGraph jobGraph,
	final BackPressureStatsTracker backPressureStatsTracker,
	final Executor ioExecutor,
	final Configuration jobMasterConfiguration,
	final SlotProvider slotProvider,
	final ScheduledExecutorService futureExecutor,
	final ClassLoader userCodeLoader,
	final CheckpointRecoveryFactory checkpointRecoveryFactory,
	final Time rpcTimeout,
	final RestartStrategyFactory restartStrategyFactory,
	final BlobWriter blobWriter,
	final JobManagerJobMetricGroup jobManagerJobMetricGroup,
	final Time slotRequestTimeout,
	final ShuffleMaster<?> shuffleMaster,
	final JobMasterPartitionTracker partitionTracker,
	final ExecutionVertexVersioner executionVertexVersioner,
	final boolean legacyScheduling) throws Exception {

	this.log = checkNotNull(log);
	this.jobGraph = checkNotNull(jobGraph);
	this.backPressureStatsTracker = checkNotNull(backPressureStatsTracker);
	this.ioExecutor = checkNotNull(ioExecutor);
	this.jobMasterConfiguration = checkNotNull(jobMasterConfiguration);
	this.slotProvider = checkNotNull(slotProvider);
	this.futureExecutor = checkNotNull(futureExecutor);
	this.userCodeLoader = checkNotNull(userCodeLoader);
	this.checkpointRecoveryFactory = checkNotNull(checkpointRecoveryFactory);
	this.rpcTimeout = checkNotNull(rpcTimeout);

	final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration =
		jobGraph.getSerializedExecutionConfig()
			.deserializeValue(userCodeLoader)
			.getRestartStrategy();

	this.restartStrategy = RestartStrategyResolving.resolve(restartStrategyConfiguration,
		restartStrategyFactory,
		jobGraph.isCheckpointingEnabled());

	if (legacyScheduling) {
		log.info("Using restart strategy {} for {} ({}).", this.restartStrategy, jobGraph.getName(), jobGraph.getJobID());
	}

	this.blobWriter = checkNotNull(blobWriter);
	this.jobManagerJobMetricGroup = checkNotNull(jobManagerJobMetricGroup);
	this.slotRequestTimeout = checkNotNull(slotRequestTimeout);
	this.executionVertexVersioner = checkNotNull(executionVertexVersioner);
	this.legacyScheduling = legacyScheduling;

	this.executionGraph = createAndRestoreExecutionGraph(jobManagerJobMetricGroup, checkNotNull(shuffleMaster), checkNotNull(partitionTracker));
	this.schedulingTopology = executionGraph.getSchedulingTopology();

	final StateLocationRetriever stateLocationRetriever =
		executionVertexId -> getExecutionVertex(executionVertexId).getPreferredLocationBasedOnState();
	final InputsLocationsRetriever inputsLocationsRetriever = new ExecutionGraphToInputsLocationsRetrieverAdapter(executionGraph);
	this.preferredLocationsRetriever = new DefaultPreferredLocationsRetriever(stateLocationRetriever, inputsLocationsRetriever);

	this.coordinatorMap = createCoordinatorMap();
}
 
Example #23
Source File: DefaultScheduler.java    From flink with Apache License 2.0 4 votes vote down vote up
DefaultScheduler(
	final Logger log,
	final JobGraph jobGraph,
	final BackPressureStatsTracker backPressureStatsTracker,
	final Executor ioExecutor,
	final Configuration jobMasterConfiguration,
	final ScheduledExecutorService futureExecutor,
	final ScheduledExecutor delayExecutor,
	final ClassLoader userCodeLoader,
	final CheckpointRecoveryFactory checkpointRecoveryFactory,
	final Time rpcTimeout,
	final BlobWriter blobWriter,
	final JobManagerJobMetricGroup jobManagerJobMetricGroup,
	final ShuffleMaster<?> shuffleMaster,
	final JobMasterPartitionTracker partitionTracker,
	final SchedulingStrategyFactory schedulingStrategyFactory,
	final FailoverStrategy.Factory failoverStrategyFactory,
	final RestartBackoffTimeStrategy restartBackoffTimeStrategy,
	final ExecutionVertexOperations executionVertexOperations,
	final ExecutionVertexVersioner executionVertexVersioner,
	final ExecutionSlotAllocatorFactory executionSlotAllocatorFactory) throws Exception {

	super(
		log,
		jobGraph,
		backPressureStatsTracker,
		ioExecutor,
		jobMasterConfiguration,
		new ThrowingSlotProvider(), // this is not used any more in the new scheduler
		futureExecutor,
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		new ThrowingRestartStrategy.ThrowingRestartStrategyFactory(),
		blobWriter,
		jobManagerJobMetricGroup,
		Time.seconds(0), // this is not used any more in the new scheduler
		shuffleMaster,
		partitionTracker,
		executionVertexVersioner,
		false);

	this.log = log;

	this.delayExecutor = checkNotNull(delayExecutor);
	this.userCodeLoader = checkNotNull(userCodeLoader);
	this.executionVertexOperations = checkNotNull(executionVertexOperations);

	final FailoverStrategy failoverStrategy = failoverStrategyFactory.create(
		getSchedulingTopology(),
		getResultPartitionAvailabilityChecker());
	log.info("Using failover strategy {} for {} ({}).", failoverStrategy, jobGraph.getName(), jobGraph.getJobID());

	this.executionFailureHandler = new ExecutionFailureHandler(
		getSchedulingTopology(),
		failoverStrategy,
		restartBackoffTimeStrategy);
	this.schedulingStrategy = schedulingStrategyFactory.createInstance(this, getSchedulingTopology());
	this.executionSlotAllocator = checkNotNull(executionSlotAllocatorFactory).createInstance(getPreferredLocationsRetriever());

	this.verticesWaitingForRestart = new HashSet<>();
}
 
Example #24
Source File: DefaultJobManagerJobMetricGroupFactory.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public JobManagerJobMetricGroup create(@Nonnull JobGraph jobGraph) {
	return jobManagerMetricGroup.addJob(jobGraph);
}
 
Example #25
Source File: DefaultJobManagerJobMetricGroupFactory.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
public JobManagerJobMetricGroup create(@Nonnull JobGraph jobGraph) {
	return jobManagerMetricGroup.addJob(jobGraph);
}
 
Example #26
Source File: LegacyScheduler.java    From flink with Apache License 2.0 4 votes vote down vote up
public LegacyScheduler(
		final Logger log,
		final JobGraph jobGraph,
		final BackPressureStatsTracker backPressureStatsTracker,
		final Executor ioExecutor,
		final Configuration jobMasterConfiguration,
		final SlotProvider slotProvider,
		final ScheduledExecutorService futureExecutor,
		final ClassLoader userCodeLoader,
		final CheckpointRecoveryFactory checkpointRecoveryFactory,
		final Time rpcTimeout,
		final RestartStrategyFactory restartStrategyFactory,
		final BlobWriter blobWriter,
		final JobManagerJobMetricGroup jobManagerJobMetricGroup,
		final Time slotRequestTimeout,
		final ShuffleMaster<?> shuffleMaster,
		final PartitionTracker partitionTracker) throws Exception {

	this.log = checkNotNull(log);
	this.jobGraph = checkNotNull(jobGraph);
	this.backPressureStatsTracker = checkNotNull(backPressureStatsTracker);
	this.ioExecutor = checkNotNull(ioExecutor);
	this.jobMasterConfiguration = checkNotNull(jobMasterConfiguration);
	this.slotProvider = checkNotNull(slotProvider);
	this.futureExecutor = checkNotNull(futureExecutor);
	this.userCodeLoader = checkNotNull(userCodeLoader);
	this.checkpointRecoveryFactory = checkNotNull(checkpointRecoveryFactory);
	this.rpcTimeout = checkNotNull(rpcTimeout);

	final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration =
		jobGraph.getSerializedExecutionConfig()
			.deserializeValue(userCodeLoader)
			.getRestartStrategy();

	this.restartStrategy = RestartStrategyResolving.resolve(restartStrategyConfiguration,
		restartStrategyFactory,
		jobGraph.isCheckpointingEnabled());

	log.info("Using restart strategy {} for {} ({}).", this.restartStrategy, jobGraph.getName(), jobGraph.getJobID());

	this.blobWriter = checkNotNull(blobWriter);
	this.slotRequestTimeout = checkNotNull(slotRequestTimeout);

	this.executionGraph = createAndRestoreExecutionGraph(jobManagerJobMetricGroup, checkNotNull(shuffleMaster), checkNotNull(partitionTracker));
}
 
Example #27
Source File: DefaultJobManagerJobMetricGroupFactory.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public JobManagerJobMetricGroup create(@Nonnull JobGraph jobGraph) {
	return jobManagerMetricGroup.addJob(jobGraph);
}
 
Example #28
Source File: JobManagerJobMetricGroupFactory.java    From flink with Apache License 2.0 2 votes vote down vote up
/**
 * Create a new {@link JobManagerJobMetricGroup}.
 *
 * @param jobGraph for which to create a new {@link JobManagerJobMetricGroup}.
 * @return {@link JobManagerJobMetricGroup}
 */
JobManagerJobMetricGroup create(@Nonnull JobGraph jobGraph);
 
Example #29
Source File: JobManagerJobMetricGroupFactory.java    From Flink-CEPplus with Apache License 2.0 2 votes vote down vote up
/**
 * Create a new {@link JobManagerJobMetricGroup}.
 *
 * @param jobGraph for which to create a new {@link JobManagerJobMetricGroup}.
 * @return {@link JobManagerJobMetricGroup}
 */
JobManagerJobMetricGroup create(@Nonnull JobGraph jobGraph);
 
Example #30
Source File: JobManagerJobMetricGroupFactory.java    From flink with Apache License 2.0 2 votes vote down vote up
/**
 * Create a new {@link JobManagerJobMetricGroup}.
 *
 * @param jobGraph for which to create a new {@link JobManagerJobMetricGroup}.
 * @return {@link JobManagerJobMetricGroup}
 */
JobManagerJobMetricGroup create(@Nonnull JobGraph jobGraph);