org.apache.flink.runtime.executiongraph.failover.FailoverStrategy Java Examples

private ExecutionGraph createSampleGraph(FailoverStrategy failoverStrategy) throws Exception {
	final JobID jid = new JobID();
	final int parallelism = new Random().nextInt(10) + 1;

	JobVertex jv = new JobVertex("test vertex");
	jv.setInvokableClass(NoOpInvokable.class);
	jv.setParallelism(parallelism);

	JobGraph jg = new JobGraph(jid, "testjob", jv);

	final SimpleSlotProvider slotProvider = new SimpleSlotProvider(parallelism);

	// build a simple execution graph with on job vertex, parallelism 2
	final ExecutionGraph graph = TestingExecutionGraphBuilder
		.newBuilder()
		.setJobGraph(jg)
		.setRestartStrategy(new InfiniteDelayRestartStrategy())
		.setFailoverStrategyFactory(new CustomStrategy(failoverStrategy))
		.setSlotProvider(slotProvider)
		.build();

	graph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	return graph;
}
 

@VisibleForTesting
ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time timeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategy,
		SlotProvider slotProvider) throws IOException {
	this(
		jobInformation,
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		failoverStrategy,
		slotProvider,
		ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		timeout);
}
 

@VisibleForTesting
ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time timeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategy,
		SlotProvider slotProvider) throws IOException {
	this(
		jobInformation,
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		failoverStrategy,
		slotProvider,
		ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		timeout);
}
 

private static ExecutionGraph createSimpleExecutionGraph(
	final RestartStrategy restartStrategy,
	final FailoverStrategy.Factory failoverStrategyFactory,
	final SlotProvider slotProvider,
	final JobGraph jobGraph) throws IOException, JobException {

	final ExecutionGraph executionGraph = new ExecutionGraph(
		new JobInformation(
			TEST_JOB_ID,
			"Test job",
			new SerializedValue<>(new ExecutionConfig()),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList()),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		restartStrategy,
		failoverStrategyFactory,
		slotProvider);

	executionGraph.start(mainThreadExecutor);
	executionGraph.attachJobGraph(jobGraph.getVerticesSortedTopologicallyFromSources());

	return executionGraph;
}
 

private ExecutionGraph createSampleGraph(FailoverStrategy failoverStrategy) throws Exception {

		final JobID jid = new JobID();
		final int parallelism = new Random().nextInt(10) + 1;

		final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jid, parallelism);

		// build a simple execution graph with on job vertex, parallelism 2
		final ExecutionGraph graph = new ExecutionGraph(
			new DummyJobInformation(
				jid,
				"test job"),
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			Time.seconds(10),
			new InfiniteDelayRestartStrategy(),
			new CustomStrategy(failoverStrategy),
			slotProvider);

		graph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

		JobVertex jv = new JobVertex("test vertex");
		jv.setInvokableClass(NoOpInvokable.class);
		jv.setParallelism(parallelism);

		JobGraph jg = new JobGraph(jid, "testjob", jv);
		graph.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());

		return graph;
	}
 

private ExecutionGraph createSampleGraph(FailoverStrategy failoverStrategy) throws Exception {

		final JobID jid = new JobID();
		final int parallelism = new Random().nextInt(10) + 1;

		final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jid, parallelism);

		// build a simple execution graph with on job vertex, parallelism 2
		final ExecutionGraph graph = new ExecutionGraph(
			new DummyJobInformation(
				jid,
				"test job"),
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			Time.seconds(10),
			new InfiniteDelayRestartStrategy(),
			new CustomStrategy(failoverStrategy),
			slotProvider);

		graph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());

		JobVertex jv = new JobVertex("test vertex");
		jv.setInvokableClass(NoOpInvokable.class);
		jv.setParallelism(parallelism);

		JobGraph jg = new JobGraph(jid, "testjob", jv);
		graph.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());

		return graph;
	}
 

@Test
public void testSkipFailoverIfExecutionStateIsNotRunning() throws Exception {
	final ExecutionGraph executionGraph = createExecutionGraph();

	final Iterator<ExecutionVertex> vertexIterator = executionGraph.getAllExecutionVertices().iterator();
	final ExecutionVertex firstVertex = vertexIterator.next();

	executionGraph.cancel();

	final FailoverStrategy failoverStrategy = executionGraph.getFailoverStrategy();
	failoverStrategy.onTaskFailure(firstVertex.getCurrentExecutionAttempt(), new Exception("Test Exception"));
	manualMainThreadExecutor.triggerAll();

	assertEquals(ExecutionState.CANCELED, firstVertex.getExecutionState());
}
 

@VisibleForTesting
public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time timeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategy,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout) throws IOException {
	this(
		jobInformation,
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		JobManagerOptions.MAX_ATTEMPTS_HISTORY_SIZE.defaultValue(),
		failoverStrategy,
		slotProvider,
		userClassLoader,
		blobWriter,
		allocationTimeout,
		new NotReleasingPartitionReleaseStrategy.Factory(),
		NettyShuffleMaster.INSTANCE,
		new PartitionTrackerImpl(
			jobInformation.getJobId(),
			NettyShuffleMaster.INSTANCE,
			ignored -> Optional.empty()),
		ScheduleMode.LAZY_FROM_SOURCES,
		false);
}
 

private ExecutionGraph createExecutionGraph(
	JobManagerJobMetricGroup currentJobManagerJobMetricGroup,
	ShuffleMaster<?> shuffleMaster,
	final JobMasterPartitionTracker partitionTracker) throws JobExecutionException, JobException {

	final FailoverStrategy.Factory failoverStrategy = legacyScheduling ?
		FailoverStrategyLoader.loadFailoverStrategy(jobMasterConfiguration, log) :
		new NoOpFailoverStrategy.Factory();

	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobMasterConfiguration,
		futureExecutor,
		ioExecutor,
		slotProvider,
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		restartStrategy,
		currentJobManagerJobMetricGroup,
		blobWriter,
		slotRequestTimeout,
		log,
		shuffleMaster,
		partitionTracker,
		failoverStrategy);
}
 

/**
 * Builds the ExecutionGraph from the JobGraph.
 * If a prior execution graph exists, the JobGraph will be attached. If no prior execution
 * graph exists, then the JobGraph will become attach to a new empty execution graph.
 */
public static ExecutionGraph buildGraph(
		@Nullable ExecutionGraph prior,
		JobGraph jobGraph,
		Configuration jobManagerConfig,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		SlotProvider slotProvider,
		ClassLoader classLoader,
		CheckpointRecoveryFactory recoveryFactory,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		MetricGroup metrics,
		BlobWriter blobWriter,
		Time allocationTimeout,
		Logger log,
		ShuffleMaster<?> shuffleMaster,
		JobMasterPartitionTracker partitionTracker) throws JobExecutionException, JobException {

	final FailoverStrategy.Factory failoverStrategy =
		FailoverStrategyLoader.loadFailoverStrategy(jobManagerConfig, log);

	return buildGraph(
		prior,
		jobGraph,
		jobManagerConfig,
		futureExecutor,
		ioExecutor,
		slotProvider,
		classLoader,
		recoveryFactory,
		rpcTimeout,
		restartStrategy,
		metrics,
		blobWriter,
		allocationTimeout,
		log,
		shuffleMaster,
		partitionTracker,
		failoverStrategy);
}
 

@Override
public FailoverStrategy create(ExecutionGraph executionGraph) {
	return failoverStrategy;
}
 

public TestingExecutionGraphBuilder setFailoverStrategyFactory(FailoverStrategy.Factory failoverStrategyFactory) {
	this.failoverStrategyFactory = failoverStrategyFactory;
	return this;
}
 

public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		int maxPriorAttemptsHistoryLength,
		FailoverStrategy.Factory failoverStrategyFactory,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout,
		PartitionReleaseStrategy.Factory partitionReleaseStrategyFactory,
		ShuffleMaster<?> shuffleMaster,
		JobMasterPartitionTracker partitionTracker,
		ScheduleMode scheduleMode) throws IOException {

	this.jobInformation = Preconditions.checkNotNull(jobInformation);

	this.blobWriter = Preconditions.checkNotNull(blobWriter);

	this.scheduleMode = checkNotNull(scheduleMode);

	this.jobInformationOrBlobKey = BlobWriter.serializeAndTryOffload(jobInformation, jobInformation.getJobId(), blobWriter);

	this.futureExecutor = Preconditions.checkNotNull(futureExecutor);
	this.ioExecutor = Preconditions.checkNotNull(ioExecutor);

	this.slotProviderStrategy = SlotProviderStrategy.from(
		scheduleMode,
		slotProvider,
		allocationTimeout);
	this.userClassLoader = Preconditions.checkNotNull(userClassLoader, "userClassLoader");

	this.tasks = new HashMap<>(16);
	this.intermediateResults = new HashMap<>(16);
	this.verticesInCreationOrder = new ArrayList<>(16);
	this.currentExecutions = new HashMap<>(16);

	this.jobStatusListeners  = new ArrayList<>();

	this.stateTimestamps = new long[JobStatus.values().length];
	this.stateTimestamps[JobStatus.CREATED.ordinal()] = System.currentTimeMillis();

	this.rpcTimeout = checkNotNull(rpcTimeout);
	this.allocationTimeout = checkNotNull(allocationTimeout);

	this.partitionReleaseStrategyFactory = checkNotNull(partitionReleaseStrategyFactory);

	this.restartStrategy = restartStrategy;
	this.kvStateLocationRegistry = new KvStateLocationRegistry(jobInformation.getJobId(), getAllVertices());

	this.globalModVersion = 1L;

	// the failover strategy must be instantiated last, so that the execution graph
	// is ready by the time the failover strategy sees it
	this.failoverStrategy = checkNotNull(failoverStrategyFactory.create(this), "null failover strategy");

	this.maxPriorAttemptsHistoryLength = maxPriorAttemptsHistoryLength;

	this.schedulingFuture = null;
	this.jobMasterMainThreadExecutor =
		new ComponentMainThreadExecutor.DummyComponentMainThreadExecutor(
			"ExecutionGraph is not initialized with proper main thread executor. " +
				"Call to ExecutionGraph.start(...) required.");

	this.shuffleMaster = checkNotNull(shuffleMaster);

	this.partitionTracker = checkNotNull(partitionTracker);

	this.resultPartitionAvailabilityChecker = new ExecutionGraphResultPartitionAvailabilityChecker(
		this::createResultPartitionId,
		partitionTracker);
}
 

/**
 * Gets the failover strategy used by the execution graph to recover from failures of tasks.
 */
public FailoverStrategy getFailoverStrategy() {
	return this.failoverStrategy;
}
 

@Override
public FailoverStrategy create(ExecutionGraph executionGraph) {
	return new RestartPipelinedRegionStrategy(executionGraph);
}
 

public TestingExecutionGraphBuilder setFailoverStrategyFactory(FailoverStrategy.Factory failoverStrategyFactory) {
	this.failoverStrategyFactory = failoverStrategyFactory;
	return this;
}
 

@Override
public FailoverStrategy create(ExecutionGraph executionGraph) {
	return new TestFailoverStrategy();
}
 

CustomStrategy(FailoverStrategy failoverStrategy) {
	this.failoverStrategy = failoverStrategy;
}
 

@Override
public FailoverStrategy create(ExecutionGraph executionGraph) {
	return failoverStrategy;
}
 

CustomStrategy(FailoverStrategy failoverStrategy) {
	this.failoverStrategy = failoverStrategy;
}
 

@Override
public FailoverStrategy create(ExecutionGraph executionGraph) {
	return new TestFailoverStrategy();
}
 

private TestingExecutionGraphBuilder setFailoverStrategyFactory(FailoverStrategy.Factory failoverStrategyFactory) {
	this.failoverStrategyFactory = failoverStrategyFactory;
	return this;
}
 

/**
 * Builds the ExecutionGraph from the JobGraph.
 * If a prior execution graph exists, the JobGraph will be attached. If no prior execution
 * graph exists, then the JobGraph will become attach to a new empty execution graph.
 */
public static ExecutionGraph buildGraph(
		@Nullable ExecutionGraph prior,
		JobGraph jobGraph,
		Configuration jobManagerConfig,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		SlotProvider slotProvider,
		ClassLoader classLoader,
		CheckpointRecoveryFactory recoveryFactory,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		MetricGroup metrics,
		BlobWriter blobWriter,
		Time allocationTimeout,
		Logger log,
		ShuffleMaster<?> shuffleMaster,
		PartitionTracker partitionTracker) throws JobExecutionException, JobException {

	final FailoverStrategy.Factory failoverStrategy =
		FailoverStrategyLoader.loadFailoverStrategy(jobManagerConfig, log);

	return buildGraph(
		prior,
		jobGraph,
		jobManagerConfig,
		futureExecutor,
		ioExecutor,
		slotProvider,
		classLoader,
		recoveryFactory,
		rpcTimeout,
		restartStrategy,
		metrics,
		blobWriter,
		allocationTimeout,
		log,
		shuffleMaster,
		partitionTracker,
		failoverStrategy);
}
 

/**
 * Gets the failover strategy used by the execution graph to recover from failures of tasks.
 */
public FailoverStrategy getFailoverStrategy() {
	return this.failoverStrategy;
}
 

public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		int maxPriorAttemptsHistoryLength,
		FailoverStrategy.Factory failoverStrategyFactory,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout,
		PartitionReleaseStrategy.Factory partitionReleaseStrategyFactory,
		ShuffleMaster<?> shuffleMaster,
		PartitionTracker partitionTracker,
		ScheduleMode scheduleMode,
		boolean allowQueuedScheduling) throws IOException {

	checkNotNull(futureExecutor);

	this.jobInformation = Preconditions.checkNotNull(jobInformation);

	this.blobWriter = Preconditions.checkNotNull(blobWriter);

	this.scheduleMode = checkNotNull(scheduleMode);

	this.allowQueuedScheduling = allowQueuedScheduling;

	this.jobInformationOrBlobKey = BlobWriter.serializeAndTryOffload(jobInformation, jobInformation.getJobId(), blobWriter);

	this.futureExecutor = Preconditions.checkNotNull(futureExecutor);
	this.ioExecutor = Preconditions.checkNotNull(ioExecutor);

	this.slotProviderStrategy = SlotProviderStrategy.from(
		scheduleMode,
		slotProvider,
		allocationTimeout,
		allowQueuedScheduling);
	this.userClassLoader = Preconditions.checkNotNull(userClassLoader, "userClassLoader");

	this.tasks = new ConcurrentHashMap<>(16);
	this.intermediateResults = new ConcurrentHashMap<>(16);
	this.verticesInCreationOrder = new ArrayList<>(16);
	this.currentExecutions = new ConcurrentHashMap<>(16);

	this.jobStatusListeners  = new CopyOnWriteArrayList<>();

	this.stateTimestamps = new long[JobStatus.values().length];
	this.stateTimestamps[JobStatus.CREATED.ordinal()] = System.currentTimeMillis();

	this.rpcTimeout = checkNotNull(rpcTimeout);
	this.allocationTimeout = checkNotNull(allocationTimeout);

	this.partitionReleaseStrategyFactory = checkNotNull(partitionReleaseStrategyFactory);

	this.restartStrategy = restartStrategy;
	this.kvStateLocationRegistry = new KvStateLocationRegistry(jobInformation.getJobId(), getAllVertices());

	this.verticesFinished = new AtomicInteger();

	this.globalModVersion = 1L;

	// the failover strategy must be instantiated last, so that the execution graph
	// is ready by the time the failover strategy sees it
	this.failoverStrategy = checkNotNull(failoverStrategyFactory.create(this), "null failover strategy");

	this.maxPriorAttemptsHistoryLength = maxPriorAttemptsHistoryLength;

	this.schedulingFuture = null;
	this.jobMasterMainThreadExecutor =
		new ComponentMainThreadExecutor.DummyComponentMainThreadExecutor(
			"ExecutionGraph is not initialized with proper main thread executor. " +
				"Call to ExecutionGraph.start(...) required.");

	this.shuffleMaster = checkNotNull(shuffleMaster);

	this.partitionTracker = checkNotNull(partitionTracker);

	this.resultPartitionAvailabilityChecker = new ExecutionGraphResultPartitionAvailabilityChecker(
		this::createResultPartitionId,
		partitionTracker);

	LOG.info("Job recovers via failover strategy: {}", failoverStrategy.getStrategyName());
}
 

@Override
public FailoverStrategy create(ExecutionGraph executionGraph) {
	return failoverStrategy;
}
 

CustomStrategy(FailoverStrategy failoverStrategy) {
	this.failoverStrategy = failoverStrategy;
}
 

@Override
public FailoverStrategy create(ExecutionGraph executionGraph) {
	return new RestartPipelinedRegionStrategy(executionGraph);
}
 

/**
 * Gets the failover strategy used by the execution graph to recover from failures of tasks.
 */
public FailoverStrategy getFailoverStrategy() {
	return this.failoverStrategy;
}
 

public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategyFactory,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout) throws IOException {

	checkNotNull(futureExecutor);

	this.jobInformation = Preconditions.checkNotNull(jobInformation);

	this.blobWriter = Preconditions.checkNotNull(blobWriter);

	this.jobInformationOrBlobKey = BlobWriter.serializeAndTryOffload(jobInformation, jobInformation.getJobId(), blobWriter);

	this.futureExecutor = Preconditions.checkNotNull(futureExecutor);
	this.ioExecutor = Preconditions.checkNotNull(ioExecutor);

	this.slotProvider = Preconditions.checkNotNull(slotProvider, "scheduler");
	this.userClassLoader = Preconditions.checkNotNull(userClassLoader, "userClassLoader");

	this.tasks = new ConcurrentHashMap<>(16);
	this.intermediateResults = new ConcurrentHashMap<>(16);
	this.verticesInCreationOrder = new ArrayList<>(16);
	this.currentExecutions = new ConcurrentHashMap<>(16);

	this.jobStatusListeners  = new CopyOnWriteArrayList<>();
	this.executionListeners = new CopyOnWriteArrayList<>();

	this.stateTimestamps = new long[JobStatus.values().length];
	this.stateTimestamps[JobStatus.CREATED.ordinal()] = System.currentTimeMillis();

	this.rpcTimeout = checkNotNull(rpcTimeout);
	this.allocationTimeout = checkNotNull(allocationTimeout);

	this.restartStrategy = restartStrategy;
	this.kvStateLocationRegistry = new KvStateLocationRegistry(jobInformation.getJobId(), getAllVertices());

	this.verticesFinished = new AtomicInteger();

	this.globalModVersion = 1L;

	// the failover strategy must be instantiated last, so that the execution graph
	// is ready by the time the failover strategy sees it
	this.failoverStrategy = checkNotNull(failoverStrategyFactory.create(this), "null failover strategy");

	this.schedulingFuture = null;
	this.jobMasterMainThreadExecutor =
		new ComponentMainThreadExecutor.DummyComponentMainThreadExecutor(
			"ExecutionGraph is not initialized with proper main thread executor. " +
				"Call to ExecutionGraph.start(...) required.");

	LOG.info("Job recovers via failover strategy: {}", failoverStrategy.getStrategyName());
}