Java Code Examples for org.apache.flink.runtime.executiongraph.failover.FailoverStrategy#Factory

The following examples show how to use org.apache.flink.runtime.executiongraph.failover.FailoverStrategy#Factory . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ExecutionGraph.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time timeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategy,
		SlotProvider slotProvider) throws IOException {
	this(
		jobInformation,
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		failoverStrategy,
		slotProvider,
		ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		timeout);
}
Example 2
Source File: ExecutionGraph.java    From flink with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time timeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategy,
		SlotProvider slotProvider) throws IOException {
	this(
		jobInformation,
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		failoverStrategy,
		slotProvider,
		ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		timeout);
}
Example 3
Source File: ExecutionGraphRestartTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static ExecutionGraph createSimpleExecutionGraph(
	final RestartStrategy restartStrategy,
	final FailoverStrategy.Factory failoverStrategyFactory,
	final SlotProvider slotProvider,
	final JobGraph jobGraph) throws IOException, JobException {

	final ExecutionGraph executionGraph = new ExecutionGraph(
		new JobInformation(
			TEST_JOB_ID,
			"Test job",
			new SerializedValue<>(new ExecutionConfig()),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList()),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		restartStrategy,
		failoverStrategyFactory,
		slotProvider);

	executionGraph.start(mainThreadExecutor);
	executionGraph.attachJobGraph(jobGraph.getVerticesSortedTopologicallyFromSources());

	return executionGraph;
}
Example 4
Source File: ExecutionGraph.java    From flink with Apache License 2.0 5 votes vote down vote up 
@VisibleForTesting
public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time timeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategy,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout) throws IOException {
	this(
		jobInformation,
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		JobManagerOptions.MAX_ATTEMPTS_HISTORY_SIZE.defaultValue(),
		failoverStrategy,
		slotProvider,
		userClassLoader,
		blobWriter,
		allocationTimeout,
		new NotReleasingPartitionReleaseStrategy.Factory(),
		NettyShuffleMaster.INSTANCE,
		new PartitionTrackerImpl(
			jobInformation.getJobId(),
			NettyShuffleMaster.INSTANCE,
			ignored -> Optional.empty()),
		ScheduleMode.LAZY_FROM_SOURCES,
		false);
}
Example 5
Source File: SchedulerBase.java    From flink with Apache License 2.0 5 votes vote down vote up 
private ExecutionGraph createExecutionGraph(
	JobManagerJobMetricGroup currentJobManagerJobMetricGroup,
	ShuffleMaster<?> shuffleMaster,
	final JobMasterPartitionTracker partitionTracker) throws JobExecutionException, JobException {

	final FailoverStrategy.Factory failoverStrategy = legacyScheduling ?
		FailoverStrategyLoader.loadFailoverStrategy(jobMasterConfiguration, log) :
		new NoOpFailoverStrategy.Factory();

	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobMasterConfiguration,
		futureExecutor,
		ioExecutor,
		slotProvider,
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		restartStrategy,
		currentJobManagerJobMetricGroup,
		blobWriter,
		slotRequestTimeout,
		log,
		shuffleMaster,
		partitionTracker,
		failoverStrategy);
}
Example 6
Source File: ExecutionGraph.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategyFactory,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout) throws IOException {

	checkNotNull(futureExecutor);

	this.jobInformation = Preconditions.checkNotNull(jobInformation);

	this.blobWriter = Preconditions.checkNotNull(blobWriter);

	this.jobInformationOrBlobKey = BlobWriter.serializeAndTryOffload(jobInformation, jobInformation.getJobId(), blobWriter);

	this.futureExecutor = Preconditions.checkNotNull(futureExecutor);
	this.ioExecutor = Preconditions.checkNotNull(ioExecutor);

	this.slotProvider = Preconditions.checkNotNull(slotProvider, "scheduler");
	this.userClassLoader = Preconditions.checkNotNull(userClassLoader, "userClassLoader");

	this.tasks = new ConcurrentHashMap<>(16);
	this.intermediateResults = new ConcurrentHashMap<>(16);
	this.verticesInCreationOrder = new ArrayList<>(16);
	this.currentExecutions = new ConcurrentHashMap<>(16);

	this.jobStatusListeners  = new CopyOnWriteArrayList<>();
	this.executionListeners = new CopyOnWriteArrayList<>();

	this.stateTimestamps = new long[JobStatus.values().length];
	this.stateTimestamps[JobStatus.CREATED.ordinal()] = System.currentTimeMillis();

	this.rpcTimeout = checkNotNull(rpcTimeout);
	this.allocationTimeout = checkNotNull(allocationTimeout);

	this.restartStrategy = restartStrategy;
	this.kvStateLocationRegistry = new KvStateLocationRegistry(jobInformation.getJobId(), getAllVertices());

	this.verticesFinished = new AtomicInteger();

	this.globalModVersion = 1L;

	// the failover strategy must be instantiated last, so that the execution graph
	// is ready by the time the failover strategy sees it
	this.failoverStrategy = checkNotNull(failoverStrategyFactory.create(this), "null failover strategy");

	this.schedulingFuture = null;
	this.jobMasterMainThreadExecutor =
		new ComponentMainThreadExecutor.DummyComponentMainThreadExecutor(
			"ExecutionGraph is not initialized with proper main thread executor. " +
				"Call to ExecutionGraph.start(...) required.");

	LOG.info("Job recovers via failover strategy: {}", failoverStrategy.getStrategyName());
}
Example 7
Source File: ExecutionGraph.java    From flink with Apache License 2.0 4 votes vote down vote up 
public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		int maxPriorAttemptsHistoryLength,
		FailoverStrategy.Factory failoverStrategyFactory,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout,
		PartitionReleaseStrategy.Factory partitionReleaseStrategyFactory,
		ShuffleMaster<?> shuffleMaster,
		PartitionTracker partitionTracker,
		ScheduleMode scheduleMode,
		boolean allowQueuedScheduling) throws IOException {

	checkNotNull(futureExecutor);

	this.jobInformation = Preconditions.checkNotNull(jobInformation);

	this.blobWriter = Preconditions.checkNotNull(blobWriter);

	this.scheduleMode = checkNotNull(scheduleMode);

	this.allowQueuedScheduling = allowQueuedScheduling;

	this.jobInformationOrBlobKey = BlobWriter.serializeAndTryOffload(jobInformation, jobInformation.getJobId(), blobWriter);

	this.futureExecutor = Preconditions.checkNotNull(futureExecutor);
	this.ioExecutor = Preconditions.checkNotNull(ioExecutor);

	this.slotProviderStrategy = SlotProviderStrategy.from(
		scheduleMode,
		slotProvider,
		allocationTimeout,
		allowQueuedScheduling);
	this.userClassLoader = Preconditions.checkNotNull(userClassLoader, "userClassLoader");

	this.tasks = new ConcurrentHashMap<>(16);
	this.intermediateResults = new ConcurrentHashMap<>(16);
	this.verticesInCreationOrder = new ArrayList<>(16);
	this.currentExecutions = new ConcurrentHashMap<>(16);

	this.jobStatusListeners  = new CopyOnWriteArrayList<>();

	this.stateTimestamps = new long[JobStatus.values().length];
	this.stateTimestamps[JobStatus.CREATED.ordinal()] = System.currentTimeMillis();

	this.rpcTimeout = checkNotNull(rpcTimeout);
	this.allocationTimeout = checkNotNull(allocationTimeout);

	this.partitionReleaseStrategyFactory = checkNotNull(partitionReleaseStrategyFactory);

	this.restartStrategy = restartStrategy;
	this.kvStateLocationRegistry = new KvStateLocationRegistry(jobInformation.getJobId(), getAllVertices());

	this.verticesFinished = new AtomicInteger();

	this.globalModVersion = 1L;

	// the failover strategy must be instantiated last, so that the execution graph
	// is ready by the time the failover strategy sees it
	this.failoverStrategy = checkNotNull(failoverStrategyFactory.create(this), "null failover strategy");

	this.maxPriorAttemptsHistoryLength = maxPriorAttemptsHistoryLength;

	this.schedulingFuture = null;
	this.jobMasterMainThreadExecutor =
		new ComponentMainThreadExecutor.DummyComponentMainThreadExecutor(
			"ExecutionGraph is not initialized with proper main thread executor. " +
				"Call to ExecutionGraph.start(...) required.");

	this.shuffleMaster = checkNotNull(shuffleMaster);

	this.partitionTracker = checkNotNull(partitionTracker);

	this.resultPartitionAvailabilityChecker = new ExecutionGraphResultPartitionAvailabilityChecker(
		this::createResultPartitionId,
		partitionTracker);

	LOG.info("Job recovers via failover strategy: {}", failoverStrategy.getStrategyName());
}
Example 8
Source File: ExecutionGraphBuilder.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Builds the ExecutionGraph from the JobGraph.
 * If a prior execution graph exists, the JobGraph will be attached. If no prior execution
 * graph exists, then the JobGraph will become attach to a new empty execution graph.
 */
public static ExecutionGraph buildGraph(
		@Nullable ExecutionGraph prior,
		JobGraph jobGraph,
		Configuration jobManagerConfig,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		SlotProvider slotProvider,
		ClassLoader classLoader,
		CheckpointRecoveryFactory recoveryFactory,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		MetricGroup metrics,
		BlobWriter blobWriter,
		Time allocationTimeout,
		Logger log,
		ShuffleMaster<?> shuffleMaster,
		PartitionTracker partitionTracker) throws JobExecutionException, JobException {

	final FailoverStrategy.Factory failoverStrategy =
		FailoverStrategyLoader.loadFailoverStrategy(jobManagerConfig, log);

	return buildGraph(
		prior,
		jobGraph,
		jobManagerConfig,
		futureExecutor,
		ioExecutor,
		slotProvider,
		classLoader,
		recoveryFactory,
		rpcTimeout,
		restartStrategy,
		metrics,
		blobWriter,
		allocationTimeout,
		log,
		shuffleMaster,
		partitionTracker,
		failoverStrategy);
}
Example 9
Source File: ExecutionGraphRestartTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private TestingExecutionGraphBuilder setFailoverStrategyFactory(FailoverStrategy.Factory failoverStrategyFactory) {
	this.failoverStrategyFactory = failoverStrategyFactory;
	return this;
}
Example 10
Source File: ExecutionGraphTestUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
public TestingExecutionGraphBuilder setFailoverStrategyFactory(FailoverStrategy.Factory failoverStrategyFactory) {
	this.failoverStrategyFactory = failoverStrategyFactory;
	return this;
}
Example 11
Source File: ExecutionGraph.java    From flink with Apache License 2.0 4 votes vote down vote up 
public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		int maxPriorAttemptsHistoryLength,
		FailoverStrategy.Factory failoverStrategyFactory,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout,
		PartitionReleaseStrategy.Factory partitionReleaseStrategyFactory,
		ShuffleMaster<?> shuffleMaster,
		JobMasterPartitionTracker partitionTracker,
		ScheduleMode scheduleMode) throws IOException {

	this.jobInformation = Preconditions.checkNotNull(jobInformation);

	this.blobWriter = Preconditions.checkNotNull(blobWriter);

	this.scheduleMode = checkNotNull(scheduleMode);

	this.jobInformationOrBlobKey = BlobWriter.serializeAndTryOffload(jobInformation, jobInformation.getJobId(), blobWriter);

	this.futureExecutor = Preconditions.checkNotNull(futureExecutor);
	this.ioExecutor = Preconditions.checkNotNull(ioExecutor);

	this.slotProviderStrategy = SlotProviderStrategy.from(
		scheduleMode,
		slotProvider,
		allocationTimeout);
	this.userClassLoader = Preconditions.checkNotNull(userClassLoader, "userClassLoader");

	this.tasks = new HashMap<>(16);
	this.intermediateResults = new HashMap<>(16);
	this.verticesInCreationOrder = new ArrayList<>(16);
	this.currentExecutions = new HashMap<>(16);

	this.jobStatusListeners  = new ArrayList<>();

	this.stateTimestamps = new long[JobStatus.values().length];
	this.stateTimestamps[JobStatus.CREATED.ordinal()] = System.currentTimeMillis();

	this.rpcTimeout = checkNotNull(rpcTimeout);
	this.allocationTimeout = checkNotNull(allocationTimeout);

	this.partitionReleaseStrategyFactory = checkNotNull(partitionReleaseStrategyFactory);

	this.restartStrategy = restartStrategy;
	this.kvStateLocationRegistry = new KvStateLocationRegistry(jobInformation.getJobId(), getAllVertices());

	this.globalModVersion = 1L;

	// the failover strategy must be instantiated last, so that the execution graph
	// is ready by the time the failover strategy sees it
	this.failoverStrategy = checkNotNull(failoverStrategyFactory.create(this), "null failover strategy");

	this.maxPriorAttemptsHistoryLength = maxPriorAttemptsHistoryLength;

	this.schedulingFuture = null;
	this.jobMasterMainThreadExecutor =
		new ComponentMainThreadExecutor.DummyComponentMainThreadExecutor(
			"ExecutionGraph is not initialized with proper main thread executor. " +
				"Call to ExecutionGraph.start(...) required.");

	this.shuffleMaster = checkNotNull(shuffleMaster);

	this.partitionTracker = checkNotNull(partitionTracker);

	this.resultPartitionAvailabilityChecker = new ExecutionGraphResultPartitionAvailabilityChecker(
		this::createResultPartitionId,
		partitionTracker);
}
Example 12
Source File: ExecutionGraphBuilder.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Builds the ExecutionGraph from the JobGraph.
 * If a prior execution graph exists, the JobGraph will be attached. If no prior execution
 * graph exists, then the JobGraph will become attach to a new empty execution graph.
 */
public static ExecutionGraph buildGraph(
		@Nullable ExecutionGraph prior,
		JobGraph jobGraph,
		Configuration jobManagerConfig,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		SlotProvider slotProvider,
		ClassLoader classLoader,
		CheckpointRecoveryFactory recoveryFactory,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		MetricGroup metrics,
		BlobWriter blobWriter,
		Time allocationTimeout,
		Logger log,
		ShuffleMaster<?> shuffleMaster,
		JobMasterPartitionTracker partitionTracker) throws JobExecutionException, JobException {

	final FailoverStrategy.Factory failoverStrategy =
		FailoverStrategyLoader.loadFailoverStrategy(jobManagerConfig, log);

	return buildGraph(
		prior,
		jobGraph,
		jobManagerConfig,
		futureExecutor,
		ioExecutor,
		slotProvider,
		classLoader,
		recoveryFactory,
		rpcTimeout,
		restartStrategy,
		metrics,
		blobWriter,
		allocationTimeout,
		log,
		shuffleMaster,
		partitionTracker,
		failoverStrategy);
}
Example 13
Source File: TestingExecutionGraphBuilder.java    From flink with Apache License 2.0 4 votes vote down vote up 
public TestingExecutionGraphBuilder setFailoverStrategyFactory(FailoverStrategy.Factory failoverStrategyFactory) {
	this.failoverStrategyFactory = failoverStrategyFactory;
	return this;
}