org.apache.flink.runtime.blob.VoidBlobWriter Java Examples

@VisibleForTesting
ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time timeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategy,
		SlotProvider slotProvider) throws IOException {
	this(
		jobInformation,
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		failoverStrategy,
		slotProvider,
		ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		timeout);
}
 

private ExecutionGraph createExecutionGraph(final JobMasterPartitionTracker partitionTracker, final JobVertex... vertices) throws Exception {
	final ExecutionGraph executionGraph = ExecutionGraphBuilder.buildGraph(
		null,
		new JobGraph(new JobID(), "test job", vertices),
		new Configuration(),
		scheduledExecutorService,
		mainThreadExecutor.getMainThreadExecutor(),
		new TestingSlotProvider(ignored -> CompletableFuture.completedFuture(new TestingLogicalSlotBuilder().createTestingLogicalSlot())),
		ExecutionGraphPartitionReleaseTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		AkkaUtils.getDefaultTimeout(),
		log,
		NettyShuffleMaster.INSTANCE,
		partitionTracker);

	executionGraph.start(mainThreadExecutor.getMainThreadExecutor());
	mainThreadExecutor.execute(executionGraph::scheduleForExecution);

	return executionGraph;
}
 

private ExecutionGraph createExecutionGraph(JobGraph jobGraph, SlotProvider slotProvider, Time timeout) throws Exception {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		new Configuration(),
		executor,
		executor,
		slotProvider,
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpJobMasterPartitionTracker.INSTANCE);
}
 

private ExecutionGraph createExecutionGraph(JobGraph jobGraph, SlotProvider slotProvider, Time timeout) throws Exception {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		new Configuration(),
		executor,
		executor,
		slotProvider,
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		1,
		VoidBlobWriter.getInstance(),
		timeout,
		log);
}
 

private ExecutionGraph createExecutionGraph(JobGraph jobGraph) throws JobException, JobExecutionException {
	// configure the pipelined failover strategy
	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setString(
			JobManagerOptions.EXECUTION_FAILOVER_STRATEGY,
			FailoverStrategyLoader.PIPELINED_REGION_RESTART_STRATEGY_NAME);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobManagerConfig,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		PipelinedFailoverRegionBuildingTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		1000,
		VoidBlobWriter.getInstance(),
		timeout,
		log);
}
 

private ExecutionGraph createExecutionGraph(final PartitionTracker partitionTracker, final JobVertex... vertices) throws Exception {
	final ExecutionGraph executionGraph = ExecutionGraphBuilder.buildGraph(
		null,
		new JobGraph(new JobID(), "test job", vertices),
		new Configuration(),
		scheduledExecutorService,
		mainThreadExecutor.getMainThreadExecutor(),
		new TestingSlotProvider(ignored -> CompletableFuture.completedFuture(new TestingLogicalSlotBuilder().createTestingLogicalSlot())),
		ExecutionGraphPartitionReleaseTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		AkkaUtils.getDefaultTimeout(),
		log,
		NettyShuffleMaster.INSTANCE,
		partitionTracker);

	executionGraph.start(mainThreadExecutor.getMainThreadExecutor());
	mainThreadExecutor.execute(executionGraph::scheduleForExecution);

	return executionGraph;
}
 

private ExecutionGraph createExecutionGraph(JobGraph jobGraph) throws JobException, JobExecutionException {
	// configure the pipelined failover strategy
	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setString(
			JobManagerOptions.EXECUTION_FAILOVER_STRATEGY,
			FailoverStrategyLoader.LEGACY_PIPELINED_REGION_RESTART_STRATEGY_NAME);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobManagerConfig,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		PipelinedFailoverRegionBuildingTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);
}
 

private ExecutionGraph createExecutionGraph(JobGraph jobGraph, SlotProvider slotProvider, Time timeout) throws Exception {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		new Configuration(),
		executor,
		executor,
		slotProvider,
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);
}
 

@VisibleForTesting
ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time timeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategy,
		SlotProvider slotProvider) throws IOException {
	this(
		jobInformation,
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		failoverStrategy,
		slotProvider,
		ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		timeout);
}
 

private LegacyScheduler createLegacyScheduler(JobGraph jobGraph, SlotPool slotPool, ComponentMainThreadExecutor mainThreadExecutor, Time slotRequestTimeout) throws Exception {
	final Scheduler scheduler = createScheduler(slotPool, mainThreadExecutor);
	final LegacyScheduler legacyScheduler = new LegacyScheduler(
		LOG,
		jobGraph,
		VoidBackPressureStatsTracker.INSTANCE,
		TestingUtils.defaultExecutor(),
		new Configuration(),
		scheduler,
		TestingUtils.defaultExecutor(),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		TestingUtils.TIMEOUT(),
		new NoRestartStrategy.NoRestartStrategyFactory(),
		VoidBlobWriter.getInstance(),
		UnregisteredMetricGroups.createUnregisteredJobManagerJobMetricGroup(),
		slotRequestTimeout,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);

	legacyScheduler.setMainThreadExecutor(mainThreadExecutor);

	return legacyScheduler;
}
 

/**
 * Verifies that building an {@link ExecutionGraph} from a {@link JobGraph} with
 * parallelism higher than the maximum parallelism fails.
 */
@Test
public void testExecutionGraphConstructionFailsRescaleDopExceedMaxParallelism() throws Exception {

	final Configuration config = new Configuration();

	final int initialParallelism = 1;
	final int maxParallelism = 10;
	final JobVertex[] jobVertices = createVerticesForSimpleBipartiteJobGraph(initialParallelism,  maxParallelism);
	final JobGraph jobGraph = new JobGraph(jobVertices);

	for (JobVertex jv : jobVertices) {
		jv.setParallelism(maxParallelism + 1);
	}

	try {
		// this should fail since we set the parallelism to maxParallelism + 1
		ExecutionGraphBuilder.buildGraph(
			null,
			jobGraph,
			config,
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			new TestingSlotProvider(ignore -> new CompletableFuture<>()),
			Thread.currentThread().getContextClassLoader(),
			new StandaloneCheckpointRecoveryFactory(),
			AkkaUtils.getDefaultTimeout(),
			new NoRestartStrategy(),
			new UnregisteredMetricsGroup(),
			VoidBlobWriter.getInstance(),
			AkkaUtils.getDefaultTimeout(),
			TEST_LOGGER,
			NettyShuffleMaster.INSTANCE,
			NoOpJobMasterPartitionTracker.INSTANCE);

		fail("Building the ExecutionGraph with a parallelism higher than the max parallelism should fail.");
	} catch (JobException e) {
		// expected, ignore
	}
}
 

/**
 * Creates a simple 2 vertex graph with a parallel source and a parallel target.
 */
private ExecutionGraph createTestGraph(int parallelism, boolean allToAll) throws Exception {

	JobVertex source = new JobVertex("source", sourceVertexId);
	source.setParallelism(parallelism);
	source.setInvokableClass(NoOpInvokable.class);

	JobVertex target = new JobVertex("source", targetVertexId);
	target.setParallelism(parallelism);
	target.setInvokableClass(NoOpInvokable.class);

	DistributionPattern connectionPattern = allToAll ? DistributionPattern.ALL_TO_ALL : DistributionPattern.POINTWISE;
	target.connectNewDataSetAsInput(source, connectionPattern, ResultPartitionType.PIPELINED);

	JobGraph testJob = new JobGraph(jobId, "test job", source, target);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		testJob,
		new Configuration(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new FixedDelayRestartStrategy(10, 0L),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpJobMasterPartitionTracker.INSTANCE);
}
 

public TestingJobManagerSharedServicesBuilder() {
	scheduledExecutorService = TestingUtils.defaultExecutor();
	libraryCacheManager = ContextClassLoaderLibraryCacheManager.INSTANCE;
	backPressureSampleCoordinator = new BackPressureRequestCoordinator(Runnable::run, 10000);
	backPressureStatsTracker = VoidBackPressureStatsTracker.INSTANCE;
	blobWriter = VoidBlobWriter.getInstance();
}
 

/**
 * Verifies that building an {@link ExecutionGraph} from a {@link JobGraph} with
 * parallelism higher than the maximum parallelism fails.
 */
@Test
public void testExecutionGraphConstructionFailsRescaleDopExceedMaxParallelism() throws Exception {

	final Configuration config = new Configuration();

	final int initialParallelism = 1;
	final int maxParallelism = 10;
	final JobVertex[] jobVertices = createVerticesForSimpleBipartiteJobGraph(initialParallelism,  maxParallelism);
	final JobGraph jobGraph = new JobGraph(jobVertices);

	for (JobVertex jv : jobVertices) {
		jv.setParallelism(maxParallelism + 1);
	}

	try {
		// this should fail since we set the parallelism to maxParallelism + 1
		ExecutionGraphBuilder.buildGraph(
			null,
			jobGraph,
			config,
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			new TestingSlotProvider(ignore -> new CompletableFuture<>()),
			Thread.currentThread().getContextClassLoader(),
			new StandaloneCheckpointRecoveryFactory(),
			AkkaUtils.getDefaultTimeout(),
			new NoRestartStrategy(),
			new UnregisteredMetricsGroup(),
			VoidBlobWriter.getInstance(),
			AkkaUtils.getDefaultTimeout(),
			TEST_LOGGER,
			NettyShuffleMaster.INSTANCE,
			NoOpPartitionTracker.INSTANCE);

		fail("Building the ExecutionGraph with a parallelism higher than the max parallelism should fail.");
	} catch (JobException e) {
		// expected, ignore
	}
}
 

/**
 * Creates a simple 2 vertex graph with a parallel source and a parallel target.
 */
private ExecutionGraph createTestGraph(int parallelism, boolean allToAll) throws Exception {

	JobVertex source = new JobVertex("source", sourceVertexId);
	source.setParallelism(parallelism);
	source.setInvokableClass(NoOpInvokable.class);

	JobVertex target = new JobVertex("source", targetVertexId);
	target.setParallelism(parallelism);
	target.setInvokableClass(NoOpInvokable.class);

	DistributionPattern connectionPattern = allToAll ? DistributionPattern.ALL_TO_ALL : DistributionPattern.POINTWISE;
	target.connectNewDataSetAsInput(source, connectionPattern, ResultPartitionType.PIPELINED);

	JobGraph testJob = new JobGraph(jobId, "test job", source, target);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		testJob,
		new Configuration(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new FixedDelayRestartStrategy(10, 0L),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);
}
 

private ExecutionGraph createSampleGraph(
	JobID jid,
	Factory failoverStrategy,
	RestartStrategy restartStrategy,
	SlotProvider slotProvider,
	int parallelism) throws Exception {

	final JobInformation jobInformation = new DummyJobInformation(
		jid,
		"test job");

	// build a simple execution graph with on job vertex, parallelism 2
	final Time timeout = Time.seconds(10L);
	final ExecutionGraph graph = new ExecutionGraph(
		jobInformation,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		timeout,
		restartStrategy,
		failoverStrategy,
		slotProvider,
		getClass().getClassLoader(),
		VoidBlobWriter.getInstance(),
		timeout);

	JobVertex jv = new JobVertex("test vertex");
	jv.setInvokableClass(NoOpInvokable.class);
	jv.setParallelism(parallelism);

	JobGraph jg = new JobGraph(jid, "testjob", jv);
	graph.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());

	return graph;
}
 

/**
 * Verifies that building an {@link ExecutionGraph} from a {@link JobGraph} with
 * parallelism higher than the maximum parallelism fails.
 */
@Test
public void testExecutionGraphConstructionFailsRescaleDopExceedMaxParallelism() throws Exception {

	final Configuration config = new Configuration();

	final int initialParallelism = 1;
	final int maxParallelism = 10;
	final JobVertex[] jobVertices = createVerticesForSimpleBipartiteJobGraph(initialParallelism,  maxParallelism);
	final JobGraph jobGraph = new JobGraph(jobVertices);

	for (JobVertex jv : jobVertices) {
		jv.setParallelism(maxParallelism + 1);
	}

	try {
		// this should fail since we set the parallelism to maxParallelism + 1
		ExecutionGraphBuilder.buildGraph(
			null,
			jobGraph,
			config,
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			new TestingSlotProvider(ignore -> new CompletableFuture<>()),
			Thread.currentThread().getContextClassLoader(),
			new StandaloneCheckpointRecoveryFactory(),
			AkkaUtils.getDefaultTimeout(),
			new NoRestartStrategy(),
			new UnregisteredMetricsGroup(),
			VoidBlobWriter.getInstance(),
			AkkaUtils.getDefaultTimeout(),
			TEST_LOGGER);

		fail("Building the ExecutionGraph with a parallelism higher than the max parallelism should fail.");
	} catch (JobException e) {
		// expected, ignore
	}
}
 

private ExecutionGraph createExecutionGraphAndEnableCheckpointing(
		CheckpointIDCounter counter,
		CompletedCheckpointStore store) throws Exception {
	final Time timeout = Time.days(1L);
	ExecutionGraph executionGraph = new ExecutionGraph(
		new DummyJobInformation(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		timeout,
		new NoRestartStrategy(),
		new RestartAllStrategy.Factory(),
		new TestingSlotProvider(slotRequestId -> CompletableFuture.completedFuture(new TestingLogicalSlot())),
		ClassLoader.getSystemClassLoader(),
		VoidBlobWriter.getInstance(),
		timeout);

	executionGraph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());

	executionGraph.enableCheckpointing(
			100,
			100,
			100,
			1,
			CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
			Collections.emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			counter,
			store,
			new MemoryStateBackend(),
			CheckpointStatsTrackerTest.createTestTracker());

	JobVertex jobVertex = new JobVertex("MockVertex");
	jobVertex.setInvokableClass(AbstractInvokable.class);
	executionGraph.attachJobGraph(Collections.singletonList(jobVertex));
	executionGraph.setQueuedSchedulingAllowed(true);

	return executionGraph;
}
 

public TestingJobManagerSharedServicesBuilder() {
	scheduledExecutorService = TestingUtils.defaultExecutor();
	libraryCacheManager = ContextClassLoaderLibraryCacheManager.INSTANCE;
	restartStrategyFactory = new NoOrFixedIfCheckpointingEnabledRestartStrategyFactory();
	stackTraceSampleCoordinator = mock(StackTraceSampleCoordinator.class);
	backPressureStatsTracker = VoidBackPressureStatsTracker.INSTANCE;
	blobWriter = VoidBlobWriter.getInstance();
}
 

private ExecutionGraph createSampleGraph(
	JobID jid,
	Factory failoverStrategy,
	RestartStrategy restartStrategy,
	SlotProvider slotProvider,
	int parallelism) throws Exception {

	final JobInformation jobInformation = new DummyJobInformation(
		jid,
		"test job");

	// build a simple execution graph with on job vertex, parallelism 2
	final Time timeout = Time.seconds(10L);
	final ExecutionGraph graph = new ExecutionGraph(
		jobInformation,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		timeout,
		restartStrategy,
		failoverStrategy,
		slotProvider,
		getClass().getClassLoader(),
		VoidBlobWriter.getInstance(),
		timeout);

	JobVertex jv = new JobVertex("test vertex");
	jv.setInvokableClass(NoOpInvokable.class);
	jv.setParallelism(parallelism);

	JobGraph jg = new JobGraph(jid, "testjob", jv);
	graph.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());

	return graph;
}
 

/**
 * Creates a simple 2 vertex graph with a parallel source and a parallel target.
 */
private ExecutionGraph createTestGraph(int parallelism, boolean allToAll) throws Exception {

	JobVertex source = new JobVertex("source", sourceVertexId);
	source.setParallelism(parallelism);
	source.setInvokableClass(NoOpInvokable.class);

	JobVertex target = new JobVertex("source", targetVertexId);
	target.setParallelism(parallelism);
	target.setInvokableClass(NoOpInvokable.class);

	DistributionPattern connectionPattern = allToAll ? DistributionPattern.ALL_TO_ALL : DistributionPattern.POINTWISE;
	target.connectNewDataSetAsInput(source, connectionPattern, ResultPartitionType.PIPELINED);

	JobGraph testJob = new JobGraph(jobId, "test job", source, target);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		testJob,
		new Configuration(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new FixedDelayRestartStrategy(10, 0L),
		new UnregisteredMetricsGroup(),
		1,
		VoidBlobWriter.getInstance(),
		timeout,
		log);
}
 

public static ExecutionGraph createExecutionGraph(
		JobID jid,
		SlotProvider slotProvider,
		RestartStrategy restartStrategy,
		ScheduledExecutorService executor,
		Time timeout,
		JobVertex... vertices) throws Exception {

	checkNotNull(jid);
	checkNotNull(restartStrategy);
	checkNotNull(vertices);
	checkNotNull(timeout);

	return ExecutionGraphBuilder.buildGraph(
		null,
		new JobGraph(jid, "test job", vertices),
		new Configuration(),
		executor,
		executor,
		slotProvider,
		ExecutionGraphTestUtils.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		restartStrategy,
		new UnregisteredMetricsGroup(),
		1,
		VoidBlobWriter.getInstance(),
		timeout,
		TEST_LOGGER);
}
 

public TestingJobManagerSharedServicesBuilder() {
	scheduledExecutorService = TestingUtils.defaultExecutor();
	libraryCacheManager = ContextClassLoaderLibraryCacheManager.INSTANCE;
	restartStrategyFactory = new NoOrFixedIfCheckpointingEnabledRestartStrategyFactory();
	stackTraceSampleCoordinator = mock(StackTraceSampleCoordinator.class);
	backPressureStatsTracker = VoidBackPressureStatsTracker.INSTANCE;
	blobWriter = VoidBlobWriter.getInstance();
}
 

/**
    * Creates a JobGraph of the following form:
    *
    * <pre>
    *  v1--->v2-->\
    *              \
    *               v4 --->|\
    *        ----->/        \
    *  v3-->/                v5
    *       \               /
    *        ------------->/
    * </pre>
    */
@Test
public void testSingleRegionWithMixedInput() throws Exception {
	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";

       JobVertex v1 = new JobVertex("vertex1");
       JobVertex v2 = new JobVertex("vertex2");
       JobVertex v3 = new JobVertex("vertex3");
       JobVertex v4 = new JobVertex("vertex4");
       JobVertex v5 = new JobVertex("vertex5");

       v1.setParallelism(3);
       v2.setParallelism(2);
       v3.setParallelism(2);
       v4.setParallelism(5);
       v5.setParallelism(2);

       v1.setInvokableClass(AbstractInvokable.class);
       v2.setInvokableClass(AbstractInvokable.class);
       v3.setInvokableClass(AbstractInvokable.class);
       v4.setInvokableClass(AbstractInvokable.class);
       v5.setInvokableClass(AbstractInvokable.class);

       v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

       List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));

	final JobInformation jobInformation = new DummyJobInformation(
		jobId,
		jobName);

	ExecutionGraph eg = new ExecutionGraph(
		jobInformation,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
           new RestartPipelinedRegionStrategy.Factory(),
		new TestingSlotProvider(ignored -> new CompletableFuture<>()),
           ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		AkkaUtils.getDefaultTimeout());
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}

       // All in one failover region
       RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
       ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
       ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
       ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
       ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
       ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
       FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
       FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
       FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);
       FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
       FailoverRegion region5 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);

       assertEquals(region1, region2);
       assertEquals(region2, region4);
       assertEquals(region3, region2);
       assertEquals(region1, region5);
   }
 

@Test
public void testDeserializationOfUserCodeWithUserClassLoader() throws Exception {
	final ClassLoader classLoader = new URLClassLoader(new URL[0], getClass().getClassLoader());
	final Serializable outOfClassPath = CommonTestUtils.createObjectForClassNotInClassPath(classLoader);

	final MasterTriggerRestoreHook.Factory[] hooks = {
			new TestFactory(outOfClassPath) };
	final SerializedValue<MasterTriggerRestoreHook.Factory[]> serHooks = new SerializedValue<>(hooks);

	final JobCheckpointingSettings checkpointingSettings = new JobCheckpointingSettings(
			Collections.<JobVertexID>emptyList(),
			Collections.<JobVertexID>emptyList(),
			Collections.<JobVertexID>emptyList(),
			new CheckpointCoordinatorConfiguration(
				1000L,
				10000L,
				0L,
				1,
				CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
				true),
			new SerializedValue<StateBackend>(new CustomStateBackend(outOfClassPath)),
			serHooks);

	final JobGraph jobGraph = new JobGraph(new JobID(), "test job");
	jobGraph.setSnapshotSettings(checkpointingSettings);

	// to serialize/deserialize the job graph to see if the behavior is correct under
	// distributed execution
	final JobGraph copy = CommonTestUtils.createCopySerializable(jobGraph);

	final Time timeout = Time.seconds(10L);
	final ExecutionGraph eg = ExecutionGraphBuilder.buildGraph(
		null,
		copy,
		new Configuration(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		classLoader,
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		10,
		VoidBlobWriter.getInstance(),
		timeout,
		log);

	assertEquals(1, eg.getCheckpointCoordinator().getNumberOfRegisteredMasterHooks());
	assertTrue(jobGraph.getCheckpointingSettings().getDefaultStateBackend().deserializeValue(classLoader) instanceof CustomStateBackend);
}
 

@Test
public void testDeserializationOfUserCodeWithUserClassLoader() throws Exception {
	final ClassLoaderUtils.ObjectAndClassLoader<Serializable> outsideClassLoading = ClassLoaderUtils.createSerializableObjectFromNewClassLoader();
	final ClassLoader classLoader = outsideClassLoading.getClassLoader();
	final Serializable outOfClassPath = outsideClassLoading.getObject();

	final MasterTriggerRestoreHook.Factory[] hooks = {
			new TestFactory(outOfClassPath) };
	final SerializedValue<MasterTriggerRestoreHook.Factory[]> serHooks = new SerializedValue<>(hooks);

	final JobCheckpointingSettings checkpointingSettings = new JobCheckpointingSettings(
			Collections.<JobVertexID>emptyList(),
			Collections.<JobVertexID>emptyList(),
			Collections.<JobVertexID>emptyList(),
			new CheckpointCoordinatorConfiguration(
				1000L,
				10000L,
				0L,
				1,
				CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
				true,
				false,
				false,
				0),
			new SerializedValue<StateBackend>(new CustomStateBackend(outOfClassPath)),
			serHooks);

	final JobGraph jobGraph = new JobGraph(new JobID(), "test job");
	jobGraph.setSnapshotSettings(checkpointingSettings);

	// to serialize/deserialize the job graph to see if the behavior is correct under
	// distributed execution
	final JobGraph copy = CommonTestUtils.createCopySerializable(jobGraph);

	final Time timeout = Time.seconds(10L);
	final ExecutionGraph eg = ExecutionGraphBuilder.buildGraph(
		null,
		copy,
		new Configuration(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		classLoader,
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpJobMasterPartitionTracker.INSTANCE);

	assertEquals(1, eg.getCheckpointCoordinator().getNumberOfRegisteredMasterHooks());
	assertTrue(jobGraph.getCheckpointingSettings().getDefaultStateBackend().deserializeValue(classLoader) instanceof CustomStateBackend);
}
 

/**
    * Creates a JobGraph of the following form:
    *
    * <pre>
    *  v1-->v2-->|\
    *              \
    *               v4
    *             /
    *  v3------>/
    * </pre>
    */
@Test
public void testMultiRegionNotAllToAll() throws Exception {
	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";

       JobVertex v1 = new JobVertex("vertex1");
       JobVertex v2 = new JobVertex("vertex2");
       JobVertex v3 = new JobVertex("vertex3");
       JobVertex v4 = new JobVertex("vertex4");
       JobVertex v5 = new JobVertex("vertex5");

       v1.setParallelism(2);
       v2.setParallelism(2);
       v3.setParallelism(5);
       v4.setParallelism(5);

       v1.setInvokableClass(AbstractInvokable.class);
       v2.setInvokableClass(AbstractInvokable.class);
       v3.setInvokableClass(AbstractInvokable.class);
       v4.setInvokableClass(AbstractInvokable.class);

       v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
       v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
       v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

       List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4));

	final JobInformation jobInformation = new DummyJobInformation(
		jobId,
		jobName);

       ExecutionGraph eg = new ExecutionGraph(
       	jobInformation,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
           new RestartPipelinedRegionStrategy.Factory(),
		new TestingSlotProvider(ignored -> new CompletableFuture<>()),
           ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		AkkaUtils.getDefaultTimeout());
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}

       // All in one failover region
       RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
       ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
       ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
       ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
       ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
       FailoverRegion region11 = strategy.getFailoverRegion(ejv1.getTaskVertices()[0]);
       FailoverRegion region12 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
       FailoverRegion region21 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
       FailoverRegion region22 = strategy.getFailoverRegion(ejv2.getTaskVertices()[1]);
       FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
       FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);

       //There should be 3 failover regions. v1 v2 in two, v3 and v4 in one
       assertEquals(region11, region21);
       assertEquals(region12, region22);
       assertFalse(region11.equals(region12));
       assertFalse(region3.equals(region4));
}
 

/**
    * Creates a JobGraph of the following form:
    *
    * <pre>
    *  v2 ------->\
    *              \
    *  v1---------> v4 --->|\
    *                        \
    *                        v5
    *                       /
    *  v3--------------->|/
    * </pre>
    */
@Test
public void testMultipleFailoverRegions() throws Exception {
	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";

       JobVertex v1 = new JobVertex("vertex1");
       JobVertex v2 = new JobVertex("vertex2");
       JobVertex v3 = new JobVertex("vertex3");
       JobVertex v4 = new JobVertex("vertex4");
       JobVertex v5 = new JobVertex("vertex5");

       v1.setParallelism(3);
       v2.setParallelism(2);
       v3.setParallelism(2);
       v4.setParallelism(5);
       v5.setParallelism(2);

       v1.setInvokableClass(AbstractInvokable.class);
       v2.setInvokableClass(AbstractInvokable.class);
       v3.setInvokableClass(AbstractInvokable.class);
       v4.setInvokableClass(AbstractInvokable.class);
       v5.setInvokableClass(AbstractInvokable.class);

       v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v4.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
       v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
       v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

       List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));

       final JobInformation jobInformation = new DummyJobInformation(
		jobId,
		jobName);

	ExecutionGraph eg = new ExecutionGraph(
		jobInformation,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
           new RestartPipelinedRegionStrategy.Factory(),
		new TestingSlotProvider(ignored -> new CompletableFuture<>()),
           ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		AkkaUtils.getDefaultTimeout());
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}

       // All in one failover region
       RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
       ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
       ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
       ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
       ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
       ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
       FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]);
       FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]);
       FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]);
       FailoverRegion region31 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
       FailoverRegion region32 = strategy.getFailoverRegion(ejv3.getTaskVertices()[1]);
       FailoverRegion region51 = strategy.getFailoverRegion(ejv5.getTaskVertices()[0]);
       FailoverRegion region52 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);

       //There should be 5 failover regions. v1 v2 v4 in one, v3 has two, v5 has two
       assertEquals(region1, region2);
       assertEquals(region2, region4);
       assertFalse(region31.equals(region32));
       assertFalse(region51.equals(region52));
}
 

/**
 * Creates a JobGraph of the following form:
 * 
 * <pre>
 *  v1--->v2-->\
 *              \
 *               v4 --->\
 *        ----->/        \
 *  v3-->/                v5
 *       \               /
 *        ------------->/
 * </pre>
 */
@Test
public void testSimpleFailoverRegion() throws Exception {
	
	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";

	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");
	JobVertex v3 = new JobVertex("vertex3");
	JobVertex v4 = new JobVertex("vertex4");
	JobVertex v5 = new JobVertex("vertex5");
	
	v1.setParallelism(5);
	v2.setParallelism(7);
	v3.setParallelism(2);
	v4.setParallelism(11);
	v5.setParallelism(4);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);
	v3.setInvokableClass(AbstractInvokable.class);
	v4.setInvokableClass(AbstractInvokable.class);
	v5.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	
	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));

       final JobInformation jobInformation = new DummyJobInformation(
		jobId,
		jobName);

	ExecutionGraph eg = new ExecutionGraph(
		jobInformation,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
           new RestartPipelinedRegionStrategy.Factory(),
		new TestingSlotProvider(ignored -> new CompletableFuture<>()),
           ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		AkkaUtils.getDefaultTimeout());
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}

       RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
       ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
       ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
       ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
       ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
       ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
       FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[2]);
       FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[3]);
       FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
       FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[4]);
       FailoverRegion region5 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);

       assertEquals(region1, region2);
       assertEquals(region3, region2);
       assertEquals(region4, region2);
       assertEquals(region5, region2);
}
 

/**
 * Creates a JobGraph of the following form:
 * 
 * <pre>
 *  v1--->v2-->\
 *              \
 *               v4 --->\
 *        ----->/        \
 *  v3-->/                v5
 *       \               /
 *        ------------->/
 * </pre>
 */
@Test
public void testSimpleFailoverRegion() throws Exception {
	
	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";

	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");
	JobVertex v3 = new JobVertex("vertex3");
	JobVertex v4 = new JobVertex("vertex4");
	JobVertex v5 = new JobVertex("vertex5");
	
	v1.setParallelism(5);
	v2.setParallelism(7);
	v3.setParallelism(2);
	v4.setParallelism(11);
	v5.setParallelism(4);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);
	v3.setInvokableClass(AbstractInvokable.class);
	v4.setInvokableClass(AbstractInvokable.class);
	v5.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	
	List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2, v3, v4, v5));

       final JobInformation jobInformation = new DummyJobInformation(
		jobId,
		jobName);

	ExecutionGraph eg = new ExecutionGraph(
		jobInformation,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
           new RestartPipelinedRegionStrategy.Factory(),
		new TestingSlotProvider(ignored -> new CompletableFuture<>()),
           ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		AkkaUtils.getDefaultTimeout());
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}

       RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();
       ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID());
       ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID());
       ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID());
       ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID());
       ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID());
       FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[2]);
       FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[3]);
       FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]);
       FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[4]);
       FailoverRegion region5 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]);

       assertEquals(region1, region2);
       assertEquals(region3, region2);
       assertEquals(region4, region2);
       assertEquals(region5, region2);
}