org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory Java Examples

The following examples show how to use org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ExecutionGraphSchedulingTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private ExecutionGraph createExecutionGraph(JobGraph jobGraph, SlotProvider slotProvider, Time timeout) throws Exception {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		new Configuration(),
		executor,
		executor,
		slotProvider,
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);
}
Example #2
Source File: ExecutionGraphPartitionReleaseTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private ExecutionGraph createExecutionGraph(final JobMasterPartitionTracker partitionTracker, final JobVertex... vertices) throws Exception {
	final ExecutionGraph executionGraph = ExecutionGraphBuilder.buildGraph(
		null,
		new JobGraph(new JobID(), "test job", vertices),
		new Configuration(),
		scheduledExecutorService,
		mainThreadExecutor.getMainThreadExecutor(),
		new TestingSlotProvider(ignored -> CompletableFuture.completedFuture(new TestingLogicalSlotBuilder().createTestingLogicalSlot())),
		ExecutionGraphPartitionReleaseTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		AkkaUtils.getDefaultTimeout(),
		log,
		NettyShuffleMaster.INSTANCE,
		partitionTracker);

	executionGraph.start(mainThreadExecutor.getMainThreadExecutor());
	mainThreadExecutor.execute(executionGraph::scheduleForExecution);

	return executionGraph;
}
Example #3
Source File: JobMasterTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Before
public void setup() throws IOException {
	configuration = new Configuration();
	haServices = new TestingHighAvailabilityServices();
	jobMasterId = JobMasterId.generate();
	jmResourceId = ResourceID.generate();

	testingFatalErrorHandler = new TestingFatalErrorHandler();

	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());

	rmLeaderRetrievalService = new SettableLeaderRetrievalService(
		null,
		null);
	haServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);

	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
}
Example #4
Source File: ExecutionGraphSchedulingTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private ExecutionGraph createExecutionGraph(JobGraph jobGraph, SlotProvider slotProvider, Time timeout) throws Exception {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		new Configuration(),
		executor,
		executor,
		slotProvider,
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpJobMasterPartitionTracker.INSTANCE);
}
Example #5
Source File: LegacySchedulerBatchSchedulingTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private LegacyScheduler createLegacyScheduler(JobGraph jobGraph, SlotPool slotPool, ComponentMainThreadExecutor mainThreadExecutor, Time slotRequestTimeout) throws Exception {
	final Scheduler scheduler = createScheduler(slotPool, mainThreadExecutor);
	final LegacyScheduler legacyScheduler = new LegacyScheduler(
		LOG,
		jobGraph,
		VoidBackPressureStatsTracker.INSTANCE,
		TestingUtils.defaultExecutor(),
		new Configuration(),
		scheduler,
		TestingUtils.defaultExecutor(),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		TestingUtils.TIMEOUT(),
		new NoRestartStrategy.NoRestartStrategyFactory(),
		VoidBlobWriter.getInstance(),
		UnregisteredMetricGroups.createUnregisteredJobManagerJobMetricGroup(),
		slotRequestTimeout,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);

	legacyScheduler.setMainThreadExecutor(mainThreadExecutor);

	return legacyScheduler;
}
Example #6
Source File: JobMasterTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Before
public void setup() throws IOException {
	configuration = new Configuration();
	haServices = new TestingHighAvailabilityServices();
	jobMasterId = JobMasterId.generate();
	jmResourceId = ResourceID.generate();

	testingFatalErrorHandler = new TestingFatalErrorHandler();

	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());

	rmLeaderRetrievalService = new SettableLeaderRetrievalService(
		null,
		null);
	haServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);

	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
}
Example #7
Source File: ExecutionGraphSchedulingTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private ExecutionGraph createExecutionGraph(JobGraph jobGraph, SlotProvider slotProvider, Time timeout) throws Exception {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		new Configuration(),
		executor,
		executor,
		slotProvider,
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		1,
		VoidBlobWriter.getInstance(),
		timeout,
		log);
}
Example #8
Source File: PipelinedFailoverRegionBuildingTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private ExecutionGraph createExecutionGraph(JobGraph jobGraph) throws JobException, JobExecutionException {
	// configure the pipelined failover strategy
	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setString(
			JobManagerOptions.EXECUTION_FAILOVER_STRATEGY,
			FailoverStrategyLoader.PIPELINED_REGION_RESTART_STRATEGY_NAME);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobManagerConfig,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		PipelinedFailoverRegionBuildingTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		1000,
		VoidBlobWriter.getInstance(),
		timeout,
		log);
}
Example #9
Source File: PipelinedFailoverRegionBuildingTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private ExecutionGraph createExecutionGraph(JobGraph jobGraph) throws JobException, JobExecutionException {
	// configure the pipelined failover strategy
	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setString(
			JobManagerOptions.EXECUTION_FAILOVER_STRATEGY,
			FailoverStrategyLoader.LEGACY_PIPELINED_REGION_RESTART_STRATEGY_NAME);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobManagerConfig,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		PipelinedFailoverRegionBuildingTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);
}
Example #10
Source File: JobMasterTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Before
public void setup() throws IOException {
	configuration = new Configuration();
	haServices = new TestingHighAvailabilityServices();
	jobMasterId = JobMasterId.generate();
	jmResourceId = ResourceID.generate();

	testingFatalErrorHandler = new TestingFatalErrorHandler();

	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());

	rmLeaderRetrievalService = new SettableLeaderRetrievalService(
		null,
		null);
	haServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);

	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
}
Example #11
Source File: DispatcherTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() throws Exception {
	final JobVertex testVertex = new JobVertex("testVertex");
	testVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(TEST_JOB_ID, "testJob", testVertex);

	heartbeatServices = new HeartbeatServices(1000L, 10000L);

	jobMasterLeaderElectionService = new TestingLeaderElectionService();

	haServices = new TestingHighAvailabilityServices();
	haServices.setJobMasterLeaderElectionService(TEST_JOB_ID, jobMasterLeaderElectionService);
	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());
	haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());

	configuration = new Configuration();

	configuration.setString(
		BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());

	createdJobManagerRunnerLatch = new CountDownLatch(2);
	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
Example #12
Source File: ExecutionGraphPartitionReleaseTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private ExecutionGraph createExecutionGraph(final PartitionTracker partitionTracker, final JobVertex... vertices) throws Exception {
	final ExecutionGraph executionGraph = ExecutionGraphBuilder.buildGraph(
		null,
		new JobGraph(new JobID(), "test job", vertices),
		new Configuration(),
		scheduledExecutorService,
		mainThreadExecutor.getMainThreadExecutor(),
		new TestingSlotProvider(ignored -> CompletableFuture.completedFuture(new TestingLogicalSlotBuilder().createTestingLogicalSlot())),
		ExecutionGraphPartitionReleaseTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		AkkaUtils.getDefaultTimeout(),
		log,
		NettyShuffleMaster.INSTANCE,
		partitionTracker);

	executionGraph.start(mainThreadExecutor.getMainThreadExecutor());
	mainThreadExecutor.execute(executionGraph::scheduleForExecution);

	return executionGraph;
}
Example #13
Source File: JobMasterBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up 
public JobMasterBuilder(JobGraph jobGraph, RpcService rpcService) {
	TestingHighAvailabilityServices testingHighAvailabilityServices = new TestingHighAvailabilityServices();
	testingHighAvailabilityServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());

	SettableLeaderRetrievalService rmLeaderRetrievalService = new SettableLeaderRetrievalService(
		null,
		null);
	testingHighAvailabilityServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);

	this.highAvailabilityServices = testingHighAvailabilityServices;
	this.jobGraph = jobGraph;
	this.rpcService = rpcService;
}
Example #14
Source File: ExecutionGraphRescalingTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Verifies that building an {@link ExecutionGraph} from a {@link JobGraph} with
 * parallelism higher than the maximum parallelism fails.
 */
@Test
public void testExecutionGraphConstructionFailsRescaleDopExceedMaxParallelism() throws Exception {

	final Configuration config = new Configuration();

	final int initialParallelism = 1;
	final int maxParallelism = 10;
	final JobVertex[] jobVertices = createVerticesForSimpleBipartiteJobGraph(initialParallelism,  maxParallelism);
	final JobGraph jobGraph = new JobGraph(jobVertices);

	for (JobVertex jv : jobVertices) {
		jv.setParallelism(maxParallelism + 1);
	}

	try {
		// this should fail since we set the parallelism to maxParallelism + 1
		ExecutionGraphBuilder.buildGraph(
			null,
			jobGraph,
			config,
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			new TestingSlotProvider(ignore -> new CompletableFuture<>()),
			Thread.currentThread().getContextClassLoader(),
			new StandaloneCheckpointRecoveryFactory(),
			AkkaUtils.getDefaultTimeout(),
			new NoRestartStrategy(),
			new UnregisteredMetricsGroup(),
			VoidBlobWriter.getInstance(),
			AkkaUtils.getDefaultTimeout(),
			TEST_LOGGER,
			NettyShuffleMaster.INSTANCE,
			NoOpPartitionTracker.INSTANCE);

		fail("Building the ExecutionGraph with a parallelism higher than the max parallelism should fail.");
	} catch (JobException e) {
		// expected, ignore
	}
}
Example #15
Source File: AbstractNonHaServices.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public CheckpointRecoveryFactory getCheckpointRecoveryFactory() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneCheckpointRecoveryFactory();
	}
}
Example #16
Source File: JobManagerRunnerImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() {
	leaderElectionService = new TestingLeaderElectionService();
	haServices = new TestingHighAvailabilityServices();
	haServices.setJobMasterLeaderElectionService(jobGraph.getJobID(), leaderElectionService);
	haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());
	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());

	fatalErrorHandler = new TestingFatalErrorHandler();
}
Example #17
Source File: AbstractNonHaServices.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public CheckpointRecoveryFactory getCheckpointRecoveryFactory() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneCheckpointRecoveryFactory();
	}
}
Example #18
Source File: ExecutionVertexLocalityTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a simple 2 vertex graph with a parallel source and a parallel target.
 */
private ExecutionGraph createTestGraph(int parallelism, boolean allToAll) throws Exception {

	JobVertex source = new JobVertex("source", sourceVertexId);
	source.setParallelism(parallelism);
	source.setInvokableClass(NoOpInvokable.class);

	JobVertex target = new JobVertex("source", targetVertexId);
	target.setParallelism(parallelism);
	target.setInvokableClass(NoOpInvokable.class);

	DistributionPattern connectionPattern = allToAll ? DistributionPattern.ALL_TO_ALL : DistributionPattern.POINTWISE;
	target.connectNewDataSetAsInput(source, connectionPattern, ResultPartitionType.PIPELINED);

	JobGraph testJob = new JobGraph(jobId, "test job", source, target);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		testJob,
		new Configuration(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new FixedDelayRestartStrategy(10, 0L),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpJobMasterPartitionTracker.INSTANCE);
}
Example #19
Source File: ExecutionGraphRescalingTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Verifies that building an {@link ExecutionGraph} from a {@link JobGraph} with
 * parallelism higher than the maximum parallelism fails.
 */
@Test
public void testExecutionGraphConstructionFailsRescaleDopExceedMaxParallelism() throws Exception {

	final Configuration config = new Configuration();

	final int initialParallelism = 1;
	final int maxParallelism = 10;
	final JobVertex[] jobVertices = createVerticesForSimpleBipartiteJobGraph(initialParallelism,  maxParallelism);
	final JobGraph jobGraph = new JobGraph(jobVertices);

	for (JobVertex jv : jobVertices) {
		jv.setParallelism(maxParallelism + 1);
	}

	try {
		// this should fail since we set the parallelism to maxParallelism + 1
		ExecutionGraphBuilder.buildGraph(
			null,
			jobGraph,
			config,
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			new TestingSlotProvider(ignore -> new CompletableFuture<>()),
			Thread.currentThread().getContextClassLoader(),
			new StandaloneCheckpointRecoveryFactory(),
			AkkaUtils.getDefaultTimeout(),
			new NoRestartStrategy(),
			new UnregisteredMetricsGroup(),
			VoidBlobWriter.getInstance(),
			AkkaUtils.getDefaultTimeout(),
			TEST_LOGGER,
			NettyShuffleMaster.INSTANCE,
			NoOpJobMasterPartitionTracker.INSTANCE);

		fail("Building the ExecutionGraph with a parallelism higher than the max parallelism should fail.");
	} catch (JobException e) {
		// expected, ignore
	}
}
Example #20
Source File: ExecutionGraphDeploymentTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private ExecutionGraph createExecutionGraph(Configuration configuration) throws Exception {
	final ScheduledExecutorService executor = TestingUtils.defaultExecutor();

	final JobID jobId = new JobID();
	final JobGraph jobGraph = new JobGraph(jobId, "test");
	jobGraph.setSnapshotSettings(
		new JobCheckpointingSettings(
			Collections.<JobVertexID>emptyList(),
			Collections.<JobVertexID>emptyList(),
			Collections.<JobVertexID>emptyList(),
			new CheckpointCoordinatorConfiguration(
				100,
				10 * 60 * 1000,
				0,
				1,
				CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
				false,
				false,
				0),
			null));

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		configuration,
		executor,
		executor,
		new ProgrammedSlotProvider(1),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		blobWriter,
		timeout,
		LoggerFactory.getLogger(getClass()),
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);
}
Example #21
Source File: AbstractYarnNonHaServices.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public CheckpointRecoveryFactory getCheckpointRecoveryFactory() {
	enter();
	try {
		return new StandaloneCheckpointRecoveryFactory();
	}
	finally {
		exit();
	}
}
Example #22
Source File: DispatcherTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws Exception {
	final JobVertex testVertex = new JobVertex("testVertex");
	testVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(TEST_JOB_ID, "testJob", testVertex);
	jobGraph.setAllowQueuedScheduling(true);

	fatalErrorHandler = new TestingFatalErrorHandler();
	heartbeatServices = new HeartbeatServices(1000L, 10000L);
	submittedJobGraphStore = new FaultySubmittedJobGraphStore();

	dispatcherLeaderElectionService = new TestingLeaderElectionService();
	jobMasterLeaderElectionService = new TestingLeaderElectionService();

	haServices = new TestingHighAvailabilityServices();
	haServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);
	haServices.setSubmittedJobGraphStore(submittedJobGraphStore);
	haServices.setJobMasterLeaderElectionService(TEST_JOB_ID, jobMasterLeaderElectionService);
	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());
	haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());
	runningJobsRegistry = haServices.getRunningJobsRegistry();

	configuration = new Configuration();

	configuration.setString(
		BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());

	createdJobManagerRunnerLatch = new CountDownLatch(2);
	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
Example #23
Source File: JobManagerRunnerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() {
	leaderElectionService = new TestingLeaderElectionService();
	haServices = new TestingHighAvailabilityServices();
	haServices.setJobMasterLeaderElectionService(jobGraph.getJobID(), leaderElectionService);
	haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());
	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());

	fatalErrorHandler = new TestingFatalErrorHandler();
}
Example #24
Source File: ExecutionGraphDeploymentTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private ExecutionGraph createExecutionGraph(Configuration configuration) throws Exception {
	final ScheduledExecutorService executor = TestingUtils.defaultExecutor();

	final JobID jobId = new JobID();
	final JobGraph jobGraph = new JobGraph(jobId, "test");
	jobGraph.setSnapshotSettings(
		new JobCheckpointingSettings(
			Collections.<JobVertexID>emptyList(),
			Collections.<JobVertexID>emptyList(),
			Collections.<JobVertexID>emptyList(),
			new CheckpointCoordinatorConfiguration(
				100,
				10 * 60 * 1000,
				0,
				1,
				CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
				false),
			null));

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		configuration,
		executor,
		executor,
		new ProgrammedSlotProvider(1),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		1,
		blobWriter,
		timeout,
		LoggerFactory.getLogger(getClass()));
}
Example #25
Source File: ExecutionVertexLocalityTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Creates a simple 2 vertex graph with a parallel source and a parallel target.
 */
private ExecutionGraph createTestGraph(int parallelism, boolean allToAll) throws Exception {

	JobVertex source = new JobVertex("source", sourceVertexId);
	source.setParallelism(parallelism);
	source.setInvokableClass(NoOpInvokable.class);

	JobVertex target = new JobVertex("source", targetVertexId);
	target.setParallelism(parallelism);
	target.setInvokableClass(NoOpInvokable.class);

	DistributionPattern connectionPattern = allToAll ? DistributionPattern.ALL_TO_ALL : DistributionPattern.POINTWISE;
	target.connectNewDataSetAsInput(source, connectionPattern, ResultPartitionType.PIPELINED);

	JobGraph testJob = new JobGraph(jobId, "test job", source, target);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		testJob,
		new Configuration(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new FixedDelayRestartStrategy(10, 0L),
		new UnregisteredMetricsGroup(),
		1,
		VoidBlobWriter.getInstance(),
		timeout,
		log);
}
Example #26
Source File: ExecutionGraphTestUtils.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public static ExecutionGraph createExecutionGraph(
		JobID jid,
		SlotProvider slotProvider,
		RestartStrategy restartStrategy,
		ScheduledExecutorService executor,
		Time timeout,
		JobVertex... vertices) throws Exception {

	checkNotNull(jid);
	checkNotNull(restartStrategy);
	checkNotNull(vertices);
	checkNotNull(timeout);

	return ExecutionGraphBuilder.buildGraph(
		null,
		new JobGraph(jid, "test job", vertices),
		new Configuration(),
		executor,
		executor,
		slotProvider,
		ExecutionGraphTestUtils.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		restartStrategy,
		new UnregisteredMetricsGroup(),
		1,
		VoidBlobWriter.getInstance(),
		timeout,
		TEST_LOGGER);
}
Example #27
Source File: ExecutionGraphRescalingTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Verifies that building an {@link ExecutionGraph} from a {@link JobGraph} with
 * parallelism higher than the maximum parallelism fails.
 */
@Test
public void testExecutionGraphConstructionFailsRescaleDopExceedMaxParallelism() throws Exception {

	final Configuration config = new Configuration();

	final int initialParallelism = 1;
	final int maxParallelism = 10;
	final JobVertex[] jobVertices = createVerticesForSimpleBipartiteJobGraph(initialParallelism,  maxParallelism);
	final JobGraph jobGraph = new JobGraph(jobVertices);

	for (JobVertex jv : jobVertices) {
		jv.setParallelism(maxParallelism + 1);
	}

	try {
		// this should fail since we set the parallelism to maxParallelism + 1
		ExecutionGraphBuilder.buildGraph(
			null,
			jobGraph,
			config,
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			new TestingSlotProvider(ignore -> new CompletableFuture<>()),
			Thread.currentThread().getContextClassLoader(),
			new StandaloneCheckpointRecoveryFactory(),
			AkkaUtils.getDefaultTimeout(),
			new NoRestartStrategy(),
			new UnregisteredMetricsGroup(),
			VoidBlobWriter.getInstance(),
			AkkaUtils.getDefaultTimeout(),
			TEST_LOGGER);

		fail("Building the ExecutionGraph with a parallelism higher than the max parallelism should fail.");
	} catch (JobException e) {
		// expected, ignore
	}
}
Example #28
Source File: AbstractYarnNonHaServices.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public CheckpointRecoveryFactory getCheckpointRecoveryFactory() {
	enter();
	try {
		return new StandaloneCheckpointRecoveryFactory();
	}
	finally {
		exit();
	}
}
Example #29
Source File: AbstractNonHaServices.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public CheckpointRecoveryFactory getCheckpointRecoveryFactory() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneCheckpointRecoveryFactory();
	}
}
Example #30
Source File: DispatcherTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws Exception {
	final JobVertex testVertex = new JobVertex("testVertex");
	testVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(TEST_JOB_ID, "testJob", testVertex);
	jobGraph.setAllowQueuedScheduling(true);

	fatalErrorHandler = new TestingFatalErrorHandler();
	heartbeatServices = new HeartbeatServices(1000L, 10000L);
	submittedJobGraphStore = new FaultySubmittedJobGraphStore();

	dispatcherLeaderElectionService = new TestingLeaderElectionService();
	jobMasterLeaderElectionService = new TestingLeaderElectionService();

	haServices = new TestingHighAvailabilityServices();
	haServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);
	haServices.setSubmittedJobGraphStore(submittedJobGraphStore);
	haServices.setJobMasterLeaderElectionService(TEST_JOB_ID, jobMasterLeaderElectionService);
	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());
	haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());
	runningJobsRegistry = haServices.getRunningJobsRegistry();

	configuration = new Configuration();

	configuration.setString(
		BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());

	createdJobManagerRunnerLatch = new CountDownLatch(2);
	blobServer = new BlobServer(configuration, new VoidBlobStore());
}