Java Code Examples for org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices#setDispatcherLeaderElectionService()

The following examples show how to use org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices#setDispatcherLeaderElectionService() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DispatcherTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws Exception {
	final JobVertex testVertex = new JobVertex("testVertex");
	testVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(TEST_JOB_ID, "testJob", testVertex);
	jobGraph.setAllowQueuedScheduling(true);

	fatalErrorHandler = new TestingFatalErrorHandler();
	heartbeatServices = new HeartbeatServices(1000L, 10000L);
	submittedJobGraphStore = new FaultySubmittedJobGraphStore();

	dispatcherLeaderElectionService = new TestingLeaderElectionService();
	jobMasterLeaderElectionService = new TestingLeaderElectionService();

	haServices = new TestingHighAvailabilityServices();
	haServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);
	haServices.setSubmittedJobGraphStore(submittedJobGraphStore);
	haServices.setJobMasterLeaderElectionService(TEST_JOB_ID, jobMasterLeaderElectionService);
	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());
	haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());
	runningJobsRegistry = haServices.getRunningJobsRegistry();

	configuration = new Configuration();

	configuration.setString(
		BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());

	createdJobManagerRunnerLatch = new CountDownLatch(2);
	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
Example 2
Source File: MiniDispatcherTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() throws Exception {
	dispatcherLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices = new TestingHighAvailabilityServices();
	testingFatalErrorHandler = new TestingFatalErrorHandler();

	highAvailabilityServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);

	jobGraphFuture = new CompletableFuture<>();
	resultFuture = new CompletableFuture<>();

	testingJobManagerRunnerFactory = new TestingJobManagerRunnerFactory(jobGraphFuture, resultFuture, CompletableFuture.completedFuture(null));
}
Example 3
Source File: DispatcherTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Before
public void setUp() throws Exception {
	final JobVertex testVertex = new JobVertex("testVertex");
	testVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(TEST_JOB_ID, "testJob", testVertex);
	jobGraph.setAllowQueuedScheduling(true);

	fatalErrorHandler = new TestingFatalErrorHandler();
	heartbeatServices = new HeartbeatServices(1000L, 10000L);
	submittedJobGraphStore = new FaultySubmittedJobGraphStore();

	dispatcherLeaderElectionService = new TestingLeaderElectionService();
	jobMasterLeaderElectionService = new TestingLeaderElectionService();

	haServices = new TestingHighAvailabilityServices();
	haServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);
	haServices.setSubmittedJobGraphStore(submittedJobGraphStore);
	haServices.setJobMasterLeaderElectionService(TEST_JOB_ID, jobMasterLeaderElectionService);
	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());
	haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());
	runningJobsRegistry = haServices.getRunningJobsRegistry();

	configuration = new Configuration();

	configuration.setString(
		BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());

	createdJobManagerRunnerLatch = new CountDownLatch(2);
	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
Example 4
Source File: MiniDispatcherTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Before
public void setup() throws Exception {
	dispatcherLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices = new TestingHighAvailabilityServices();
	testingFatalErrorHandler = new TestingFatalErrorHandler();

	highAvailabilityServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);

	jobGraphFuture = new CompletableFuture<>();
	resultFuture = new CompletableFuture<>();

	testingJobManagerRunnerFactory = new TestingJobManagerRunnerFactory(jobGraphFuture, resultFuture, CompletableFuture.completedFuture(null));
}
Example 5
Source File: DispatcherResourceCleanupTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Before
public void setup() throws Exception {
	final JobVertex testVertex = new JobVertex("testVertex");
	testVertex.setInvokableClass(NoOpInvokable.class);
	jobId = new JobID();
	jobGraph = new JobGraph(jobId, "testJob", testVertex);
	jobGraph.setAllowQueuedScheduling(true);

	configuration = new Configuration();
	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());

	highAvailabilityServices = new TestingHighAvailabilityServices();
	dispatcherLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);
	clearedJobLatch = new OneShotLatch();
	runningJobsRegistry = new SingleRunningJobsRegistry(jobId, clearedJobLatch);
	highAvailabilityServices.setRunningJobsRegistry(runningJobsRegistry);
	submittedJobGraphStore = new FaultySubmittedJobGraphStore();
	highAvailabilityServices.setSubmittedJobGraphStore(submittedJobGraphStore);

	storedHABlobFuture = new CompletableFuture<>();
	deleteAllHABlobsFuture = new CompletableFuture<>();

	final TestingBlobStore testingBlobStore = new TestingBlobStoreBuilder()
		.setPutFunction(
			putArguments -> storedHABlobFuture.complete(putArguments.f2))
		.setDeleteAllFunction(deleteAllHABlobsFuture::complete)
		.createTestingBlobStore();

	cleanupJobFuture = new CompletableFuture<>();
	terminationFuture = new CompletableFuture<>();

	blobServer = new TestingBlobServer(configuration, testingBlobStore, cleanupJobFuture);

	// upload a blob to the blob server
	permanentBlobKey = blobServer.putPermanent(jobId, new byte[256]);
	jobGraph.addUserJarBlobKey(permanentBlobKey);
	blobFile = blobServer.getStorageLocation(jobId, permanentBlobKey);

	resultFuture = new CompletableFuture<>();

	fatalErrorHandler = new TestingFatalErrorHandler();

	failJobMasterCreationWith = new AtomicReference<>();

	TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	dispatcher = new TestingDispatcher(
		rpcService,
		Dispatcher.DISPATCHER_NAME + UUID.randomUUID(),
		configuration,
		highAvailabilityServices,
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		new HeartbeatServices(1000L, 1000L),
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(),
		null,
		new MemoryArchivedExecutionGraphStore(),
		new TestingJobManagerRunnerFactory(new CompletableFuture<>(), resultFuture, terminationFuture, failJobMasterCreationWith),
		fatalErrorHandler);

	dispatcher.start();

	dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

	dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get();

	assertThat(blobFile.exists(), is(true));

	// verify that we stored the blob also in the BlobStore
	assertThat(storedHABlobFuture.get(), equalTo(permanentBlobKey));
}
Example 6
Source File: DispatcherHATest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that interleaved granting and revoking of the leadership won't interfere
 * with the job recovery and the resulting internal state of the Dispatcher.
 */
@Test
public void testGrantingRevokingLeadership() throws Exception {
	final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
	final JobGraph nonEmptyJobGraph = createNonEmptyJobGraph();
	final SubmittedJobGraph submittedJobGraph = new SubmittedJobGraph(nonEmptyJobGraph);

	final OneShotLatch enterGetJobIdsLatch = new OneShotLatch();
	final OneShotLatch proceedGetJobIdsLatch = new OneShotLatch();
	highAvailabilityServices.setSubmittedJobGraphStore(new BlockingSubmittedJobGraphStore(submittedJobGraph, enterGetJobIdsLatch, proceedGetJobIdsLatch));
	final TestingLeaderElectionService dispatcherLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);

	final BlockingQueue<DispatcherId> fencingTokens = new ArrayBlockingQueue<>(2);

	final HATestingDispatcher dispatcher = createDispatcherWithObservableFencingTokens(highAvailabilityServices, fencingTokens);

	dispatcher.start();

	try {
		// wait until the election service has been started
		dispatcherLeaderElectionService.getStartFuture().get();

		final UUID leaderId = UUID.randomUUID();
		dispatcherLeaderElectionService.isLeader(leaderId);

		dispatcherLeaderElectionService.notLeader();

		final DispatcherId firstFencingToken = fencingTokens.take();

		assertThat(firstFencingToken, equalTo(NULL_FENCING_TOKEN));

		enterGetJobIdsLatch.await();
		proceedGetJobIdsLatch.trigger();

		assertThat(dispatcher.getNumberJobs(timeout).get(), is(0));

	} finally {
		RpcUtils.terminateRpcEndpoint(dispatcher, timeout);
	}
}
Example 7
Source File: DispatcherResourceCleanupTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Before
public void setup() throws Exception {
	final JobVertex testVertex = new JobVertex("testVertex");
	testVertex.setInvokableClass(NoOpInvokable.class);
	jobId = new JobID();
	jobGraph = new JobGraph(jobId, "testJob", testVertex);
	jobGraph.setAllowQueuedScheduling(true);

	configuration = new Configuration();
	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());

	highAvailabilityServices = new TestingHighAvailabilityServices();
	dispatcherLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);
	clearedJobLatch = new OneShotLatch();
	runningJobsRegistry = new SingleRunningJobsRegistry(jobId, clearedJobLatch);
	highAvailabilityServices.setRunningJobsRegistry(runningJobsRegistry);
	submittedJobGraphStore = new FaultySubmittedJobGraphStore();
	highAvailabilityServices.setSubmittedJobGraphStore(submittedJobGraphStore);

	storedHABlobFuture = new CompletableFuture<>();
	deleteAllHABlobsFuture = new CompletableFuture<>();

	final TestingBlobStore testingBlobStore = new TestingBlobStoreBuilder()
		.setPutFunction(
			putArguments -> storedHABlobFuture.complete(putArguments.f2))
		.setDeleteAllFunction(deleteAllHABlobsFuture::complete)
		.createTestingBlobStore();

	cleanupJobFuture = new CompletableFuture<>();
	terminationFuture = new CompletableFuture<>();

	blobServer = new TestingBlobServer(configuration, testingBlobStore, cleanupJobFuture);

	// upload a blob to the blob server
	permanentBlobKey = blobServer.putPermanent(jobId, new byte[256]);
	jobGraph.addUserJarBlobKey(permanentBlobKey);
	blobFile = blobServer.getStorageLocation(jobId, permanentBlobKey);

	resultFuture = new CompletableFuture<>();

	fatalErrorHandler = new TestingFatalErrorHandler();

	failJobMasterCreationWith = new AtomicReference<>();

	TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	dispatcher = new TestingDispatcher(
		rpcService,
		Dispatcher.DISPATCHER_NAME + UUID.randomUUID(),
		configuration,
		highAvailabilityServices,
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		new HeartbeatServices(1000L, 1000L),
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(),
		null,
		new MemoryArchivedExecutionGraphStore(),
		new TestingJobManagerRunnerFactory(new CompletableFuture<>(), resultFuture, terminationFuture, failJobMasterCreationWith),
		fatalErrorHandler);

	dispatcher.start();

	dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

	dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get();

	assertThat(blobFile.exists(), is(true));

	// verify that we stored the blob also in the BlobStore
	assertThat(storedHABlobFuture.get(), equalTo(permanentBlobKey));
}
Example 8
Source File: DispatcherHATest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that interleaved granting and revoking of the leadership won't interfere
 * with the job recovery and the resulting internal state of the Dispatcher.
 */
@Test
public void testGrantingRevokingLeadership() throws Exception {
	final TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
	final JobGraph nonEmptyJobGraph = createNonEmptyJobGraph();
	final SubmittedJobGraph submittedJobGraph = new SubmittedJobGraph(nonEmptyJobGraph);

	final OneShotLatch enterGetJobIdsLatch = new OneShotLatch();
	final OneShotLatch proceedGetJobIdsLatch = new OneShotLatch();
	highAvailabilityServices.setSubmittedJobGraphStore(new BlockingSubmittedJobGraphStore(submittedJobGraph, enterGetJobIdsLatch, proceedGetJobIdsLatch));
	final TestingLeaderElectionService dispatcherLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);

	final BlockingQueue<DispatcherId> fencingTokens = new ArrayBlockingQueue<>(2);

	final HATestingDispatcher dispatcher = createDispatcherWithObservableFencingTokens(highAvailabilityServices, fencingTokens);

	dispatcher.start();

	try {
		// wait until the election service has been started
		dispatcherLeaderElectionService.getStartFuture().get();

		final UUID leaderId = UUID.randomUUID();
		dispatcherLeaderElectionService.isLeader(leaderId);

		dispatcherLeaderElectionService.notLeader();

		final DispatcherId firstFencingToken = fencingTokens.take();

		assertThat(firstFencingToken, equalTo(NULL_FENCING_TOKEN));

		enterGetJobIdsLatch.await();
		proceedGetJobIdsLatch.trigger();

		assertThat(dispatcher.getNumberJobs(timeout).get(), is(0));

	} finally {
		RpcUtils.terminateRpcEndpoint(dispatcher, timeout);
	}
}