org.apache.flink.runtime.leaderelection.LeaderElectionService Java Examples

@Override
public DispatcherRunner createDispatcherRunner(
		LeaderElectionService leaderElectionService,
		FatalErrorHandler fatalErrorHandler,
		JobGraphStoreFactory jobGraphStoreFactory,
		Executor ioExecutor,
		RpcService rpcService,
		PartialDispatcherServices partialDispatcherServices) throws Exception {

	final DispatcherLeaderProcessFactory dispatcherLeaderProcessFactory = dispatcherLeaderProcessFactoryFactory.createFactory(
		jobGraphStoreFactory,
		ioExecutor,
		rpcService,
		partialDispatcherServices,
		fatalErrorHandler);

	return DefaultDispatcherRunner.create(
		leaderElectionService,
		fatalErrorHandler,
		dispatcherLeaderProcessFactory);
}
 

public DispatcherRestEndpoint(
		RestServerEndpointConfiguration endpointConfiguration,
		GatewayRetriever<DispatcherGateway> leaderRetriever,
		Configuration clusterConfiguration,
		RestHandlerConfiguration restConfiguration,
		GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,
		TransientBlobService transientBlobService,
		ExecutorService executor,
		MetricFetcher metricFetcher,
		LeaderElectionService leaderElectionService,
		FatalErrorHandler fatalErrorHandler) throws IOException {

	super(
		endpointConfiguration,
		leaderRetriever,
		clusterConfiguration,
		restConfiguration,
		resourceManagerRetriever,
		transientBlobService,
		executor,
		metricFetcher,
		leaderElectionService,
		fatalErrorHandler);

	webSubmissionExtension = WebMonitorExtension.empty();
}
 

/**
 * Tests that the {@link EmbeddedLeaderService} can handle a concurrent grant
 * leadership call and a shutdown.
 */
@Test
public void testConcurrentGrantLeadershipAndShutdown() throws Exception {
	final EmbeddedLeaderService embeddedLeaderService = new EmbeddedLeaderService(TestingUtils.defaultExecutor());

	try {
		final LeaderElectionService leaderElectionService = embeddedLeaderService.createLeaderElectionService();

		final TestingLeaderContender contender = new TestingLeaderContender();

		leaderElectionService.start(contender);
		leaderElectionService.stop();

		try {
			// check that no exception occurred
			contender.getLeaderSessionFuture().get(10L, TimeUnit.MILLISECONDS);
		} catch (TimeoutException ignored) {
			// we haven't participated in the leader election
		}

		// the election service should still be running
		Assert.assertThat(embeddedLeaderService.isShutdown(), is(false));
	} finally {
		embeddedLeaderService.shutdown();
	}
}
 

public MiniDispatcherRestEndpoint(
		RestServerEndpointConfiguration endpointConfiguration,
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Configuration clusterConfiguration,
		RestHandlerConfiguration restConfiguration,
		GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,
		TransientBlobService transientBlobService,
		ExecutorService executor,
		MetricFetcher metricFetcher,
		LeaderElectionService leaderElectionService,
		FatalErrorHandler fatalErrorHandler) throws IOException {
	super(
		endpointConfiguration,
		leaderRetriever,
		clusterConfiguration,
		restConfiguration,
		resourceManagerRetriever,
		transientBlobService,
		executor,
		metricFetcher,
		leaderElectionService,
		fatalErrorHandler);
}
 

/**
 * Tests that the standalone leader election services return a fixed address and leader session
 * id.
 */
@Test
public void testLeaderElection() throws Exception {
	JobID jobId = new JobID();
	LeaderContender jmLeaderContender = mock(LeaderContender.class);
	LeaderContender rmLeaderContender = mock(LeaderContender.class);

	LeaderElectionService jmLeaderElectionService = standaloneHaServices.getJobManagerLeaderElectionService(jobId);
	LeaderElectionService rmLeaderElectionService = standaloneHaServices.getResourceManagerLeaderElectionService();

	jmLeaderElectionService.start(jmLeaderContender);
	rmLeaderElectionService.start(rmLeaderContender);

	verify(jmLeaderContender).grantLeadership(eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(rmLeaderContender).grantLeadership(eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 

/**
 * Tests that the {@link EmbeddedLeaderService} can handle a concurrent grant
 * leadership call and a shutdown.
 */
@Test
public void testConcurrentGrantLeadershipAndShutdown() throws Exception {
	final EmbeddedLeaderService embeddedLeaderService = new EmbeddedLeaderService(TestingUtils.defaultExecutor());

	try {
		final LeaderElectionService leaderElectionService = embeddedLeaderService.createLeaderElectionService();

		final TestingLeaderContender contender = new TestingLeaderContender();

		leaderElectionService.start(contender);
		leaderElectionService.stop();

		try {
			// check that no exception occurred
			contender.getLeaderSessionFuture().get(10L, TimeUnit.MILLISECONDS);
		} catch (TimeoutException ignored) {
			// we haven't participated in the leader election
		}

		// the election service should still be running
		Assert.assertThat(embeddedLeaderService.isShutdown(), is(false));
	} finally {
		embeddedLeaderService.shutdown();
	}
}
 

/**
 * Tests that exactly one ResourceManager is elected as the leader.
 */
@Test
public void testResourceManagerLeaderElection() throws Exception {
	LeaderContender leaderContender1 = mock(LeaderContender.class);
	LeaderContender leaderContender2 = mock(LeaderContender.class);

	LeaderElectionService leaderElectionService1 = embeddedHaServices.getResourceManagerLeaderElectionService();
	LeaderElectionService leaderElectionService2 = embeddedHaServices.getResourceManagerLeaderElectionService();

	leaderElectionService1.start(leaderContender1);
	leaderElectionService2.start(leaderContender2);

	ArgumentCaptor<UUID> leaderIdArgumentCaptor1 = ArgumentCaptor.forClass(UUID.class);
	ArgumentCaptor<UUID> leaderIdArgumentCaptor2 = ArgumentCaptor.forClass(UUID.class);
	verify(leaderContender1, atLeast(0)).grantLeadership(leaderIdArgumentCaptor1.capture());
	verify(leaderContender2, atLeast(0)).grantLeadership(leaderIdArgumentCaptor2.capture());

	assertTrue(leaderIdArgumentCaptor1.getAllValues().isEmpty() ^ leaderIdArgumentCaptor2.getAllValues().isEmpty());
}
 

/**
 * Tests the JobManager leader retrieval for a given job.
 */
@Test
public void testJobManagerLeaderRetrieval() throws Exception {
	final String address = "foobar";
	JobID jobId = new JobID();
	LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class);
	LeaderContender leaderContender = mock(LeaderContender.class);
	when(leaderContender.getAddress()).thenReturn(address);

	LeaderElectionService leaderElectionService = embeddedHaServices.getJobManagerLeaderElectionService(jobId);
	LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getJobManagerLeaderRetriever(jobId);

	leaderRetrievalService.start(leaderRetrievalListener);
	leaderElectionService.start(leaderContender);

	ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);
	verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture());

	final UUID leaderId = leaderIdArgumentCaptor.getValue();

	leaderElectionService.confirmLeaderSessionID(leaderId);

	verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId));
}
 

/**
 * Tests the ResourceManager leader retrieval for a given job.
 */
@Test
public void testResourceManagerLeaderRetrieval() throws Exception {
	final String address = "foobar";
	LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class);
	LeaderContender leaderContender = mock(LeaderContender.class);
	when(leaderContender.getAddress()).thenReturn(address);

	LeaderElectionService leaderElectionService = embeddedHaServices.getResourceManagerLeaderElectionService();
	LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getResourceManagerLeaderRetriever();

	leaderRetrievalService.start(leaderRetrievalListener);
	leaderElectionService.start(leaderContender);

	ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);
	verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture());

	final UUID leaderId = leaderIdArgumentCaptor.getValue();

	leaderElectionService.confirmLeaderSessionID(leaderId);

	verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId));
}
 

/**
 * Tests that the standalone leader election services return a fixed address and leader session
 * id.
 */
@Test
public void testLeaderElection() throws Exception {
	JobID jobId = new JobID();
	LeaderContender jmLeaderContender = mock(LeaderContender.class);
	LeaderContender rmLeaderContender = mock(LeaderContender.class);

	LeaderElectionService jmLeaderElectionService = standaloneHaServices.getJobManagerLeaderElectionService(jobId);
	LeaderElectionService rmLeaderElectionService = standaloneHaServices.getResourceManagerLeaderElectionService();

	jmLeaderElectionService.start(jmLeaderContender);
	rmLeaderElectionService.start(rmLeaderContender);

	verify(jmLeaderContender).grantLeadership(eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(rmLeaderContender).grantLeadership(eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 

@Test
public void testClosingReportsToLeader() throws Exception {
	final Configuration flinkConfig = new Configuration();

	try (YarnHighAvailabilityServices services = new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig)) {
		final LeaderElectionService elector = services.getResourceManagerLeaderElectionService();
		final LeaderRetrievalService retrieval = services.getResourceManagerLeaderRetriever();
		final LeaderContender contender = mockContender(elector);
		final LeaderRetrievalListener listener = mock(LeaderRetrievalListener.class);

		elector.start(contender);
		retrieval.start(listener);

		// wait until the contender has become the leader
		verify(listener, timeout(1000L).times(1)).notifyLeaderAddress(anyString(), any(UUID.class));

		// now we can close the election service
		services.close();

		verify(contender, timeout(1000L).times(1)).handleError(any(Exception.class));
	}
}
 

public DispatcherRestEndpoint(
		RestServerEndpointConfiguration endpointConfiguration,
		GatewayRetriever<DispatcherGateway> leaderRetriever,
		Configuration clusterConfiguration,
		RestHandlerConfiguration restConfiguration,
		GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,
		TransientBlobService transientBlobService,
		ExecutorService executor,
		MetricFetcher metricFetcher,
		LeaderElectionService leaderElectionService,
		FatalErrorHandler fatalErrorHandler) throws IOException {

	super(
		endpointConfiguration,
		leaderRetriever,
		clusterConfiguration,
		restConfiguration,
		resourceManagerRetriever,
		transientBlobService,
		executor,
		metricFetcher,
		leaderElectionService,
		fatalErrorHandler);

	webSubmissionExtension = WebMonitorExtension.empty();
}
 

@Override
public LeaderElectionService getResourceManagerLeaderElectionService() {
	enter();
	try {
		return resourceManagerLeaderElectionService;
	}
	finally {
		exit();
	}
}
 

private StandaloneResourceManager createAndStartResourceManager(LeaderElectionService rmLeaderElectionService, FatalErrorHandler fatalErrorHandler) throws Exception {
	TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
	HeartbeatServices heartbeatServices = new HeartbeatServices(1000L, HEARTBEAT_TIMEOUT);
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);

	SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();

	JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	StandaloneResourceManager resourceManager =
		new StandaloneResourceManager(
			rpcService,
			ResourceManager.RESOURCE_MANAGER_NAME + UUID.randomUUID(),
			resourceManagerResourceID,
			highAvailabilityServices,
			heartbeatServices,
			slotManager,
			NoOpMetricRegistry.INSTANCE,
			jobLeaderIdService,
			new ClusterInformation("localhost", 1234),
			fatalErrorHandler,
			UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());

	resourceManager.start();

	return resourceManager;
}
 

@Override
public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
	enter();
	try {
		throw new UnsupportedOperationException("needs refactoring to accept default address");
	}
	finally {
		exit();
	}
}
 

@Override
public LeaderElectionService getDispatcherLeaderElectionService() {
	LeaderElectionService service = dispatcherLeaderElectionService;

	if (service != null) {
		return service;
	} else {
		throw new IllegalStateException("DispatcherLeaderElectionService has not been set");
	}
}
 

private StandaloneResourceManager createAndStartResourceManager(LeaderElectionService rmLeaderElectionService, FatalErrorHandler fatalErrorHandler) throws Exception {
	TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
	HeartbeatServices heartbeatServices = new HeartbeatServices(1000L, HEARTBEAT_TIMEOUT);
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);

	SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();

	JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	StandaloneResourceManager resourceManager =
		new StandaloneResourceManager(
			rpcService,
			resourceManagerResourceID,
			highAvailabilityServices,
			heartbeatServices,
			slotManager,
			NoOpResourceManagerPartitionTracker::get,
			jobLeaderIdService,
			new ClusterInformation("localhost", 1234),
			fatalErrorHandler,
			UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup(),
			Time.minutes(5L),
			RpcUtils.INF_TIMEOUT);

	resourceManager.start();

	return resourceManager;
}
 

@Override
public LeaderElectionService getWebMonitorLeaderElectionService() {
	enter();
	try {
		throw new UnsupportedOperationException();
	}
	finally {
		exit();
	}
}
 

@Override
public LeaderElectionService getDispatcherLeaderElectionService() {
	enter();
	try {
		return dispatcherLeaderElectionService;
	} finally {
		exit();
	}
}
 

@Override
public LeaderElectionService getResourceManagerLeaderElectionService() {
	enter();
	try {
		return resourceManagerLeaderElectionService;
	}
	finally {
		exit();
	}
}
 

@Override
public LeaderElectionService getResourceManagerLeaderElectionService() {
	LeaderElectionService service = resourceManagerLeaderElectionService;

	if (service != null) {
		return service;
	} else {
		throw new IllegalStateException("ResourceManagerLeaderElectionService has not been set");
	}
}
 

@Override
public LeaderElectionService getResourceManagerLeaderElectionService() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderElectionService();
	}
}
 

@Override
public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
	LeaderElectionService service = jobManagerLeaderElectionServices.computeIfAbsent(jobID, jobMasterLeaderElectionServiceFunction);

	if (service != null) {
		return service;
	} else {
		throw new IllegalStateException("JobMasterLeaderElectionService has not been set");
	}
}
 

@Override
public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderElectionService();
	}
}
 

@Override
public LeaderElectionService getResourceManagerLeaderElectionService() {
	enter();
	try {
		throw new UnsupportedOperationException("Not supported on the TaskManager side");
	}
	finally {
		exit();
	}
}
 

@Override
public LeaderElectionService getWebMonitorLeaderElectionService() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderElectionService();
	}
}
 

/**
 * Tests that the {@link EmbeddedLeaderService} can handle a concurrent revoke
 * leadership call and a shutdown.
 */
@Test
public void testConcurrentRevokeLeadershipAndShutdown() throws Exception {
	final EmbeddedLeaderService embeddedLeaderService = new EmbeddedLeaderService(TestingUtils.defaultExecutor());

	try {
		final LeaderElectionService leaderElectionService = embeddedLeaderService.createLeaderElectionService();

		final TestingLeaderContender contender = new TestingLeaderContender();

		leaderElectionService.start(contender);

		// wait for the leadership
		contender.getLeaderSessionFuture().get();

		final CompletableFuture<Void> revokeLeadershipFuture = embeddedLeaderService.revokeLeadership();
		leaderElectionService.stop();

		try {
			// check that no exception occurred
			revokeLeadershipFuture.get(10L, TimeUnit.MILLISECONDS);
		} catch (TimeoutException ignored) {
			// the leader election service has been stopped before revoking could be executed
		}

		// the election service should still be running
		Assert.assertThat(embeddedLeaderService.isShutdown(), is(false));
	} finally {
		embeddedLeaderService.shutdown();
	}
}
 

@Override
public LeaderElectionService getDispatcherLeaderElectionService() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderElectionService();
	}
}
 

@Override
public LeaderElectionService getWebMonitorLeaderElectionService() {
	enter();
	try {
		throw new UnsupportedOperationException();
	}
	finally {
		exit();
	}
}
 

@Override
public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
	enter();
	try {
		throw new UnsupportedOperationException("needs refactoring to accept default address");
	}
	finally {
		exit();
	}
}