Java Code Examples for org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService#start()
The following examples show how to use
org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService#start() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StandaloneHaServicesTest.java From flink with Apache License 2.0 | 6 votes |
/** * Tests that the standalone leader retrieval services return the specified address and the * fixed leader session id. */ @Test public void testJobManagerLeaderRetrieval() throws Exception { JobID jobId1 = new JobID(); JobID jobId2 = new JobID(); LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class); LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class); LeaderRetrievalListener rmListener = mock(LeaderRetrievalListener.class); LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1); LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2); LeaderRetrievalService rmLeaderRetrievalService = standaloneHaServices.getResourceManagerLeaderRetriever(); jmLeaderRetrievalService1.start(jmListener1); jmLeaderRetrievalService2.start(jmListener2); rmLeaderRetrievalService.start(rmListener); verify(jmListener1).notifyLeaderAddress(eq("UNKNOWN"), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); verify(jmListener2).notifyLeaderAddress(eq("UNKNOWN"), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); verify(rmListener).notifyLeaderAddress(eq(resourceManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); }
Example 2
Source File: DefaultJobLeaderService.java From flink with Apache License 2.0 | 6 votes |
@Override public void addJob(final JobID jobId, final String defaultTargetAddress) throws Exception { Preconditions.checkState(DefaultJobLeaderService.State.STARTED == state, "The service is currently not running."); LOG.info("Add job {} for job leader monitoring.", jobId); final LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever( jobId, defaultTargetAddress); DefaultJobLeaderService.JobManagerLeaderListener jobManagerLeaderListener = new JobManagerLeaderListener(jobId); final Tuple2<LeaderRetrievalService, JobManagerLeaderListener> oldEntry = jobLeaderServices.put(jobId, Tuple2.of(leaderRetrievalService, jobManagerLeaderListener)); if (oldEntry != null) { oldEntry.f0.stop(); oldEntry.f1.stop(); } leaderRetrievalService.start(jobManagerLeaderListener); }
Example 3
Source File: EmbeddedHaServicesTest.java From flink with Apache License 2.0 | 6 votes |
/** * Tests the JobManager leader retrieval for a given job. */ @Test public void testJobManagerLeaderRetrieval() throws Exception { final String address = "foobar"; JobID jobId = new JobID(); LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class); LeaderContender leaderContender = mock(LeaderContender.class); when(leaderContender.getAddress()).thenReturn(address); LeaderElectionService leaderElectionService = embeddedHaServices.getJobManagerLeaderElectionService(jobId); LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getJobManagerLeaderRetriever(jobId); leaderRetrievalService.start(leaderRetrievalListener); leaderElectionService.start(leaderContender); ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class); verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture()); final UUID leaderId = leaderIdArgumentCaptor.getValue(); leaderElectionService.confirmLeaderSessionID(leaderId); verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId)); }
Example 4
Source File: StandaloneHaServicesTest.java From flink with Apache License 2.0 | 6 votes |
/** * Tests that the standalone leader retrieval services return the given address and the * fixed leader session id. */ @Test public void testJobMasterLeaderRetrieval() throws Exception { JobID jobId1 = new JobID(); JobID jobId2 = new JobID(); final String jobManagerAddress1 = "foobar"; final String jobManagerAddress2 = "barfoo"; LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class); LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class); LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1, jobManagerAddress1); LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2, jobManagerAddress2); jmLeaderRetrievalService1.start(jmListener1); jmLeaderRetrievalService2.start(jmListener2); verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress1), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress2), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); }
Example 5
Source File: LeaderRetrievalUtils.java From flink with Apache License 2.0 | 6 votes |
/** * Retrieves the leader akka url and the current leader session ID. The values are stored in a * {@link LeaderConnectionInfo} instance. * * @param leaderRetrievalService Leader retrieval service to retrieve the leader connection * information * @param timeout Timeout when to give up looking for the leader * @return LeaderConnectionInfo containing the leader's akka URL and the current leader session * ID * @throws LeaderRetrievalException */ public static LeaderConnectionInfo retrieveLeaderConnectionInfo( LeaderRetrievalService leaderRetrievalService, FiniteDuration timeout ) throws LeaderRetrievalException { LeaderConnectionInfoListener listener = new LeaderConnectionInfoListener(); try { leaderRetrievalService.start(listener); Future<LeaderConnectionInfo> connectionInfoFuture = listener.getLeaderConnectionInfoFuture(); return Await.result(connectionInfoFuture, timeout); } catch (Exception e) { throw new LeaderRetrievalException("Could not retrieve the leader address and leader " + "session ID.", e); } finally { try { leaderRetrievalService.stop(); } catch (Exception fe) { LOG.warn("Could not stop the leader retrieval service.", fe); } } }
Example 6
Source File: YarnIntraNonHaMasterServicesTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testClosingReportsToLeader() throws Exception { final Configuration flinkConfig = new Configuration(); try (YarnHighAvailabilityServices services = new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig)) { final LeaderElectionService elector = services.getResourceManagerLeaderElectionService(); final LeaderRetrievalService retrieval = services.getResourceManagerLeaderRetriever(); final LeaderContender contender = mockContender(elector); final LeaderRetrievalListener listener = mock(LeaderRetrievalListener.class); elector.start(contender); retrieval.start(listener); // wait until the contender has become the leader verify(listener, timeout(1000L).times(1)).notifyLeaderAddress(anyString(), any(UUID.class)); // now we can close the election service services.close(); verify(contender, timeout(1000L).times(1)).handleError(any(Exception.class)); } }
Example 7
Source File: YarnIntraNonHaMasterServicesTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testClosingReportsToLeader() throws Exception { final Configuration flinkConfig = new Configuration(); try (YarnHighAvailabilityServices services = new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig)) { final LeaderElectionService elector = services.getResourceManagerLeaderElectionService(); final LeaderRetrievalService retrieval = services.getResourceManagerLeaderRetriever(); final LeaderContender contender = mockContender(elector); final LeaderRetrievalListener listener = mock(LeaderRetrievalListener.class); elector.start(contender); retrieval.start(listener); // wait until the contender has become the leader verify(listener, timeout(1000L).times(1)).notifyLeaderAddress(anyString(), any(UUID.class)); // now we can close the election service services.close(); verify(contender, timeout(1000L).times(1)).handleError(any(Exception.class)); } }
Example 8
Source File: StandaloneHaServicesTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Tests that the standalone leader retrieval services return the specified address and the * fixed leader session id. */ @Test public void testJobManagerLeaderRetrieval() throws Exception { JobID jobId1 = new JobID(); JobID jobId2 = new JobID(); LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class); LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class); LeaderRetrievalListener rmListener = mock(LeaderRetrievalListener.class); LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1); LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2); LeaderRetrievalService rmLeaderRetrievalService = standaloneHaServices.getResourceManagerLeaderRetriever(); jmLeaderRetrievalService1.start(jmListener1); jmLeaderRetrievalService2.start(jmListener2); rmLeaderRetrievalService.start(rmListener); verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); verify(rmListener).notifyLeaderAddress(eq(resourceManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); }
Example 9
Source File: EmbeddedHaServicesTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Tests the ResourceManager leader retrieval for a given job. */ @Test public void testResourceManagerLeaderRetrieval() throws Exception { final String address = "foobar"; LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class); LeaderContender leaderContender = mock(LeaderContender.class); when(leaderContender.getAddress()).thenReturn(address); LeaderElectionService leaderElectionService = embeddedHaServices.getResourceManagerLeaderElectionService(); LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getResourceManagerLeaderRetriever(); leaderRetrievalService.start(leaderRetrievalListener); leaderElectionService.start(leaderContender); ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class); verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture()); final UUID leaderId = leaderIdArgumentCaptor.getValue(); leaderElectionService.confirmLeaderSessionID(leaderId); verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId)); }
Example 10
Source File: EmbeddedHaServicesTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Tests the JobManager leader retrieval for a given job. */ @Test public void testJobManagerLeaderRetrieval() throws Exception { final String address = "foobar"; JobID jobId = new JobID(); LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class); LeaderContender leaderContender = mock(LeaderContender.class); when(leaderContender.getAddress()).thenReturn(address); LeaderElectionService leaderElectionService = embeddedHaServices.getJobManagerLeaderElectionService(jobId); LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getJobManagerLeaderRetriever(jobId); leaderRetrievalService.start(leaderRetrievalListener); leaderElectionService.start(leaderContender); ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class); verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture()); final UUID leaderId = leaderIdArgumentCaptor.getValue(); leaderElectionService.confirmLeaderSessionID(leaderId); verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId)); }
Example 11
Source File: StandaloneHaServicesTest.java From flink with Apache License 2.0 | 6 votes |
/** * Tests that the standalone leader retrieval services return the specified address and the * fixed leader session id. */ @Test public void testJobManagerLeaderRetrieval() throws Exception { JobID jobId1 = new JobID(); JobID jobId2 = new JobID(); LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class); LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class); LeaderRetrievalListener rmListener = mock(LeaderRetrievalListener.class); LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1); LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2); LeaderRetrievalService rmLeaderRetrievalService = standaloneHaServices.getResourceManagerLeaderRetriever(); jmLeaderRetrievalService1.start(jmListener1); jmLeaderRetrievalService2.start(jmListener2); rmLeaderRetrievalService.start(rmListener); verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); verify(rmListener).notifyLeaderAddress(eq(resourceManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); }
Example 12
Source File: LeaderRetrievalUtils.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Retrieves the leader akka url and the current leader session ID. The values are stored in a * {@link LeaderConnectionInfo} instance. * * @param leaderRetrievalService Leader retrieval service to retrieve the leader connection * information * @param timeout Timeout when to give up looking for the leader * @return LeaderConnectionInfo containing the leader's akka URL and the current leader session * ID * @throws LeaderRetrievalException */ public static LeaderConnectionInfo retrieveLeaderConnectionInfo( LeaderRetrievalService leaderRetrievalService, FiniteDuration timeout ) throws LeaderRetrievalException { LeaderConnectionInfoListener listener = new LeaderConnectionInfoListener(); try { leaderRetrievalService.start(listener); Future<LeaderConnectionInfo> connectionInfoFuture = listener.getLeaderConnectionInfoFuture(); return Await.result(connectionInfoFuture, timeout); } catch (Exception e) { throw new LeaderRetrievalException("Could not retrieve the leader address and leader " + "session ID.", e); } finally { try { leaderRetrievalService.stop(); } catch (Exception fe) { LOG.warn("Could not stop the leader retrieval service.", fe); } } }
Example 13
Source File: LeaderRetrievalUtils.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Retrieves the current leader gateway using the given {@link LeaderRetrievalService}. If the * current leader could not be retrieved after the given timeout, then a * {@link LeaderRetrievalException} is thrown. * * @param leaderRetrievalService {@link LeaderRetrievalService} which is used for the leader retrieval * @param actorSystem ActorSystem which is used for the {@link LeaderRetrievalListener} implementation * @param timeout Timeout value for the retrieval call * @return The current leader gateway * @throws LeaderRetrievalException If the actor gateway could not be retrieved or the timeout has been exceeded */ public static ActorGateway retrieveLeaderGateway( LeaderRetrievalService leaderRetrievalService, ActorSystem actorSystem, FiniteDuration timeout) throws LeaderRetrievalException { LeaderGatewayListener listener = new LeaderGatewayListener(actorSystem, timeout); try { leaderRetrievalService.start(listener); Future<ActorGateway> actorGatewayFuture = listener.getActorGatewayFuture(); return Await.result(actorGatewayFuture, timeout); } catch (Exception e) { throw new LeaderRetrievalException("Could not retrieve the leader gateway.", e); } finally { try { leaderRetrievalService.stop(); } catch (Exception fe) { LOG.warn("Could not stop the leader retrieval service.", fe); } } }
Example 14
Source File: JobLeaderService.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Add the given job to be monitored. This means that the service tries to detect leaders for * this job and then tries to establish a connection to it. * * @param jobId identifying the job to monitor * @param defaultTargetAddress of the job leader * @throws Exception if an error occurs while starting the leader retrieval service */ public void addJob(final JobID jobId, final String defaultTargetAddress) throws Exception { Preconditions.checkState(JobLeaderService.State.STARTED == state, "The service is currently not running."); LOG.info("Add job {} for job leader monitoring.", jobId); final LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever( jobId, defaultTargetAddress); JobLeaderService.JobManagerLeaderListener jobManagerLeaderListener = new JobManagerLeaderListener(jobId); final Tuple2<LeaderRetrievalService, JobManagerLeaderListener> oldEntry = jobLeaderServices.put(jobId, Tuple2.of(leaderRetrievalService, jobManagerLeaderListener)); if (oldEntry != null) { oldEntry.f0.stop(); oldEntry.f1.stop(); } leaderRetrievalService.start(jobManagerLeaderListener); }
Example 15
Source File: ZooKeeperHaServicesTest.java From flink with Apache License 2.0 | 5 votes |
private void runCleanupTest( Configuration configuration, TestingBlobStoreService blobStoreService, ThrowingConsumer<ZooKeeperHaServices, Exception> zooKeeperHaServicesConsumer) throws Exception { try (ZooKeeperHaServices zooKeeperHaServices = new ZooKeeperHaServices( ZooKeeperUtils.startCuratorFramework(configuration), Executors.directExecutor(), configuration, blobStoreService)) { // create some Zk services to trigger the generation of paths final LeaderRetrievalService resourceManagerLeaderRetriever = zooKeeperHaServices.getResourceManagerLeaderRetriever(); final LeaderElectionService resourceManagerLeaderElectionService = zooKeeperHaServices.getResourceManagerLeaderElectionService(); final RunningJobsRegistry runningJobsRegistry = zooKeeperHaServices.getRunningJobsRegistry(); final TestingListener listener = new TestingListener(); resourceManagerLeaderRetriever.start(listener); resourceManagerLeaderElectionService.start(new TestingContender("foobar", resourceManagerLeaderElectionService)); final JobID jobId = new JobID(); runningJobsRegistry.setJobRunning(jobId); listener.waitForNewLeader(2000L); resourceManagerLeaderRetriever.stop(); resourceManagerLeaderElectionService.stop(); runningJobsRegistry.clearJob(jobId); zooKeeperHaServicesConsumer.accept(zooKeeperHaServices); } }
Example 16
Source File: JobLeaderIdService.java From flink with Apache License 2.0 | 5 votes |
private JobLeaderIdListener( JobID jobId, JobLeaderIdActions listenerJobLeaderIdActions, LeaderRetrievalService leaderRetrievalService) throws Exception { this.jobId = Preconditions.checkNotNull(jobId); this.listenerJobLeaderIdActions = Preconditions.checkNotNull(listenerJobLeaderIdActions); this.leaderRetrievalService = Preconditions.checkNotNull(leaderRetrievalService); leaderIdFuture = new CompletableFuture<>(); activateTimeout(); // start the leader service we're listening to leaderRetrievalService.start(this); }
Example 17
Source File: ZooKeeperHaServicesTest.java From flink with Apache License 2.0 | 5 votes |
private void runCleanupTest( Configuration configuration, TestingBlobStoreService blobStoreService, ThrowingConsumer<ZooKeeperHaServices, Exception> zooKeeperHaServicesConsumer) throws Exception { try (ZooKeeperHaServices zooKeeperHaServices = new ZooKeeperHaServices( ZooKeeperUtils.startCuratorFramework(configuration), Executors.directExecutor(), configuration, blobStoreService)) { // create some Zk services to trigger the generation of paths final LeaderRetrievalService resourceManagerLeaderRetriever = zooKeeperHaServices.getResourceManagerLeaderRetriever(); final LeaderElectionService resourceManagerLeaderElectionService = zooKeeperHaServices.getResourceManagerLeaderElectionService(); final RunningJobsRegistry runningJobsRegistry = zooKeeperHaServices.getRunningJobsRegistry(); final TestingListener listener = new TestingListener(); resourceManagerLeaderRetriever.start(listener); resourceManagerLeaderElectionService.start(new TestingContender("foobar", resourceManagerLeaderElectionService)); final JobID jobId = new JobID(); runningJobsRegistry.setJobRunning(jobId); listener.waitForNewLeader(2000L); resourceManagerLeaderRetriever.stop(); resourceManagerLeaderElectionService.stop(); runningJobsRegistry.clearJob(jobId); zooKeeperHaServicesConsumer.accept(zooKeeperHaServices); } }
Example 18
Source File: JobLeaderIdService.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private JobLeaderIdListener( JobID jobId, JobLeaderIdActions listenerJobLeaderIdActions, LeaderRetrievalService leaderRetrievalService) throws Exception { this.jobId = Preconditions.checkNotNull(jobId); this.listenerJobLeaderIdActions = Preconditions.checkNotNull(listenerJobLeaderIdActions); this.leaderRetrievalService = Preconditions.checkNotNull(leaderRetrievalService); leaderIdFuture = new CompletableFuture<>(); activateTimeout(); // start the leader service we're listening to leaderRetrievalService.start(this); }
Example 19
Source File: SingleLeaderElectionServiceTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testShutdown() throws Exception { final UUID uuid = UUID.randomUUID(); final SingleLeaderElectionService service = new SingleLeaderElectionService(executor, uuid); // create a leader contender and let it grab leadership final LeaderContender contender = mockContender(service); service.start(contender); verify(contender, times(1)).grantLeadership(uuid); // some leader listeners final LeaderRetrievalListener listener1 = mock(LeaderRetrievalListener.class); final LeaderRetrievalListener listener2 = mock(LeaderRetrievalListener.class); LeaderRetrievalService listenerService1 = service.createLeaderRetrievalService(); LeaderRetrievalService listenerService2 = service.createLeaderRetrievalService(); listenerService1.start(listener1); listenerService2.start(listener2); // one listener stops listenerService1.stop(); // shut down the service service.shutdown(); // the leader contender and running listener should get error notifications verify(contender, times(1)).handleError(any(Exception.class)); verify(listener2, times(1)).handleError(any(Exception.class)); // the stopped listener gets no notification verify(listener1, times(0)).handleError(any(Exception.class)); // should not be possible to start again after shutdown try { service.start(contender); fail("should fail with an exception"); } catch (IllegalStateException e) { // expected } // no additional leadership grant verify(contender, times(1)).grantLeadership(any(UUID.class)); }
Example 20
Source File: EmbeddedHaServicesTest.java From flink with Apache License 2.0 | 4 votes |
private void runLeaderRetrievalTest(LeaderElectionService leaderElectionService, LeaderRetrievalService leaderRetrievalService) throws Exception { LeaderRetrievalUtils.LeaderConnectionInfoListener leaderRetrievalListener = new LeaderRetrievalUtils.LeaderConnectionInfoListener(); TestingLeaderContender leaderContender = new TestingLeaderContender(); leaderRetrievalService.start(leaderRetrievalListener); leaderElectionService.start(leaderContender); final UUID leaderId = leaderContender.getLeaderSessionFuture().get(); leaderElectionService.confirmLeadership(leaderId, ADDRESS); final LeaderConnectionInfo leaderConnectionInfo = leaderRetrievalListener.getLeaderConnectionInfoFuture().get(); assertThat(leaderConnectionInfo.getAddress(), is(ADDRESS)); assertThat(leaderConnectionInfo.getLeaderSessionId(), is(leaderId)); }