org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService Java Examples
The following examples show how to use
org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SessionDispatcherResourceManagerComponentFactory.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override protected DispatcherResourceManagerComponent<Dispatcher> createDispatcherResourceManagerComponent( Dispatcher dispatcher, ResourceManager<?> resourceManager, LeaderRetrievalService dispatcherLeaderRetrievalService, LeaderRetrievalService resourceManagerRetrievalService, WebMonitorEndpoint<?> webMonitorEndpoint, JobManagerMetricGroup jobManagerMetricGroup) { return new SessionDispatcherResourceManagerComponent( dispatcher, resourceManager, dispatcherLeaderRetrievalService, resourceManagerRetrievalService, webMonitorEndpoint, jobManagerMetricGroup); }
Example #2
Source File: StandaloneHaServicesTest.java From flink with Apache License 2.0 | 6 votes |
/** * Tests that the standalone leader retrieval services return the given address and the * fixed leader session id. */ @Test public void testJobMasterLeaderRetrieval() throws Exception { JobID jobId1 = new JobID(); JobID jobId2 = new JobID(); final String jobManagerAddress1 = "foobar"; final String jobManagerAddress2 = "barfoo"; LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class); LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class); LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1, jobManagerAddress1); LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2, jobManagerAddress2); jmLeaderRetrievalService1.start(jmListener1); jmLeaderRetrievalService2.start(jmListener2); verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress1), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress2), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); }
Example #3
Source File: StandaloneHaServicesTest.java From flink with Apache License 2.0 | 6 votes |
/** * Tests that the standalone leader retrieval services return the specified address and the * fixed leader session id. */ @Test public void testJobManagerLeaderRetrieval() throws Exception { JobID jobId1 = new JobID(); JobID jobId2 = new JobID(); LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class); LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class); LeaderRetrievalListener rmListener = mock(LeaderRetrievalListener.class); LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1); LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2); LeaderRetrievalService rmLeaderRetrievalService = standaloneHaServices.getResourceManagerLeaderRetriever(); jmLeaderRetrievalService1.start(jmListener1); jmLeaderRetrievalService2.start(jmListener2); rmLeaderRetrievalService.start(rmListener); verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); verify(rmListener).notifyLeaderAddress(eq(resourceManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); }
Example #4
Source File: LeaderRetrievalUtils.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Retrieves the leader akka url and the current leader session ID. The values are stored in a * {@link LeaderConnectionInfo} instance. * * @param leaderRetrievalService Leader retrieval service to retrieve the leader connection * information * @param timeout Timeout when to give up looking for the leader * @return LeaderConnectionInfo containing the leader's akka URL and the current leader session * ID * @throws LeaderRetrievalException */ public static LeaderConnectionInfo retrieveLeaderConnectionInfo( LeaderRetrievalService leaderRetrievalService, FiniteDuration timeout ) throws LeaderRetrievalException { LeaderConnectionInfoListener listener = new LeaderConnectionInfoListener(); try { leaderRetrievalService.start(listener); Future<LeaderConnectionInfo> connectionInfoFuture = listener.getLeaderConnectionInfoFuture(); return Await.result(connectionInfoFuture, timeout); } catch (Exception e) { throw new LeaderRetrievalException("Could not retrieve the leader address and leader " + "session ID.", e); } finally { try { leaderRetrievalService.stop(); } catch (Exception fe) { LOG.warn("Could not stop the leader retrieval service.", fe); } } }
Example #5
Source File: EmbeddedHaServicesTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Tests the ResourceManager leader retrieval for a given job. */ @Test public void testResourceManagerLeaderRetrieval() throws Exception { final String address = "foobar"; LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class); LeaderContender leaderContender = mock(LeaderContender.class); when(leaderContender.getAddress()).thenReturn(address); LeaderElectionService leaderElectionService = embeddedHaServices.getResourceManagerLeaderElectionService(); LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getResourceManagerLeaderRetriever(); leaderRetrievalService.start(leaderRetrievalListener); leaderElectionService.start(leaderContender); ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class); verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture()); final UUID leaderId = leaderIdArgumentCaptor.getValue(); leaderElectionService.confirmLeaderSessionID(leaderId); verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId)); }
Example #6
Source File: LeaderRetrievalUtils.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Retrieves the current leader gateway using the given {@link LeaderRetrievalService}. If the * current leader could not be retrieved after the given timeout, then a * {@link LeaderRetrievalException} is thrown. * * @param leaderRetrievalService {@link LeaderRetrievalService} which is used for the leader retrieval * @param actorSystem ActorSystem which is used for the {@link LeaderRetrievalListener} implementation * @param timeout Timeout value for the retrieval call * @return The current leader gateway * @throws LeaderRetrievalException If the actor gateway could not be retrieved or the timeout has been exceeded */ public static ActorGateway retrieveLeaderGateway( LeaderRetrievalService leaderRetrievalService, ActorSystem actorSystem, FiniteDuration timeout) throws LeaderRetrievalException { LeaderGatewayListener listener = new LeaderGatewayListener(actorSystem, timeout); try { leaderRetrievalService.start(listener); Future<ActorGateway> actorGatewayFuture = listener.getActorGatewayFuture(); return Await.result(actorGatewayFuture, timeout); } catch (Exception e) { throw new LeaderRetrievalException("Could not retrieve the leader gateway.", e); } finally { try { leaderRetrievalService.stop(); } catch (Exception fe) { LOG.warn("Could not stop the leader retrieval service.", fe); } } }
Example #7
Source File: JobLeaderService.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Add the given job to be monitored. This means that the service tries to detect leaders for * this job and then tries to establish a connection to it. * * @param jobId identifying the job to monitor * @param defaultTargetAddress of the job leader * @throws Exception if an error occurs while starting the leader retrieval service */ public void addJob(final JobID jobId, final String defaultTargetAddress) throws Exception { Preconditions.checkState(JobLeaderService.State.STARTED == state, "The service is currently not running."); LOG.info("Add job {} for job leader monitoring.", jobId); final LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever( jobId, defaultTargetAddress); JobLeaderService.JobManagerLeaderListener jobManagerLeaderListener = new JobManagerLeaderListener(jobId); final Tuple2<LeaderRetrievalService, JobManagerLeaderListener> oldEntry = jobLeaderServices.put(jobId, Tuple2.of(leaderRetrievalService, jobManagerLeaderListener)); if (oldEntry != null) { oldEntry.f0.stop(); oldEntry.f1.stop(); } leaderRetrievalService.start(jobManagerLeaderListener); }
Example #8
Source File: JobLeaderService.java From flink with Apache License 2.0 | 6 votes |
/** * Stop the job leader services. This implies stopping all leader retrieval services for the * different jobs and their leader retrieval listeners. * * @throws Exception if an error occurs while stopping the service */ public void stop() throws Exception { LOG.info("Stop job leader service."); if (JobLeaderService.State.STARTED == state) { for (Tuple2<LeaderRetrievalService, JobLeaderService.JobManagerLeaderListener> leaderRetrievalServiceEntry: jobLeaderServices.values()) { LeaderRetrievalService leaderRetrievalService = leaderRetrievalServiceEntry.f0; JobLeaderService.JobManagerLeaderListener jobManagerLeaderListener = leaderRetrievalServiceEntry.f1; jobManagerLeaderListener.stop(); leaderRetrievalService.stop(); } jobLeaderServices.clear(); } state = JobLeaderService.State.STOPPED; }
Example #9
Source File: EmbeddedHaServicesTest.java From flink with Apache License 2.0 | 6 votes |
/** * Tests the JobManager leader retrieval for a given job. */ @Test public void testJobManagerLeaderRetrieval() throws Exception { final String address = "foobar"; JobID jobId = new JobID(); LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class); LeaderContender leaderContender = mock(LeaderContender.class); when(leaderContender.getAddress()).thenReturn(address); LeaderElectionService leaderElectionService = embeddedHaServices.getJobManagerLeaderElectionService(jobId); LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getJobManagerLeaderRetriever(jobId); leaderRetrievalService.start(leaderRetrievalListener); leaderElectionService.start(leaderContender); ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class); verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture()); final UUID leaderId = leaderIdArgumentCaptor.getValue(); leaderElectionService.confirmLeaderSessionID(leaderId); verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId)); }
Example #10
Source File: StandaloneHaServicesTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Tests that the standalone leader retrieval services return the given address and the * fixed leader session id. */ @Test public void testJobMasterLeaderRetrieval() throws Exception { JobID jobId1 = new JobID(); JobID jobId2 = new JobID(); final String jobManagerAddress1 = "foobar"; final String jobManagerAddress2 = "barfoo"; LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class); LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class); LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1, jobManagerAddress1); LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2, jobManagerAddress2); jmLeaderRetrievalService1.start(jmListener1); jmLeaderRetrievalService2.start(jmListener2); verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress1), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress2), eq(HighAvailabilityServices.DEFAULT_LEADER_ID)); }
Example #11
Source File: HighAvailabilityServices.java From flink with Apache License 2.0 | 5 votes |
/** * This retriever should no longer be used on the cluster side. The web monitor retriever * is only required on the client-side and we have a dedicated high-availability services * for the client, named {@link ClientHighAvailabilityServices}. See also FLINK-13750. * * @return the leader retriever for web monitor * @deprecated just use {@link #getClusterRestEndpointLeaderRetriever()} instead of this method. */ @Deprecated default LeaderRetrievalService getWebMonitorLeaderRetriever() { throw new UnsupportedOperationException( "getWebMonitorLeaderRetriever should no longer be used. Instead use " + "#getClusterRestEndpointLeaderRetriever to instantiate the cluster " + "rest endpoint leader retriever. If you called this method, then " + "make sure that #getClusterRestEndpointLeaderRetriever has been " + "implemented by your HighAvailabilityServices implementation."); }
Example #12
Source File: TestingHighAvailabilityServices.java From flink with Apache License 2.0 | 5 votes |
@Override public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) { LeaderRetrievalService service = jobMasterLeaderRetrievers.computeIfAbsent(jobID, jobMasterLeaderRetrieverFunction); if (service != null) { return service; } else { throw new IllegalStateException("JobMasterLeaderRetriever has not been set"); } }
Example #13
Source File: TestingHighAvailabilityServices.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public LeaderRetrievalService getDispatcherLeaderRetriever() { LeaderRetrievalService service = this.dispatcherLeaderRetriever; if (service != null) { return service; } else { throw new IllegalStateException("ResourceManagerLeaderRetriever has not been set"); } }
Example #14
Source File: YarnPreConfiguredMasterNonHaServices.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public LeaderRetrievalService getResourceManagerLeaderRetriever() { enter(); try { return new StandaloneLeaderRetrievalService(resourceManagerRpcUrl, DEFAULT_LEADER_ID); } finally { exit(); } }
Example #15
Source File: StandaloneHaServices.java From flink with Apache License 2.0 | 5 votes |
@Override public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) { synchronized (lock) { checkNotShutdown(); return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID); } }
Example #16
Source File: ZooKeeperLeaderRetrievalTest.java From flink with Apache License 2.0 | 5 votes |
/** * Tests that the LeaderRetrievalUtils.findConnectingAddress stops trying to find the * connecting address if no leader address has been specified. The call should return * then InetAddress.getLocalHost(). */ @Test public void testTimeoutOfFindConnectingAddress() throws Exception { Duration timeout = Duration.ofSeconds(1L); LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID); InetAddress result = LeaderRetrievalUtils.findConnectingAddress(leaderRetrievalService, timeout); assertEquals(InetAddress.getLocalHost(), result); }
Example #17
Source File: YarnPreConfiguredMasterNonHaServices.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public LeaderRetrievalService getDispatcherLeaderRetriever() { enter(); try { return new StandaloneLeaderRetrievalService(dispatcherRpcUrl, DEFAULT_LEADER_ID); } finally { exit(); } }
Example #18
Source File: YarnIntraNonHaMasterServices.java From flink with Apache License 2.0 | 5 votes |
@Override public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) { enter(); try { throw new UnsupportedOperationException("needs refactoring to accept default address"); } finally { exit(); } }
Example #19
Source File: ResourceManagerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testHeartbeatTimeoutWithJobMaster() throws Exception { final CompletableFuture<ResourceID> heartbeatRequestFuture = new CompletableFuture<>(); final CompletableFuture<ResourceManagerId> disconnectFuture = new CompletableFuture<>(); final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder() .setResourceManagerHeartbeatConsumer(heartbeatRequestFuture::complete) .setDisconnectResourceManagerConsumer(disconnectFuture::complete) .build(); rpcService.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway); final JobID jobId = new JobID(); final ResourceID jobMasterResourceId = ResourceID.generate(); final LeaderRetrievalService jobMasterLeaderRetrievalService = new SettableLeaderRetrievalService(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID()); highAvailabilityServices.setJobMasterLeaderRetrieverFunction(requestedJobId -> { assertThat(requestedJobId, is(equalTo(jobId))); return jobMasterLeaderRetrievalService; }); runHeartbeatTimeoutTest( resourceManagerGateway -> { final CompletableFuture<RegistrationResponse> registrationFuture = resourceManagerGateway.registerJobManager( jobMasterGateway.getFencingToken(), jobMasterResourceId, jobMasterGateway.getAddress(), jobId, TIMEOUT); assertThat(registrationFuture.get(), instanceOf(RegistrationResponse.Success.class)); }, resourceManagerResourceId -> { // might have been completed or not depending whether the timeout was triggered first final ResourceID optionalHeartbeatRequestOrigin = heartbeatRequestFuture.getNow(null); assertThat(optionalHeartbeatRequestOrigin, anyOf(is(resourceManagerResourceId), is(nullValue()))); assertThat(disconnectFuture.get(), is(equalTo(resourceManagerId))); }); }
Example #20
Source File: StandaloneHaServices.java From flink with Apache License 2.0 | 5 votes |
@Override public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) { synchronized (lock) { checkNotShutdown(); return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID); } }
Example #21
Source File: StandaloneHaServices.java From flink with Apache License 2.0 | 5 votes |
@Override public LeaderRetrievalService getResourceManagerLeaderRetriever() { synchronized (lock) { checkNotShutdown(); return new StandaloneLeaderRetrievalService(resourceManagerAddress, DEFAULT_LEADER_ID); } }
Example #22
Source File: YarnIntraNonHaMasterServices.java From flink with Apache License 2.0 | 5 votes |
@Override public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) { enter(); try { throw new UnsupportedOperationException("needs refactoring to accept default address"); } finally { exit(); } }
Example #23
Source File: StandaloneHaServices.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) { synchronized (lock) { checkNotShutdown(); return new StandaloneLeaderRetrievalService(jobManagerAddress, DEFAULT_LEADER_ID); } }
Example #24
Source File: StandaloneClientHAServices.java From flink with Apache License 2.0 | 5 votes |
@Override public LeaderRetrievalService getClusterRestEndpointLeaderRetriever() { synchronized (lock) { checkState(running, "ClientHaService has already been closed."); return new StandaloneLeaderRetrievalService(webMonitorAddress, DEFAULT_LEADER_ID); } }
Example #25
Source File: StandaloneHaServices.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public LeaderRetrievalService getWebMonitorLeaderRetriever() { synchronized (lock) { checkNotShutdown(); return new StandaloneLeaderRetrievalService(webMonitorAddress, DEFAULT_LEADER_ID); } }
Example #26
Source File: ManualLeaderService.java From flink with Apache License 2.0 | 5 votes |
public LeaderRetrievalService createLeaderRetrievalService() { final SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService( getLeaderAddress(currentLeaderIndex), currentLeaderId); leaderRetrievalServices.add(settableLeaderRetrievalService); return settableLeaderRetrievalService; }
Example #27
Source File: TestingHighAvailabilityServices.java From flink with Apache License 2.0 | 5 votes |
@Override public LeaderRetrievalService getResourceManagerLeaderRetriever() { LeaderRetrievalService service = this.resourceManagerLeaderRetriever; if (service != null) { return service; } else { throw new IllegalStateException("ResourceManagerLeaderRetriever has not been set"); } }
Example #28
Source File: EmbeddedHaServicesTest.java From flink with Apache License 2.0 | 5 votes |
/** * Tests the JobManager leader retrieval for a given job. */ @Test public void testJobManagerLeaderRetrieval() throws Exception { JobID jobId = new JobID(); LeaderElectionService leaderElectionService = embeddedHaServices.getJobManagerLeaderElectionService(jobId); LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getJobManagerLeaderRetriever(jobId); runLeaderRetrievalTest(leaderElectionService, leaderRetrievalService); }
Example #29
Source File: RestClusterClient.java From flink with Apache License 2.0 | 5 votes |
public RestClusterClient( Configuration config, T clusterId, LeaderRetrievalService webMonitorRetrievalService) throws Exception { this( config, null, clusterId, new ExponentialWaitStrategy(10L, 2000L), webMonitorRetrievalService); }
Example #30
Source File: RestClusterClient.java From flink with Apache License 2.0 | 5 votes |
@VisibleForTesting RestClusterClient( Configuration configuration, @Nullable RestClient restClient, T clusterId, WaitStrategy waitStrategy, @Nullable LeaderRetrievalService webMonitorRetrievalService) throws Exception { super(configuration); this.restClusterClientConfiguration = RestClusterClientConfiguration.fromConfiguration(configuration); if (restClient != null) { this.restClient = restClient; } else { this.restClient = new RestClient(restClusterClientConfiguration.getRestClientConfiguration(), executorService); } this.waitStrategy = Preconditions.checkNotNull(waitStrategy); this.clusterId = Preconditions.checkNotNull(clusterId); if (webMonitorRetrievalService == null) { this.webMonitorRetrievalService = highAvailabilityServices.getWebMonitorLeaderRetriever(); } else { this.webMonitorRetrievalService = webMonitorRetrievalService; } this.retryExecutorService = Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-RestClusterClient-Retry")); startLeaderRetrievers(); }