org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService Java Examples

/**
 * Tests that the standalone leader election and retrieval service return the same leader
 * URL.
 */
@Test
public void testStandaloneLeaderElectionRetrieval() throws Exception {
	StandaloneLeaderElectionService leaderElectionService = new StandaloneLeaderElectionService();
	StandaloneLeaderRetrievalService leaderRetrievalService = new StandaloneLeaderRetrievalService(TEST_URL);
	TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
	TestingListener testingListener = new TestingListener();

	try {
		leaderElectionService.start(contender);
		leaderRetrievalService.start(testingListener);

		contender.waitForLeader(1000l);

		assertTrue(contender.isLeader());
		assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, contender.getLeaderSessionID());

		testingListener.waitForNewLeader(1000l);

		assertEquals(TEST_URL, testingListener.getAddress());
		assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, testingListener.getLeaderSessionID());
	} finally {
		leaderElectionService.stop();
		leaderRetrievalService.stop();
	}
}
 

/**
 * Creates a {@link StandaloneLeaderRetrievalService} form the given configuration and the
 * JobManager name. The host and port for the remote Akka URL are retrieved from the provided
 * configuration. Instead of using the standard JobManager Akka name, the provided one is used
 * for the remote Akka URL.
 *
 * @param configuration Configuration instance containing hte host and port information
 * @param resolveInitialHostName If true, resolves the hostname of the StandaloneLeaderRetrievalService
 * @param jobManagerName Name of the JobManager actor
 * @return StandaloneLeaderRetrievalService
 * @throws ConfigurationException if the job manager address cannot be retrieved from the configuration
 * @throws UnknownHostException if the job manager address cannot be resolved
 */
public static StandaloneLeaderRetrievalService createLeaderRetrievalService(
		Configuration configuration,
		boolean resolveInitialHostName,
		String jobManagerName)
	throws ConfigurationException, UnknownHostException {
	Tuple2<String, Integer> hostnamePort = HighAvailabilityServicesUtils.getJobManagerAddress(configuration);

	String jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
		hostnamePort.f0,
		hostnamePort.f1,
		jobManagerName != null ? jobManagerName : JobMaster.JOB_MANAGER_NAME,
		resolveInitialHostName ? AddressResolution.TRY_ADDRESS_RESOLUTION : AddressResolution.NO_ADDRESS_RESOLUTION,
		configuration);

	return new StandaloneLeaderRetrievalService(jobManagerAkkaUrl);
}
 

/**
 * Tests that the standalone leader election and retrieval service return the same leader
 * URL.
 */
@Test
public void testStandaloneLeaderElectionRetrieval() throws Exception {
	StandaloneLeaderElectionService leaderElectionService = new StandaloneLeaderElectionService();
	StandaloneLeaderRetrievalService leaderRetrievalService = new StandaloneLeaderRetrievalService(TEST_URL);
	TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
	TestingListener testingListener = new TestingListener();

	try {
		leaderElectionService.start(contender);
		leaderRetrievalService.start(testingListener);

		contender.waitForLeader(1000l);

		assertTrue(contender.isLeader());
		assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, contender.getLeaderSessionID());

		testingListener.waitForNewLeader(1000l);

		assertEquals(TEST_URL, testingListener.getAddress());
		assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, testingListener.getLeaderSessionID());
	} finally {
		leaderElectionService.stop();
		leaderRetrievalService.stop();
	}
}
 

/**
 * Tests that the standalone leader election and retrieval service return the same leader
 * URL.
 */
@Test
public void testStandaloneLeaderElectionRetrieval() throws Exception {
	StandaloneLeaderElectionService leaderElectionService = new StandaloneLeaderElectionService();
	StandaloneLeaderRetrievalService leaderRetrievalService = new StandaloneLeaderRetrievalService(TEST_URL);
	TestingContender contender = new TestingContender(TEST_URL, leaderElectionService);
	TestingListener testingListener = new TestingListener();

	try {
		leaderElectionService.start(contender);
		leaderRetrievalService.start(testingListener);

		contender.waitForLeader(1000l);

		assertTrue(contender.isLeader());
		assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, contender.getLeaderSessionID());

		testingListener.waitForNewLeader(1000l);

		assertEquals(TEST_URL, testingListener.getAddress());
		assertEquals(HighAvailabilityServices.DEFAULT_LEADER_ID, testingListener.getLeaderSessionID());
	} finally {
		leaderElectionService.stop();
		leaderRetrievalService.stop();
	}
}
 

/**
 * Creates a {@link StandaloneLeaderRetrievalService} form the given configuration and the
 * JobManager name. The host and port for the remote Akka URL are retrieved from the provided
 * configuration. Instead of using the standard JobManager Akka name, the provided one is used
 * for the remote Akka URL.
 *
 * @param configuration Configuration instance containing hte host and port information
 * @param resolveInitialHostName If true, resolves the hostname of the StandaloneLeaderRetrievalService
 * @param jobManagerName Name of the JobManager actor
 * @return StandaloneLeaderRetrievalService
 * @throws ConfigurationException if the job manager address cannot be retrieved from the configuration
 * @throws UnknownHostException if the job manager address cannot be resolved
 */
public static StandaloneLeaderRetrievalService createLeaderRetrievalService(
		Configuration configuration,
		boolean resolveInitialHostName,
		String jobManagerName)
	throws ConfigurationException, UnknownHostException {
	Tuple2<String, Integer> hostnamePort = HighAvailabilityServicesUtils.getJobManagerAddress(configuration);

	String jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
		hostnamePort.f0,
		hostnamePort.f1,
		jobManagerName != null ? jobManagerName : JobMaster.JOB_MANAGER_NAME,
		resolveInitialHostName ? AddressResolution.TRY_ADDRESS_RESOLUTION : AddressResolution.NO_ADDRESS_RESOLUTION,
		configuration);

	return new StandaloneLeaderRetrievalService(jobManagerAkkaUrl);
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService("UNKNOWN", DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(dispatcherAddress, DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(resourceManagerAddress, DEFAULT_LEADER_ID);
	}

}
 

@Override
public LeaderRetrievalService getClusterRestEndpointLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(clusterRestEndpointAddress, DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getClusterRestEndpointLeaderRetriever() {
	synchronized (lock) {
		checkState(running, "ClientHaService has already been closed.");
		return new StandaloneLeaderRetrievalService(webMonitorAddress, DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getWebMonitorLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(webMonitorAddress, DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(jobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(dispatcherAddress, DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(resourceManagerAddress, DEFAULT_LEADER_ID);
	}

}
 

@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	enter();
	try {
		return new StandaloneLeaderRetrievalService(resourceManagerRpcUrl, DEFAULT_LEADER_ID);
	}
	finally {
		exit();
	}
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	enter();
	try {
		return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
	} finally {
		exit();
	}
}
 

@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	enter();

	try {
		return new StandaloneLeaderRetrievalService(dispatcherRpcUrl, DEFAULT_LEADER_ID);
	} finally {
		exit();
	}
}
 

@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	enter();

	try {
		return new StandaloneLeaderRetrievalService(dispatcherRpcUrl, DEFAULT_LEADER_ID);
	} finally {
		exit();
	}
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	enter();
	try {
		return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
	} finally {
		exit();
	}
}
 

@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(resourceManagerAddress, DEFAULT_LEADER_ID);
	}

}
 

@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(dispatcherAddress, DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(jobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getWebMonitorLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(webMonitorAddress, DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	enter();
	try {
		return new StandaloneLeaderRetrievalService(resourceManagerRpcUrl, DEFAULT_LEADER_ID);
	}
	finally {
		exit();
	}
}
 

/**
 *  Tests that repeated local {@link PartitionNotFoundException}s ultimately fail the receiver.
 */
@Test
public void testLocalPartitionNotFound() throws Exception {

	new JavaTestKit(system){{

		ActorGateway jobManager = null;
		ActorGateway taskManager = null;

		final ActorGateway testActorGateway = new AkkaActorGateway(
				getTestActor(),
			LEADER_SESSION_ID);

		try {
			final IntermediateDataSetID resultId = new IntermediateDataSetID();

			// Create the JM
			ActorRef jm = system.actorOf(Props.create(
					new SimplePartitionStateLookupJobManagerCreator(LEADER_SESSION_ID, getTestActor())));

			jobManager = new AkkaActorGateway(jm, LEADER_SESSION_ID);

			highAvailabilityServices.setJobMasterLeaderRetriever(
				HighAvailabilityServices.DEFAULT_JOB_ID,
				new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

			final Configuration config = new Configuration();
			config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
			config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);

			taskManager = TestingUtils.createTaskManager(
					system,
					highAvailabilityServices,
					config,
					true,
					true);

			// ---------------------------------------------------------------------------------

			final ActorGateway tm = taskManager;

			final JobID jid = new JobID();
			final JobVertexID vid = new JobVertexID();
			final ExecutionAttemptID eid = new ExecutionAttemptID();

			final ResultPartitionID partitionId = new ResultPartitionID();

			// Local location (on the same TM though) for the partition
			final ResultPartitionLocation loc = ResultPartitionLocation.createLocal();

			final InputChannelDeploymentDescriptor[] icdd =
					new InputChannelDeploymentDescriptor[] {
							new InputChannelDeploymentDescriptor(partitionId, loc)};

			final InputGateDeploymentDescriptor igdd =
					new InputGateDeploymentDescriptor(resultId, ResultPartitionType.PIPELINED, 0, icdd);

			final TaskDeploymentDescriptor tdd = createTaskDeploymentDescriptor(
					jid, "TestJob", vid, eid,
					new SerializedValue<>(new ExecutionConfig()),
					"Receiver", 1, 0, 1, 0,
					new Configuration(), new Configuration(),
					Tasks.AgnosticReceiver.class.getName(),
					Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
					Collections.singletonList(igdd),
					Collections.emptyList(),
					Collections.emptyList(), 0);

			new Within(new FiniteDuration(120, TimeUnit.SECONDS)) {
				@Override
				protected void run() {
					// Submit the task
					tm.tell(new SubmitTask(tdd), testActorGateway);
					expectMsgClass(Acknowledge.get().getClass());

					// Wait to be notified about the final execution state by the mock JM
					TaskExecutionState msg = expectMsgClass(TaskExecutionState.class);

					// The task should fail after repeated requests
					assertEquals(msg.getExecutionState(), ExecutionState.FAILED);

					Throwable error = msg.getError(getClass().getClassLoader());
					if (error.getClass() != PartitionNotFoundException.class) {
						error.printStackTrace();
						fail("Wrong exception: " + error.getMessage());
					}
				}
			};
		}
		catch (Exception e) {
			e.printStackTrace();
			fail(e.getMessage());
		}
		finally {
			TestingUtils.stopActor(taskManager);
			TestingUtils.stopActor(jobManager);
		}
	}};
}
 

@Test
public void testLogNotFoundHandling() throws Exception {

	new JavaTestKit(system){{

		// we require a JobManager so that the BlobService is also started
		ActorGateway jobManager = null;
		ActorGateway taskManager = null;

		try {

			// Create the JM
			ActorRef jm = system.actorOf(Props.create(
				new SimplePartitionStateLookupJobManagerCreator(LEADER_SESSION_ID, getTestActor())));

			jobManager = new AkkaActorGateway(jm, LEADER_SESSION_ID);

			final int dataPort = NetUtils.getAvailablePort();
			Configuration config = new Configuration();
			config.setInteger(TaskManagerOptions.DATA_PORT, dataPort);
			config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
			config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);
			config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, "/i/dont/exist");

			highAvailabilityServices.setJobMasterLeaderRetriever(
				HighAvailabilityServices.DEFAULT_JOB_ID,
				new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

			taskManager = TestingUtils.createTaskManager(
				system,
				highAvailabilityServices,
				config,
				false,
				true);

			// ---------------------------------------------------------------------------------

			final ActorGateway tm = taskManager;

			new Within(d) {
				@Override
				protected void run() {
					Future<Object> logFuture = tm.ask(TaskManagerMessages.getRequestTaskManagerLog(), timeout);
					try {
						Await.result(logFuture, timeout);
						Assert.fail();
					} catch (Exception e) {
						Assert.assertTrue(e.getMessage().startsWith("TaskManager log files are unavailable. Log file could not be found at"));
					}
				}
			};
		} finally {
			TestingUtils.stopActor(taskManager);
			TestingUtils.stopActor(jobManager);
		}
	}};
}
 

/**
 * Test that a failing schedule or update consumers call leads to the failing of the respective
 * task.
 *
 * <p>IMPORTANT: We have to make sure that the invokable's cancel method is called, because only
 * then the future is completed. We do this by not eagerly deploy consumer tasks and requiring
 * the invokable to fill one memory segment. The completed memory segment will trigger the
 * scheduling of the downstream operator since it is in pipeline mode. After we've filled the
 * memory segment, we'll block the invokable and wait for the task failure due to the failed
 * schedule or update consumers call.
 */
@Test(timeout = 10000L)
public void testFailingScheduleOrUpdateConsumersMessage() throws Exception {
	new JavaTestKit(system) {{
		final Configuration configuration = new Configuration();

		// set the memory segment to the smallest size possible, because we have to fill one
		// memory buffer to trigger the schedule or update consumers message to the downstream
		// operators
		configuration.setString(TaskManagerOptions.MEMORY_SEGMENT_SIZE, "4096");

		final JobID jid = new JobID();
		final JobVertexID vid = new JobVertexID();
		final ExecutionAttemptID eid = new ExecutionAttemptID();
		final SerializedValue<ExecutionConfig> executionConfig = new SerializedValue<>(new ExecutionConfig());

		final ResultPartitionDeploymentDescriptor resultPartitionDeploymentDescriptor = new ResultPartitionDeploymentDescriptor(
			new IntermediateDataSetID(),
			new IntermediateResultPartitionID(),
			ResultPartitionType.PIPELINED,
			1,
			1,
			true);

		final TaskDeploymentDescriptor tdd = createTaskDeploymentDescriptor(jid, "TestJob", vid, eid, executionConfig,
			"TestTask", 1, 0, 1, 0, new Configuration(), new Configuration(),
			TestInvokableRecordCancel.class.getName(),
			Collections.singletonList(resultPartitionDeploymentDescriptor),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			new ArrayList<>(), Collections.emptyList(), 0);

		ActorRef jmActorRef = system.actorOf(Props.create(FailingScheduleOrUpdateConsumersJobManager.class, LEADER_SESSION_ID), "jobmanager");
		ActorGateway jobManager = new AkkaActorGateway(jmActorRef, LEADER_SESSION_ID);

		highAvailabilityServices.setJobMasterLeaderRetriever(
			HighAvailabilityServices.DEFAULT_JOB_ID,
			new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

		final ActorGateway taskManager = TestingUtils.createTaskManager(
			system,
			highAvailabilityServices,
			configuration,
			true,
			true);

		try {
			TestInvokableRecordCancel.resetGotCanceledFuture();

			Future<Object> result = taskManager.ask(new SubmitTask(tdd), timeout);

			Await.result(result, timeout);

			CompletableFuture<Boolean> cancelFuture = TestInvokableRecordCancel.gotCanceled();

			assertEquals(true, cancelFuture.get());
		} finally {
			TestingUtils.stopActor(taskManager);
			TestingUtils.stopActor(jobManager);
		}
	}};
}