org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerInfo Java Examples

The following examples show how to use org.apache.flink.runtime.rest.messages.taskmanager.TaskManagerInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ResourceManagerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that we can retrieve the correct {@link TaskManagerInfo} from the {@link ResourceManager}.
 */
@Test
public void testRequestTaskManagerInfo() throws Exception {
	final ResourceID taskManagerId = ResourceID.generate();
	final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setAddress(UUID.randomUUID().toString()).createTestingTaskExecutorGateway();
	rpcService.registerGateway(taskExecutorGateway.getAddress(), taskExecutorGateway);

	resourceManager = createAndStartResourceManager(heartbeatServices);
	final ResourceManagerGateway resourceManagerGateway = resourceManager.getSelfGateway(ResourceManagerGateway.class);

	registerTaskExecutor(resourceManagerGateway, taskManagerId, taskExecutorGateway.getAddress());

	CompletableFuture<TaskManagerInfo> taskManagerInfoFuture = resourceManagerGateway.requestTaskManagerInfo(
		taskManagerId,
		TestingUtils.TIMEOUT());

	TaskManagerInfo taskManagerInfo = taskManagerInfoFuture.get();

	Assert.assertEquals(taskManagerId, taskManagerInfo.getResourceId());
	Assert.assertEquals(hardwareDescription, taskManagerInfo.getHardwareDescription());
	Assert.assertEquals(taskExecutorGateway.getAddress(), taskManagerInfo.getAddress());
	Assert.assertEquals(dataPort, taskManagerInfo.getDataPort());
	Assert.assertEquals(0, taskManagerInfo.getNumberSlots());
	Assert.assertEquals(0, taskManagerInfo.getNumberAvailableSlots());
}
 
Example #2
Source File: ResourceManagerTaskExecutorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Test receive normal registration from task executor and receive duplicate registration
 * from task executor.
 */
@Test
public void testRegisterTaskExecutor() throws Exception {
	// test response successful
	CompletableFuture<RegistrationResponse> successfulFuture = registerTaskExecutor(rmGateway, taskExecutorGateway.getAddress());

	RegistrationResponse response = successfulFuture.get(TIMEOUT.toMilliseconds(), TimeUnit.MILLISECONDS);
	assertTrue(response instanceof TaskExecutorRegistrationSuccess);
	final TaskManagerInfo taskManagerInfo = rmGateway.requestTaskManagerInfo(
		taskExecutorResourceID,
		TIMEOUT).get();
	assertThat(taskManagerInfo.getResourceId(), equalTo(taskExecutorResourceID));

	// test response successful with instanceID not equal to previous when receive duplicate registration from taskExecutor
	CompletableFuture<RegistrationResponse> duplicateFuture = registerTaskExecutor(rmGateway, taskExecutorGateway.getAddress());

	RegistrationResponse duplicateResponse = duplicateFuture.get();
	assertTrue(duplicateResponse instanceof TaskExecutorRegistrationSuccess);
	assertNotEquals(((TaskExecutorRegistrationSuccess) response).getRegistrationId(), ((TaskExecutorRegistrationSuccess) duplicateResponse).getRegistrationId());

	assertThat(rmGateway.requestResourceOverview(TIMEOUT).get().getNumberTaskManagers(), is(1));
}
 
Example #3
Source File: ResourceManager.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<TaskManagerInfo> requestTaskManagerInfo(ResourceID resourceId, Time timeout) {

	final WorkerRegistration<WorkerType> taskExecutor = taskExecutors.get(resourceId);

	if (taskExecutor == null) {
		return FutureUtils.completedExceptionally(new UnknownTaskExecutorException(resourceId));
	} else {
		final InstanceID instanceId = taskExecutor.getInstanceID();
		final TaskManagerInfo taskManagerInfo = new TaskManagerInfo(
			resourceId,
			taskExecutor.getTaskExecutorGateway().getAddress(),
			taskExecutor.getDataPort(),
			taskManagerHeartbeatManager.getLastHeartbeatFrom(resourceId),
			slotManager.getNumberRegisteredSlotsOf(instanceId),
			slotManager.getNumberFreeSlotsOf(instanceId),
			taskExecutor.getHardwareDescription());

		return CompletableFuture.completedFuture(taskManagerInfo);
	}
}
 
Example #4
Source File: ResourceManager.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Collection<TaskManagerInfo>> requestTaskManagerInfo(Time timeout) {

	final ArrayList<TaskManagerInfo> taskManagerInfos = new ArrayList<>(taskExecutors.size());

	for (Map.Entry<ResourceID, WorkerRegistration<WorkerType>> taskExecutorEntry : taskExecutors.entrySet()) {
		final ResourceID resourceId = taskExecutorEntry.getKey();
		final WorkerRegistration<WorkerType> taskExecutor = taskExecutorEntry.getValue();

		taskManagerInfos.add(
			new TaskManagerInfo(
				resourceId,
				taskExecutor.getTaskExecutorGateway().getAddress(),
				taskExecutor.getDataPort(),
				taskManagerHeartbeatManager.getLastHeartbeatFrom(resourceId),
				slotManager.getNumberRegisteredSlotsOf(taskExecutor.getInstanceID()),
				slotManager.getNumberFreeSlotsOf(taskExecutor.getInstanceID()),
				taskExecutor.getHardwareDescription()));
	}

	return CompletableFuture.completedFuture(taskManagerInfos);
}
 
Example #5
Source File: ResourceManagerTaskExecutorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Test receive normal registration from task executor and receive duplicate registration
 * from task executor.
 */
@Test
public void testRegisterTaskExecutor() throws Exception {
	// test response successful
	CompletableFuture<RegistrationResponse> successfulFuture = registerTaskExecutor(rmGateway, taskExecutorGateway.getAddress());

	RegistrationResponse response = successfulFuture.get(TIMEOUT.toMilliseconds(), TimeUnit.MILLISECONDS);
	assertTrue(response instanceof TaskExecutorRegistrationSuccess);
	final TaskManagerInfo taskManagerInfo = rmGateway.requestTaskManagerInfo(
		taskExecutorResourceID,
		TIMEOUT).get();
	assertThat(taskManagerInfo.getResourceId(), equalTo(taskExecutorResourceID));

	// test response successful with instanceID not equal to previous when receive duplicate registration from taskExecutor
	CompletableFuture<RegistrationResponse> duplicateFuture = registerTaskExecutor(rmGateway, taskExecutorGateway.getAddress());

	RegistrationResponse duplicateResponse = duplicateFuture.get();
	assertTrue(duplicateResponse instanceof TaskExecutorRegistrationSuccess);
	assertNotEquals(((TaskExecutorRegistrationSuccess) response).getRegistrationId(), ((TaskExecutorRegistrationSuccess) duplicateResponse).getRegistrationId());

	assertThat(rmGateway.requestResourceOverview(TIMEOUT).get().getNumberTaskManagers(), is(1));
}
 
Example #6
Source File: ResourceManager.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<TaskManagerInfo> requestTaskManagerInfo(ResourceID resourceId, Time timeout) {

	final WorkerRegistration<WorkerType> taskExecutor = taskExecutors.get(resourceId);

	if (taskExecutor == null) {
		return FutureUtils.completedExceptionally(new UnknownTaskExecutorException(resourceId));
	} else {
		final InstanceID instanceId = taskExecutor.getInstanceID();
		final TaskManagerInfo taskManagerInfo = new TaskManagerInfo(
			resourceId,
			taskExecutor.getTaskExecutorGateway().getAddress(),
			taskExecutor.getDataPort(),
			taskManagerHeartbeatManager.getLastHeartbeatFrom(resourceId),
			slotManager.getNumberRegisteredSlotsOf(instanceId),
			slotManager.getNumberFreeSlotsOf(instanceId),
			slotManager.getRegisteredResourceOf(instanceId),
			slotManager.getFreeResourceOf(instanceId),
			taskExecutor.getHardwareDescription());

		return CompletableFuture.completedFuture(taskManagerInfo);
	}
}
 
Example #7
Source File: ResourceManager.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Collection<TaskManagerInfo>> requestTaskManagerInfo(Time timeout) {

	final ArrayList<TaskManagerInfo> taskManagerInfos = new ArrayList<>(taskExecutors.size());

	for (Map.Entry<ResourceID, WorkerRegistration<WorkerType>> taskExecutorEntry : taskExecutors.entrySet()) {
		final ResourceID resourceId = taskExecutorEntry.getKey();
		final WorkerRegistration<WorkerType> taskExecutor = taskExecutorEntry.getValue();

		taskManagerInfos.add(
			new TaskManagerInfo(
				resourceId,
				taskExecutor.getTaskExecutorGateway().getAddress(),
				taskExecutor.getDataPort(),
				taskManagerHeartbeatManager.getLastHeartbeatFrom(resourceId),
				slotManager.getNumberRegisteredSlotsOf(taskExecutor.getInstanceID()),
				slotManager.getNumberFreeSlotsOf(taskExecutor.getInstanceID()),
				slotManager.getRegisteredResourceOf(taskExecutor.getInstanceID()),
				slotManager.getFreeResourceOf(taskExecutor.getInstanceID()),
				taskExecutor.getHardwareDescription()));
	}

	return CompletableFuture.completedFuture(taskManagerInfos);
}
 
Example #8
Source File: ResourceManagerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that we can retrieve the correct {@link TaskManagerInfo} from the {@link ResourceManager}.
 */
@Test
public void testRequestTaskManagerInfo() throws Exception {
	final ResourceID taskManagerId = ResourceID.generate();
	final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setAddress(UUID.randomUUID().toString()).createTestingTaskExecutorGateway();
	rpcService.registerGateway(taskExecutorGateway.getAddress(), taskExecutorGateway);

	resourceManager = createAndStartResourceManager(heartbeatServices);
	final ResourceManagerGateway resourceManagerGateway = resourceManager.getSelfGateway(ResourceManagerGateway.class);

	registerTaskExecutor(resourceManagerGateway, taskManagerId, taskExecutorGateway.getAddress());

	CompletableFuture<TaskManagerInfo> taskManagerInfoFuture = resourceManagerGateway.requestTaskManagerInfo(
		taskManagerId,
		TestingUtils.TIMEOUT());

	TaskManagerInfo taskManagerInfo = taskManagerInfoFuture.get();

	assertEquals(taskManagerId, taskManagerInfo.getResourceId());
	assertEquals(hardwareDescription, taskManagerInfo.getHardwareDescription());
	assertEquals(taskExecutorGateway.getAddress(), taskManagerInfo.getAddress());
	assertEquals(dataPort, taskManagerInfo.getDataPort());
	assertEquals(0, taskManagerInfo.getNumberSlots());
	assertEquals(0, taskManagerInfo.getNumberAvailableSlots());
}
 
Example #9
Source File: MetricsAvailabilityITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static Collection<ResourceID> getTaskManagerIds(final RestClient restClient) throws Exception {
	final TaskManagersHeaders headers = TaskManagersHeaders.getInstance();

	final TaskManagersInfo response = fetchMetric(() ->
			restClient.sendRequest(
				HOST,
				PORT,
				headers,
				EmptyMessageParameters.getInstance(),
				EmptyRequestBody.getInstance()),
		taskManagersInfo -> !taskManagersInfo.getTaskManagerInfos().isEmpty());

	return response.getTaskManagerInfos().stream()
		.map(TaskManagerInfo::getResourceId)
		.collect(Collectors.toList());
}
 
Example #10
Source File: MetricsAvailabilityITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private static Collection<ResourceID> getTaskManagerIds(final RestClient restClient) throws Exception {
	final TaskManagersHeaders headers = TaskManagersHeaders.getInstance();

	final TaskManagersInfo response = fetchMetric(() ->
			restClient.sendRequest(
				HOST,
				PORT,
				headers,
				EmptyMessageParameters.getInstance(),
				EmptyRequestBody.getInstance()),
		taskManagersInfo -> !taskManagersInfo.getTaskManagerInfos().isEmpty());

	return response.getTaskManagerInfos().stream()
		.map(TaskManagerInfo::getResourceId)
		.collect(Collectors.toList());
}
 
Example #11
Source File: ResourceManagerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that we can retrieve the correct {@link TaskManagerInfo} from the {@link ResourceManager}.
 */
@Test
public void testRequestTaskManagerInfo() throws Exception {
	final ResourceID taskManagerId = ResourceID.generate();
	final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setAddress(UUID.randomUUID().toString()).createTestingTaskExecutorGateway();
	rpcService.registerGateway(taskExecutorGateway.getAddress(), taskExecutorGateway);

	resourceManager = createAndStartResourceManager(heartbeatServices);
	final ResourceManagerGateway resourceManagerGateway = resourceManager.getSelfGateway(ResourceManagerGateway.class);

	registerTaskExecutor(resourceManagerGateway, taskManagerId, taskExecutorGateway.getAddress());

	CompletableFuture<TaskManagerInfo> taskManagerInfoFuture = resourceManagerGateway.requestTaskManagerInfo(
		taskManagerId,
		TestingUtils.TIMEOUT());

	TaskManagerInfo taskManagerInfo = taskManagerInfoFuture.get();

	assertEquals(taskManagerId, taskManagerInfo.getResourceId());
	assertEquals(hardwareDescription, taskManagerInfo.getHardwareDescription());
	assertEquals(taskExecutorGateway.getAddress(), taskManagerInfo.getAddress());
	assertEquals(dataPort, taskManagerInfo.getDataPort());
	assertEquals(0, taskManagerInfo.getNumberSlots());
	assertEquals(0, taskManagerInfo.getNumberAvailableSlots());
}
 
Example #12
Source File: MetricsAvailabilityITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private static Collection<ResourceID> getTaskManagerIds(final RestClient restClient) throws Exception {
	final TaskManagersHeaders headers = TaskManagersHeaders.getInstance();

	final TaskManagersInfo response = fetchMetric(() ->
			restClient.sendRequest(
				HOST,
				PORT,
				headers,
				EmptyMessageParameters.getInstance(),
				EmptyRequestBody.getInstance()),
		taskManagersInfo -> !taskManagersInfo.getTaskManagerInfos().isEmpty());

	return response.getTaskManagerInfos().stream()
		.map(TaskManagerInfo::getResourceId)
		.collect(Collectors.toList());
}
 
Example #13
Source File: ResourceManagerTaskExecutorTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Test receive normal registration from task executor and receive duplicate registration
 * from task executor.
 */
@Test
public void testRegisterTaskExecutor() throws Exception {
	// test response successful
	CompletableFuture<RegistrationResponse> successfulFuture = registerTaskExecutor(rmGateway, taskExecutorGateway.getAddress());

	RegistrationResponse response = successfulFuture.get(TIMEOUT.toMilliseconds(), TimeUnit.MILLISECONDS);
	assertTrue(response instanceof TaskExecutorRegistrationSuccess);
	final TaskManagerInfo taskManagerInfo = rmGateway.requestTaskManagerInfo(
		taskExecutorResourceID,
		TIMEOUT).get();
	assertThat(taskManagerInfo.getResourceId(), equalTo(taskExecutorResourceID));

	// test response successful with instanceID not equal to previous when receive duplicate registration from taskExecutor
	CompletableFuture<RegistrationResponse> duplicateFuture = registerTaskExecutor(rmGateway, taskExecutorGateway.getAddress());

	RegistrationResponse duplicateResponse = duplicateFuture.get();
	assertTrue(duplicateResponse instanceof TaskExecutorRegistrationSuccess);
	assertNotEquals(((TaskExecutorRegistrationSuccess) response).getRegistrationId(), ((TaskExecutorRegistrationSuccess) duplicateResponse).getRegistrationId());

	assertThat(rmGateway.requestResourceOverview(TIMEOUT).get().getNumberTaskManagers(), is(1));
}
 
Example #14
Source File: ResourceManager.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<TaskManagerInfo> requestTaskManagerInfo(ResourceID resourceId, Time timeout) {

	final WorkerRegistration<WorkerType> taskExecutor = taskExecutors.get(resourceId);

	if (taskExecutor == null) {
		return FutureUtils.completedExceptionally(new UnknownTaskExecutorException(resourceId));
	} else {
		final InstanceID instanceId = taskExecutor.getInstanceID();
		final TaskManagerInfo taskManagerInfo = new TaskManagerInfo(
			resourceId,
			taskExecutor.getTaskExecutorGateway().getAddress(),
			taskExecutor.getDataPort(),
			taskManagerHeartbeatManager.getLastHeartbeatFrom(resourceId),
			slotManager.getNumberRegisteredSlotsOf(instanceId),
			slotManager.getNumberFreeSlotsOf(instanceId),
			taskExecutor.getHardwareDescription());

		return CompletableFuture.completedFuture(taskManagerInfo);
	}
}
 
Example #15
Source File: ResourceManager.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Collection<TaskManagerInfo>> requestTaskManagerInfo(Time timeout) {

	final ArrayList<TaskManagerInfo> taskManagerInfos = new ArrayList<>(taskExecutors.size());

	for (Map.Entry<ResourceID, WorkerRegistration<WorkerType>> taskExecutorEntry : taskExecutors.entrySet()) {
		final ResourceID resourceId = taskExecutorEntry.getKey();
		final WorkerRegistration<WorkerType> taskExecutor = taskExecutorEntry.getValue();

		taskManagerInfos.add(
			new TaskManagerInfo(
				resourceId,
				taskExecutor.getTaskExecutorGateway().getAddress(),
				taskExecutor.getDataPort(),
				taskManagerHeartbeatManager.getLastHeartbeatFrom(resourceId),
				slotManager.getNumberRegisteredSlotsOf(taskExecutor.getInstanceID()),
				slotManager.getNumberFreeSlotsOf(taskExecutor.getInstanceID()),
				taskExecutor.getHardwareDescription()));
	}

	return CompletableFuture.completedFuture(taskManagerInfos);
}
 
Example #16
Source File: YARNSessionCapacitySchedulerITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
private static int getNumberOfSlotsPerTaskManager(final String host, final int port) throws Exception {
	final TaskManagersInfo taskManagersInfo = restClient.sendRequest(
		host,
		port,
		TaskManagersHeaders.getInstance()).get();

	return taskManagersInfo.getTaskManagerInfos()
		.stream()
		.map(TaskManagerInfo::getNumberSlots)
		.findFirst()
		.orElse(0);
}
 
Example #17
Source File: YarnConfigurationITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
private boolean hasTaskManagerConnectedAndReportedSlots(Collection<TaskManagerInfo> taskManagerInfos) {
	if (taskManagerInfos.isEmpty()) {
		return false;
	} else {
		final TaskManagerInfo taskManagerInfo = taskManagerInfos.iterator().next();
		return taskManagerInfo.getNumberSlots() > 0;
	}
}
 
Example #18
Source File: YARNSessionCapacitySchedulerITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
private static int getNumberOfSlotsPerTaskManager(final String host, final int port) throws Exception {
	final TaskManagersInfo taskManagersInfo = restClient.sendRequest(
		host,
		port,
		TaskManagersHeaders.getInstance()).get();

	return taskManagersInfo.getTaskManagerInfos()
		.stream()
		.map(TaskManagerInfo::getNumberSlots)
		.findFirst()
		.orElse(0);
}
 
Example #19
Source File: YARNSessionCapacitySchedulerITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private static int getNumberOfSlotsPerTaskManager(final String host, final int port) throws Exception {
	final TaskManagersInfo taskManagersInfo = restClient.sendRequest(
		host,
		port,
		TaskManagersHeaders.getInstance()).get();

	return taskManagersInfo.getTaskManagerInfos()
		.stream()
		.map(TaskManagerInfo::getNumberSlots)
		.findFirst()
		.orElse(0);
}
 
Example #20
Source File: YarnConfigurationITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private boolean hasTaskManagerConnectedAndReportedSlots(Collection<TaskManagerInfo> taskManagerInfos) {
	if (taskManagerInfos.isEmpty()) {
		return false;
	} else {
		final TaskManagerInfo taskManagerInfo = taskManagerInfos.iterator().next();
		return taskManagerInfo.getNumberSlots() > 0;
	}
}
 
Example #21
Source File: YarnConfigurationITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
private boolean hasTaskManagerConnectedAndReportedSlots(Collection<TaskManagerInfo> taskManagerInfos) {
	if (taskManagerInfos.isEmpty()) {
		return false;
	} else {
		final TaskManagerInfo taskManagerInfo = taskManagerInfos.iterator().next();
		return taskManagerInfo.getNumberSlots() > 0;
	}
}
 
Example #22
Source File: ResourceManagerTaskExecutorTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Test delayed registration of task executor where the delay is introduced during connection from resource manager
 * to the registering task executor.
 */
@Test
public void testDelayedRegisterTaskExecutor() throws Exception {
	final Time fastTimeout = Time.milliseconds(1L);
	try {
		final OneShotLatch startConnection = new OneShotLatch();
		final OneShotLatch finishConnection = new OneShotLatch();

		// first registration is with blocking connection
		rpcService.setRpcGatewayFutureFunction(rpcGateway ->
			CompletableFuture.supplyAsync(
				() -> {
					startConnection.trigger();
					try {
						finishConnection.await();
					} catch (InterruptedException ignored) {}
					return rpcGateway;
				},
				TestingUtils.defaultExecutor()));

		TaskExecutorRegistration taskExecutorRegistration = new TaskExecutorRegistration(
			taskExecutorGateway.getAddress(),
			taskExecutorResourceID,
			dataPort,
			hardwareDescription,
			ResourceProfile.ZERO,
			ResourceProfile.ZERO);

		CompletableFuture<RegistrationResponse> firstFuture =
			rmGateway.registerTaskExecutor(taskExecutorRegistration, fastTimeout);
		try {
			firstFuture.get();
			fail("Should have failed because connection to taskmanager is delayed beyond timeout");
		} catch (Exception e) {
			final Throwable cause = ExceptionUtils.stripExecutionException(e);
			assertThat(cause, instanceOf(TimeoutException.class));
			assertThat(cause.getMessage(), containsString("ResourceManagerGateway.registerTaskExecutor"));
		}

		startConnection.await();

		// second registration after timeout is with no delay, expecting it to be succeeded
		rpcService.resetRpcGatewayFutureFunction();
		CompletableFuture<RegistrationResponse> secondFuture =
			rmGateway.registerTaskExecutor(taskExecutorRegistration, TIMEOUT);
		RegistrationResponse response = secondFuture.get();
		assertTrue(response instanceof TaskExecutorRegistrationSuccess);

		// on success, send slot report for taskmanager registration
		final SlotReport slotReport = new SlotReport(new SlotStatus(new SlotID(taskExecutorResourceID, 0), ResourceProfile.ANY));
		rmGateway.sendSlotReport(taskExecutorResourceID,
			((TaskExecutorRegistrationSuccess) response).getRegistrationId(), slotReport, TIMEOUT).get();

		// let the remaining part of the first registration proceed
		finishConnection.trigger();
		Thread.sleep(1L);

		// verify that the latest registration is valid not being unregistered by the delayed one
		final TaskManagerInfo taskManagerInfo = rmGateway.requestTaskManagerInfo(
			taskExecutorResourceID,
			TIMEOUT).get();
		assertThat(taskManagerInfo.getResourceId(), equalTo(taskExecutorResourceID));
		assertThat(taskManagerInfo.getNumberSlots(), equalTo(1));
	} finally {
		rpcService.resetRpcGatewayFutureFunction();
	}
}
 
Example #23
Source File: TaskManagerDetailsHandler.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
protected CompletableFuture<TaskManagerDetailsInfo> handleRequest(
		@Nonnull HandlerRequest<EmptyRequestBody, TaskManagerMessageParameters> request,
		@Nonnull ResourceManagerGateway gateway) throws RestHandlerException {
	final ResourceID taskManagerResourceId = request.getPathParameter(TaskManagerIdPathParameter
		.class);

	CompletableFuture<TaskManagerInfo> taskManagerInfoFuture = gateway.requestTaskManagerInfo(taskManagerResourceId, timeout);

	metricFetcher.update();

	return taskManagerInfoFuture.thenApply(
		(TaskManagerInfo taskManagerInfo) -> {
			final MetricStore.TaskManagerMetricStore tmMetrics =
				metricStore.getTaskManagerMetricStore(taskManagerResourceId.getResourceIdString());

			final TaskManagerMetricsInfo taskManagerMetricsInfo;

			if (tmMetrics != null) {
				log.debug("Create metrics info for TaskManager {}.", taskManagerResourceId);
				taskManagerMetricsInfo = createTaskManagerMetricsInfo(tmMetrics);
			} else {
				log.debug("No metrics for TaskManager {}.", taskManagerResourceId);
				taskManagerMetricsInfo = TaskManagerMetricsInfo.empty();
			}

			return new TaskManagerDetailsInfo(
				taskManagerInfo,
				taskManagerMetricsInfo);
		})
		.exceptionally(
			(Throwable throwable) -> {
				final Throwable strippedThrowable = ExceptionUtils.stripExecutionException(throwable);

				if (strippedThrowable instanceof UnknownTaskExecutorException) {
					throw new CompletionException(
						new RestHandlerException(
							"Could not find TaskExecutor " + taskManagerResourceId + '.',
							HttpResponseStatus.NOT_FOUND,
							strippedThrowable));
				} else {
					throw new CompletionException(strippedThrowable);
				}
			}
		);
}
 
Example #24
Source File: TestingResourceManagerGateway.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public CompletableFuture<Collection<TaskManagerInfo>> requestTaskManagerInfo(Time timeout) {
	return CompletableFuture.completedFuture(Collections.emptyList());
}
 
Example #25
Source File: TestingResourceManagerGateway.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public CompletableFuture<TaskManagerInfo> requestTaskManagerInfo(ResourceID resourceId, Time timeout) {
	return FutureUtils.completedExceptionally(new UnsupportedOperationException("Not yet implemented"));
}
 
Example #26
Source File: ResourceManagerTaskExecutorTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Test delayed registration of task executor where the delay is introduced during connection from resource manager
 * to the registering task executor.
 */
@Test
public void testDelayedRegisterTaskExecutor() throws Exception {
	final Time fastTimeout = Time.milliseconds(1L);
	try {
		final OneShotLatch startConnection = new OneShotLatch();
		final OneShotLatch finishConnection = new OneShotLatch();

		// first registration is with blocking connection
		rpcService.setRpcGatewayFutureFunction(rpcGateway ->
			CompletableFuture.supplyAsync(
				() -> {
					startConnection.trigger();
					try {
						finishConnection.await();
					} catch (InterruptedException ignored) {}
					return rpcGateway;
				},
				TestingUtils.defaultExecutor()));

		CompletableFuture<RegistrationResponse> firstFuture =
			rmGateway.registerTaskExecutor(taskExecutorGateway.getAddress(), taskExecutorResourceID, dataPort, hardwareDescription, fastTimeout);
		try {
			firstFuture.get();
			fail("Should have failed because connection to taskmanager is delayed beyond timeout");
		} catch (Exception e) {
			assertThat(ExceptionUtils.stripExecutionException(e), instanceOf(AskTimeoutException.class));
		}

		startConnection.await();

		// second registration after timeout is with no delay, expecting it to be succeeded
		rpcService.resetRpcGatewayFutureFunction();
		CompletableFuture<RegistrationResponse> secondFuture =
			rmGateway.registerTaskExecutor(taskExecutorGateway.getAddress(), taskExecutorResourceID, dataPort, hardwareDescription, TIMEOUT);
		RegistrationResponse response = secondFuture.get();
		assertTrue(response instanceof TaskExecutorRegistrationSuccess);

		// on success, send slot report for taskmanager registration
		final SlotReport slotReport = new SlotReport(new SlotStatus(new SlotID(taskExecutorResourceID, 0), ResourceProfile.UNKNOWN));
		rmGateway.sendSlotReport(taskExecutorResourceID,
			((TaskExecutorRegistrationSuccess) response).getRegistrationId(), slotReport, TIMEOUT).get();

		// let the remaining part of the first registration proceed
		finishConnection.trigger();
		Thread.sleep(1L);

		// verify that the latest registration is valid not being unregistered by the delayed one
		final TaskManagerInfo taskManagerInfo = rmGateway.requestTaskManagerInfo(
			taskExecutorResourceID,
			TIMEOUT).get();
		assertThat(taskManagerInfo.getResourceId(), equalTo(taskExecutorResourceID));
		assertThat(taskManagerInfo.getNumberSlots(), equalTo(1));
	} finally {
		rpcService.resetRpcGatewayFutureFunction();
	}
}
 
Example #27
Source File: TestingResourceManagerGateway.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public CompletableFuture<TaskManagerInfo> requestTaskManagerInfo(ResourceID resourceId, Time timeout) {
	return FutureUtils.completedExceptionally(new UnsupportedOperationException("Not yet implemented"));
}
 
Example #28
Source File: TestingResourceManagerGateway.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public CompletableFuture<Collection<TaskManagerInfo>> requestTaskManagerInfo(Time timeout) {
	return CompletableFuture.completedFuture(Collections.emptyList());
}
 
Example #29
Source File: TaskManagerDetailsHandler.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
protected CompletableFuture<TaskManagerDetailsInfo> handleRequest(
		@Nonnull HandlerRequest<EmptyRequestBody, TaskManagerMessageParameters> request,
		@Nonnull ResourceManagerGateway gateway) throws RestHandlerException {
	final ResourceID taskManagerResourceId = request.getPathParameter(TaskManagerIdPathParameter
		.class);

	CompletableFuture<TaskManagerInfo> taskManagerInfoFuture = gateway.requestTaskManagerInfo(taskManagerResourceId, timeout);

	metricFetcher.update();

	return taskManagerInfoFuture.thenApply(
		(TaskManagerInfo taskManagerInfo) -> {
			final MetricStore.TaskManagerMetricStore tmMetrics =
				metricStore.getTaskManagerMetricStore(taskManagerResourceId.getResourceIdString());

			final TaskManagerMetricsInfo taskManagerMetricsInfo;

			if (tmMetrics != null) {
				log.debug("Create metrics info for TaskManager {}.", taskManagerResourceId);
				taskManagerMetricsInfo = createTaskManagerMetricsInfo(tmMetrics);
			} else {
				log.debug("No metrics for TaskManager {}.", taskManagerResourceId);
				taskManagerMetricsInfo = TaskManagerMetricsInfo.empty();
			}

			return new TaskManagerDetailsInfo(
				taskManagerInfo,
				taskManagerMetricsInfo);
		})
		.exceptionally(
			(Throwable throwable) -> {
				final Throwable strippedThrowable = ExceptionUtils.stripExecutionException(throwable);

				if (strippedThrowable instanceof UnknownTaskExecutorException) {
					throw new CompletionException(
						new RestHandlerException(
							"Could not find TaskExecutor " + taskManagerResourceId + '.',
							HttpResponseStatus.NOT_FOUND,
							strippedThrowable));
				} else {
					throw new CompletionException(strippedThrowable);
				}
			}
		);
}
 
Example #30
Source File: TestingResourceManagerGateway.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
public CompletableFuture<TaskManagerInfo> requestTaskManagerInfo(ResourceID resourceId, Time timeout) {
	return FutureUtils.completedExceptionally(new UnsupportedOperationException("Not yet implemented"));
}