Java Code Examples for org.apache.flink.runtime.concurrent.FutureUtils#whenCompleteAsyncIfNotDone()

The following examples show how to use org.apache.flink.runtime.concurrent.FutureUtils#whenCompleteAsyncIfNotDone() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SlotPoolImpl.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void requestSlotFromResourceManager(
		final ResourceManagerGateway resourceManagerGateway,
		final PendingRequest pendingRequest) {

	checkNotNull(resourceManagerGateway);
	checkNotNull(pendingRequest);

	log.info("Requesting new slot [{}] and profile {} from resource manager.", pendingRequest.getSlotRequestId(), pendingRequest.getResourceProfile());

	final AllocationID allocationId = new AllocationID();

	pendingRequests.put(pendingRequest.getSlotRequestId(), allocationId, pendingRequest);

	pendingRequest.getAllocatedSlotFuture().whenComplete(
		(AllocatedSlot allocatedSlot, Throwable throwable) -> {
			if (throwable != null || !allocationId.equals(allocatedSlot.getAllocationId())) {
				// cancel the slot request if there is a failure or if the pending request has
				// been completed with another allocated slot
				resourceManagerGateway.cancelSlotRequest(allocationId);
			}
		});

	CompletableFuture<Acknowledge> rmResponse = resourceManagerGateway.requestSlot(
		jobMasterId,
		new SlotRequest(jobId, allocationId, pendingRequest.getResourceProfile(), jobManagerAddress),
		rpcTimeout);

	FutureUtils.whenCompleteAsyncIfNotDone(
		rmResponse,
		componentMainThreadExecutor,
		(Acknowledge ignored, Throwable failure) -> {
			// on failure, fail the request future
			if (failure != null) {
				slotRequestToResourceManagerFailed(pendingRequest.getSlotRequestId(), failure);
			}
		});
}
Example 2
Source File: SlotPoolImpl.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void requestSlotFromResourceManager(
		final ResourceManagerGateway resourceManagerGateway,
		final PendingRequest pendingRequest) {

	checkNotNull(resourceManagerGateway);
	checkNotNull(pendingRequest);

	log.info("Requesting new slot [{}] and profile {} from resource manager.", pendingRequest.getSlotRequestId(), pendingRequest.getResourceProfile());

	final AllocationID allocationId = new AllocationID();

	pendingRequests.put(pendingRequest.getSlotRequestId(), allocationId, pendingRequest);

	pendingRequest.getAllocatedSlotFuture().whenComplete(
		(AllocatedSlot allocatedSlot, Throwable throwable) -> {
			if (throwable != null || !allocationId.equals(allocatedSlot.getAllocationId())) {
				// cancel the slot request if there is a failure or if the pending request has
				// been completed with another allocated slot
				resourceManagerGateway.cancelSlotRequest(allocationId);
			}
		});

	CompletableFuture<Acknowledge> rmResponse = resourceManagerGateway.requestSlot(
		jobMasterId,
		new SlotRequest(jobId, allocationId, pendingRequest.getResourceProfile(), jobManagerAddress),
		rpcTimeout);

	FutureUtils.whenCompleteAsyncIfNotDone(
		rmResponse,
		componentMainThreadExecutor,
		(Acknowledge ignored, Throwable failure) -> {
			// on failure, fail the request future
			if (failure != null) {
				slotRequestToResourceManagerFailed(pendingRequest.getSlotRequestId(), failure);
			}
		});
}
Example 3
Source File: SlotPoolImpl.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Check the available slots, release the slot that is idle for a long time.
 */
private void checkIdleSlot() {

	// The timestamp in SlotAndTimestamp is relative
	final long currentRelativeTimeMillis = clock.relativeTimeMillis();

	final List<AllocatedSlot> expiredSlots = new ArrayList<>(availableSlots.size());

	for (SlotAndTimestamp slotAndTimestamp : availableSlots.availableSlots.values()) {
		if (currentRelativeTimeMillis - slotAndTimestamp.timestamp > idleSlotTimeout.toMilliseconds()) {
			expiredSlots.add(slotAndTimestamp.slot);
		}
	}

	final FlinkException cause = new FlinkException("Releasing idle slot.");

	for (AllocatedSlot expiredSlot : expiredSlots) {
		final AllocationID allocationID = expiredSlot.getAllocationId();
		if (availableSlots.tryRemove(allocationID) != null) {

			log.info("Releasing idle slot [{}].", allocationID);
			final CompletableFuture<Acknowledge> freeSlotFuture = expiredSlot.getTaskManagerGateway().freeSlot(
				allocationID,
				cause,
				rpcTimeout);

			FutureUtils.whenCompleteAsyncIfNotDone(
				freeSlotFuture,
				componentMainThreadExecutor,
				(Acknowledge ignored, Throwable throwable) -> {
					if (throwable != null) {
						// The slot status will be synced to task manager in next heartbeat.
						log.debug("Releasing slot [{}] of registered TaskExecutor {} failed. Discarding slot.",
									allocationID, expiredSlot.getTaskManagerId(), throwable);
					}
				});
		}
	}

	scheduleRunAsync(this::checkIdleSlot, idleSlotTimeout);
}
Example 4
Source File: SlotPoolImpl.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Check the available slots, release the slot that is idle for a long time.
 */
protected void checkIdleSlot() {

	// The timestamp in SlotAndTimestamp is relative
	final long currentRelativeTimeMillis = clock.relativeTimeMillis();

	final List<AllocatedSlot> expiredSlots = new ArrayList<>(availableSlots.size());

	for (SlotAndTimestamp slotAndTimestamp : availableSlots.availableSlots.values()) {
		if (currentRelativeTimeMillis - slotAndTimestamp.timestamp > idleSlotTimeout.toMilliseconds()) {
			expiredSlots.add(slotAndTimestamp.slot);
		}
	}

	final FlinkException cause = new FlinkException("Releasing idle slot.");

	for (AllocatedSlot expiredSlot : expiredSlots) {
		final AllocationID allocationID = expiredSlot.getAllocationId();
		if (availableSlots.tryRemove(allocationID) != null) {

			log.info("Releasing idle slot [{}].", allocationID);
			final CompletableFuture<Acknowledge> freeSlotFuture = expiredSlot.getTaskManagerGateway().freeSlot(
				allocationID,
				cause,
				rpcTimeout);

			FutureUtils.whenCompleteAsyncIfNotDone(
				freeSlotFuture,
				componentMainThreadExecutor,
				(Acknowledge ignored, Throwable throwable) -> {
					if (throwable != null) {
						// The slot status will be synced to task manager in next heartbeat.
						log.debug("Releasing slot [{}] of registered TaskExecutor {} failed. Discarding slot.",
									allocationID, expiredSlot.getTaskManagerId(), throwable);
					}
				});
		}
	}

	scheduleRunAsync(this::checkIdleSlot, idleSlotTimeout);
}
Example 5
Source File: SlotPoolImpl.java    From flink with Apache License 2.0 4 votes vote down vote up 
private void requestSlotFromResourceManager(
		final ResourceManagerGateway resourceManagerGateway,
		final PendingRequest pendingRequest) {

	checkNotNull(resourceManagerGateway);
	checkNotNull(pendingRequest);

	final AllocationID allocationId = new AllocationID();
	pendingRequest.setAllocationId(allocationId);

	pendingRequests.put(pendingRequest.getSlotRequestId(), allocationId, pendingRequest);

	pendingRequest.getAllocatedSlotFuture().whenComplete(
		(AllocatedSlot allocatedSlot, Throwable throwable) -> {
			if (throwable != null) {
				// the allocation id can be remapped so we need to get it from the pendingRequest
				// where it will be updated timely
				final Optional<AllocationID> updatedAllocationId = pendingRequest.getAllocationId();

				if (updatedAllocationId.isPresent()) {
					// cancel the slot request if there is a failure
					resourceManagerGateway.cancelSlotRequest(updatedAllocationId.get());
				}
			}
		});

	log.info("Requesting new slot [{}] and profile {} with allocation id {} from resource manager.",
		pendingRequest.getSlotRequestId(), pendingRequest.getResourceProfile(), allocationId);

	CompletableFuture<Acknowledge> rmResponse = resourceManagerGateway.requestSlot(
		jobMasterId,
		new SlotRequest(jobId, allocationId, pendingRequest.getResourceProfile(), jobManagerAddress),
		rpcTimeout);

	FutureUtils.whenCompleteAsyncIfNotDone(
		rmResponse,
		componentMainThreadExecutor,
		(Acknowledge ignored, Throwable failure) -> {
			// on failure, fail the request future
			if (failure != null) {
				slotRequestToResourceManagerFailed(pendingRequest.getSlotRequestId(), failure);
			}
		});
}
Example 6
Source File: SlotPoolImpl.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Check the available slots, release the slot that is idle for a long time.
 */
protected void checkIdleSlot() {

	// The timestamp in SlotAndTimestamp is relative
	final long currentRelativeTimeMillis = clock.relativeTimeMillis();

	final List<AllocatedSlot> expiredSlots = new ArrayList<>(availableSlots.size());

	for (SlotAndTimestamp slotAndTimestamp : availableSlots.availableSlots.values()) {
		if (currentRelativeTimeMillis - slotAndTimestamp.timestamp > idleSlotTimeout.toMilliseconds()) {
			expiredSlots.add(slotAndTimestamp.slot);
		}
	}

	final FlinkException cause = new FlinkException("Releasing idle slot.");

	for (AllocatedSlot expiredSlot : expiredSlots) {
		final AllocationID allocationID = expiredSlot.getAllocationId();
		if (availableSlots.tryRemove(allocationID) != null) {

			log.info("Releasing idle slot [{}].", allocationID);
			final CompletableFuture<Acknowledge> freeSlotFuture = expiredSlot.getTaskManagerGateway().freeSlot(
				allocationID,
				cause,
				rpcTimeout);

			FutureUtils.whenCompleteAsyncIfNotDone(
				freeSlotFuture,
				componentMainThreadExecutor,
				(Acknowledge ignored, Throwable throwable) -> {
					if (throwable != null) {
						// The slot status will be synced to task manager in next heartbeat.
						log.debug("Releasing slot [{}] of registered TaskExecutor {} failed. Discarding slot.",
									allocationID, expiredSlot.getTaskManagerId(), throwable);
					}
				});
		}
	}

	scheduleRunAsync(this::checkIdleSlot, idleSlotTimeout);
}