Java Code Examples for org.apache.flink.runtime.clusterframework.types.SlotID#equals()

/**
 * Slot offering by TaskExecutor with AllocationID. The AllocationID is originally generated by this pool and
 * transfer through the ResourceManager to TaskManager. We use it to distinguish the different allocation
 * we issued. Slot offering may be rejected if we find something mismatching or there is actually no pending
 * request waiting for this slot (maybe fulfilled by some other returned slot).
 *
 * @param taskManagerLocation location from where the offer comes from
 * @param taskManagerGateway TaskManager gateway
 * @param slotOffer the offered slot
 * @return True if we accept the offering
 */
boolean offerSlot(
		final TaskManagerLocation taskManagerLocation,
		final TaskManagerGateway taskManagerGateway,
		final SlotOffer slotOffer) {

	componentMainThreadExecutor.assertRunningInMainThread();

	// check if this TaskManager is valid
	final ResourceID resourceID = taskManagerLocation.getResourceID();
	final AllocationID allocationID = slotOffer.getAllocationId();

	if (!registeredTaskManagers.contains(resourceID)) {
		log.debug("Received outdated slot offering [{}] from unregistered TaskManager: {}",
				slotOffer.getAllocationId(), taskManagerLocation);
		return false;
	}

	// check whether we have already using this slot
	AllocatedSlot existingSlot;
	if ((existingSlot = allocatedSlots.get(allocationID)) != null ||
		(existingSlot = availableSlots.get(allocationID)) != null) {

		// we need to figure out if this is a repeated offer for the exact same slot,
		// or another offer that comes from a different TaskManager after the ResourceManager
		// re-tried the request

		// we write this in terms of comparing slot IDs, because the Slot IDs are the identifiers of
		// the actual slots on the TaskManagers
		// Note: The slotOffer should have the SlotID
		final SlotID existingSlotId = existingSlot.getSlotId();
		final SlotID newSlotId = new SlotID(taskManagerLocation.getResourceID(), slotOffer.getSlotIndex());

		if (existingSlotId.equals(newSlotId)) {
			log.info("Received repeated offer for slot [{}]. Ignoring.", allocationID);

			// return true here so that the sender will get a positive acknowledgement to the retry
			// and mark the offering as a success
			return true;
		} else {
			// the allocation has been fulfilled by another slot, reject the offer so the task executor
			// will offer the slot to the resource manager
			return false;
		}
	}

	final AllocatedSlot allocatedSlot = new AllocatedSlot(
		allocationID,
		taskManagerLocation,
		slotOffer.getSlotIndex(),
		slotOffer.getResourceProfile(),
		taskManagerGateway);

	// check whether we have request waiting for this slot
	PendingRequest pendingRequest = pendingRequests.removeKeyB(allocationID);
	if (pendingRequest != null) {
		// we were waiting for this!
		allocatedSlots.add(pendingRequest.getSlotRequestId(), allocatedSlot);

		if (!pendingRequest.getAllocatedSlotFuture().complete(allocatedSlot)) {
			// we could not complete the pending slot future --> try to fulfill another pending request
			allocatedSlots.remove(pendingRequest.getSlotRequestId());
			tryFulfillSlotRequestOrMakeAvailable(allocatedSlot);
		} else {
			log.debug("Fulfilled slot request [{}] with allocated slot [{}].", pendingRequest.getSlotRequestId(), allocationID);
		}
	}
	else {
		// we were actually not waiting for this:
		//   - could be that this request had been fulfilled
		//   - we are receiving the slots from TaskManagers after becoming leaders
		tryFulfillSlotRequestOrMakeAvailable(allocatedSlot);
	}

	// we accepted the request in any case. slot will be released after it idled for
	// too long and timed out
	return true;
}
 

/**
 * Slot offering by TaskExecutor with AllocationID. The AllocationID is originally generated by this pool and
 * transfer through the ResourceManager to TaskManager. We use it to distinguish the different allocation
 * we issued. Slot offering may be rejected if we find something mismatching or there is actually no pending
 * request waiting for this slot (maybe fulfilled by some other returned slot).
 *
 * @param taskManagerLocation location from where the offer comes from
 * @param taskManagerGateway TaskManager gateway
 * @param slotOffer the offered slot
 * @return True if we accept the offering
 */
boolean offerSlot(
		final TaskManagerLocation taskManagerLocation,
		final TaskManagerGateway taskManagerGateway,
		final SlotOffer slotOffer) {

	componentMainThreadExecutor.assertRunningInMainThread();

	// check if this TaskManager is valid
	final ResourceID resourceID = taskManagerLocation.getResourceID();
	final AllocationID allocationID = slotOffer.getAllocationId();

	if (!registeredTaskManagers.contains(resourceID)) {
		log.debug("Received outdated slot offering [{}] from unregistered TaskManager: {}",
				slotOffer.getAllocationId(), taskManagerLocation);
		return false;
	}

	// check whether we have already using this slot
	AllocatedSlot existingSlot;
	if ((existingSlot = allocatedSlots.get(allocationID)) != null ||
		(existingSlot = availableSlots.get(allocationID)) != null) {

		// we need to figure out if this is a repeated offer for the exact same slot,
		// or another offer that comes from a different TaskManager after the ResourceManager
		// re-tried the request

		// we write this in terms of comparing slot IDs, because the Slot IDs are the identifiers of
		// the actual slots on the TaskManagers
		// Note: The slotOffer should have the SlotID
		final SlotID existingSlotId = existingSlot.getSlotId();
		final SlotID newSlotId = new SlotID(taskManagerLocation.getResourceID(), slotOffer.getSlotIndex());

		if (existingSlotId.equals(newSlotId)) {
			log.info("Received repeated offer for slot [{}]. Ignoring.", allocationID);

			// return true here so that the sender will get a positive acknowledgement to the retry
			// and mark the offering as a success
			return true;
		} else {
			// the allocation has been fulfilled by another slot, reject the offer so the task executor
			// will offer the slot to the resource manager
			return false;
		}
	}

	final AllocatedSlot allocatedSlot = new AllocatedSlot(
		allocationID,
		taskManagerLocation,
		slotOffer.getSlotIndex(),
		slotOffer.getResourceProfile(),
		taskManagerGateway);

	// check whether we have request waiting for this slot
	PendingRequest pendingRequest = pendingRequests.removeKeyB(allocationID);
	if (pendingRequest != null) {
		// we were waiting for this!
		allocatedSlots.add(pendingRequest.getSlotRequestId(), allocatedSlot);

		if (!pendingRequest.getAllocatedSlotFuture().complete(allocatedSlot)) {
			// we could not complete the pending slot future --> try to fulfill another pending request
			allocatedSlots.remove(pendingRequest.getSlotRequestId());
			tryFulfillSlotRequestOrMakeAvailable(allocatedSlot);
		} else {
			log.debug("Fulfilled slot request [{}] with allocated slot [{}].", pendingRequest.getSlotRequestId(), allocationID);
		}
	}
	else {
		// we were actually not waiting for this:
		//   - could be that this request had been fulfilled
		//   - we are receiving the slots from TaskManagers after becoming leaders
		tryFulfillSlotRequestOrMakeAvailable(allocatedSlot);
	}

	// we accepted the request in any case. slot will be released after it idled for
	// too long and timed out
	return true;
}
 

/**
 * Slot offering by TaskExecutor with AllocationID. The AllocationID is originally generated by this pool and
 * transfer through the ResourceManager to TaskManager. We use it to distinguish the different allocation
 * we issued. Slot offering may be rejected if we find something mismatching or there is actually no pending
 * request waiting for this slot (maybe fulfilled by some other returned slot).
 *
 * @param taskManagerLocation location from where the offer comes from
 * @param taskManagerGateway TaskManager gateway
 * @param slotOffer the offered slot
 * @return True if we accept the offering
 */
boolean offerSlot(
		final TaskManagerLocation taskManagerLocation,
		final TaskManagerGateway taskManagerGateway,
		final SlotOffer slotOffer) {

	componentMainThreadExecutor.assertRunningInMainThread();

	// check if this TaskManager is valid
	final ResourceID resourceID = taskManagerLocation.getResourceID();
	final AllocationID allocationID = slotOffer.getAllocationId();

	if (!registeredTaskManagers.contains(resourceID)) {
		log.debug("Received outdated slot offering [{}] from unregistered TaskManager: {}",
				slotOffer.getAllocationId(), taskManagerLocation);
		return false;
	}

	// check whether we have already using this slot
	AllocatedSlot existingSlot;
	if ((existingSlot = allocatedSlots.get(allocationID)) != null ||
		(existingSlot = availableSlots.get(allocationID)) != null) {

		// we need to figure out if this is a repeated offer for the exact same slot,
		// or another offer that comes from a different TaskManager after the ResourceManager
		// re-tried the request

		// we write this in terms of comparing slot IDs, because the Slot IDs are the identifiers of
		// the actual slots on the TaskManagers
		// Note: The slotOffer should have the SlotID
		final SlotID existingSlotId = existingSlot.getSlotId();
		final SlotID newSlotId = new SlotID(taskManagerLocation.getResourceID(), slotOffer.getSlotIndex());

		if (existingSlotId.equals(newSlotId)) {
			log.info("Received repeated offer for slot [{}]. Ignoring.", allocationID);

			// return true here so that the sender will get a positive acknowledgement to the retry
			// and mark the offering as a success
			return true;
		} else {
			// the allocation has been fulfilled by another slot, reject the offer so the task executor
			// will offer the slot to the resource manager
			return false;
		}
	}

	final AllocatedSlot allocatedSlot = new AllocatedSlot(
		allocationID,
		taskManagerLocation,
		slotOffer.getSlotIndex(),
		slotOffer.getResourceProfile(),
		taskManagerGateway);

	// use the slot to fulfill pending request, in requested order
	tryFulfillSlotRequestOrMakeAvailable(allocatedSlot);

	// we accepted the request in any case. slot will be released after it idled for
	// too long and timed out
	return true;
}
 

/**
 * Tests that pending slot requests are rejected if a slot report with a different allocation
 * is received.
 */
@Test
public void testSlotReportWhileActiveSlotRequest() throws Exception {
	final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
	final ResourceActions resourceManagerActions = new TestingResourceActionsBuilder().build();

	final JobID jobId = new JobID();
	final AllocationID allocationId = new AllocationID();
	final ResourceProfile resourceProfile = ResourceProfile.fromResources(42.0, 1337);
	final SlotRequest slotRequest = new SlotRequest(jobId, allocationId, resourceProfile, "foobar");
	final CompletableFuture<Acknowledge> slotRequestFuture1 = new CompletableFuture<>();

	final Iterator<CompletableFuture<Acknowledge>> slotRequestFutureIterator = Arrays.asList(
		slotRequestFuture1,
		CompletableFuture.completedFuture(Acknowledge.get())).iterator();
	final ArrayBlockingQueue<SlotID> slotIds = new ArrayBlockingQueue<>(2);

	final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder()
		.setRequestSlotFunction(FunctionUtils.uncheckedFunction(
			requestSlotParameters -> {
				slotIds.put(requestSlotParameters.f0);
				return slotRequestFutureIterator.next();
			}))
		.createTestingTaskExecutorGateway();

	final ResourceID resourceId = ResourceID.generate();
	final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(resourceId, taskExecutorGateway);

	final SlotID slotId1 = new SlotID(resourceId, 0);
	final SlotID slotId2 = new SlotID(resourceId, 1);
	final SlotStatus slotStatus1 = new SlotStatus(slotId1, resourceProfile);
	final SlotStatus slotStatus2 = new SlotStatus(slotId2, resourceProfile);
	final SlotReport slotReport = new SlotReport(Arrays.asList(slotStatus1, slotStatus2));

	final ScheduledExecutor mainThreadExecutor = TestingUtils.defaultScheduledExecutor();

	final SlotManagerImpl slotManager = createSlotManagerBuilder()
		.setScheduledExecutor(mainThreadExecutor)
		.build();

	try {

		slotManager.start(resourceManagerId, mainThreadExecutor, resourceManagerActions);

		CompletableFuture<Void> registrationFuture = CompletableFuture.supplyAsync(
			() -> {
				slotManager.registerTaskManager(taskManagerConnection, slotReport);

				return null;
			},
			mainThreadExecutor)
		.thenAccept(
			(Object value) -> {
				try {
					slotManager.registerSlotRequest(slotRequest);
				} catch (ResourceManagerException e) {
					throw new RuntimeException("Could not register slots.", e);
				}
			});

		// check that no exception has been thrown
		registrationFuture.get();

		final SlotID requestedSlotId = slotIds.take();
		final SlotID freeSlotId = requestedSlotId.equals(slotId1) ? slotId2 : slotId1;

		final SlotStatus newSlotStatus1 = new SlotStatus(requestedSlotId, resourceProfile, new JobID(), new AllocationID());
		final SlotStatus newSlotStatus2 = new SlotStatus(freeSlotId, resourceProfile);
		final SlotReport newSlotReport = new SlotReport(Arrays.asList(newSlotStatus1, newSlotStatus2));

		CompletableFuture<Boolean> reportSlotStatusFuture = CompletableFuture.supplyAsync(
			// this should update the slot with the pending slot request triggering the reassignment of it
			() -> slotManager.reportSlotStatus(taskManagerConnection.getInstanceID(), newSlotReport),
			mainThreadExecutor);

		assertTrue(reportSlotStatusFuture.get());

		final SlotID requestedSlotId2 = slotIds.take();

		assertEquals(freeSlotId, requestedSlotId2);
	} finally {
		CompletableFuture.runAsync(
			ThrowingRunnable.unchecked(slotManager::close),
			mainThreadExecutor);
	}
}