org.apache.flink.runtime.jobmaster.SlotInfo Java Examples

private Optional<SlotAndLocality> tryAllocateFromAvailable(
	@Nonnull SlotRequestId slotRequestId,
	@Nonnull SlotProfile slotProfile) {

	Collection<SlotInfo> slotInfoList = slotPool.getAvailableSlotsInformation();

	Optional<SlotSelectionStrategy.SlotInfoAndLocality> selectedAvailableSlot =
		slotSelectionStrategy.selectBestSlotForProfile(slotInfoList, slotProfile);

	return selectedAvailableSlot.flatMap(slotInfoAndLocality -> {
		Optional<PhysicalSlot> optionalAllocatedSlot = slotPool.allocateAvailableSlot(
			slotRequestId,
			slotInfoAndLocality.getSlotInfo().getAllocationId());

		return optionalAllocatedSlot.map(
			allocatedSlot -> new SlotAndLocality(allocatedSlot, slotInfoAndLocality.getLocality()));
	});
}
 

private static boolean areRequestsFulfillableWithSlots(
		final Collection<ResourceProfile> requestResourceProfiles,
		final Set<SlotInfo> slots) {

	final Set<SlotInfo> remainingSlots = new HashSet<>(slots);
	for (ResourceProfile requestResourceProfile : requestResourceProfiles) {
		final Optional<SlotInfo> matchedSlot = findMatchingSlotForRequest(requestResourceProfile, remainingSlots);
		if (matchedSlot.isPresent()) {
			remainingSlots.remove(matchedSlot.get());
		} else {
			return false;
		}
	}
	return true;
}
 

private static Set<SlotInfo> getReusableSlots(
		final Supplier<Set<SlotInfo>> slotsRetriever,
		final Set<AllocationID> slotsToExclude) {

	return slotsRetriever.get().stream()
		.filter(slotInfo -> !slotInfo.willBeOccupiedIndefinitely())
		.filter(slotInfo -> !slotsToExclude.contains(slotInfo.getAllocationId()))
		.collect(Collectors.toSet());
}
 

/**
 * Returns whether the given bulk of slot requests are possible to be fulfilled at the same time
 * with all the reusable slots in the slot pool. A reusable slot means the slot is available or
 * will not be occupied indefinitely.
 *
 * @param slotRequestBulk bulk of slot requests to check
 * @param slotsRetriever supplies slots to be used for the fulfill-ability check
 * @return true if the slot requests are possible to be fulfilled, otherwise false
 */
@VisibleForTesting
static boolean isSlotRequestBulkFulfillable(
		final PhysicalSlotRequestBulk slotRequestBulk,
		final Supplier<Set<SlotInfo>> slotsRetriever) {

	final Set<AllocationID> assignedSlots = new HashSet<>(slotRequestBulk.getFulfilledRequests().values());
	final Set<SlotInfo> reusableSlots = getReusableSlots(slotsRetriever, assignedSlots);
	return areRequestsFulfillableWithSlots(slotRequestBulk.getPendingRequests().values(), reusableSlots);
}
 

private Set<SlotInfo> getAllSlotInfos() {
	return Stream
		.concat(
			slotPool.getAvailableSlotsInformation().stream(),
			slotPool.getAllocatedSlotsInformation().stream())
		.collect(Collectors.toSet());
}
 

@Nonnull
public Collection<SlotSelectionStrategy.SlotInfoAndResources> listResolvedRootSlotInfo(@Nullable AbstractID groupId) {
	return resolvedRootSlots
		.values()
		.stream()
			.flatMap((Map<AllocationID, MultiTaskSlot> map) -> map.values().stream())
			.filter(validMultiTaskSlotAndDoesNotContain(groupId))
			.map((MultiTaskSlot multiTaskSlot) -> {
				SlotInfo slotInfo = multiTaskSlot.getSlotContextFuture().join();
				return new SlotSelectionStrategy.SlotInfoAndResources(
						slotInfo,
						slotInfo.getResourceProfile().subtract(multiTaskSlot.getReservedResources()));
			}).collect(Collectors.toList());
}
 

@Nonnull
List<SlotInfo> listSlotInfo() {
	return availableSlots
		.values()
		.stream()
		.map(SlotAndTimestamp::slot)
		.collect(Collectors.toList());
}
 

private Set<ResourceProfile> getAllocatedResourceProfiles() {
	return Stream
		.concat(
			getAvailableSlotsInformation().stream(),
			getAllocatedSlotsInformation().stream())
		.map(SlotInfo::getResourceProfile)
		.collect(Collectors.toSet());
}
 

private Set<ResourceProfile> getAllocatedResourceProfiles() {
	return Stream
		.concat(
			getAvailableSlotsInformation().stream(),
			getAllocatedSlotsInformation().stream())
		.map(SlotInfo::getResourceProfile)
		.collect(Collectors.toSet());
}
 

@Nonnull
List<SlotInfo> listSlotInfo() {
	return availableSlots
		.values()
		.stream()
		.map(SlotAndTimestamp::slot)
		.collect(Collectors.toList());
}
 

@Nonnull
public Collection<SlotSelectionStrategy.SlotInfoAndResources> listResolvedRootSlotInfo(@Nullable AbstractID groupId) {
	return resolvedRootSlots
		.values()
		.stream()
			.flatMap((Map<AllocationID, MultiTaskSlot> map) -> createValidMultiTaskSlotInfos(map, groupId))
			.map((MultiTaskSlotInfo multiTaskSlotInfo) -> {
				SlotInfo slotInfo = multiTaskSlotInfo.getSlotInfo();
				return new SlotSelectionStrategy.SlotInfoAndResources(
					slotInfo,
					slotInfo.getResourceProfile().subtract(multiTaskSlotInfo.getReservedResources()),
					multiTaskSlotInfo.getTaskExecutorUtilization());
			}).collect(Collectors.toList());
}
 

private void tryMarkSlotAsResolved(SlotRequestId slotRequestId, SlotInfo slotInfo) {
	final MultiTaskSlot resolvedRootNode = unresolvedRootSlots.remove(slotRequestId);

	if (resolvedRootNode != null) {
		final AllocationID allocationId = slotInfo.getAllocationId();
		LOG.trace("Fulfill multi task slot [{}] with slot [{}].", slotRequestId, allocationId);

		final Map<AllocationID, MultiTaskSlot> innerMap = resolvedRootSlots.computeIfAbsent(
			slotInfo.getTaskManagerLocation(),
			taskManagerLocation -> new HashMap<>());

		MultiTaskSlot previousValue = innerMap.put(allocationId, resolvedRootNode);
		Preconditions.checkState(previousValue == null);
	}
}
 

@Test
public void returnsHostLocalMatchingIfExactTMLocationCannotBeFulfilled() {

	SlotProfile slotProfile = SlotProfile.preferredLocality(resourceProfile, Collections.singletonList(tmlX));
	Optional<SlotSelectionStrategy.SlotInfoAndLocality> match = runMatching(slotProfile);

	Assert.assertTrue(match.isPresent());
	final SlotSelectionStrategy.SlotInfoAndLocality slotInfoAndLocality = match.get();
	final SlotInfo slotInfo = slotInfoAndLocality.getSlotInfo();
	assertThat(candidates, hasItem(withSlotInfo(slotInfo)));
	assertThat(slotInfoAndLocality, hasLocality(Locality.HOST_LOCAL));
}
 

@Nonnull
List<SlotInfo> listSlotInfo() {
	return availableSlots
		.values()
		.stream()
		.map(SlotAndTimestamp::slot)
		.collect(Collectors.toList());
}
 

@Override
protected SlotInfo featureValueOf(SlotSelectionStrategy.SlotInfoAndResources actual) {
	return actual.getSlotInfo();
}
 

SlotInfoFeatureMatcher(SlotInfo slotInfo) {
	super(is(slotInfo), "Slot info of a SlotInfoAndResources instance", "slotInfo");
}
 

private Matcher<SlotSelectionStrategy.SlotInfoAndResources> withSlotInfo(SlotInfo slotInfo) {
	return new SlotInfoFeatureMatcher(slotInfo);
}
 

private SlotInfoWithUtilization(SlotInfo slotInfo, double taskExecutorUtilization) {
	this.slotInfoDelegate = slotInfo;
	this.taskExecutorUtilization = taskExecutorUtilization;
}
 

private SlotInfo getSlotInfo() {
	return slotInfo;
}
 

private MultiTaskSlotInfo(SlotInfo slotInfo, ResourceProfile reservedResources, double taskExecutorUtilization) {
	this.slotInfo = slotInfo;
	this.reservedResources = reservedResources;
	this.taskExecutorUtilization = taskExecutorUtilization;
}
 

public static SlotInfoWithUtilization from(SlotInfo slotInfo, double taskExecutorUtilization) {
	return new SlotInfoWithUtilization(slotInfo, taskExecutorUtilization);
}
 

public SlotInfoAndResources(@Nonnull SlotInfo slotInfo, @Nonnull ResourceProfile remainingResources, double taskExecutorUtilization) {
	this.slotInfo = slotInfo;
	this.remainingResources = remainingResources;
	this.taskExecutorUtilization = taskExecutorUtilization;
}
 

@Nonnull
public SlotInfo getSlotInfo() {
	return slotInfo;
}
 

private SlotInfoAndLocality(@Nonnull SlotInfo slotInfo, @Nonnull Locality locality) {
	this.slotInfo = slotInfo;
	this.locality = locality;
}
 

@Nonnull
public SlotInfo getSlotInfo() {
	return slotInfo;
}
 

public static SlotInfoAndLocality of(@Nonnull SlotInfo slotInfo, @Nonnull Locality locality) {
	return new SlotInfoAndLocality(slotInfo, locality);
}
 

@Override
public Collection<SlotInfo> getAllocatedSlotsInformation() {
	return allocatedSlots.listSlotInfo();
}
 

@Nullable
public MultiTaskSlot getResolvedRootSlot(@Nonnull SlotInfo slotInfo) {
	Map<AllocationID, MultiTaskSlot> forLocationEntry = resolvedRootSlots.get(slotInfo.getTaskManagerLocation());
	return forLocationEntry != null ? forLocationEntry.get(slotInfo.getAllocationId()) : null;
}
 

Collection<SlotInfo> listSlotInfo() {
	return new ArrayList<>(allocatedSlotsById.values());
}
 

PhysicalSlotRequestBulkChecker(final Supplier<Set<SlotInfo>> slotsRetriever, final Clock clock) {
	this.slotsRetriever = checkNotNull(slotsRetriever);
	this.clock = checkNotNull(clock);
}