org.apache.flink.runtime.instance.SlotSharingGroupId Java Examples

TestingLogicalSlot(
		TaskManagerLocation taskManagerLocation,
		TaskManagerGateway taskManagerGateway,
		int slotNumber,
		AllocationID allocationId,
		SlotRequestId slotRequestId,
		SlotSharingGroupId slotSharingGroupId,
		boolean automaticallyCompleteReleaseFuture,
		SlotOwner slotOwner) {

	this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
	this.taskManagerGateway = Preconditions.checkNotNull(taskManagerGateway);
	this.payloadReference = new AtomicReference<>();
	this.slotNumber = slotNumber;
	this.allocationId = Preconditions.checkNotNull(allocationId);
	this.slotRequestId = Preconditions.checkNotNull(slotRequestId);
	this.slotSharingGroupId = Preconditions.checkNotNull(slotSharingGroupId);
	this.releaseFuture = new CompletableFuture<>();
	this.automaticallyCompleteReleaseFuture = automaticallyCompleteReleaseFuture;
	this.slotOwner = slotOwner;
}
 

public SingleLogicalSlot(
		SlotRequestId slotRequestId,
		SlotContext slotContext,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		Locality locality,
		SlotOwner slotOwner) {
	this.slotRequestId = Preconditions.checkNotNull(slotRequestId);
	this.slotContext = Preconditions.checkNotNull(slotContext);
	this.slotSharingGroupId = slotSharingGroupId;
	this.locality = Preconditions.checkNotNull(locality);
	this.slotOwner = Preconditions.checkNotNull(slotOwner);
	this.releaseFuture = new CompletableFuture<>();

	this.state = State.ALIVE;
	this.payload = null;
}
 

private void releaseSharedSlot(
	@Nonnull SlotRequestId slotRequestId,
	@Nonnull SlotSharingGroupId slotSharingGroupId,
	Throwable cause) {

	final SlotSharingManager multiTaskSlotManager = slotSharingManagers.get(slotSharingGroupId);

	if (multiTaskSlotManager != null) {
		final SlotSharingManager.TaskSlot taskSlot = multiTaskSlotManager.getTaskSlot(slotRequestId);

		if (taskSlot != null) {
			taskSlot.release(cause);
		} else {
			log.debug("Could not find slot [{}] in slot sharing group {}. Ignoring release slot request.", slotRequestId, slotSharingGroupId);
		}
	} else {
		log.debug("Could not find slot sharing group {}. Ignoring release slot request.", slotSharingGroupId);
	}
}
 

private void releaseSharedSlot(
	@Nonnull SlotRequestId slotRequestId,
	@Nonnull SlotSharingGroupId slotSharingGroupId,
	Throwable cause) {

	final SlotSharingManager multiTaskSlotManager = slotSharingManagers.get(slotSharingGroupId);

	if (multiTaskSlotManager != null) {
		final SlotSharingManager.TaskSlot taskSlot = multiTaskSlotManager.getTaskSlot(slotRequestId);

		if (taskSlot != null) {
			taskSlot.release(cause);
		} else {
			log.debug("Could not find slot [{}] in slot sharing group {}. Ignoring release slot request.", slotRequestId, slotSharingGroupId);
		}
	} else {
		log.debug("Could not find slot sharing group {}. Ignoring release slot request.", slotSharingGroupId);
	}
}
 

public SingleLogicalSlot(
		SlotRequestId slotRequestId,
		SlotContext slotContext,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		Locality locality,
		SlotOwner slotOwner) {
	this.slotRequestId = Preconditions.checkNotNull(slotRequestId);
	this.slotContext = Preconditions.checkNotNull(slotContext);
	this.slotSharingGroupId = slotSharingGroupId;
	this.locality = Preconditions.checkNotNull(locality);
	this.slotOwner = Preconditions.checkNotNull(slotOwner);
	this.releaseFuture = new CompletableFuture<>();

	this.state = State.ALIVE;
	this.payload = null;
}
 

public int getNumberOfAvailableSlotsForGroup(SlotSharingGroupId slotSharingGroupId, JobVertexID jobVertexId) {
	final SlotSharingManager multiTaskSlotManager = slotSharingManagersMap.get(slotSharingGroupId);

	if (multiTaskSlotManager != null) {
		int availableSlots = 0;

		for (SlotSharingManager.MultiTaskSlot multiTaskSlot : multiTaskSlotManager.getResolvedRootSlots()) {
			if (!multiTaskSlot.contains(jobVertexId)) {
				availableSlots++;
			}
		}

		return availableSlots;
	} else {
		throw new FlinkRuntimeException("No MultiTaskSlotmanager registered under " + slotSharingGroupId + '.');
	}
}
 

private void releaseSharedSlot(
	@Nonnull SlotRequestId slotRequestId,
	@Nonnull SlotSharingGroupId slotSharingGroupId,
	Throwable cause) {

	final SlotSharingManager multiTaskSlotManager = slotSharingManagers.get(slotSharingGroupId);

	if (multiTaskSlotManager != null) {
		final SlotSharingManager.TaskSlot taskSlot = multiTaskSlotManager.getTaskSlot(slotRequestId);

		if (taskSlot != null) {
			taskSlot.release(cause);
		} else {
			log.debug("Could not find slot [{}] in slot sharing group {}. Ignoring release slot request.", slotRequestId, slotSharingGroupId);
		}
	} else {
		log.debug("Could not find slot sharing group {}. Ignoring release slot request.", slotSharingGroupId);
	}
}
 

/**
 * Tests that returned slot futures are failed if the allocation request is failed.
 */
@Test
public void testFailingQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			new SlotSharingGroupId(),
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	final AllocationID allocationId = allocationIdFuture.get();

	// this should fail the returned logical slot future
	final SlotPool slotPoolGateway = slotPoolResource.getSlotPool();
	slotPoolGateway.failAllocation(allocationId, new FlinkException("Testing Exception"));

	try {
		logicalSlotFuture.get();
		fail("The slot future should have failed.");
	} catch (ExecutionException ee) {
		assertTrue(ExceptionUtils.findThrowable(ee, FlinkException.class).isPresent());
	}
}
 

@Override
public void cancelSlotRequest(
		SlotRequestId slotRequestId,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		Throwable cause) {
	canceledSlotRequests.add(slotRequestId);
}
 

@Override
public void cancelSlotRequest(
	SlotRequestId slotRequestId,
	@Nullable SlotSharingGroupId slotSharingGroupId,
	Throwable cause) {

	componentMainThreadExecutor.assertRunningInMainThread();

	if (slotSharingGroupId != null) {
		releaseSharedSlot(slotRequestId, slotSharingGroupId, cause);
	} else {
		slotPool.releaseSlot(slotRequestId, cause);
	}
}
 

public int getNumberOfSharedSlots(SlotSharingGroupId slotSharingGroupId) {
	final SlotSharingManager multiTaskSlotManager = slotSharingManagersMap.get(slotSharingGroupId);

	if (multiTaskSlotManager != null) {
		return multiTaskSlotManager.getResolvedRootSlots().size();
	} else {
		throw new FlinkRuntimeException("No MultiTaskSlotManager registered under " + slotSharingGroupId + '.');
	}
}
 

public TestingScheduler(
	@Nonnull Map<SlotSharingGroupId, SlotSharingManager> slotSharingManagersMap,
	@Nonnull SlotSelectionStrategy slotSelectionStrategy,
	@Nonnull SlotPool slotPoolGateway) {

	super(slotSelectionStrategy, slotPoolGateway, slotSharingManagersMap);
	this.slotSharingManagersMap = slotSharingManagersMap;
}
 

@Override
public void cancelSlotRequest(SlotRequestId slotRequestId, @Nullable SlotSharingGroupId slotSharingGroupId, Throwable cause) {
	synchronized (lock) {
		final SlotContext slotContext = allocatedSlots.remove(slotRequestId);

		if (slotContext != null) {
			slots.add(slotContext);
		} else {
			throw new FlinkRuntimeException("Unknown slot request id " + slotRequestId + '.');
		}
	}
}
 

@Override
public void cancelSlotRequest(SlotRequestId slotRequestId, @Nullable SlotSharingGroupId slotSharingGroupId, Throwable cause) {
	final CompletableFuture<LogicalSlot> slotFuture = slotFutures.remove(slotRequestId);
	slotFuture.cancel(false);

	slotCanceller.accept(slotRequestId);
}
 

@Test
public void testSingleQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	final SlotPoolImpl slotPool = slotPoolResource.getSlotPool();
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();
	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			slotSharingGroupId,
			null),
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture.isDone());

	final AllocationID allocationId = allocationIdFuture.get();

	boolean booleanCompletableFuture = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId,
			0,
			ResourceProfile.ANY));

	assertTrue(booleanCompletableFuture);

	final LogicalSlot logicalSlot = logicalSlotFuture.get();

	assertEquals(slotSharingGroupId, logicalSlot.getSlotSharingGroupId());
}
 

public ScheduledUnit(
		JobVertexID jobVertexId,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		@Nullable CoLocationConstraint coLocationConstraint) {
	this(
		null,
		jobVertexId,
		slotSharingGroupId,
		coLocationConstraint);
}
 

private ExecutionVertexSchedulingRequirements(
		ExecutionVertexID executionVertexId,
		@Nullable AllocationID previousAllocationId,
		ResourceProfile taskResourceProfile,
		ResourceProfile physicalSlotResourceProfile,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		@Nullable CoLocationConstraint coLocationConstraint) {
	this.executionVertexId = checkNotNull(executionVertexId);
	this.previousAllocationId = previousAllocationId;
	this.taskResourceProfile = checkNotNull(taskResourceProfile);
	this.physicalSlotResourceProfile = checkNotNull(physicalSlotResourceProfile);
	this.slotSharingGroupId = slotSharingGroupId;
	this.coLocationConstraint = coLocationConstraint;
}
 

@Override
public void cancelSlotRequest(
		final SlotRequestId slotRequestId,
		@Nullable final SlotSharingGroupId slotSharingGroupId,
		final Throwable cause) {
	cancelledSlotRequestIds.add(slotRequestId);
}
 

@Override
public void returnLogicalSlot(LogicalSlot logicalSlot) {
	SlotRequestId slotRequestId = logicalSlot.getSlotRequestId();
	SlotSharingGroupId slotSharingGroupId = logicalSlot.getSlotSharingGroupId();
	FlinkException cause = new FlinkException("Slot is being returned to the SlotPool.");
	cancelSlotRequest(slotRequestId, slotSharingGroupId, cause);
}
 

@Test
public void allocateSharedSlot_withNoRequirements_selectsSlotsSoThatWorkloadIsSpreadOut() {
	final int numberSlotsPerTaskExecutor = 2;
	final int numberTaskExecutors = 2;
	final int numberSlots = numberTaskExecutors * numberSlotsPerTaskExecutor;

	registerTaskExecutors(numberTaskExecutors, numberSlotsPerTaskExecutor);

	final JobVertexID sourceJobVertexId = new JobVertexID();
	final JobVertexID sinkJobVertexId = new JobVertexID();
	final SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();

	final List<ScheduledUnit> sourceScheduledUnits = IntStream.range(0, numberSlots)
		.mapToObj(ignored -> createSharedSlotRequest(sourceJobVertexId, slotSharingGroupId))
		.collect(Collectors.toList());

	final List<ScheduledUnit> sinkScheduledUnits = IntStream.range(0, numberTaskExecutors)
		.mapToObj(ignored -> createSharedSlotRequest(sinkJobVertexId, slotSharingGroupId))
		.collect(Collectors.toList());

	sourceScheduledUnits.forEach(this::allocateSlot);
	final Set<TaskManagerLocation> sinkLocations = sinkScheduledUnits.stream()
		.map(this::allocateSlot)
		.map(this::getTaskManagerLocation)
		.collect(Collectors.toSet());

	// verify that the sinks have been evenly spread across the available TaskExecutors
	assertThat(sinkLocations, hasSize(numberTaskExecutors));
}
 

@Override
public void cancelSlotRequest(
	SlotRequestId slotRequestId,
	@Nullable SlotSharingGroupId slotSharingGroupId,
	Throwable cause) {

	componentMainThreadExecutor.assertRunningInMainThread();

	if (slotSharingGroupId != null) {
		releaseSharedSlot(slotRequestId, slotSharingGroupId, cause);
	} else {
		slotPool.releaseSlot(slotRequestId, cause);
	}
}
 

@Override
public void cancelSlotRequest(
		final SlotRequestId slotRequestId,
		@Nullable final SlotSharingGroupId slotSharingGroupId,
		final Throwable cause) {
	cancelledSlotRequestIds.add(slotRequestId);
}
 

SlotSharingManager(
		SlotSharingGroupId slotSharingGroupId,
		AllocatedSlotActions allocatedSlotActions,
		SlotOwner slotOwner) {
	this.slotSharingGroupId = Preconditions.checkNotNull(slotSharingGroupId);
	this.allocatedSlotActions = Preconditions.checkNotNull(allocatedSlotActions);
	this.slotOwner = Preconditions.checkNotNull(slotOwner);

	allTaskSlots = new HashMap<>(16);
	unresolvedRootSlots = new HashMap<>(16);
	resolvedRootSlots = new HashMap<>(16);
}
 

private ExecutionVertexSchedulingRequirements(
		ExecutionVertexID executionVertexId,
		@Nullable AllocationID previousAllocationId,
		ResourceProfile resourceProfile,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		@Nullable CoLocationConstraint coLocationConstraint,
		Collection<TaskManagerLocation> preferredLocations) {
	this.executionVertexId = checkNotNull(executionVertexId);
	this.previousAllocationId = previousAllocationId;
	this.resourceProfile = checkNotNull(resourceProfile);
	this.slotSharingGroupId = slotSharingGroupId;
	this.coLocationConstraint = coLocationConstraint;
	this.preferredLocations = checkNotNull(preferredLocations);
}
 

@Override
public void cancelSlotRequest(
	SlotRequestId slotRequestId,
	@Nullable SlotSharingGroupId slotSharingGroupId,
	Throwable cause) {

	componentMainThreadExecutor.assertRunningInMainThread();

	if (slotSharingGroupId != null) {
		releaseSharedSlot(slotRequestId, slotSharingGroupId, cause);
	} else {
		slotPool.releaseSlot(slotRequestId, cause);
	}
}
 

public ScheduledUnit(Execution task, @Nullable SlotSharingGroupId slotSharingGroupId) {
	this(
		Preconditions.checkNotNull(task),
		task.getVertex().getJobvertexId(),
		slotSharingGroupId,
		null);
}
 

public ScheduledUnit(
		Execution task,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		@Nullable CoLocationConstraint coLocationConstraint) {
	this(
		Preconditions.checkNotNull(task),
		task.getVertex().getJobvertexId(),
		slotSharingGroupId,
		coLocationConstraint);
}
 

public ScheduledUnit(
		JobVertexID jobVertexId,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		@Nullable CoLocationConstraint coLocationConstraint) {
	this(
		null,
		jobVertexId,
		slotSharingGroupId,
		coLocationConstraint);
}
 

public ScheduledUnit(
	@Nullable Execution task,
	JobVertexID jobVertexId,
	@Nullable SlotSharingGroupId slotSharingGroupId,
	@Nullable CoLocationConstraint coLocationConstraint) {

	this.vertexExecution = task;
	this.jobVertexId = Preconditions.checkNotNull(jobVertexId);
	this.slotSharingGroupId = slotSharingGroupId;
	this.coLocationConstraint = coLocationConstraint;

}
 

public TestingScheduler(
	@Nonnull Map<SlotSharingGroupId, SlotSharingManager> slotSharingManagersMap,
	@Nonnull SlotSelectionStrategy slotSelectionStrategy,
	@Nonnull SlotPool slotPoolGateway) {

	super(slotSelectionStrategy, slotPoolGateway, slotSharingManagersMap);
	this.slotSharingManagersMap = slotSharingManagersMap;
}