org.apache.flink.runtime.jobmaster.SlotRequestId Java Examples

The following examples show how to use org.apache.flink.runtime.jobmaster.SlotRequestId. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SimpleSlotProvider.java    From Flink-CEPplus with Apache License 2.0 7 votes vote down vote up
@Override
public CompletableFuture<LogicalSlot> allocateSlot(
		SlotRequestId slotRequestId,
		ScheduledUnit task,
		SlotProfile slotProfile,
		boolean allowQueued,
		Time allocationTimeout) {
	final SlotContext slot;

	synchronized (lock) {
		if (slots.isEmpty()) {
			slot = null;
		} else {
			slot = slots.removeFirst();
		}
		if (slot != null) {
			SimpleSlot result = new SimpleSlot(slot, this, 0);
			allocatedSlots.put(slotRequestId, slot);
			return CompletableFuture.completedFuture(result);
		}
		else {
			return FutureUtils.completedExceptionally(new NoResourceAvailableException());
		}
	}
}
 
Example #2
Source File: SchedulerImpl.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private LogicalSlot completeAllocationByAssigningPayload(
	@Nonnull SlotRequestId slotRequestId,
	@Nonnull SlotAndLocality slotAndLocality) throws FlinkException {

	final PhysicalSlot allocatedSlot = slotAndLocality.getSlot();

	final SingleLogicalSlot singleTaskSlot = new SingleLogicalSlot(
		slotRequestId,
		allocatedSlot,
		null,
		slotAndLocality.getLocality(),
		this);

	if (allocatedSlot.tryAssignPayload(singleTaskSlot)) {
		return singleTaskSlot;
	} else {
		final FlinkException flinkException =
			new FlinkException("Could not assign payload to allocated slot " + allocatedSlot.getAllocationId() + '.');
		slotPool.releaseSlot(slotRequestId, flinkException);
		throw flinkException;
	}
}
 
Example #3
Source File: ExecutionVertexLocalityTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private void initializeLocation(ExecutionVertex vertex, TaskManagerLocation location) throws Exception {
	// we need a bit of reflection magic to initialize the location without going through
	// scheduling paths. we choose to do that, rather than the alternatives:
	//  - mocking the scheduler created fragile tests that break whenever the scheduler is adjusted
	//  - exposing test methods in the ExecutionVertex leads to undesirable setters 

	SlotContext slotContext = new SimpleSlotContext(
		new AllocationID(),
		location,
		0,
		mock(TaskManagerGateway.class));



	LogicalSlot slot = new SingleLogicalSlot(
		new SlotRequestId(),
		slotContext,
		null,
		Locality.LOCAL,
		mock(SlotOwner.class));

	if (!vertex.getCurrentExecutionAttempt().tryAssignResource(slot)) {
		throw new FlinkException("Could not assign resource.");
	}
}
 
Example #4
Source File: SchedulerImpl.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private CompletableFuture<LogicalSlot> allocateSlotInternal(
	SlotRequestId slotRequestId,
	ScheduledUnit scheduledUnit,
	SlotProfile slotProfile,
	boolean allowQueuedScheduling,
	@Nullable Time allocationTimeout) {
	log.debug("Received slot request [{}] for task: {}", slotRequestId, scheduledUnit.getTaskToExecute());

	componentMainThreadExecutor.assertRunningInMainThread();

	final CompletableFuture<LogicalSlot> allocationResultFuture = new CompletableFuture<>();
	internalAllocateSlot(
			allocationResultFuture,
			slotRequestId,
			scheduledUnit,
			slotProfile,
			allowQueuedScheduling,
			allocationTimeout);
	return allocationResultFuture;
}
 
Example #5
Source File: SchedulerImpl.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private void releaseSharedSlot(
	@Nonnull SlotRequestId slotRequestId,
	@Nonnull SlotSharingGroupId slotSharingGroupId,
	Throwable cause) {

	final SlotSharingManager multiTaskSlotManager = slotSharingManagers.get(slotSharingGroupId);

	if (multiTaskSlotManager != null) {
		final SlotSharingManager.TaskSlot taskSlot = multiTaskSlotManager.getTaskSlot(slotRequestId);

		if (taskSlot != null) {
			taskSlot.release(cause);
		} else {
			log.debug("Could not find slot [{}] in slot sharing group {}. Ignoring release slot request.", slotRequestId, slotSharingGroupId);
		}
	} else {
		log.debug("Could not find slot sharing group {}. Ignoring release slot request.", slotSharingGroupId);
	}
}
 
Example #6
Source File: ProgrammedSlotProvider.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<LogicalSlot> allocateSlot(
		SlotRequestId slotRequestId,
		ScheduledUnit task,
		SlotProfile slotProfile,
		boolean allowQueued,
		Time allocationTimeout) {
	JobVertexID vertexId = task.getTaskToExecute().getVertex().getJobvertexId();
	int subtask = task.getTaskToExecute().getParallelSubtaskIndex();

	CompletableFuture<LogicalSlot>[] forTask = slotFutures.get(vertexId);
	if (forTask != null) {
		CompletableFuture<LogicalSlot> future = forTask[subtask];
		if (future != null) {
			slotFutureRequested.get(vertexId)[subtask].complete(true);
			slotRequests.add(slotRequestId);

			return future;
		}
	}

	throw new IllegalArgumentException("No registered slot future for task " + vertexId + " (" + subtask + ')');
}
 
Example #7
Source File: SlotSharingManager.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Allocates a {@link SingleTaskSlot} and registers it under the given groupId at
 * this MultiTaskSlot.
 *
 * @param slotRequestId of the new single task slot
 * @param groupId under which the new single task slot is registered
 * @param locality of the allocation
 * @return the newly allocated {@link SingleTaskSlot}
 */
SingleTaskSlot allocateSingleTaskSlot(
		SlotRequestId slotRequestId,
		AbstractID groupId,
		Locality locality) {
	Preconditions.checkState(!super.contains(groupId));

	LOG.debug("Create single task slot [{}] in multi task slot [{}] for group {}.", slotRequestId, getSlotRequestId(), groupId);

	final SingleTaskSlot leaf = new SingleTaskSlot(
		slotRequestId,
		groupId,
		this,
		locality);

	children.put(groupId, leaf);

	// register the newly allocated slot also at the SlotSharingManager
	allTaskSlots.put(slotRequestId, leaf);

	return leaf;
}
 
Example #8
Source File: SlotSharingManager.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private SingleTaskSlot(
		SlotRequestId slotRequestId,
		AbstractID groupId,
		MultiTaskSlot parent,
		Locality locality) {
	super(slotRequestId, groupId);

	this.parent = Preconditions.checkNotNull(parent);

	Preconditions.checkNotNull(locality);
	singleLogicalSlotFuture = parent.getSlotContextFuture()
		.thenApply(
			(SlotContext slotContext) -> {
				LOG.trace("Fulfill single task slot [{}] with slot [{}].", slotRequestId, slotContext.getAllocationId());
				return new SingleLogicalSlot(
					slotRequestId,
					slotContext,
					slotSharingGroupId,
					locality,
					slotOwner);
			});
}
 
Example #9
Source File: ExecutionGraphSchedulingTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Nonnull
static SingleLogicalSlot createSingleLogicalSlot(SlotOwner slotOwner, TaskManagerGateway taskManagerGateway, SlotRequestId slotRequestId) {
	TaskManagerLocation location = new TaskManagerLocation(
		ResourceID.generate(), InetAddress.getLoopbackAddress(), 12345);

	SimpleSlotContext slotContext = new SimpleSlotContext(
		new AllocationID(),
		location,
		0,
		taskManagerGateway);

	return new SingleLogicalSlot(
		slotRequestId,
		slotContext,
		null,
		Locality.LOCAL,
		slotOwner);
}
 
Example #10
Source File: SingleLogicalSlot.java    From flink with Apache License 2.0 6 votes vote down vote up
public SingleLogicalSlot(
		SlotRequestId slotRequestId,
		SlotContext slotContext,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		Locality locality,
		SlotOwner slotOwner) {
	this.slotRequestId = Preconditions.checkNotNull(slotRequestId);
	this.slotContext = Preconditions.checkNotNull(slotContext);
	this.slotSharingGroupId = slotSharingGroupId;
	this.locality = Preconditions.checkNotNull(locality);
	this.slotOwner = Preconditions.checkNotNull(slotOwner);
	this.releaseFuture = new CompletableFuture<>();

	this.state = State.ALIVE;
	this.payload = null;
}
 
Example #11
Source File: SlotPoolImpl.java    From flink with Apache License 2.0 6 votes vote down vote up
private void slotRequestToResourceManagerFailed(SlotRequestId slotRequestID, Throwable failure) {
	final PendingRequest request = pendingRequests.getKeyA(slotRequestID);
	if (request != null) {
		if (isBatchRequestAndFailureCanBeIgnored(request, failure)) {
			log.debug("Ignoring failed request to the resource manager for a batch slot request.");
		} else {
			pendingRequests.removeKeyA(slotRequestID);
			request.getAllocatedSlotFuture().completeExceptionally(new NoResourceAvailableException(
				"No pooled slot available and request to ResourceManager for new slot failed", failure));
		}
	} else {
		if (log.isDebugEnabled()) {
			log.debug("Unregistered slot request [{}] failed.", slotRequestID, failure);
		}
	}
}
 
Example #12
Source File: SlotPoolInteractionsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * This case make sure when allocateSlot in ProviderAndOwner timeout,
 * it will automatically call cancelSlotAllocation as will inject future.whenComplete in ProviderAndOwner.
 */
@Test
public void testProviderAndOwnerSlotAllocationTimeout() throws Exception {
	final JobID jid = new JobID();

	try (TestingSlotPool pool = createTestingSlotPool(jid)) {

		final CompletableFuture<SlotRequestId> releaseSlotFuture = new CompletableFuture<>();

		pool.setReleaseSlotConsumer(releaseSlotFuture::complete);

		pool.start(JobMasterId.generate(), "foobar", testMainThreadExecutor.getMainThreadExecutor());
		ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
		pool.connectToResourceManager(resourceManagerGateway);

		Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.INSTANCE, pool);
		scheduler.start(testMainThreadExecutor.getMainThreadExecutor());

		// test the pending request is clear when timed out
		CompletableFuture<LogicalSlot> future = testMainThreadExecutor.execute(() -> scheduler.allocateSlot(
			new DummyScheduledUnit(),
			true,
			SlotProfile.noRequirements(),
			fastTimeout));
		try {
			future.get();
			fail("We expected a TimeoutException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException);
		}

		// wait for the cancel call on the SlotPoolImpl
		releaseSlotFuture.get();

		assertEquals(0L, pool.getNumberOfPendingRequests());
	}
}
 
Example #13
Source File: SlotPoolInteractionsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a slot allocation times out wrt to the specified time out.
 */
@Test
public void testSlotAllocationTimeout() throws Exception {
	final JobID jid = new JobID();

	try (TestingSlotPool pool = createTestingSlotPool(jid)) {

		pool.start(JobMasterId.generate(), "foobar", testMainThreadExecutor.getMainThreadExecutor());

		final CompletableFuture<SlotRequestId> slotRequestTimeoutFuture = new CompletableFuture<>();
		pool.setTimeoutPendingSlotRequestConsumer(slotRequestTimeoutFuture::complete);

		ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
		pool.connectToResourceManager(resourceManagerGateway);

		Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.INSTANCE, pool);
		scheduler.start(testMainThreadExecutor.getMainThreadExecutor());

		SlotRequestId requestId = new SlotRequestId();
		CompletableFuture<LogicalSlot> future = testMainThreadExecutor.execute(() -> scheduler.allocateSlot(
			requestId,
			new DummyScheduledUnit(),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			true,
			fastTimeout));

		try {
			future.get();
			fail("We expected a TimeoutException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException);
		}

		// wait until we have timed out the slot request
		slotRequestTimeoutFuture.get();

		assertEquals(0L, pool.getNumberOfPendingRequests());
	}
}
 
Example #14
Source File: TestingAllocatedSlotActions.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void releaseSlot(@Nonnull SlotRequestId slotRequestId, @Nullable Throwable cause) {
	Consumer<Tuple2<SlotRequestId, Throwable>> currentReleaseSlotConsumer = this.releaseSlotConsumer;

	if (currentReleaseSlotConsumer != null) {
		currentReleaseSlotConsumer.accept(Tuple2.of(slotRequestId, cause));
	}
}
 
Example #15
Source File: SlotPoolImpl.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void releaseSlot(@Nonnull SlotRequestId slotRequestId, @Nullable Throwable cause) {

	componentMainThreadExecutor.assertRunningInMainThread();

	log.debug("Releasing slot [{}] because: {}", slotRequestId, cause != null ? cause.getMessage() : "null");
	releaseSingleSlot(slotRequestId, cause);
}
 
Example #16
Source File: SlotPoolInteractionsTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testCancelSlotAllocationWithoutResourceManager() throws Exception {
	final JobID jid = new JobID();

	try (TestingSlotPool pool = new TestingSlotPool(
		jid,
		SystemClock.getInstance(),
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime())) {

		final CompletableFuture<SlotRequestId> timeoutFuture = new CompletableFuture<>();
		pool.setTimeoutPendingSlotRequestConsumer(timeoutFuture::complete);
		pool.start(JobMasterId.generate(), "foobar", testMainThreadExecutor.getMainThreadExecutor());
		Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.INSTANCE, pool);
		scheduler.start(testMainThreadExecutor.getMainThreadExecutor());

		SlotRequestId requestId = new SlotRequestId();
		CompletableFuture<LogicalSlot> future = testMainThreadExecutor.execute(() -> scheduler.allocateSlot(
			requestId,
			new ScheduledUnit(SchedulerTestUtils.getDummyTask()),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			true,
			fastTimeout));

		try {
			future.get();
			fail("We expected a TimeoutException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException);
		}

		// wait for the timeout of the pending slot request
		timeoutFuture.get();

		assertEquals(0L, pool.getNumberOfWaitingForResourceRequests());
	}
}
 
Example #17
Source File: SlotPoolImplTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that failing an allocation fails the pending slot request.
 */
@Test
public void testFailingAllocationFailsPendingSlotRequests() throws Exception {

	try (SlotPoolImpl slotPool = new SlotPoolImpl(jobId)) {
		final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
		resourceManagerGateway.setRequestSlotConsumer(slotRequest -> allocationIdFuture.complete(slotRequest.getAllocationId()));

		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);

		final CompletableFuture<LogicalSlot> slotFuture = allocateSlot(scheduler, new SlotRequestId());

		final AllocationID allocationId = allocationIdFuture.get();

		assertThat(slotFuture.isDone(), is(false));

		final FlinkException cause = new FlinkException("Fail pending slot request failure.");
		final Optional<ResourceID> responseFuture = slotPool.failAllocation(allocationId, cause);

		assertThat(responseFuture.isPresent(), is(false));

		try {
			slotFuture.get();
			fail("Expected a slot allocation failure.");
		} catch (ExecutionException ee) {
			assertThat(ExceptionUtils.stripExecutionException(ee), equalTo(cause));
		}
	}
}
 
Example #18
Source File: SlotPoolInteractionsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void releaseSlot(
	@Nonnull SlotRequestId slotRequestId,
	@Nullable Throwable cause) {
	final Consumer<SlotRequestId> currentReleaseSlotConsumer = releaseSlotConsumer;

	super.releaseSlot(slotRequestId, cause);

	if (currentReleaseSlotConsumer != null) {
		currentReleaseSlotConsumer.accept(slotRequestId);
	}
}
 
Example #19
Source File: SlotPoolImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that create report of allocated slots on a {@link TaskExecutor}.
 */
@Test
public void testCreateAllocatedSlotReport() throws Exception {

	try (SlotPoolImpl slotPool = createSlotPoolImpl()) {

		final ArrayBlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(1);
		resourceManagerGateway.setRequestSlotConsumer(
				slotRequest -> allocationIds.offer(slotRequest.getAllocationId()));

		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);

		final SlotRequestId slotRequestId = new SlotRequestId();
		final CompletableFuture<LogicalSlot> slotRequestFuture = allocateSlot(scheduler, slotRequestId);

		final List<AllocatedSlotInfo> allocatedSlotInfos = new ArrayList<>(2);
		final List<SlotOffer> slotOffers = new ArrayList<>(2);

		final AllocationID allocatedId = allocationIds.take();
		slotOffers.add(new SlotOffer(allocatedId, 0, ResourceProfile.UNKNOWN));
		allocatedSlotInfos.add(new AllocatedSlotInfo(0, allocatedId));

		final AllocationID availableId = new AllocationID();
		slotOffers.add(new SlotOffer(availableId, 1, ResourceProfile.UNKNOWN));
		allocatedSlotInfos.add(new AllocatedSlotInfo(1, availableId));

		slotPool.registerTaskManager(taskManagerLocation.getResourceID());
		slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

		// wait for the completion of slot future
		slotRequestFuture.get();

		final AllocatedSlotReport slotReport = slotPool.createAllocatedSlotReport(taskManagerLocation.getResourceID());
		assertThat(jobId, is(slotReport.getJobId()));
		assertThat(slotReport.getAllocatedSlotInfos(), containsInAnyOrder(isEachEqual(allocatedSlotInfos)));
	}
}
 
Example #20
Source File: SchedulerImpl.java    From flink with Apache License 2.0 5 votes vote down vote up
@Nonnull
private CompletableFuture<PhysicalSlot> requestNewAllocatedSlot(
		SlotRequestId slotRequestId,
		SlotProfile slotProfile,
		@Nullable Time allocationTimeout) {
	if (allocationTimeout == null) {
		return slotPool.requestNewAllocatedBatchSlot(slotRequestId, slotProfile.getResourceProfile());
	} else {
		return slotPool.requestNewAllocatedSlot(slotRequestId, slotProfile.getResourceProfile(), allocationTimeout);
	}
}
 
Example #21
Source File: SlotPoolImplTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<LogicalSlot> allocateSlot(Scheduler scheduler, SlotRequestId slotRequestId) {
	return scheduler.allocateSlot(
		slotRequestId,
		new DummyScheduledUnit(),
		SlotProfile.noRequirements(),
		true,
		timeout);
}
 
Example #22
Source File: SlotSharingManagerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetUnresolvedSlot() {
	final TestingAllocatedSlotActions allocatedSlotActions = new TestingAllocatedSlotActions();

	SlotSharingManager slotSharingManager = new SlotSharingManager(
		SLOT_SHARING_GROUP_ID,
		allocatedSlotActions,
		SLOT_OWNER);

	SlotSharingManager.MultiTaskSlot rootSlot1 = slotSharingManager.createRootSlot(
		new SlotRequestId(),
		new CompletableFuture<>(),
		new SlotRequestId());

	final AbstractID groupId = new AbstractID();
	SlotSharingManager.MultiTaskSlot unresolvedRootSlot = slotSharingManager.getUnresolvedRootSlot(groupId);

	assertNotNull(unresolvedRootSlot);
	assertEquals(rootSlot1.getSlotRequestId(), unresolvedRootSlot.getSlotRequestId());

	// occupy the unresolved slot
	unresolvedRootSlot.allocateSingleTaskSlot(
		new SlotRequestId(),
		groupId,
		Locality.UNKNOWN);

	SlotSharingManager.MultiTaskSlot unresolvedRootSlot1 = slotSharingManager.getUnresolvedRootSlot(groupId);

	// we should no longer have a free unresolved root slot
	assertNull(unresolvedRootSlot1);
}
 
Example #23
Source File: SchedulerImpl.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<LogicalSlot> allocateBatchSlot(
		SlotRequestId slotRequestId,
		ScheduledUnit scheduledUnit,
		SlotProfile slotProfile,
		boolean allowQueuedScheduling) {
	return allocateSlotInternal(
		slotRequestId,
		scheduledUnit,
		slotProfile,
		allowQueuedScheduling,
		null);
}
 
Example #24
Source File: ExecutionGraphDeploymentTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<LogicalSlot> apply(SlotRequestId slotRequestId) {
	if (slotIterator.hasNext()) {
		return slotIterator.next();
	} else {
		return FutureUtils.completedExceptionally(new FlinkException("No more slots available."));
	}
}
 
Example #25
Source File: SlotPoolInteractionsTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
protected void timeoutPendingSlotRequest(SlotRequestId slotRequestId) {
	final Consumer<SlotRequestId> currentTimeoutPendingSlotRequestConsumer = timeoutPendingSlotRequestConsumer;

	if (currentTimeoutPendingSlotRequestConsumer != null) {
		currentTimeoutPendingSlotRequestConsumer.accept(slotRequestId);
	}

	super.timeoutPendingSlotRequest(slotRequestId);
}
 
Example #26
Source File: SlotSharingManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that we can release inner slots and that this triggers the slot release for all
 * its children.
 */
@Test
public void testInnerSlotRelease() {
	final TestingAllocatedSlotActions allocatedSlotActions = new TestingAllocatedSlotActions();

	final SlotSharingManager slotSharingManager = new SlotSharingManager(
		SLOT_SHARING_GROUP_ID,
		allocatedSlotActions,
		SLOT_OWNER);

	SlotSharingManager.MultiTaskSlot rootSlot = slotSharingManager.createRootSlot(
		new SlotRequestId(),
		new CompletableFuture<>(),
		new SlotRequestId());

	SlotSharingManager.MultiTaskSlot multiTaskSlot = rootSlot.allocateMultiTaskSlot(
		new SlotRequestId(),
		new AbstractID());

	SlotSharingManager.SingleTaskSlot singleTaskSlot1 = multiTaskSlot.allocateSingleTaskSlot(
		new SlotRequestId(),
		ResourceProfile.UNKNOWN,
		new AbstractID(),
		Locality.LOCAL);

	SlotSharingManager.MultiTaskSlot multiTaskSlot1 = multiTaskSlot.allocateMultiTaskSlot(
		new SlotRequestId(),
		new AbstractID());

	assertTrue(slotSharingManager.contains(multiTaskSlot1.getSlotRequestId()));
	assertTrue(slotSharingManager.contains(singleTaskSlot1.getSlotRequestId()));
	assertTrue(slotSharingManager.contains(multiTaskSlot.getSlotRequestId()));

	multiTaskSlot.release(new FlinkException("Test exception"));

	assertFalse(slotSharingManager.contains(multiTaskSlot1.getSlotRequestId()));
	assertFalse(slotSharingManager.contains(singleTaskSlot1.getSlotRequestId()));
	assertFalse(slotSharingManager.contains(multiTaskSlot.getSlotRequestId()));
	assertTrue(singleTaskSlot1.getLogicalSlotFuture().isCompletedExceptionally());
}
 
Example #27
Source File: SchedulerImpl.java    From flink with Apache License 2.0 5 votes vote down vote up
private void internalAllocateSlot(
		CompletableFuture<LogicalSlot> allocationResultFuture,
		SlotRequestId slotRequestId,
		ScheduledUnit scheduledUnit,
		SlotProfile slotProfile,
		boolean allowQueuedScheduling,
		Time allocationTimeout) {
	CompletableFuture<LogicalSlot> allocationFuture = scheduledUnit.getSlotSharingGroupId() == null ?
		allocateSingleSlot(slotRequestId, slotProfile, allowQueuedScheduling, allocationTimeout) :
		allocateSharedSlot(slotRequestId, scheduledUnit, slotProfile, allowQueuedScheduling, allocationTimeout);

	allocationFuture.whenComplete((LogicalSlot slot, Throwable failure) -> {
		if (failure != null) {
			Optional<SharedSlotOversubscribedException> sharedSlotOverAllocatedException =
					ExceptionUtils.findThrowable(failure, SharedSlotOversubscribedException.class);
			if (sharedSlotOverAllocatedException.isPresent() &&
					sharedSlotOverAllocatedException.get().canRetry()) {

				// Retry the allocation
				internalAllocateSlot(
						allocationResultFuture,
						slotRequestId,
						scheduledUnit,
						slotProfile,
						allowQueuedScheduling,
						allocationTimeout);
			} else {
				cancelSlotRequest(
						slotRequestId,
						scheduledUnit.getSlotSharingGroupId(),
						failure);
				allocationResultFuture.completeExceptionally(failure);
			}
		} else {
			allocationResultFuture.complete(slot);
		}
	});
}
 
Example #28
Source File: SlotSharingManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the root slot are moved from unresolved to resolved once the
 * slot context future is successfully completed
 */
@Test
public void testRootSlotTransition() {
	final TestingAllocatedSlotActions allocatedSlotActions = new TestingAllocatedSlotActions();

	SlotSharingManager slotSharingManager = new SlotSharingManager(
		SLOT_SHARING_GROUP_ID,
		allocatedSlotActions,
		SLOT_OWNER);

	CompletableFuture<SlotContext> slotContextFuture = new CompletableFuture<>();
	SlotSharingManager.MultiTaskSlot rootSlot = slotSharingManager.createRootSlot(
		new SlotRequestId(),
		slotContextFuture,
		new SlotRequestId());

	assertTrue(slotSharingManager.getUnresolvedRootSlots().contains(rootSlot));
	assertFalse(slotSharingManager.getResolvedRootSlots().contains(rootSlot));

	// now complete the slotContextFuture
	slotContextFuture.complete(
		new SimpleSlotContext(
			new AllocationID(),
			new LocalTaskManagerLocation(),
			0,
			new SimpleAckingTaskManagerGateway()));

	assertFalse(slotSharingManager.getUnresolvedRootSlots().contains(rootSlot));
	assertTrue(slotSharingManager.getResolvedRootSlots().contains(rootSlot));
}
 
Example #29
Source File: SchedulerTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<LogicalSlot> allocateSlot(
	SlotRequestId slotRequestId,
	ScheduledUnit task,
	SlotProfile slotProfile,
	boolean allowQueued,
	Time allocationTimeout) {
	return scheduler.allocateSlot(task, allowQueued, slotProfile, allocationTimeout).thenApply(
		(LogicalSlot logicalSlot) -> {
			switch (logicalSlot.getLocality()) {
				case LOCAL:
					numberOfLocalizedAssignments.incrementAndGet();
					break;
				case UNCONSTRAINED:
					numberOfUnconstrainedAssignments.incrementAndGet();
					break;
				case NON_LOCAL:
					numberOfNonLocalizedAssignments.incrementAndGet();
					break;
				case HOST_LOCAL:
					numberOfHostLocalizedAssignments.incrementAndGet();
					break;
				default:
					// ignore
			}

			return logicalSlot;
		});
}
 
Example #30
Source File: ExecutionGraphDeploymentTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<LogicalSlot> apply(SlotRequestId slotRequestId) {
	if (slotIterator.hasNext()) {
		return slotIterator.next();
	} else {
		return FutureUtils.completedExceptionally(new FlinkException("No more slots available."));
	}
}