Java Code Examples for org.apache.flink.runtime.jobmanager.scheduler.Locality#NON_LOCAL

The following examples show how to use org.apache.flink.runtime.jobmanager.scheduler.Locality#NON_LOCAL . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SlotSharingManagerTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that the logical task slot futures are completed once the slot context
 * future is completed.
 */
@Test
public void testSlotContextFutureCompletion() throws Exception {
	final TestingAllocatedSlotActions allocatedSlotActions = new TestingAllocatedSlotActions();

	final SlotSharingManager slotSharingManager = new SlotSharingManager(
		SLOT_SHARING_GROUP_ID,
		allocatedSlotActions,
		SLOT_OWNER);

	final SlotContext slotContext = new SimpleSlotContext(
		new AllocationID(),
		new LocalTaskManagerLocation(),
		0,
		new SimpleAckingTaskManagerGateway());

	CompletableFuture<SlotContext> slotContextFuture = new CompletableFuture<>();
	SlotSharingManager.MultiTaskSlot rootSlot = slotSharingManager.createRootSlot(
		new SlotRequestId(),
		slotContextFuture,
		new SlotRequestId());

	Locality locality1 = Locality.LOCAL;
	SlotSharingManager.SingleTaskSlot singleTaskSlot1 = rootSlot.allocateSingleTaskSlot(
		new SlotRequestId(),
		new AbstractID(),
		locality1);

	Locality locality2 = Locality.HOST_LOCAL;
	SlotSharingManager.SingleTaskSlot singleTaskSlot2 = rootSlot.allocateSingleTaskSlot(
		new SlotRequestId(),
		new AbstractID(),
		locality2);

	CompletableFuture<LogicalSlot> logicalSlotFuture1 = singleTaskSlot1.getLogicalSlotFuture();
	CompletableFuture<LogicalSlot> logicalSlotFuture2 = singleTaskSlot2.getLogicalSlotFuture();
	assertFalse(logicalSlotFuture1.isDone());
	assertFalse(logicalSlotFuture2.isDone());

	slotContextFuture.complete(slotContext);

	assertTrue(logicalSlotFuture1.isDone());
	assertTrue(logicalSlotFuture2.isDone());

	final LogicalSlot logicalSlot1 = logicalSlotFuture1.get();
	final LogicalSlot logicalSlot2 = logicalSlotFuture2.get();

	assertEquals(logicalSlot1.getAllocationId(), slotContext.getAllocationId());
	assertEquals(logicalSlot2.getAllocationId(), slotContext.getAllocationId());
	assertEquals(locality1, logicalSlot1.getLocality());
	assertEquals(locality2, logicalSlot2.getLocality());

	Locality locality3 = Locality.NON_LOCAL;
	SlotSharingManager.SingleTaskSlot singleTaskSlot3 = rootSlot.allocateSingleTaskSlot(
		new SlotRequestId(),
		new AbstractID(),
		locality3);

	CompletableFuture<LogicalSlot> logicalSlotFuture3 = singleTaskSlot3.getLogicalSlotFuture();

	assertTrue(logicalSlotFuture3.isDone());
	LogicalSlot logicalSlot3 = logicalSlotFuture3.get();

	assertEquals(locality3, logicalSlot3.getLocality());
	assertEquals(slotContext.getAllocationId(), logicalSlot3.getAllocationId());
}
Example 2
Source File: SlotSharingManagerTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that the logical task slot futures are completed once the slot context
 * future is completed.
 */
@Test
public void testSlotContextFutureCompletion() throws Exception {
	final TestingAllocatedSlotActions allocatedSlotActions = new TestingAllocatedSlotActions();

	final SlotSharingManager slotSharingManager = new SlotSharingManager(
		SLOT_SHARING_GROUP_ID,
		allocatedSlotActions,
		SLOT_OWNER);

	final SlotContext slotContext = new SimpleSlotContext(
		new AllocationID(),
		new LocalTaskManagerLocation(),
		0,
		new SimpleAckingTaskManagerGateway());

	CompletableFuture<SlotContext> slotContextFuture = new CompletableFuture<>();
	SlotSharingManager.MultiTaskSlot rootSlot = slotSharingManager.createRootSlot(
		new SlotRequestId(),
		slotContextFuture,
		new SlotRequestId());

	Locality locality1 = Locality.LOCAL;
	SlotSharingManager.SingleTaskSlot singleTaskSlot1 = rootSlot.allocateSingleTaskSlot(
		new SlotRequestId(),
		ResourceProfile.UNKNOWN,
		new AbstractID(),
		locality1);

	Locality locality2 = Locality.HOST_LOCAL;
	SlotSharingManager.SingleTaskSlot singleTaskSlot2 = rootSlot.allocateSingleTaskSlot(
		new SlotRequestId(),
		ResourceProfile.UNKNOWN,
		new AbstractID(),
		locality2);

	CompletableFuture<LogicalSlot> logicalSlotFuture1 = singleTaskSlot1.getLogicalSlotFuture();
	CompletableFuture<LogicalSlot> logicalSlotFuture2 = singleTaskSlot2.getLogicalSlotFuture();
	assertFalse(logicalSlotFuture1.isDone());
	assertFalse(logicalSlotFuture2.isDone());

	slotContextFuture.complete(slotContext);

	assertTrue(logicalSlotFuture1.isDone());
	assertTrue(logicalSlotFuture2.isDone());

	final LogicalSlot logicalSlot1 = logicalSlotFuture1.get();
	final LogicalSlot logicalSlot2 = logicalSlotFuture2.get();

	assertEquals(logicalSlot1.getAllocationId(), slotContext.getAllocationId());
	assertEquals(logicalSlot2.getAllocationId(), slotContext.getAllocationId());
	assertEquals(locality1, logicalSlot1.getLocality());
	assertEquals(locality2, logicalSlot2.getLocality());

	Locality locality3 = Locality.NON_LOCAL;
	SlotSharingManager.SingleTaskSlot singleTaskSlot3 = rootSlot.allocateSingleTaskSlot(
		new SlotRequestId(),
		ResourceProfile.UNKNOWN,
		new AbstractID(),
		locality3);

	CompletableFuture<LogicalSlot> logicalSlotFuture3 = singleTaskSlot3.getLogicalSlotFuture();

	assertTrue(logicalSlotFuture3.isDone());
	LogicalSlot logicalSlot3 = logicalSlotFuture3.get();

	assertEquals(locality3, logicalSlot3.getLocality());
	assertEquals(slotContext.getAllocationId(), logicalSlot3.getAllocationId());
}
Example 3
Source File: LocationPreferenceSlotSelectionStrategy.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Nonnull
private Optional<SlotInfoAndLocality> selectWitLocationPreference(
	@Nonnull Collection<SlotInfoAndResources> availableSlots,
	@Nonnull Collection<TaskManagerLocation> locationPreferences,
	@Nonnull ResourceProfile resourceProfile) {

	// we build up two indexes, one for resource id and one for host names of the preferred locations.
	final Map<ResourceID, Integer> preferredResourceIDs = new HashMap<>(locationPreferences.size());
	final Map<String, Integer> preferredFQHostNames = new HashMap<>(locationPreferences.size());

	for (TaskManagerLocation locationPreference : locationPreferences) {
		preferredResourceIDs.merge(locationPreference.getResourceID(), 1, Integer::sum);
		preferredFQHostNames.merge(locationPreference.getFQDNHostname(), 1, Integer::sum);
	}

	SlotInfoAndResources bestCandidate = null;
	Locality bestCandidateLocality = Locality.UNKNOWN;
	double bestCandidateScore = Double.NEGATIVE_INFINITY;

	for (SlotInfoAndResources candidate : availableSlots) {

		if (candidate.getRemainingResources().isMatching(resourceProfile)) {

			// this gets candidate is local-weigh
			int localWeigh = preferredResourceIDs.getOrDefault(
				candidate.getSlotInfo().getTaskManagerLocation().getResourceID(), 0);

			// this gets candidate is host-local-weigh
			int hostLocalWeigh = preferredFQHostNames.getOrDefault(
				candidate.getSlotInfo().getTaskManagerLocation().getFQDNHostname(), 0);

			double candidateScore = calculateCandidateScore(localWeigh, hostLocalWeigh, candidate.getTaskExecutorUtilization());
			if (candidateScore > bestCandidateScore) {
				bestCandidateScore = candidateScore;
				bestCandidate = candidate;
				bestCandidateLocality = localWeigh > 0 ?
					Locality.LOCAL : hostLocalWeigh > 0 ?
					Locality.HOST_LOCAL : Locality.NON_LOCAL;
			}
		}
	}

	// at the end of the iteration, we return the candidate with best possible locality or null.
	return bestCandidate != null ?
		Optional.of(SlotInfoAndLocality.of(bestCandidate.getSlotInfo(), bestCandidateLocality)) :
		Optional.empty();
}
Example 4
Source File: SlotSharingManagerTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that the logical task slot futures are completed once the slot context
 * future is completed.
 */
@Test
public void testSlotContextFutureCompletion() throws Exception {
	final SlotSharingManager slotSharingManager = createTestingSlotSharingManager();

	final SlotContext slotContext = createSimpleSlotContext();

	CompletableFuture<SlotContext> slotContextFuture = new CompletableFuture<>();
	SlotSharingManager.MultiTaskSlot rootSlot = slotSharingManager.createRootSlot(
		new SlotRequestId(),
		slotContextFuture,
		new SlotRequestId());

	Locality locality1 = Locality.LOCAL;
	SlotSharingManager.SingleTaskSlot singleTaskSlot1 = rootSlot.allocateSingleTaskSlot(
		new SlotRequestId(),
		ResourceProfile.UNKNOWN,
		new AbstractID(),
		locality1);

	Locality locality2 = Locality.HOST_LOCAL;
	SlotSharingManager.SingleTaskSlot singleTaskSlot2 = rootSlot.allocateSingleTaskSlot(
		new SlotRequestId(),
		ResourceProfile.UNKNOWN,
		new AbstractID(),
		locality2);

	CompletableFuture<LogicalSlot> logicalSlotFuture1 = singleTaskSlot1.getLogicalSlotFuture();
	CompletableFuture<LogicalSlot> logicalSlotFuture2 = singleTaskSlot2.getLogicalSlotFuture();
	assertFalse(logicalSlotFuture1.isDone());
	assertFalse(logicalSlotFuture2.isDone());

	slotContextFuture.complete(slotContext);

	assertTrue(logicalSlotFuture1.isDone());
	assertTrue(logicalSlotFuture2.isDone());

	final LogicalSlot logicalSlot1 = logicalSlotFuture1.get();
	final LogicalSlot logicalSlot2 = logicalSlotFuture2.get();

	assertEquals(logicalSlot1.getAllocationId(), slotContext.getAllocationId());
	assertEquals(logicalSlot2.getAllocationId(), slotContext.getAllocationId());
	assertEquals(locality1, logicalSlot1.getLocality());
	assertEquals(locality2, logicalSlot2.getLocality());

	Locality locality3 = Locality.NON_LOCAL;
	SlotSharingManager.SingleTaskSlot singleTaskSlot3 = rootSlot.allocateSingleTaskSlot(
		new SlotRequestId(),
		ResourceProfile.UNKNOWN,
		new AbstractID(),
		locality3);

	CompletableFuture<LogicalSlot> logicalSlotFuture3 = singleTaskSlot3.getLogicalSlotFuture();

	assertTrue(logicalSlotFuture3.isDone());
	LogicalSlot logicalSlot3 = logicalSlotFuture3.get();

	assertEquals(locality3, logicalSlot3.getLocality());
	assertEquals(slotContext.getAllocationId(), logicalSlot3.getAllocationId());
}