Java Code Examples for org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter#forMainThread()

The following examples show how to use org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter#forMainThread() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that a batch slot request does not react to {@link SlotPool#failAllocation(AllocationID, Exception)}
 * signals whose exception is not {@link UnfulfillableSlotRequestException}.
 */
@Test
public void testPendingBatchSlotRequestDoesNotFailIfAllocationFails() throws Exception {
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	testingResourceManagerGateway.setRequestSlotConsumer(slotRequest -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();

	final Time batchSlotTimeout = Time.milliseconds(1000L);
	try (final SlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
		.setBatchSlotTimeout(batchSlotTimeout)
		.setResourceManagerGateway(testingResourceManagerGateway)
		.build()) {

		final CompletableFuture<PhysicalSlot> slotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);

		SlotPoolUtils.failAllocation(slotPool, directMainThreadExecutor, allocationIdFuture.get(), new FlinkException("Failed request"));

		assertThat(slotFuture.isDone(), is(false));
	}
}
Example 2
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that a batch slot request does react to {@link SlotPool#failAllocation(AllocationID, Exception)}
 * signals whose exception is {@link UnfulfillableSlotRequestException}.
 */
@Test
public void testPendingBatchSlotRequestFailsIfAllocationFailsUnfulfillably() throws Exception {
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	testingResourceManagerGateway.setRequestSlotConsumer(slotRequest -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();

	try (final SlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
		.setResourceManagerGateway(testingResourceManagerGateway)
		.build()) {

		final CompletableFuture<PhysicalSlot> slotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);

		SlotPoolUtils.failAllocation(slotPool, directMainThreadExecutor, allocationIdFuture.get(),
			new UnfulfillableSlotRequestException(new AllocationID(), ResourceProfile.UNKNOWN));

		assertThat(slotFuture.isCompletedExceptionally(), is(true));
	}
}
Example 3
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that a batch slot request won't fail if its resource manager request fails with exceptions other than
 * {@link UnfulfillableSlotRequestException}.
 */
@Test
public void testPendingBatchSlotRequestDoesNotFailIfRMRequestFails() throws Exception {
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotFuture(FutureUtils.completedExceptionally(new FlinkException("Failed request")));

	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();

	final Time batchSlotTimeout = Time.milliseconds(1000L);
	try (final SlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
		.setBatchSlotTimeout(batchSlotTimeout)
		.setResourceManagerGateway(testingResourceManagerGateway)
		.build()) {

		final CompletableFuture<PhysicalSlot> slotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);

		assertThat(slotFuture.isDone(), is(false));
	}
}
Example 4
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that a batch slot request fails if its resource manager request fails with {@link UnfulfillableSlotRequestException}.
 */
@Test
public void testPendingBatchSlotRequestFailsIfRMRequestFailsUnfulfillably() throws Exception {
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotFuture(FutureUtils.completedExceptionally(
		new UnfulfillableSlotRequestException(new AllocationID(), ResourceProfile.UNKNOWN)));

	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();

	try (final SlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
		.setResourceManagerGateway(testingResourceManagerGateway)
		.build()) {

		final CompletableFuture<PhysicalSlot> slotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);

		assertThat(slotFuture.isCompletedExceptionally(), is(true));
	}
}
Example 5
Source File: SchedulerTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Before
public void setup() throws Exception {
	final JobID jobId = new JobID();
	final SlotPool slotPool = new TestingSlotPoolImpl(jobId);
	final TestingScheduler testingScheduler = new TestingScheduler(
		new HashMap<>(16),
		LocationPreferenceSlotSelectionStrategy.INSTANCE,
		slotPool);

	testingSlotProvider = new TestingSlotPoolSlotProvider(slotPool, testingScheduler);

	final JobMasterId jobMasterId = JobMasterId.generate();
	final String jobManagerAddress = "localhost";
	ComponentMainThreadExecutor executor = ComponentMainThreadExecutorServiceAdapter.forMainThread();
	slotPool.start(jobMasterId, jobManagerAddress, executor);
	testingScheduler.start(executor);
}
Example 6
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that a batch slot request does not react to {@link SlotPool#failAllocation(AllocationID, Exception)}
 * signals whose exception is not {@link UnfulfillableSlotRequestException}.
 */
@Test
public void testPendingBatchSlotRequestDoesNotFailIfAllocationFails() throws Exception {
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	testingResourceManagerGateway.setRequestSlotConsumer(slotRequest -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();

	final Time batchSlotTimeout = Time.milliseconds(1000L);
	try (final SlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
		.setBatchSlotTimeout(batchSlotTimeout)
		.setResourceManagerGateway(testingResourceManagerGateway)
		.build()) {

		final CompletableFuture<PhysicalSlot> slotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);

		SlotPoolUtils.failAllocation(slotPool, directMainThreadExecutor, allocationIdFuture.get(), new FlinkException("Failed request"));

		assertThat(slotFuture.isDone(), is(false));
	}
}
Example 7
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that a batch slot request does react to {@link SlotPool#failAllocation(AllocationID, Exception)}
 * signals whose exception is {@link UnfulfillableSlotRequestException}.
 */
@Test
public void testPendingBatchSlotRequestFailsIfAllocationFailsUnfulfillably() throws Exception {
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	testingResourceManagerGateway.setRequestSlotConsumer(slotRequest -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();

	try (final SlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
		.setResourceManagerGateway(testingResourceManagerGateway)
		.build()) {

		final CompletableFuture<PhysicalSlot> slotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);

		SlotPoolUtils.failAllocation(slotPool, directMainThreadExecutor, allocationIdFuture.get(),
			new UnfulfillableSlotRequestException(new AllocationID(), ResourceProfile.UNKNOWN));

		assertThat(slotFuture.isCompletedExceptionally(), is(true));
	}
}
Example 8
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that a batch slot request won't fail if its resource manager request fails with exceptions other than
 * {@link UnfulfillableSlotRequestException}.
 */
@Test
public void testPendingBatchSlotRequestDoesNotFailIfRMRequestFails() throws Exception {
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotFuture(FutureUtils.completedExceptionally(new FlinkException("Failed request")));

	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();

	final Time batchSlotTimeout = Time.milliseconds(1000L);
	try (final SlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
		.setBatchSlotTimeout(batchSlotTimeout)
		.setResourceManagerGateway(testingResourceManagerGateway)
		.build()) {

		final CompletableFuture<PhysicalSlot> slotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);

		assertThat(slotFuture.isDone(), is(false));
	}
}
Example 9
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that a batch slot request fails if its resource manager request fails with {@link UnfulfillableSlotRequestException}.
 */
@Test
public void testPendingBatchSlotRequestFailsIfRMRequestFailsUnfulfillably() throws Exception {
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotFuture(FutureUtils.completedExceptionally(
		new UnfulfillableSlotRequestException(new AllocationID(), ResourceProfile.UNKNOWN)));

	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();

	try (final SlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
		.setResourceManagerGateway(testingResourceManagerGateway)
		.build()) {

		final CompletableFuture<PhysicalSlot> slotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);

		assertThat(slotFuture.isCompletedExceptionally(), is(true));
	}
}
Example 10
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that a batch slot request won't time out if there exists a slot in the
 * SlotPool which fulfills the requested {@link ResourceProfile}.
 */
@Test
public void testPendingBatchSlotRequestDoesNotTimeoutIfFulfillingSlotExists() throws Exception {
	final Time batchSlotTimeout = Time.milliseconds(2L);
	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();
	final ManualClock clock = new ManualClock();

	try (final TestingSlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
			.setClock(clock)
			.setBatchSlotTimeout(batchSlotTimeout)
			.build()) {

		SlotPoolUtils.offerSlots(slotPool, directMainThreadExecutor, Collections.singletonList(resourceProfile));

		final CompletableFuture<PhysicalSlot> firstSlotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);
		final CompletableFuture<PhysicalSlot> secondSlotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, ResourceProfile.UNKNOWN);
		final CompletableFuture<PhysicalSlot> thirdSlotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, smallerResourceProfile);

		final List<CompletableFuture<PhysicalSlot>> slotFutures = Arrays.asList(firstSlotFuture, secondSlotFuture, thirdSlotFuture);
		advanceTimeAndTriggerCheckBatchSlotTimeout(slotPool, clock, batchSlotTimeout);

		for (CompletableFuture<PhysicalSlot> slotFuture : slotFutures) {
			assertThat(slotFuture.isDone(), is(false));
		}
	}

}
Example 11
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that a pending batch slot request times out after the last fulfilling slot gets
 * released.
 */
@Test
public void testPendingBatchSlotRequestTimeoutAfterSlotRelease() throws Exception {
	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();
	final ManualClock clock = new ManualClock();
	final Time batchSlotTimeout = Time.milliseconds(1000L);

	try (final TestingSlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
			.setClock(clock)
			.setBatchSlotTimeout(batchSlotTimeout)
			.build()) {
		final ResourceID taskManagerResourceId = SlotPoolUtils.offerSlots(slotPool, directMainThreadExecutor, Collections.singletonList(resourceProfile));
		final CompletableFuture<PhysicalSlot> firstSlotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);
		final CompletableFuture<PhysicalSlot> secondSlotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, ResourceProfile.UNKNOWN);
		final CompletableFuture<PhysicalSlot> thirdSlotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, smallerResourceProfile);

		final List<CompletableFuture<PhysicalSlot>> slotFutures = Arrays.asList(firstSlotFuture, secondSlotFuture, thirdSlotFuture);

		// initial batch slot timeout check
		advanceTimeAndTriggerCheckBatchSlotTimeout(slotPool, clock, batchSlotTimeout);

		assertThat(CompletableFuture.anyOf(slotFutures.toArray(COMPLETABLE_FUTURES_EMPTY_ARRAY)).isDone(), is(false));

		SlotPoolUtils.releaseTaskManager(slotPool, directMainThreadExecutor, taskManagerResourceId);

		advanceTimeAndTriggerCheckBatchSlotTimeout(slotPool, clock, batchSlotTimeout);

		for (CompletableFuture<PhysicalSlot> slotFuture : slotFutures) {
			assertThat(slotFuture.isCompletedExceptionally(), is(true));

			try {
				slotFuture.get();
				fail("Expected that the slot future times out.");
			} catch (ExecutionException ee) {
				assertThat(ExceptionUtils.stripExecutionException(ee), instanceOf(TimeoutException.class));
			}
		}
	}
}
Example 12
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that a batch slot request won't time out if there exists a slot in the
 * SlotPool which fulfills the requested {@link ResourceProfile}.
 */
@Test
public void testPendingBatchSlotRequestDoesNotTimeoutIfFulfillingSlotExists() throws Exception {
	final Time batchSlotTimeout = Time.milliseconds(2L);
	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();
	final ManualClock clock = new ManualClock();

	try (final TestingSlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
			.setClock(clock)
			.setBatchSlotTimeout(batchSlotTimeout)
			.build()) {

		SlotPoolUtils.offerSlots(slotPool, directMainThreadExecutor, Collections.singletonList(resourceProfile));

		final CompletableFuture<PhysicalSlot> firstSlotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);
		final CompletableFuture<PhysicalSlot> secondSlotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, ResourceProfile.UNKNOWN);
		final CompletableFuture<PhysicalSlot> thirdSlotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, smallerResourceProfile);

		final List<CompletableFuture<PhysicalSlot>> slotFutures = Arrays.asList(firstSlotFuture, secondSlotFuture, thirdSlotFuture);
		advanceTimeAndTriggerCheckBatchSlotTimeout(slotPool, clock, batchSlotTimeout);

		for (CompletableFuture<PhysicalSlot> slotFuture : slotFutures) {
			assertThat(slotFuture.isDone(), is(false));
		}
	}

}
Example 13
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that a pending batch slot request times out after the last fulfilling slot gets
 * released.
 */
@Test
public void testPendingBatchSlotRequestTimeoutAfterSlotRelease() throws Exception {
	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();
	final ManualClock clock = new ManualClock();
	final Time batchSlotTimeout = Time.milliseconds(1000L);

	try (final TestingSlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
			.setClock(clock)
			.setBatchSlotTimeout(batchSlotTimeout)
			.build()) {
		final ResourceID taskManagerResourceId = SlotPoolUtils.offerSlots(slotPool, directMainThreadExecutor, Collections.singletonList(resourceProfile));
		final CompletableFuture<PhysicalSlot> firstSlotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);
		final CompletableFuture<PhysicalSlot> secondSlotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, ResourceProfile.UNKNOWN);
		final CompletableFuture<PhysicalSlot> thirdSlotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, smallerResourceProfile);

		final List<CompletableFuture<PhysicalSlot>> slotFutures = Arrays.asList(firstSlotFuture, secondSlotFuture, thirdSlotFuture);

		// initial batch slot timeout check
		advanceTimeAndTriggerCheckBatchSlotTimeout(slotPool, clock, batchSlotTimeout);

		assertThat(CompletableFuture.anyOf(slotFutures.toArray(COMPLETABLE_FUTURES_EMPTY_ARRAY)).isDone(), is(false));

		SlotPoolUtils.releaseTaskManager(slotPool, directMainThreadExecutor, taskManagerResourceId);

		advanceTimeAndTriggerCheckBatchSlotTimeout(slotPool, clock, batchSlotTimeout);

		for (CompletableFuture<PhysicalSlot> slotFuture : slotFutures) {
			assertThat(slotFuture.isCompletedExceptionally(), is(true));

			try {
				slotFuture.get();
				fail("Expected that the slot future times out.");
			} catch (ExecutionException ee) {
				assertThat(ExceptionUtils.stripExecutionException(ee), instanceOf(TimeoutException.class));
			}
		}
	}
}
Example 14
Source File: ScheduleWithCoLocationHintTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
protected ComponentMainThreadExecutor getComponentMainThreadExecutor() {
	return ComponentMainThreadExecutorServiceAdapter.forMainThread();
}
Example 15
Source File: SlotPoolResource.java    From flink with Apache License 2.0 4 votes vote down vote up 
public SlotPoolResource(@Nonnull SlotSelectionStrategy schedulingStrategy) {
	this.schedulingStrategy = schedulingStrategy;
	this.mainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();
	slotPool = null;
	testingResourceManagerGateway = null;
}
Example 16
Source File: ExecutionGraphCoLocationRestartTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
protected ComponentMainThreadExecutor getComponentMainThreadExecutor() {
	return ComponentMainThreadExecutorServiceAdapter.forMainThread();
}
Example 17
Source File: SlotPoolResource.java    From flink with Apache License 2.0 4 votes vote down vote up 
public SlotPoolResource(@Nonnull SlotSelectionStrategy schedulingStrategy) {
	this.schedulingStrategy = schedulingStrategy;
	this.mainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();
	slotPool = null;
	testingResourceManagerGateway = null;
}