Java Code Examples for org.apache.flink.runtime.testingUtils.TestingUtils#infiniteTime()

The following examples show how to use org.apache.flink.runtime.testingUtils.TestingUtils#infiniteTime() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AbstractTaskManagerFileHandlerTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private AbstractTaskManagerFileHandlerTest.TestTaskManagerFileHandler createTestTaskManagerFileHandler(
	Time cacheEntryDuration,
	Queue<CompletableFuture<TransientBlobKey>> requestFileUploads,
	ResourceID expectedTaskManagerId) {
	final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();

	return new TestTaskManagerFileHandler(
		() -> CompletableFuture.completedFuture(null),
		TestingUtils.infiniteTime(),
		Collections.emptyMap(),
		new TestUntypedMessageHeaders(),
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		cacheEntryDuration,
		requestFileUploads,
		expectedTaskManagerId);
}
Example 2
Source File: AbstractTaskManagerFileHandlerTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
private AbstractTaskManagerFileHandlerTest.TestTaskManagerFileHandler createTestTaskManagerFileHandler(
	Time cacheEntryDuration,
	Queue<CompletableFuture<TransientBlobKey>> requestFileUploads,
	ResourceID expectedTaskManagerId) {
	final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();

	return new TestTaskManagerFileHandler(
		() -> CompletableFuture.completedFuture(null),
		TestingUtils.infiniteTime(),
		Collections.emptyMap(),
		new TestUntypedMessageHeaders(),
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		cacheEntryDuration,
		requestFileUploads,
		expectedTaskManagerId);
}
Example 3
Source File: AbstractTaskManagerFileHandlerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private AbstractTaskManagerFileHandlerTest.TestTaskManagerFileHandler createTestTaskManagerFileHandler(
	Time cacheEntryDuration,
	Queue<CompletableFuture<TransientBlobKey>> requestFileUploads,
	ResourceID expectedTaskManagerId) {
	final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();

	return new TestTaskManagerFileHandler(
		() -> CompletableFuture.completedFuture(null),
		TestingUtils.infiniteTime(),
		Collections.emptyMap(),
		new TestUntypedMessageHeaders(),
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		cacheEntryDuration,
		requestFileUploads,
		expectedTaskManagerId);
}
Example 4
Source File: SlotPoolBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up 
public TestingSlotPoolImpl build() throws Exception {
	final TestingSlotPoolImpl slotPool = new TestingSlotPoolImpl(
		new JobID(),
		clock,
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime(),
		batchSlotTimeout);

	slotPool.start(JobMasterId.generate(), "foobar", componentMainThreadExecutor);

	CompletableFuture.runAsync(() -> slotPool.connectToResourceManager(resourceManagerGateway), componentMainThreadExecutor).join();

	return slotPool;
}
Example 5
Source File: SlotPoolInteractionsTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSlotAllocationNoResourceManager() throws Exception {
	final JobID jid = new JobID();

	try (SlotPool pool = new SlotPoolImpl(
		jid,
		SystemClock.getInstance(),
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime()
	)) {

		pool.start(JobMasterId.generate(), "foobar", testMainThreadExecutor.getMainThreadExecutor());
		Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), pool);
		scheduler.start(testMainThreadExecutor.getMainThreadExecutor());

		CompletableFuture<LogicalSlot> future = testMainThreadExecutor.execute(() -> scheduler.allocateSlot(
			new SlotRequestId(),
			new ScheduledUnit(getExecution()),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			fastTimeout));

		try {
			future.get();
			fail("We expected an ExecutionException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException);
		}
	}
}
Example 6
Source File: ResourceManagerPartitionLifecycleTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private ResourceManagerGateway createAndStartResourceManager() throws Exception {
	final SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();
	final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		TestingUtils.infiniteTime());

	final TestingResourceManager resourceManager = new TestingResourceManager(
		rpcService,
		ResourceID.generate(),
		highAvailabilityServices,
		new HeartbeatServices(100000L, 1000000L),
		slotManager,
		ResourceManagerPartitionTrackerImpl::new,
		jobLeaderIdService,
		testingFatalErrorHandler,
		UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup());

	resourceManager.start();

	// first make the ResourceManager the leader
	resourceManagerLeaderElectionService.isLeader(ResourceManagerId.generate().toUUID()).get();

	this.resourceManager = resourceManager;

	return resourceManager.getSelfGateway(ResourceManagerGateway.class);
}
Example 7
Source File: SlotPoolBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up 
public TestingSlotPoolImpl build() throws Exception {
	final TestingSlotPoolImpl slotPool = new TestingSlotPoolImpl(
		new JobID(),
		clock,
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime(),
		batchSlotTimeout);

	slotPool.start(JobMasterId.generate(), "foobar", componentMainThreadExecutor);

	CompletableFuture.runAsync(() -> slotPool.connectToResourceManager(resourceManagerGateway), componentMainThreadExecutor).join();

	return slotPool;
}
Example 8
Source File: SlotManagerBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up 
private SlotManagerBuilder() {
	this.scheduledExecutor = TestingUtils.defaultScheduledExecutor();
	this.taskManagerRequestTimeout = TestingUtils.infiniteTime();
	this.slotRequestTimeout = TestingUtils.infiniteTime();
	this.taskManagerTimeout = TestingUtils.infiniteTime();
	this.waitResultConsumedBeforeRelease = true;
}
Example 9
Source File: SlotPoolInteractionsTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Nonnull
private TestingSlotPool createTestingSlotPool(JobID jid) {
	return new TestingSlotPool(
		jid,
		SystemClock.getInstance(),
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime());
}
Example 10
Source File: SlotPoolInteractionsTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSlotAllocationNoResourceManager() throws Exception {
	final JobID jid = new JobID();

	try (SlotPool pool = new SlotPoolImpl(
		jid,
		SystemClock.getInstance(),
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime()
	)) {

		pool.start(JobMasterId.generate(), "foobar", testMainThreadExecutor.getMainThreadExecutor());
		Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.INSTANCE, pool);
		scheduler.start(testMainThreadExecutor.getMainThreadExecutor());

		CompletableFuture<LogicalSlot> future = testMainThreadExecutor.execute(() -> scheduler.allocateSlot(
			new SlotRequestId(),
			new ScheduledUnit(SchedulerTestUtils.getDummyTask()),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			true,
			fastTimeout));

		try {
			future.get();
			fail("We expected an ExecutionException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException);
		}
	}
}
Example 11
Source File: SlotPoolImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Nonnull
private TestingSlotPoolImpl createSlotPoolImpl(ManualClock clock) {
	return new TestingSlotPoolImpl(
		jobId,
		clock,
		TestingUtils.infiniteTime(),
		timeout,
		TestingUtils.infiniteTime());
}
Example 12
Source File: SlotManagerBuilder.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private SlotManagerBuilder() {
	this.scheduledExecutor = TestingUtils.defaultScheduledExecutor();
	this.taskManagerRequestTimeout = TestingUtils.infiniteTime();
	this.slotRequestTimeout = TestingUtils.infiniteTime();
	this.taskManagerTimeout = TestingUtils.infiniteTime();
	this.waitResultConsumedBeforeRelease = true;
}
Example 13
Source File: ResourceManagerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private TestingResourceManager createAndStartResourceManager(HeartbeatServices heartbeatServices) throws Exception {
	final SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();
	final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		TestingUtils.infiniteTime());

	final TestingResourceManager resourceManager = new TestingResourceManager(
		rpcService,
		ResourceManager.RESOURCE_MANAGER_NAME + UUID.randomUUID(),
		resourceManagerResourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		NoOpMetricRegistry.INSTANCE,
		jobLeaderIdService,
		testingFatalErrorHandler,
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());

	resourceManager.start();

	// first make the ResourceManager the leader
	resourceManagerId = ResourceManagerId.generate();
	resourceManagerLeaderElectionService.isLeader(resourceManagerId.toUUID()).get();

	return resourceManager;
}
Example 14
Source File: SlotManagerBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up 
private SlotManagerBuilder() {
	this.slotMatchingStrategy = AnyMatchingSlotMatchingStrategy.INSTANCE;
	this.scheduledExecutor = TestingUtils.defaultScheduledExecutor();
	this.taskManagerRequestTimeout = TestingUtils.infiniteTime();
	this.slotRequestTimeout = TestingUtils.infiniteTime();
	this.taskManagerTimeout = TestingUtils.infiniteTime();
	this.waitResultConsumedBeforeRelease = true;
	this.defaultWorkerResourceSpec = WorkerResourceSpec.ZERO;
	this.numSlotsPerWorker = 1;
	this.slotManagerMetricGroup = UnregisteredMetricGroups.createUnregisteredSlotManagerMetricGroup();
	this.maxSlotNum = ResourceManagerOptions.MAX_SLOT_NUM.defaultValue();
}
Example 15
Source File: ResourceManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private TestingResourceManager createAndStartResourceManager(HeartbeatServices heartbeatServices) throws Exception {
	final SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();
	final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		TestingUtils.infiniteTime());

	final TestingResourceManager resourceManager = new TestingResourceManager(
		rpcService,
		resourceManagerResourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		NoOpResourceManagerPartitionTracker::get,
		jobLeaderIdService,
		testingFatalErrorHandler,
		UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup());

	resourceManager.start();

	// first make the ResourceManager the leader
	resourceManagerId = ResourceManagerId.generate();
	resourceManagerLeaderElectionService.isLeader(resourceManagerId.toUUID()).get();

	return resourceManager;
}
Example 16
Source File: SlotPoolInteractionsTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCancelSlotAllocationWithoutResourceManager() throws Exception {
	final JobID jid = new JobID();

	try (TestingSlotPool pool = new TestingSlotPool(
		jid,
		SystemClock.getInstance(),
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime())) {

		final CompletableFuture<SlotRequestId> timeoutFuture = new CompletableFuture<>();
		pool.setTimeoutPendingSlotRequestConsumer(timeoutFuture::complete);
		pool.start(JobMasterId.generate(), "foobar", testMainThreadExecutor.getMainThreadExecutor());
		Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.INSTANCE, pool);
		scheduler.start(testMainThreadExecutor.getMainThreadExecutor());

		SlotRequestId requestId = new SlotRequestId();
		CompletableFuture<LogicalSlot> future = testMainThreadExecutor.execute(() -> scheduler.allocateSlot(
			requestId,
			new ScheduledUnit(SchedulerTestUtils.getDummyTask()),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			true,
			fastTimeout));

		try {
			future.get();
			fail("We expected a TimeoutException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException);
		}

		// wait for the timeout of the pending slot request
		timeoutFuture.get();

		assertEquals(0L, pool.getNumberOfWaitingForResourceRequests());
	}
}
Example 17
Source File: SlotPoolInteractionsTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSlotAllocationNoResourceManager() throws Exception {
	final JobID jid = new JobID();

	try (SlotPool pool = new SlotPoolImpl(
		jid,
		SystemClock.getInstance(),
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime()
	)) {

		pool.start(JobMasterId.generate(), "foobar", testMainThreadExecutor.getMainThreadExecutor());
		Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.INSTANCE, pool);
		scheduler.start(testMainThreadExecutor.getMainThreadExecutor());

		CompletableFuture<LogicalSlot> future = testMainThreadExecutor.execute(() -> scheduler.allocateSlot(
			new SlotRequestId(),
			new ScheduledUnit(SchedulerTestUtils.getDummyTask()),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			true,
			fastTimeout));

		try {
			future.get();
			fail("We expected an ExecutionException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException);
		}
	}
}
Example 18
Source File: SlotPoolImplTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testCheckIdleSlot() throws Exception {
	final ManualClock clock = new ManualClock();

	try (SlotPoolImpl slotPool = new SlotPoolImpl(
		jobId,
		clock,
		TestingUtils.infiniteTime(),
		timeout)) {
		final BlockingQueue<AllocationID> freedSlots = new ArrayBlockingQueue<>(1);
		taskManagerGateway.setFreeSlotFunction(
			(AllocationID allocationId, Throwable cause) -> {
				try {
					freedSlots.put(allocationId);
					return CompletableFuture.completedFuture(Acknowledge.get());
				} catch (InterruptedException e) {
					return FutureUtils.completedExceptionally(e);
				}
			});

		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);

		final AllocationID expiredSlotID = new AllocationID();
		final AllocationID freshSlotID = new AllocationID();
		final SlotOffer slotToExpire = new SlotOffer(expiredSlotID, 0, ResourceProfile.UNKNOWN);
		final SlotOffer slotToNotExpire = new SlotOffer(freshSlotID, 1, ResourceProfile.UNKNOWN);

		assertThat(slotPool.registerTaskManager(taskManagerLocation.getResourceID()),
			Matchers.is(true));

		assertThat(
			slotPool.offerSlot(taskManagerLocation, taskManagerGateway, slotToExpire),
			Matchers.is(true));

		clock.advanceTime(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		assertThat(slotPool.offerSlot(taskManagerLocation, taskManagerGateway, slotToNotExpire),
			Matchers.is(true));

		clock.advanceTime(1L, TimeUnit.MILLISECONDS);

		slotPool.triggerCheckIdleSlot();

		final AllocationID freedSlot = freedSlots.poll(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		assertThat(freedSlot, Matchers.is(expiredSlotID));
		assertThat(freedSlots.isEmpty(), Matchers.is(true));
	}
}
Example 19
Source File: SlotPoolInteractionsTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that extra slots are kept by the {@link SlotPoolImpl}.
 */
@Test
public void testExtraSlotsAreKept() throws Exception {
	final JobID jid = new JobID();

	try (TestingSlotPool pool = new TestingSlotPool(
		jid,
		SystemClock.getInstance(),
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime())) {

		pool.start(JobMasterId.generate(), "foobar", testMainThreadExecutor.getMainThreadExecutor());

		Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.INSTANCE, pool);
		scheduler.start(testMainThreadExecutor.getMainThreadExecutor());

		final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();

		TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
		resourceManagerGateway.setRequestSlotConsumer(
			(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

		final CompletableFuture<SlotRequestId> slotRequestTimeoutFuture = new CompletableFuture<>();
		pool.setTimeoutPendingSlotRequestConsumer(slotRequestTimeoutFuture::complete);

		pool.connectToResourceManager(resourceManagerGateway);

		SlotRequestId requestId = new SlotRequestId();
		CompletableFuture<LogicalSlot> future = testMainThreadExecutor.execute(() -> scheduler.allocateSlot(
			requestId,
			new ScheduledUnit(SchedulerTestUtils.getDummyTask()),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			true,
			fastTimeout));

		try {
			future.get();
			fail("We expected a TimeoutException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException);
		}

		// wait until we have timed out the slot request
		slotRequestTimeoutFuture.get();

		assertEquals(0L, pool.getNumberOfPendingRequests());

		AllocationID allocationId = allocationIdFuture.get();
		final SlotOffer slotOffer = new SlotOffer(
			allocationId,
			0,
			DEFAULT_TESTING_PROFILE);
		final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
		final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();

		testMainThreadExecutor.execute(() -> pool.registerTaskManager(taskManagerLocation.getResourceID()));

		assertTrue(testMainThreadExecutor.execute(() -> pool.offerSlot(taskManagerLocation, taskManagerGateway, slotOffer)));

		assertTrue(pool.containsAvailableSlot(allocationId));
	}
}
Example 20
Source File: SlotPoolImplTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that idle slots which cannot be released will be discarded. See FLINK-11059.
 */
@Test
public void testDiscardIdleSlotIfReleasingFailed() throws Exception {
	final ManualClock clock = new ManualClock();

	try (SlotPoolImpl slotPool = new SlotPoolImpl(
		jobId,
		clock,
		TestingUtils.infiniteTime(),
		timeout)) {

		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);

		final AllocationID expiredAllocationId = new AllocationID();
		final SlotOffer slotToExpire = new SlotOffer(expiredAllocationId, 0, ResourceProfile.UNKNOWN);

		OneShotLatch freeSlotLatch = new OneShotLatch();
		taskManagerGateway.setFreeSlotFunction((AllocationID allocationId, Throwable cause) -> {
			freeSlotLatch.trigger();
			return FutureUtils.completedExceptionally(new TimeoutException("Test failure"));
		});

		assertThat(slotPool.registerTaskManager(taskManagerLocation.getResourceID()), Matchers.is(true));

		assertThat(slotPool.offerSlot(taskManagerLocation, taskManagerGateway, slotToExpire), Matchers.is(true));

		clock.advanceTime(timeout.toMilliseconds() + 1, TimeUnit.MILLISECONDS);

		slotPool.triggerCheckIdleSlot();

		freeSlotLatch.await();

		CompletableFuture<LogicalSlot> allocatedSlotFuture = allocateSlot(scheduler, new SlotRequestId());

		try {
			// since the slot must have been discarded, we cannot fulfill the slot request
			allocatedSlotFuture.get(10L, TimeUnit.MILLISECONDS);
			fail("Expected to fail with a timeout.");
		} catch (TimeoutException ignored) {
			// expected
			assertEquals(0, slotPool.getAvailableSlots().size());
		}
	}
}