org.apache.flink.runtime.concurrent.ScheduledExecutorServiceAdapter Java Examples

MockResourceManagerRuntimeServices() throws Exception {
	highAvailabilityServices = new TestingHighAvailabilityServices();
	rmLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
	heartbeatServices = new TestingHeartbeatServices();
	metricRegistry = NoOpMetricRegistry.INSTANCE;
	slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(new ScheduledExecutorServiceAdapter(new DirectScheduledExecutorService()))
		.setTaskManagerRequestTimeout(Time.seconds(10))
		.setSlotRequestTimeout(Time.seconds(10))
		.setTaskManagerTimeout(Time.minutes(1))
		.build();
	jobLeaderIdService = new JobLeaderIdService(
			highAvailabilityServices,
			rpcService.getScheduledExecutor(),
			Time.minutes(5L));
}
 

@Test
public void testFailedJobResult() throws ExecutionException, InterruptedException {
	final int maxAttemptCounter = 1;
	final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
	try {
		final ScheduledExecutor scheduledExecutor = new ScheduledExecutorServiceAdapter(executor);

		final CallCountingJobStatusSupplier jobStatusSupplier = new CallCountingJobStatusSupplier(maxAttemptCounter);

		final CompletableFuture<JobResult> result = JobStatusPollingUtils.pollJobResultAsync(
				jobStatusSupplier,
				() -> CompletableFuture.completedFuture(createFailedJobResult(new JobID(0, 0))),
				scheduledExecutor,
				10
		);

		result.join();

		assertThat(jobStatusSupplier.getAttemptCounter(), is(equalTo(maxAttemptCounter)));
		assertTrue(result.isDone() && result.get().getSerializedThrowable().isPresent());

	} finally {
		ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, executor);
	}
}
 

@Test
public void testHappyPath() throws ExecutionException, InterruptedException {
	final int maxAttemptCounter = 1;
	final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
	try {
		final ScheduledExecutor scheduledExecutor = new ScheduledExecutorServiceAdapter(executor);

		final CallCountingJobStatusSupplier jobStatusSupplier = new CallCountingJobStatusSupplier(maxAttemptCounter);

		final CompletableFuture<JobResult> result = JobStatusPollingUtils.pollJobResultAsync(
				jobStatusSupplier,
				() -> CompletableFuture.completedFuture(createSuccessfulJobResult(new JobID(0, 0))),
				scheduledExecutor,
				10
		);

		result.join();

		assertThat(jobStatusSupplier.getAttemptCounter(), is(equalTo(maxAttemptCounter)));
		assertTrue(result.isDone() && result.get().isSuccess());

	} finally {
		ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, executor);
	}
}
 

@Test
public void testPolling() {
	final int maxAttemptCounter = 3;
	final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
	try {
		final ScheduledExecutor scheduledExecutor = new ScheduledExecutorServiceAdapter(executor);

		final CallCountingJobStatusSupplier jobStatusSupplier = new CallCountingJobStatusSupplier(maxAttemptCounter);

		final CompletableFuture<JobResult> result = JobStatusPollingUtils.pollJobResultAsync(
				jobStatusSupplier,
				() -> CompletableFuture.completedFuture(createSuccessfulJobResult(new JobID(0, 0))),
				scheduledExecutor,
				10
		);

		result.join();

		assertThat(jobStatusSupplier.getAttemptCounter(), is(equalTo(maxAttemptCounter)));

	} finally {
		ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, executor);
	}
}
 

private static <X> X fetchMetric(final SupplierWithException<CompletableFuture<X>, IOException> clientOperation, final Predicate<X> predicate) throws InterruptedException, ExecutionException, TimeoutException {
	final CompletableFuture<X> responseFuture = FutureUtils.retrySuccessfulWithDelay(() -> {
			try {
				return clientOperation.get();
			} catch (IOException e) {
				throw new RuntimeException(e);
			}
		},
		Time.seconds(1),
		Deadline.fromNow(Duration.ofSeconds(5)),
		predicate,
		new ScheduledExecutorServiceAdapter(scheduledExecutorService));

	return responseFuture.get(30, TimeUnit.SECONDS);
}
 

private void waitForTaskManagers(int numberOfTaskManagers, DispatcherGateway dispatcherGateway, FiniteDuration timeLeft) throws ExecutionException, InterruptedException {
	FutureUtils.retrySuccessfulWithDelay(
		() -> dispatcherGateway.requestClusterOverview(Time.milliseconds(timeLeft.toMillis())),
		Time.milliseconds(50L),
		org.apache.flink.api.common.time.Deadline.fromNow(Duration.ofMillis(timeLeft.toMillis())),
		clusterOverview -> clusterOverview.getNumTaskManagersConnected() >= numberOfTaskManagers,
		new ScheduledExecutorServiceAdapter(Executors.newSingleThreadScheduledExecutor()))
		.get();
}
 

private <C> CompletableFuture<C> retry(
		CheckedSupplier<CompletableFuture<C>> operation,
		Predicate<Throwable> retryPredicate) {
	return FutureUtils.retryWithDelay(
		CheckedSupplier.unchecked(operation),
		restClusterClientConfiguration.getRetryMaxAttempts(),
		Time.milliseconds(restClusterClientConfiguration.getRetryDelay()),
		retryPredicate,
		new ScheduledExecutorServiceAdapter(retryExecutorService));
}
 

public MockResourceManagerRuntimeServices(RpcService rpcService, Time timeout) {
	this(rpcService, timeout, SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(new ScheduledExecutorServiceAdapter(new DirectScheduledExecutorService()))
		.setTaskManagerRequestTimeout(Time.seconds(10))
		.setSlotRequestTimeout(Time.seconds(10))
		.setTaskManagerTimeout(Time.minutes(1))
		.build());
}
 

@BeforeClass
public static void setUp() throws Exception {
	scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
	final ScheduledExecutor scheduledExecutor = new ScheduledExecutorServiceAdapter(scheduledExecutorService);

	backPressureSampleService = new BackPressureSampleService(10, Time.milliseconds(10), scheduledExecutor);
}
 

/**
 * This test only fails eventually.
 */
@Test
public void discardingTriggeringCheckpointWillExecuteNextCheckpointRequest() throws Exception {
	final ExecutionVertex executionVertex = mockExecutionVertex(new ExecutionAttemptID());

	final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
	final CheckpointCoordinator checkpointCoordinator = new CheckpointCoordinatorTestingUtils.CheckpointCoordinatorBuilder()
		.setTasks(new ExecutionVertex[]{executionVertex})
		.setTimer(new ScheduledExecutorServiceAdapter(scheduledExecutorService))
		.setCheckpointCoordinatorConfiguration(CheckpointCoordinatorConfiguration.builder()
			.build())
		.build();

	final CompletableFuture<String> masterHookCheckpointFuture = new CompletableFuture<>();
	final OneShotLatch triggerCheckpointLatch = new OneShotLatch();
	checkpointCoordinator.addMasterHook(new TestingMasterHook(masterHookCheckpointFuture, triggerCheckpointLatch));

	try {
		checkpointCoordinator.triggerCheckpoint(false);
		final CompletableFuture<CompletedCheckpoint> secondCheckpoint = checkpointCoordinator.triggerCheckpoint(false);

		triggerCheckpointLatch.await();
		masterHookCheckpointFuture.complete("Completed");

		// discard triggering checkpoint
		checkpointCoordinator.abortPendingCheckpoints(new CheckpointException(CheckpointFailureReason.CHECKPOINT_DECLINED));

		try {
			// verify that the second checkpoint request will be executed and eventually times out
			secondCheckpoint.get();
			fail("Expected the second checkpoint to fail.");
		} catch (ExecutionException ee) {
			assertThat(ExceptionUtils.stripExecutionException(ee), instanceOf(CheckpointException.class));
		}
	} finally {
		checkpointCoordinator.shutdown(JobStatus.FINISHED);
		ExecutorUtils.gracefulShutdown(10L, TimeUnit.SECONDS, scheduledExecutorService);
	}
}
 

private void waitForTaskManagers(int numberOfTaskManagers, DispatcherGateway dispatcherGateway, Duration timeLeft) throws ExecutionException, InterruptedException {
	FutureUtils.retrySuccessfulWithDelay(
		() -> dispatcherGateway.requestClusterOverview(Time.milliseconds(timeLeft.toMillis())),
		Time.milliseconds(50L),
		org.apache.flink.api.common.time.Deadline.fromNow(Duration.ofMillis(timeLeft.toMillis())),
		clusterOverview -> clusterOverview.getNumTaskManagersConnected() >= numberOfTaskManagers,
		new ScheduledExecutorServiceAdapter(Executors.newSingleThreadScheduledExecutor()))
		.get();
}
 

private static <X> X fetchMetric(final SupplierWithException<CompletableFuture<X>, IOException> clientOperation, final Predicate<X> predicate) throws InterruptedException, ExecutionException, TimeoutException {
	final CompletableFuture<X> responseFuture = FutureUtils.retrySuccessfulWithDelay(() -> {
			try {
				return clientOperation.get();
			} catch (IOException e) {
				throw new RuntimeException(e);
			}
		},
		Time.seconds(1),
		Deadline.fromNow(Duration.ofSeconds(5)),
		predicate,
		new ScheduledExecutorServiceAdapter(scheduledExecutorService));

	return responseFuture.get(30, TimeUnit.SECONDS);
}
 

private <C> CompletableFuture<C> retry(
		CheckedSupplier<CompletableFuture<C>> operation,
		Predicate<Throwable> retryPredicate) {
	return FutureUtils.retryWithDelay(
		CheckedSupplier.unchecked(operation),
		restClusterClientConfiguration.getRetryMaxAttempts(),
		Time.milliseconds(restClusterClientConfiguration.getRetryDelay()),
		retryPredicate,
		new ScheduledExecutorServiceAdapter(retryExecutorService));
}
 

public MockResourceManagerRuntimeServices(RpcService rpcService, Time timeout) {
	this(rpcService, timeout, SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(new ScheduledExecutorServiceAdapter(new DirectScheduledExecutorService()))
		.setTaskManagerRequestTimeout(Time.seconds(10))
		.setSlotRequestTimeout(Time.seconds(10))
		.setTaskManagerTimeout(Time.minutes(1))
		.build());
}
 

@Before
public void setUp() throws Exception {
	scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
	final ScheduledExecutor scheduledExecutor = new ScheduledExecutorServiceAdapter(scheduledExecutorService);

	stackTraceSampleService = new StackTraceSampleService(scheduledExecutor);
}
 

private void waitForTaskManagers(int numberOfTaskManagers, DispatcherGateway dispatcherGateway, FiniteDuration timeLeft) throws ExecutionException, InterruptedException {
	FutureUtils.retrySuccessfulWithDelay(
		() -> dispatcherGateway.requestClusterOverview(Time.milliseconds(timeLeft.toMillis())),
		Time.milliseconds(50L),
		org.apache.flink.api.common.time.Deadline.fromNow(Duration.ofMillis(timeLeft.toMillis())),
		clusterOverview -> clusterOverview.getNumTaskManagersConnected() >= numberOfTaskManagers,
		new ScheduledExecutorServiceAdapter(Executors.newSingleThreadScheduledExecutor()))
		.get();
}
 

private static <X> X fetchMetric(final SupplierWithException<CompletableFuture<X>, IOException> clientOperation, final Predicate<X> predicate) throws InterruptedException, ExecutionException, TimeoutException {
	final CompletableFuture<X> responseFuture = FutureUtils.retrySuccessfulWithDelay(() -> {
			try {
				return clientOperation.get();
			} catch (IOException e) {
				throw new RuntimeException(e);
			}
		},
		Time.seconds(1),
		Deadline.fromNow(Duration.ofSeconds(5)),
		predicate,
		new ScheduledExecutorServiceAdapter(scheduledExecutorService));

	return responseFuture.get(30, TimeUnit.SECONDS);
}
 

private <C> CompletableFuture<C> retry(
		CheckedSupplier<CompletableFuture<C>> operation,
		Predicate<Throwable> retryPredicate) {
	return FutureUtils.retryWithDelay(
		CheckedSupplier.unchecked(operation),
		restClusterClientConfiguration.getRetryMaxAttempts(),
		Time.milliseconds(restClusterClientConfiguration.getRetryDelay()),
		retryPredicate,
		new ScheduledExecutorServiceAdapter(retryExecutorService));
}
 

@Before
public void setUp() throws Exception {
	scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
	final ScheduledExecutor scheduledExecutor = new ScheduledExecutorServiceAdapter(scheduledExecutorService);

	stackTraceSampleService = new StackTraceSampleService(scheduledExecutor);
}
 

@Override
public SchedulerNG createInstance(
		final Logger log,
		final JobGraph jobGraph,
		final BackPressureStatsTracker backPressureStatsTracker,
		final Executor ioExecutor,
		final Configuration jobMasterConfiguration,
		final SlotProvider slotProvider,
		final ScheduledExecutorService futureExecutor,
		final ClassLoader userCodeLoader,
		final CheckpointRecoveryFactory checkpointRecoveryFactory,
		final Time rpcTimeout,
		final BlobWriter blobWriter,
		final JobManagerJobMetricGroup jobManagerJobMetricGroup,
		final Time slotRequestTimeout,
		final ShuffleMaster<?> shuffleMaster,
		final JobMasterPartitionTracker partitionTracker) throws Exception {

	final SchedulingStrategyFactory schedulingStrategyFactory = createSchedulingStrategyFactory(jobGraph.getScheduleMode());
	final RestartBackoffTimeStrategy restartBackoffTimeStrategy = RestartBackoffTimeStrategyFactoryLoader
		.createRestartBackoffTimeStrategyFactory(
			jobGraph
				.getSerializedExecutionConfig()
				.deserializeValue(userCodeLoader)
				.getRestartStrategy(),
			jobMasterConfiguration,
			jobGraph.isCheckpointingEnabled())
		.create();
	log.info("Using restart back off time strategy {} for {} ({}).", restartBackoffTimeStrategy, jobGraph.getName(), jobGraph.getJobID());

	final ExecutionSlotAllocatorFactory slotAllocatorFactory =
		createExecutionSlotAllocatorFactory(
			jobGraph.getScheduleMode(),
			slotProvider,
			slotRequestTimeout,
			schedulingStrategyFactory);

	return new DefaultScheduler(
		log,
		jobGraph,
		backPressureStatsTracker,
		ioExecutor,
		jobMasterConfiguration,
		futureExecutor,
		new ScheduledExecutorServiceAdapter(futureExecutor),
		userCodeLoader,
		checkpointRecoveryFactory,
		rpcTimeout,
		blobWriter,
		jobManagerJobMetricGroup,
		shuffleMaster,
		partitionTracker,
		schedulingStrategyFactory,
		FailoverStrategyFactoryLoader.loadFailoverStrategyFactory(jobMasterConfiguration),
		restartBackoffTimeStrategy,
		new DefaultExecutionVertexOperations(),
		new ExecutionVertexVersioner(),
		slotAllocatorFactory);
}
 

/**
 * Tests that the heartbeat target {@link ResourceID} is properly passed to the {@link HeartbeatListener} by the
 * {@link HeartbeatManagerSenderImpl}.
 */
@Test
public void testHeartbeatManagerSenderTargetPayload() throws Exception {
	final long heartbeatTimeout = 100L;
	final long heartbeatPeriod = 2000L;

	final ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(1);

	final ResourceID someTargetId = ResourceID.generate();
	final ResourceID specialTargetId = ResourceID.generate();

	final OneShotLatch someTargetReceivedLatch = new OneShotLatch();
	final OneShotLatch specialTargetReceivedLatch = new OneShotLatch();

	final TargetDependentHeartbeatReceiver someHeartbeatTarget = new TargetDependentHeartbeatReceiver(someTargetReceivedLatch);
	final TargetDependentHeartbeatReceiver specialHeartbeatTarget = new TargetDependentHeartbeatReceiver(specialTargetReceivedLatch);

	final int defaultResponse = 0;
	final int specialResponse = 1;

	HeartbeatManager<?, Integer> heartbeatManager = new HeartbeatManagerSenderImpl<>(
		heartbeatPeriod,
		heartbeatTimeout,
		ResourceID.generate(),
		new TargetDependentHeartbeatSender(specialTargetId, specialResponse, defaultResponse),
		new ScheduledExecutorServiceAdapter(scheduledThreadPoolExecutor),
		LOG);

	try {
		heartbeatManager.monitorTarget(someTargetId, someHeartbeatTarget);
		heartbeatManager.monitorTarget(specialTargetId, specialHeartbeatTarget);

		someTargetReceivedLatch.await(5, TimeUnit.SECONDS);
		specialTargetReceivedLatch.await(5, TimeUnit.SECONDS);

		assertEquals(defaultResponse, someHeartbeatTarget.getLastRequestedHeartbeatPayload());
		assertEquals(specialResponse, specialHeartbeatTarget.getLastRequestedHeartbeatPayload());
	} finally {
		heartbeatManager.stop();
		scheduledThreadPoolExecutor.shutdown();
	}
}
 

/**
 * Tests that the heartbeat target {@link ResourceID} is properly passed to the {@link HeartbeatListener} by the
 * {@link HeartbeatManagerSenderImpl}.
 */
@Test
public void testHeartbeatManagerSenderTargetPayload() throws Exception {
	final long heartbeatTimeout = 100L;
	final long heartbeatPeriod = 2000L;

	final ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(1);

	final ResourceID someTargetId = ResourceID.generate();
	final ResourceID specialTargetId = ResourceID.generate();

	final OneShotLatch someTargetReceivedLatch = new OneShotLatch();
	final OneShotLatch specialTargetReceivedLatch = new OneShotLatch();

	final TargetDependentHeartbeatReceiver someHeartbeatTarget = new TargetDependentHeartbeatReceiver(someTargetReceivedLatch);
	final TargetDependentHeartbeatReceiver specialHeartbeatTarget = new TargetDependentHeartbeatReceiver(specialTargetReceivedLatch);

	final int defaultResponse = 0;
	final int specialResponse = 1;

	HeartbeatManager<?, Integer> heartbeatManager = new HeartbeatManagerSenderImpl<>(
		heartbeatPeriod,
		heartbeatTimeout,
		ResourceID.generate(),
		new TargetDependentHeartbeatSender(specialTargetId, specialResponse, defaultResponse),
		new ScheduledExecutorServiceAdapter(scheduledThreadPoolExecutor),
		LOG);

	try {
		heartbeatManager.monitorTarget(someTargetId, someHeartbeatTarget);
		heartbeatManager.monitorTarget(specialTargetId, specialHeartbeatTarget);

		someTargetReceivedLatch.await(5, TimeUnit.SECONDS);
		specialTargetReceivedLatch.await(5, TimeUnit.SECONDS);

		assertEquals(defaultResponse, someHeartbeatTarget.getLastRequestedHeartbeatPayload());
		assertEquals(specialResponse, specialHeartbeatTarget.getLastRequestedHeartbeatPayload());
	} finally {
		heartbeatManager.stop();
		scheduledThreadPoolExecutor.shutdown();
	}
}
 

/**
 * Tests that the heartbeat target {@link ResourceID} is properly passed to the {@link HeartbeatListener} by the
 * {@link HeartbeatManagerSenderImpl}.
 */
@Test
public void testHeartbeatManagerSenderTargetPayload() throws Exception {
	final long heartbeatTimeout = 100L;
	final long heartbeatPeriod = 2000L;

	final ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(1);

	final ResourceID someTargetId = ResourceID.generate();
	final ResourceID specialTargetId = ResourceID.generate();

	final OneShotLatch someTargetReceivedLatch = new OneShotLatch();
	final OneShotLatch specialTargetReceivedLatch = new OneShotLatch();

	final TargetDependentHeartbeatReceiver someHeartbeatTarget = new TargetDependentHeartbeatReceiver(someTargetReceivedLatch);
	final TargetDependentHeartbeatReceiver specialHeartbeatTarget = new TargetDependentHeartbeatReceiver(specialTargetReceivedLatch);

	final int defaultResponse = 0;
	final int specialResponse = 1;

	HeartbeatManager<?, Integer> heartbeatManager = new HeartbeatManagerSenderImpl<>(
		heartbeatPeriod,
		heartbeatTimeout,
		ResourceID.generate(),
		new TargetDependentHeartbeatSender(specialTargetId, specialResponse, defaultResponse),
		new ScheduledExecutorServiceAdapter(scheduledThreadPoolExecutor),
		LOG);

	try {
		heartbeatManager.monitorTarget(someTargetId, someHeartbeatTarget);
		heartbeatManager.monitorTarget(specialTargetId, specialHeartbeatTarget);

		someTargetReceivedLatch.await(5, TimeUnit.SECONDS);
		specialTargetReceivedLatch.await(5, TimeUnit.SECONDS);

		assertEquals(defaultResponse, someHeartbeatTarget.getLastRequestedHeartbeatPayload());
		assertEquals(specialResponse, specialHeartbeatTarget.getLastRequestedHeartbeatPayload());
	} finally {
		heartbeatManager.stop();
		scheduledThreadPoolExecutor.shutdown();
	}
}