org.apache.flink.runtime.concurrent.ScheduledExecutor Java Examples

@VisibleForTesting
CompletableFuture<Void> fixJobIdAndRunApplicationAsync(
		final DispatcherGateway dispatcher,
		final ScheduledExecutor scheduledExecutor) {

	final Optional<String> configuredJobId =
			configuration.getOptional(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID);

	if (!HighAvailabilityMode.isHighAvailabilityModeActivated(configuration) && !configuredJobId.isPresent()) {
		return runApplicationAsync(dispatcher, scheduledExecutor, false);
	}

	if (!configuredJobId.isPresent()) {
		configuration.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, ZERO_JOB_ID.toHexString());
	}
	return runApplicationAsync(dispatcher, scheduledExecutor, true);
}
 

public HeartbeatManagerImpl(
		long heartbeatTimeoutIntervalMs,
		ResourceID ownResourceID,
		HeartbeatListener<I, O> heartbeatListener,
		ScheduledExecutor mainThreadExecutor,
		Logger log) {
	Preconditions.checkArgument(heartbeatTimeoutIntervalMs > 0L, "The heartbeat timeout has to be larger than 0.");

	this.heartbeatTimeoutIntervalMs = heartbeatTimeoutIntervalMs;
	this.ownResourceID = Preconditions.checkNotNull(ownResourceID);
	this.heartbeatListener = Preconditions.checkNotNull(heartbeatListener, "heartbeatListener");
	this.mainThreadExecutor = Preconditions.checkNotNull(mainThreadExecutor);
	this.log = Preconditions.checkNotNull(log);
	this.heartbeatTargets = new ConcurrentHashMap<>(16);

	stopped = false;
}
 

/**
 * Tests that the last heartbeat from an unregistered target equals -1.
 */
@Test
public void testLastHeartbeatFromUnregisteredTarget() {
	final long heartbeatTimeout = 100L;
	final ResourceID resourceId = ResourceID.generate();
	@SuppressWarnings("unchecked")
	final HeartbeatListener<Object, Object> heartbeatListener = mock(HeartbeatListener.class);

	HeartbeatManager<?, ?> heartbeatManager = new HeartbeatManagerImpl<>(
		heartbeatTimeout,
		resourceId,
		heartbeatListener,
		mock(ScheduledExecutor.class),
		LOG);

	try {
		assertEquals(-1L, heartbeatManager.getLastHeartbeatFrom(ResourceID.generate()));
	} finally {
		heartbeatManager.stop();
	}
}
 

HeartbeatMonitor(
	ResourceID resourceID,
	HeartbeatTarget<O> heartbeatTarget,
	ScheduledExecutor scheduledExecutor,
	HeartbeatListener<?, O> heartbeatListener,
	long heartbeatTimeoutIntervalMs) {

	this.resourceID = Preconditions.checkNotNull(resourceID);
	this.heartbeatTarget = Preconditions.checkNotNull(heartbeatTarget);
	this.scheduledExecutor = Preconditions.checkNotNull(scheduledExecutor);
	this.heartbeatListener = Preconditions.checkNotNull(heartbeatListener);

	Preconditions.checkArgument(heartbeatTimeoutIntervalMs > 0L, "The heartbeat timeout interval has to be larger than 0.");
	this.heartbeatTimeoutIntervalMs = heartbeatTimeoutIntervalMs;

	lastHeartbeat = 0L;

	resetHeartbeatTimeout(heartbeatTimeoutIntervalMs);
}
 

public static ResourceManagerRuntimeServices fromConfiguration(
		ResourceManagerRuntimeServicesConfiguration configuration,
		HighAvailabilityServices highAvailabilityServices,
		ScheduledExecutor scheduledExecutor,
		SlotManagerMetricGroup slotManagerMetricGroup) {

	final SlotManager slotManager = new SlotManagerImpl(
		scheduledExecutor,
		configuration.getSlotManagerConfiguration(),
		slotManagerMetricGroup);

	final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		scheduledExecutor,
		configuration.getJobTimeout());

	return new ResourceManagerRuntimeServices(slotManager, jobLeaderIdService);
}
 

HeartbeatManagerSenderImpl(
		long heartbeatPeriod,
		long heartbeatTimeout,
		ResourceID ownResourceID,
		HeartbeatListener<I, O> heartbeatListener,
		ScheduledExecutor mainThreadExecutor,
		Logger log) {
	super(
		heartbeatTimeout,
		ownResourceID,
		heartbeatListener,
		mainThreadExecutor,
		log);

	this.heartbeatPeriod = heartbeatPeriod;
	mainThreadExecutor.schedule(this, 0L, TimeUnit.MILLISECONDS);
}
 

public static ResourceManagerRuntimeServices fromConfiguration(
		ResourceManagerRuntimeServicesConfiguration configuration,
		HighAvailabilityServices highAvailabilityServices,
		ScheduledExecutor scheduledExecutor) throws Exception {

	final SlotManagerConfiguration slotManagerConfiguration = configuration.getSlotManagerConfiguration();

	final SlotManager slotManager = new SlotManager(
		scheduledExecutor,
		slotManagerConfiguration.getTaskManagerRequestTimeout(),
		slotManagerConfiguration.getSlotRequestTimeout(),
		slotManagerConfiguration.getTaskManagerTimeout(),
		slotManagerConfiguration.isWaitResultConsumedBeforeRelease());

	final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		scheduledExecutor,
		configuration.getJobTimeout());

	return new ResourceManagerRuntimeServices(slotManager, jobLeaderIdService);
}
 

/**
 * Tests a simple scheduled runnable being executed by the RPC services scheduled executor
 * service.
 */
@Test(timeout = 60000)
public void testScheduledExecutorServiceSimpleSchedule() throws Exception {
	ScheduledExecutor scheduledExecutor = akkaRpcService.getScheduledExecutor();

	final OneShotLatch latch = new OneShotLatch();

	ScheduledFuture<?> future = scheduledExecutor.schedule(
		latch::trigger,
		10L,
		TimeUnit.MILLISECONDS);

	future.get();

	// once the future is completed, then the latch should have been triggered
	assertTrue(latch.isTriggered());
}
 

HeartbeatMonitorImpl(
	ResourceID resourceID,
	HeartbeatTarget<O> heartbeatTarget,
	ScheduledExecutor scheduledExecutor,
	HeartbeatListener<?, O> heartbeatListener,
	long heartbeatTimeoutIntervalMs) {

	this.resourceID = Preconditions.checkNotNull(resourceID);
	this.heartbeatTarget = Preconditions.checkNotNull(heartbeatTarget);
	this.scheduledExecutor = Preconditions.checkNotNull(scheduledExecutor);
	this.heartbeatListener = Preconditions.checkNotNull(heartbeatListener);

	Preconditions.checkArgument(heartbeatTimeoutIntervalMs > 0L, "The heartbeat timeout interval has to be larger than 0.");
	this.heartbeatTimeoutIntervalMs = heartbeatTimeoutIntervalMs;

	lastHeartbeat = 0L;

	resetHeartbeatTimeout(heartbeatTimeoutIntervalMs);
}
 

HeartbeatMonitor(
	ResourceID resourceID,
	HeartbeatTarget<O> heartbeatTarget,
	ScheduledExecutor scheduledExecutor,
	HeartbeatListener<?, O> heartbeatListener,
	long heartbeatTimeoutIntervalMs) {

	this.resourceID = Preconditions.checkNotNull(resourceID);
	this.heartbeatTarget = Preconditions.checkNotNull(heartbeatTarget);
	this.scheduledExecutor = Preconditions.checkNotNull(scheduledExecutor);
	this.heartbeatListener = Preconditions.checkNotNull(heartbeatListener);

	Preconditions.checkArgument(heartbeatTimeoutIntervalMs > 0L, "The heartbeat timeout interval has to be larger than 0.");
	this.heartbeatTimeoutIntervalMs = heartbeatTimeoutIntervalMs;

	lastHeartbeat = 0L;

	resetHeartbeatTimeout(heartbeatTimeoutIntervalMs);
}
 

private static <K, S extends State, V> CompletableFuture<S> getKvState(
		final Deadline deadline,
		final QueryableStateClient client,
		final JobID jobId,
		final String queryName,
		final K key,
		final TypeInformation<K> keyTypeInfo,
		final StateDescriptor<S, V> stateDescriptor,
		final boolean failForUnknownKeyOrNamespace,
		final ScheduledExecutor executor) {

	final CompletableFuture<S> resultFuture = new CompletableFuture<>();
	getKvStateIgnoringCertainExceptions(
			deadline, resultFuture, client, jobId, queryName, key, keyTypeInfo,
			stateDescriptor, failForUnknownKeyOrNamespace, executor);
	return resultFuture;
}
 

HeartbeatManagerSenderImpl(
		long heartbeatPeriod,
		long heartbeatTimeout,
		ResourceID ownResourceID,
		HeartbeatListener<I, O> heartbeatListener,
		ScheduledExecutor mainThreadExecutor,
		Logger log) {
	this(
		heartbeatPeriod,
		heartbeatTimeout,
		ownResourceID,
		heartbeatListener,
		mainThreadExecutor,
		log,
		new HeartbeatMonitorImpl.Factory<>());
}
 

public HeartbeatManagerImpl(
		long heartbeatTimeoutIntervalMs,
		ResourceID ownResourceID,
		HeartbeatListener<I, O> heartbeatListener,
		ScheduledExecutor mainThreadExecutor,
		Logger log,
		HeartbeatMonitor.Factory<O> heartbeatMonitorFactory) {

	Preconditions.checkArgument(heartbeatTimeoutIntervalMs > 0L, "The heartbeat timeout has to be larger than 0.");

	this.heartbeatTimeoutIntervalMs = heartbeatTimeoutIntervalMs;
	this.ownResourceID = Preconditions.checkNotNull(ownResourceID);
	this.heartbeatListener = Preconditions.checkNotNull(heartbeatListener, "heartbeatListener");
	this.mainThreadExecutor = Preconditions.checkNotNull(mainThreadExecutor);
	this.log = Preconditions.checkNotNull(log);
	this.heartbeatMonitorFactory = heartbeatMonitorFactory;
	this.heartbeatTargets = new ConcurrentHashMap<>(16);

	stopped = false;
}
 

@Test
public void testFailedJobResult() throws ExecutionException, InterruptedException {
	final int maxAttemptCounter = 1;
	final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
	try {
		final ScheduledExecutor scheduledExecutor = new ScheduledExecutorServiceAdapter(executor);

		final CallCountingJobStatusSupplier jobStatusSupplier = new CallCountingJobStatusSupplier(maxAttemptCounter);

		final CompletableFuture<JobResult> result = JobStatusPollingUtils.pollJobResultAsync(
				jobStatusSupplier,
				() -> CompletableFuture.completedFuture(createFailedJobResult(new JobID(0, 0))),
				scheduledExecutor,
				10
		);

		result.join();

		assertThat(jobStatusSupplier.getAttemptCounter(), is(equalTo(maxAttemptCounter)));
		assertTrue(result.isDone() && result.get().getSerializedThrowable().isPresent());

	} finally {
		ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, executor);
	}
}
 

/**
 * Tests that the last heartbeat from an unregistered target equals -1.
 */
@Test
public void testLastHeartbeatFromUnregisteredTarget() {
	final long heartbeatTimeout = 100L;
	final ResourceID resourceId = ResourceID.generate();
	@SuppressWarnings("unchecked")
	final HeartbeatListener<Object, Object> heartbeatListener = mock(HeartbeatListener.class);

	HeartbeatManager<?, ?> heartbeatManager = new HeartbeatManagerImpl<>(
		heartbeatTimeout,
		resourceId,
		heartbeatListener,
		mock(ScheduledExecutor.class),
		LOG);

	try {
		assertEquals(-1L, heartbeatManager.getLastHeartbeatFrom(ResourceID.generate()));
	} finally {
		heartbeatManager.stop();
	}
}
 

private static <K, S extends State, V> CompletableFuture<S> getKvState(
		final Deadline deadline,
		final QueryableStateClient client,
		final JobID jobId,
		final String queryName,
		final K key,
		final TypeInformation<K> keyTypeInfo,
		final StateDescriptor<S, V> stateDescriptor,
		final boolean failForUnknownKeyOrNamespace,
		final ScheduledExecutor executor) {

	final CompletableFuture<S> resultFuture = new CompletableFuture<>();
	getKvStateIgnoringCertainExceptions(
			deadline, resultFuture, client, jobId, queryName, key, keyTypeInfo,
			stateDescriptor, failForUnknownKeyOrNamespace, executor);
	return resultFuture;
}
 

public RecordingHeartbeatManagerImpl(
		long heartbeatTimeoutIntervalMs,
		ResourceID ownResourceID,
		HeartbeatListener<I, O> heartbeatListener,
		ScheduledExecutor mainThreadExecutor,
		Logger log,
		BlockingQueue<ResourceID> unmonitoredTargets,
		BlockingQueue<ResourceID> monitoredTargets) {
	super(heartbeatTimeoutIntervalMs, ownResourceID, heartbeatListener, mainThreadExecutor, log);
	this.unmonitoredTargets = unmonitoredTargets;
	this.monitoredTargets = monitoredTargets;
}
 

static BackPressureSampleService createBackPressureSampleService(
		Configuration configuration,
		ScheduledExecutor scheduledExecutor) {
	return new BackPressureSampleService(
		configuration.getInteger(WebOptions.BACKPRESSURE_NUM_SAMPLES),
		Time.milliseconds(configuration.getInteger(WebOptions.BACKPRESSURE_DELAY)),
		scheduledExecutor);
}
 

public JobLeaderIdService(
		HighAvailabilityServices highAvailabilityServices,
		ScheduledExecutor scheduledExecutor,
		Time jobTimeout) {
	this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices, "highAvailabilityServices");
	this.scheduledExecutor = Preconditions.checkNotNull(scheduledExecutor, "scheduledExecutor");
	this.jobTimeout = Preconditions.checkNotNull(jobTimeout, "jobTimeout");

	jobLeaderIdListeners = new HashMap<>(4);

	jobLeaderIdActions = null;
}
 

public JobLeaderIdService(
		HighAvailabilityServices highAvailabilityServices,
		ScheduledExecutor scheduledExecutor,
		Time jobTimeout) {
	this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices, "highAvailabilityServices");
	this.scheduledExecutor = Preconditions.checkNotNull(scheduledExecutor, "scheduledExecutor");
	this.jobTimeout = Preconditions.checkNotNull(jobTimeout, "jobTimeout");

	jobLeaderIdListeners = new HashMap<>(4);

	jobLeaderIdActions = null;
}
 

/**
 * Tests that we can correctly retrieve the last heartbeat for registered targets.
 */
@Test
public void testLastHeartbeatFrom() {
	final long heartbeatTimeout = 100L;
	final ResourceID resourceId = ResourceID.generate();
	@SuppressWarnings("unchecked")
	final HeartbeatListener<Object, Object> heartbeatListener = mock(HeartbeatListener.class);
	@SuppressWarnings("unchecked")
	final HeartbeatTarget<Object> heartbeatTarget = mock(HeartbeatTarget.class);
	final ResourceID target = ResourceID.generate();

	HeartbeatManager<Object, Object> heartbeatManager = new HeartbeatManagerImpl<>(
		heartbeatTimeout,
		resourceId,
		heartbeatListener,
		mock(ScheduledExecutor.class),
		LOG);

	try {
		heartbeatManager.monitorTarget(target, heartbeatTarget);

		assertEquals(0L, heartbeatManager.getLastHeartbeatFrom(target));

		final long currentTime = System.currentTimeMillis();

		heartbeatManager.receiveHeartbeat(target, null);

		assertTrue(heartbeatManager.getLastHeartbeatFrom(target) >= currentTime);
	} finally {
		heartbeatManager.stop();
	}
}
 

public SlotManagerImpl(
		ScheduledExecutor scheduledExecutor,
		Time taskManagerRequestTimeout,
		Time slotRequestTimeout,
		Time taskManagerTimeout,
		boolean waitResultConsumedBeforeRelease) {

	this.scheduledExecutor = Preconditions.checkNotNull(scheduledExecutor);
	this.taskManagerRequestTimeout = Preconditions.checkNotNull(taskManagerRequestTimeout);
	this.slotRequestTimeout = Preconditions.checkNotNull(slotRequestTimeout);
	this.taskManagerTimeout = Preconditions.checkNotNull(taskManagerTimeout);
	this.waitResultConsumedBeforeRelease = waitResultConsumedBeforeRelease;

	slots = new HashMap<>(16);
	freeSlots = new LinkedHashMap<>(16);
	taskManagerRegistrations = new HashMap<>(4);
	fulfilledSlotRequests = new HashMap<>(16);
	pendingSlotRequests = new HashMap<>(16);
	pendingSlots = new HashMap<>(16);

	resourceManagerId = null;
	resourceActions = null;
	mainThreadExecutor = null;
	taskManagerTimeoutCheck = null;
	slotRequestTimeoutCheck = null;

	started = false;
}
 

@VisibleForTesting
static CompletableFuture<JobResult> pollJobResultAsync(
		final Supplier<CompletableFuture<JobStatus>> jobStatusSupplier,
		final Supplier<CompletableFuture<JobResult>> jobResultSupplier,
		final ScheduledExecutor scheduledExecutor,
		final long retryMsTimeout) {
	return pollJobResultAsync(jobStatusSupplier, jobResultSupplier, scheduledExecutor, new CompletableFuture<>(), retryMsTimeout, 0);
}
 

/**
 * Tests that the initial job registration registers a timeout which will call
 * {@link JobLeaderIdActions#notifyJobTimeout(JobID, UUID)} when executed.
 */
@Test
public void testInitialJobTimeout() throws Exception {
	final JobID jobId = new JobID();
	TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
	SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService(
		null,
		null);

	highAvailabilityServices.setJobMasterLeaderRetriever(jobId, leaderRetrievalService);

	ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
	Time timeout = Time.milliseconds(5000L);
	JobLeaderIdActions jobLeaderIdActions = mock(JobLeaderIdActions.class);

	JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		scheduledExecutor,
		timeout);

	jobLeaderIdService.start(jobLeaderIdActions);

	jobLeaderIdService.addJob(jobId);

	assertTrue(jobLeaderIdService.containsJob(jobId));

	ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
	verify(scheduledExecutor).schedule(runnableArgumentCaptor.capture(), anyLong(), any(TimeUnit.class));

	Runnable timeoutRunnable = runnableArgumentCaptor.getValue();
	timeoutRunnable.run();

	ArgumentCaptor<UUID> timeoutIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);

	verify(jobLeaderIdActions, times(1)).notifyJobTimeout(eq(jobId), timeoutIdArgumentCaptor.capture());

	assertTrue(jobLeaderIdService.isValidTimeout(jobId, timeoutIdArgumentCaptor.getValue()));
}
 

@Override
public CompletableFuture<Void> restart(RestartCallback restarter, ScheduledExecutor executor) {
	++restartedTimes;

	if (restartedTimes <= numberOfFailures) {
		return FutureUtils.completedExceptionally(new FlinkRuntimeException("Fail to restart for " + restartedTimes + " time(s)."));
	} else {
		return FutureUtils.scheduleWithDelay(restarter::triggerFullRecovery, Time.milliseconds(0L), executor);
	}
}
 

public YarnApplicationStatusMonitor(
		YarnClient yarnClient,
		ApplicationId yarnApplicationId,
		ScheduledExecutor scheduledExecutor) {
	this.yarnClient = Preconditions.checkNotNull(yarnClient);
	this.yarnApplicationId = Preconditions.checkNotNull(yarnApplicationId);

	applicationStatusUpdateFuture = scheduledExecutor.scheduleWithFixedDelay(
		this::updateApplicationStatus,
		0L,
		UPDATE_INTERVAL,
		TimeUnit.MILLISECONDS);

	applicationStatus = ApplicationStatus.UNKNOWN;
}
 

TestingHeartbeatMonitor(
	ResourceID resourceID,
	HeartbeatTarget<O> heartbeatTarget,
	ScheduledExecutor scheduledExecutor,
	HeartbeatListener<?, O> heartbeatListener,
	long heartbeatTimeoutIntervalMs) {

	super(resourceID, heartbeatTarget, scheduledExecutor, heartbeatListener, heartbeatTimeoutIntervalMs);
}
 

/**
 * Tests that the RPC service's scheduled executor service can execute runnable with a fixed
 * delay.
 */
@Test(timeout = 60000)
public void testScheduledExecutorServiceWithFixedDelaySchedule() throws Exception {
	ScheduledExecutor scheduledExecutor = akkaRpcService.getScheduledExecutor();

	final int tries = 4;
	final long delay = 10L;
	final CountDownLatch countDownLatch = new CountDownLatch(tries);

	long currentTime = System.nanoTime();

	ScheduledFuture<?> future = scheduledExecutor.scheduleWithFixedDelay(
		countDownLatch::countDown,
		delay,
		delay,
		TimeUnit.MILLISECONDS);

	assertTrue(!future.isDone());

	countDownLatch.await();

	// the future should not complete since we have a periodic task
	assertTrue(!future.isDone());

	long finalTime = System.nanoTime() - currentTime;

	// the processing should have taken at least delay times the number of count downs.
	assertTrue(finalTime >= tries * delay);

	future.cancel(true);
}
 

@Override
public void restart(RestartCallback restarter, ScheduledExecutor executor) {

	++restartAttempts;
	ExecutorAction executorAction = new ExecutorAction(restarter::triggerFullRecovery, executor);
	if (manuallyTriggeredExecution) {
		synchronized (actionsQueue) {
			actionsQueue.add(executorAction);
		}
	} else {
		executorAction.trigger();
	}
}
 

/**
 * Creates a heartbeat manager which does not actively send heartbeats.
 *
 * @param resourceId Resource Id which identifies the owner of the heartbeat manager
 * @param heartbeatListener Listener which will be notified upon heartbeat timeouts for registered
 *                          targets
 * @param mainThreadExecutor Scheduled executor to be used for scheduling heartbeat timeouts
 * @param log Logger to be used for the logging
 * @param <I> Type of the incoming payload
 * @param <O> Type of the outgoing payload
 * @return A new HeartbeatManager instance
 */
public <I, O> HeartbeatManager<I, O> createHeartbeatManager(
	ResourceID resourceId,
	HeartbeatListener<I, O> heartbeatListener,
	ScheduledExecutor mainThreadExecutor,
	Logger log) {

	return new HeartbeatManagerImpl<>(
		heartbeatTimeout,
		resourceId,
		heartbeatListener,
		mainThreadExecutor,
		log);
}