org.apache.flink.runtime.concurrent.FutureUtils Java Examples

@Override
public CompletableFuture<Acknowledge> confirmCheckpoint(
		ExecutionAttemptID executionAttemptID,
		long checkpointId,
		long checkpointTimestamp) {
	log.debug("Confirm checkpoint {}@{} for {}.", checkpointId, checkpointTimestamp, executionAttemptID);

	final Task task = taskSlotTable.getTask(executionAttemptID);

	if (task != null) {
		task.notifyCheckpointComplete(checkpointId);

		return CompletableFuture.completedFuture(Acknowledge.get());
	} else {
		final String message = "TaskManager received a checkpoint confirmation for unknown task " + executionAttemptID + '.';

		log.debug(message);
		return FutureUtils.completedExceptionally(new CheckpointException(message));
	}
}
 

/**
 * Copies all the files from the given stream state handles to the given path, renaming the files w.r.t. their
 * {@link StateHandleID}.
 */
private void downloadDataForAllStateHandles(
	Map<StateHandleID, StreamStateHandle> stateHandleMap,
	Path restoreInstancePath,
	CloseableRegistry closeableRegistry) throws Exception {

	try {
		List<Runnable> runnables = createDownloadRunnables(stateHandleMap, restoreInstancePath, closeableRegistry);
		List<CompletableFuture<Void>> futures = new ArrayList<>(runnables.size());
		for (Runnable runnable : runnables) {
			futures.add(CompletableFuture.runAsync(runnable, executorService));
		}
		FutureUtils.waitForAll(futures).get();
	} catch (ExecutionException e) {
		Throwable throwable = ExceptionUtils.stripExecutionException(e);
		throwable = ExceptionUtils.stripException(throwable, RuntimeException.class);
		if (throwable instanceof IOException) {
			throw (IOException) throwable;
		} else {
			throw new FlinkRuntimeException("Failed to download data for state handles.", e);
		}
	}
}
 

@Test
public void testSnapshotEmpty() throws Exception {
	final AbstractStateBackend abstractStateBackend = new MemoryStateBackend(4096);
	CloseableRegistry cancelStreamRegistry = new CloseableRegistry();

	final OperatorStateBackend operatorStateBackend =
			abstractStateBackend.createOperatorStateBackend(createMockEnvironment(), "testOperator", emptyStateHandles, cancelStreamRegistry);

	CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(4096);

	RunnableFuture<SnapshotResult<OperatorStateHandle>> snapshot =
			operatorStateBackend.snapshot(0L, 0L, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation());

	SnapshotResult<OperatorStateHandle> snapshotResult = FutureUtils.runIfNotDoneAndGet(snapshot);
	OperatorStateHandle stateHandle = snapshotResult.getJobManagerOwnedSnapshot();
	assertNull(stateHandle);
}
 

@Override
public CompletableFuture<Acknowledge> confirmCheckpoint(
		ExecutionAttemptID executionAttemptID,
		long checkpointId,
		long checkpointTimestamp) {
	log.debug("Confirm checkpoint {}@{} for {}.", checkpointId, checkpointTimestamp, executionAttemptID);

	final Task task = taskSlotTable.getTask(executionAttemptID);

	if (task != null) {
		task.notifyCheckpointComplete(checkpointId);

		return CompletableFuture.completedFuture(Acknowledge.get());
	} else {
		final String message = "TaskManager received a checkpoint confirmation for unknown task " + executionAttemptID + '.';

		log.debug(message);
		return FutureUtils.completedExceptionally(new CheckpointException(message, CheckpointFailureReason.UNKNOWN_TASK_CHECKPOINT_NOTIFICATION_FAILURE));
	}
}
 

/**
 * Triggers a savepoint with the given savepoint directory as a target.
 *
 * @param timestamp The timestamp for the savepoint.
 * @param targetLocation Target location for the savepoint, optional. If null, the
 *                       state backend's configured default will be used.
 * @return A future to the completed checkpoint
 * @throws IllegalStateException If no savepoint directory has been
 *                               specified and no default savepoint directory has been
 *                               configured
 */
public CompletableFuture<CompletedCheckpoint> triggerSavepoint(
		long timestamp,
		@Nullable String targetLocation) {

	CheckpointProperties props = CheckpointProperties.forSavepoint();

	CheckpointTriggerResult triggerResult = triggerCheckpoint(
		timestamp,
		props,
		targetLocation,
		false);

	if (triggerResult.isSuccess()) {
		return triggerResult.getPendingCheckpoint().getCompletionFuture();
	} else {
		Throwable cause = new CheckpointTriggerException("Failed to trigger savepoint.", triggerResult.getFailureReason());
		return FutureUtils.completedExceptionally(cause);
	}
}
 

@Nonnull
private RestClient createRestClient() throws ConfigurationException {
	return new RestClient(RestClientConfiguration.fromConfiguration(restConfig), executor) {
		@Override
		public <M extends MessageHeaders<R, P, U>, U extends MessageParameters, R extends RequestBody, P extends ResponseBody> CompletableFuture<P>
		sendRequest(
			final String targetAddress,
			final int targetPort,
			final M messageHeaders,
			final U messageParameters,
			final R request,
			final Collection<FileUpload> files) throws IOException {
			if (failHttpRequest.test(messageHeaders, messageParameters, request)) {
				return FutureUtils.completedExceptionally(new IOException("expected"));
			} else {
				return super.sendRequest(targetAddress, targetPort, messageHeaders, messageParameters, request, files);
			}
		}
	};
}
 

private CompletableFuture<Boolean> tryAcceptLeadershipAndRunJobs(UUID newLeaderSessionID, Collection<JobGraph> recoveredJobs) {
	final DispatcherId dispatcherId = DispatcherId.fromUuid(newLeaderSessionID);

	if (leaderElectionService.hasLeadership(newLeaderSessionID)) {
		log.debug("Dispatcher {} accepted leadership with fencing token {}. Start recovered jobs.", getAddress(), dispatcherId);
		setNewFencingToken(dispatcherId);

		Collection<CompletableFuture<?>> runFutures = new ArrayList<>(recoveredJobs.size());

		for (JobGraph recoveredJob : recoveredJobs) {
			final CompletableFuture<?> runFuture = waitForTerminatingJobManager(recoveredJob.getJobID(), recoveredJob, this::runJob);
			runFutures.add(runFuture);
		}

		return FutureUtils.waitForAll(runFutures).thenApply(ignored -> true);
	} else {
		log.debug("Dispatcher {} lost leadership before accepting it. Stop recovering jobs for fencing token {}.", getAddress(), dispatcherId);
		return CompletableFuture.completedFuture(false);
	}
}
 

@Override
public State terminate(AkkaRpcActor<?> akkaRpcActor) {
	akkaRpcActor.mainThreadValidator.enterMainThread();

	CompletableFuture<Void> terminationFuture;
	try {
		terminationFuture = akkaRpcActor.rpcEndpoint.onStop();
	} catch (Throwable t) {
		terminationFuture = FutureUtils.completedExceptionally(
			new AkkaRpcException(
				String.format("Failure while stopping RpcEndpoint %s.", akkaRpcActor.rpcEndpoint.getEndpointId()),
				t));
	} finally {
		akkaRpcActor.mainThreadValidator.exitMainThread();
	}

	// IMPORTANT: This only works if we don't use a restarting supervisor strategy. Otherwise
	// we would complete the future and let the actor system restart the actor with a completed
	// future.
	// Complete the termination future so that others know that we've stopped.

	terminationFuture.whenComplete((ignored, throwable) -> akkaRpcActor.stop(RpcEndpointTerminationResult.of(throwable)));

	return TerminatingState.INSTANCE;
}
 

@Test
public void testSavepointCompletedWithException() throws Exception {
	TestingRestfulGateway testingRestfulGateway = new TestingRestfulGateway.Builder()
			.setStopWithSavepointFunction((JobID jobId, String directory) -> FutureUtils.completedExceptionally(new RuntimeException("expected")))
			.build();

	final TriggerId triggerId = savepointTriggerHandler.handleRequest(
			triggerSavepointRequest(),
			testingRestfulGateway).get().getTriggerId();

	final AsynchronousOperationResult<SavepointInfo> savepointResponseBody = savepointStatusHandler.handleRequest(
			savepointStatusRequest(triggerId),
			testingRestfulGateway).get();

	assertThat(savepointResponseBody.queueStatus().getId(), equalTo(QueueStatus.Id.COMPLETED));
	assertThat(savepointResponseBody.resource(), notNullValue());
	assertThat(savepointResponseBody.resource().getFailureCause(), notNullValue());
	final Throwable savepointError = savepointResponseBody.resource()
			.getFailureCause()
			.deserializeError(ClassLoader.getSystemClassLoader());
	assertThat(savepointError.getMessage(), equalTo("expected"));
	assertThat(savepointError, instanceOf(RuntimeException.class));
}
 

@Test
public void testSavepointCompletedWithException() throws Exception {
	TestingRestfulGateway testingRestfulGateway = new TestingRestfulGateway.Builder()
		.setTriggerSavepointFunction((JobID jobId, String directory) -> FutureUtils.completedExceptionally(new RuntimeException("expected")))
		.build();

	final TriggerId triggerId = savepointTriggerHandler.handleRequest(
		triggerSavepointRequest(),
		testingRestfulGateway).get().getTriggerId();

	final AsynchronousOperationResult<SavepointInfo> savepointResponseBody = savepointStatusHandler.handleRequest(
		savepointStatusRequest(triggerId),
		testingRestfulGateway).get();

	assertThat(savepointResponseBody.queueStatus().getId(), equalTo(QueueStatus.Id.COMPLETED));
	assertThat(savepointResponseBody.resource(), notNullValue());
	assertThat(savepointResponseBody.resource().getFailureCause(), notNullValue());
	final Throwable savepointError = savepointResponseBody.resource()
		.getFailureCause()
		.deserializeError(ClassLoader.getSystemClassLoader());
	assertThat(savepointError.getMessage(), equalTo("expected"));
	assertThat(savepointError, instanceOf(RuntimeException.class));
}
 

private CompletableFuture<KvStateResponse> getState(
		final KvStateRequest request,
		final boolean forceUpdate) {

	return getKvStateLookupInfo(request.getJobId(), request.getStateName(), forceUpdate)
			.thenComposeAsync((Function<KvStateLocation, CompletableFuture<KvStateResponse>>) location -> {
				final int keyGroupIndex = KeyGroupRangeAssignment.computeKeyGroupForKeyHash(
						request.getKeyHashCode(), location.getNumKeyGroups());

				final InetSocketAddress serverAddress = location.getKvStateServerAddress(keyGroupIndex);
				if (serverAddress == null) {
					return FutureUtils.completedExceptionally(new UnknownKvStateKeyGroupLocationException(getServerName()));
				} else {
					// Query server
					final KvStateID kvStateId = location.getKvStateID(keyGroupIndex);
					final KvStateInternalRequest internalRequest = new KvStateInternalRequest(
							kvStateId, request.getSerializedKeyAndNamespace());
					return kvStateClient.sendRequest(serverAddress, internalRequest);
				}
			}, queryExecutor);
}
 

@Override
public CompletableFuture<TaskManagerInfo> requestTaskManagerInfo(ResourceID resourceId, Time timeout) {

	final WorkerRegistration<WorkerType> taskExecutor = taskExecutors.get(resourceId);

	if (taskExecutor == null) {
		return FutureUtils.completedExceptionally(new UnknownTaskExecutorException(resourceId));
	} else {
		final InstanceID instanceId = taskExecutor.getInstanceID();
		final TaskManagerInfo taskManagerInfo = new TaskManagerInfo(
			resourceId,
			taskExecutor.getTaskExecutorGateway().getAddress(),
			taskExecutor.getDataPort(),
			taskManagerHeartbeatManager.getLastHeartbeatFrom(resourceId),
			slotManager.getNumberRegisteredSlotsOf(instanceId),
			slotManager.getNumberFreeSlotsOf(instanceId),
			slotManager.getRegisteredResourceOf(instanceId),
			slotManager.getFreeResourceOf(instanceId),
			taskExecutor.getHardwareDescription());

		return CompletableFuture.completedFuture(taskManagerInfo);
	}
}
 

@Override
public CompletableFuture<Acknowledge> updatePartitions(ExecutionAttemptID executionAttemptID, Iterable<PartitionInfo> partitionInfos, Time timeout) {
	Preconditions.checkNotNull(executionAttemptID);
	Preconditions.checkNotNull(partitionInfos);

	TaskMessages.UpdatePartitionInfo updatePartitionInfoMessage = new TaskMessages.UpdateTaskMultiplePartitionInfos(
		executionAttemptID,
		partitionInfos);

	scala.concurrent.Future<Acknowledge> updatePartitionsResult = actorGateway.ask(
		updatePartitionInfoMessage,
		new FiniteDuration(timeout.getSize(), timeout.getUnit()))
		.mapTo(ClassTag$.MODULE$.<Acknowledge>apply(Acknowledge.class));

	return FutureUtils.toJava(updatePartitionsResult);
}
 

@Nonnull
private CompletionStage<Void> terminateRpcServices() {
	synchronized (lock) {
		final int numRpcServices = 1 + rpcServices.size();

		final Collection<CompletableFuture<?>> rpcTerminationFutures = new ArrayList<>(numRpcServices);

		rpcTerminationFutures.add(commonRpcService.stopService());

		for (RpcService rpcService : rpcServices) {
			rpcTerminationFutures.add(rpcService.stopService());
		}

		commonRpcService = null;
		rpcServices.clear();

		return FutureUtils.completeAll(rpcTerminationFutures);
	}
}
 

@Override
public CompletableFuture<Collection<PhysicalSlotRequest.Result>> allocatePhysicalSlots(
		final Collection<PhysicalSlotRequest> physicalSlotRequests,
		final Time timeout) {

	slotRequests.addAll(physicalSlotRequests);

	if (forceFailingSlotAllocation) {
		return FutureUtils.completedExceptionally(new Exception("Forced failure"));
	}

	final List<PhysicalSlotRequest.Result> results = new ArrayList<>(physicalSlotRequests.size());
	for (PhysicalSlotRequest request : physicalSlotRequests) {
		final PhysicalSlotRequest.Result result = new PhysicalSlotRequest.Result(
			request.getSlotRequestId(),
			createPhysicalSlot());
		results.add(result);
	}
	return CompletableFuture.completedFuture(results);
}
 

@Override
public CompletableFuture<JobResult> requestJobResult(JobID jobId, Time timeout) {
	final CompletableFuture<JobManagerRunner> jobManagerRunnerFuture = jobManagerRunnerFutures.get(jobId);

	if (jobManagerRunnerFuture == null) {
		final ArchivedExecutionGraph archivedExecutionGraph = archivedExecutionGraphStore.get(jobId);

		if (archivedExecutionGraph == null) {
			return FutureUtils.completedExceptionally(new FlinkJobNotFoundException(jobId));
		} else {
			return CompletableFuture.completedFuture(JobResult.createFrom(archivedExecutionGraph));
		}
	} else {
		return jobManagerRunnerFuture.thenCompose(JobManagerRunner::getResultFuture).thenApply(JobResult::createFrom);
	}
}
 

@Override
public CompletableFuture<Collection<SlotOffer>> offerSlots(
		final ResourceID taskManagerId,
		final Collection<SlotOffer> slots,
		final Time timeout) {

	Tuple2<TaskManagerLocation, TaskExecutorGateway> taskManager = registeredTaskManagers.get(taskManagerId);

	if (taskManager == null) {
		return FutureUtils.completedExceptionally(new Exception("Unknown TaskManager " + taskManagerId));
	}

	final TaskManagerLocation taskManagerLocation = taskManager.f0;
	final TaskExecutorGateway taskExecutorGateway = taskManager.f1;

	final RpcTaskManagerGateway rpcTaskManagerGateway = new RpcTaskManagerGateway(taskExecutorGateway, getFencingToken());

	return CompletableFuture.completedFuture(
		slotPool.offerSlots(
			taskManagerLocation,
			rpcTaskManagerGateway,
			slots));
}
 

@Override
public CompletableFuture<Void> closeAsync() {
	synchronized (lock) {
		if (!shutdown) {
			shutdown = true;

			final CompletableFuture<Void> taskManagerTerminationFuture = taskManager.closeAsync();

			final CompletableFuture<Void> serviceTerminationFuture = FutureUtils.composeAfterwards(
				taskManagerTerminationFuture,
				this::shutDownServices);

			serviceTerminationFuture.whenComplete(
				(Void ignored, Throwable throwable) -> {
					if (throwable != null) {
						terminationFuture.completeExceptionally(throwable);
					} else {
						terminationFuture.complete(null);
					}
				});
		}
	}

	return terminationFuture;
}
 

@Override
public CompletableFuture<KvStateLocation> requestKvStateLocation(final JobID jobId, final String registrationName) {
	// sanity check for the correct JobID
	if (jobGraph.getJobID().equals(jobId)) {
		if (log.isDebugEnabled()) {
			log.debug("Lookup key-value state for job {} with registration " +
				"name {}.", jobGraph.getJobID(), registrationName);
		}

		final KvStateLocationRegistry registry = executionGraph.getKvStateLocationRegistry();
		final KvStateLocation location = registry.getKvStateLocation(registrationName);
		if (location != null) {
			return CompletableFuture.completedFuture(location);
		} else {
			return FutureUtils.completedExceptionally(new UnknownKvStateLocation(registrationName));
		}
	} else {
		if (log.isDebugEnabled()) {
			log.debug("Request of key-value state location for unknown job {} received.", jobId);
		}
		return FutureUtils.completedExceptionally(new FlinkJobNotFoundException(jobId));
	}
}
 

@Override
public CompletableFuture<Collection<Tuple2<ResourceID, String>>> requestTaskManagerMetricQueryServicePaths(Time timeout) {
	final ArrayList<CompletableFuture<Optional<Tuple2<ResourceID, String>>>> metricQueryServicePathFutures = new ArrayList<>(taskExecutors.size());

	for (Map.Entry<ResourceID, WorkerRegistration<WorkerType>> workerRegistrationEntry : taskExecutors.entrySet()) {
		final ResourceID tmResourceId = workerRegistrationEntry.getKey();
		final WorkerRegistration<WorkerType> workerRegistration = workerRegistrationEntry.getValue();
		final TaskExecutorGateway taskExecutorGateway = workerRegistration.getTaskExecutorGateway();

		final CompletableFuture<Optional<Tuple2<ResourceID, String>>> metricQueryServicePathFuture = taskExecutorGateway
			.requestMetricQueryServiceAddress(timeout)
			.thenApply(optional -> optional.map(path -> Tuple2.of(tmResourceId, path)));

		metricQueryServicePathFutures.add(metricQueryServicePathFuture);
	}

	return FutureUtils.combineAll(metricQueryServicePathFutures).thenApply(
		collection -> collection
			.stream()
			.filter(Optional::isPresent)
			.map(Optional::get)
			.collect(Collectors.toList()));
}
 

/**
 * Tests that the send operation is being retried.
 */
@Test
public void testRetriableSendOperationIfConnectionErrorOrServiceUnavailable() throws Exception {
	final PingRestHandler pingRestHandler = new PingRestHandler(
		FutureUtils.completedExceptionally(new RestHandlerException("test exception", HttpResponseStatus.SERVICE_UNAVAILABLE)),
		CompletableFuture.completedFuture(EmptyResponseBody.getInstance()));

	try (final TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint(pingRestHandler)) {
		RestClusterClient<?> restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort());

		try {
			final AtomicBoolean firstPollFailed = new AtomicBoolean();
			failHttpRequest = (messageHeaders, messageParameters, requestBody) ->
				messageHeaders instanceof PingRestHandlerHeaders && !firstPollFailed.getAndSet(true);

			restClusterClient.sendRequest(PingRestHandlerHeaders.INSTANCE).get();
		} finally {
			restClusterClient.close();
		}
	}
}
 

public CompletableFuture<Execution> registerProducedPartitions(
		TaskManagerLocation location,
		boolean sendScheduleOrUpdateConsumersMessage) {

	assertRunningInJobMasterMainThread();

	return FutureUtils.thenApplyAsyncIfNotDone(
		registerProducedPartitions(vertex, location, attemptId, sendScheduleOrUpdateConsumersMessage),
		vertex.getExecutionGraph().getJobMasterMainThreadExecutor(),
		producedPartitionsCache -> {
			producedPartitions = producedPartitionsCache;
			startTrackingPartitions(location.getResourceID(), producedPartitionsCache.values());
			return this;
		});
}
 

private void requestSlotFromResourceManager(
		final ResourceManagerGateway resourceManagerGateway,
		final PendingRequest pendingRequest) {

	checkNotNull(resourceManagerGateway);
	checkNotNull(pendingRequest);

	log.info("Requesting new slot [{}] and profile {} from resource manager.", pendingRequest.getSlotRequestId(), pendingRequest.getResourceProfile());

	final AllocationID allocationId = new AllocationID();

	pendingRequests.put(pendingRequest.getSlotRequestId(), allocationId, pendingRequest);

	pendingRequest.getAllocatedSlotFuture().whenComplete(
		(AllocatedSlot allocatedSlot, Throwable throwable) -> {
			if (throwable != null || !allocationId.equals(allocatedSlot.getAllocationId())) {
				// cancel the slot request if there is a failure or if the pending request has
				// been completed with another allocated slot
				resourceManagerGateway.cancelSlotRequest(allocationId);
			}
		});

	CompletableFuture<Acknowledge> rmResponse = resourceManagerGateway.requestSlot(
		jobMasterId,
		new SlotRequest(jobId, allocationId, pendingRequest.getResourceProfile(), jobManagerAddress),
		rpcTimeout);

	FutureUtils.whenCompleteAsyncIfNotDone(
		rmResponse,
		componentMainThreadExecutor,
		(Acknowledge ignored, Throwable failure) -> {
			// on failure, fail the request future
			if (failure != null) {
				slotRequestToResourceManagerFailed(pendingRequest.getSlotRequestId(), failure);
			}
		});
}
 

@Test
public void testDisposeSavepointFailure() throws Exception {
	replaceStdOutAndStdErr();

	String savepointPath = "expectedSavepointPath";

	Exception testException = new Exception("expectedTestException");

	DisposeSavepointClusterClient clusterClient = new DisposeSavepointClusterClient((String path) -> FutureUtils.completedExceptionally(testException), getConfiguration());

	try {
		CliFrontend frontend = new MockedCliFrontend(clusterClient);

		String[] parameters = { "-d", savepointPath };

		try {
			frontend.savepoint(parameters);

			fail("Savepoint should have failed.");
		} catch (Exception e) {
			assertTrue(ExceptionUtils.findThrowableWithMessage(e, testException.getMessage()).isPresent());
		}
	}
	finally {
		clusterClient.shutdown();
		restoreStdOutAndStdErr();
	}
}
 

private CompletableFuture<TransientBlobKey> requestTaskManagerLog(TaskManagerMessages.RequestTaskManagerLog request, Time timeout) {
	Preconditions.checkNotNull(request);
	Preconditions.checkNotNull(timeout);

	scala.concurrent.Future<TransientBlobKey> blobKeyFuture = actorGateway
		.ask(
			request,
			new FiniteDuration(timeout.getSize(), timeout.getUnit()))
		.mapTo(ClassTag$.MODULE$.<TransientBlobKey>apply(TransientBlobKey.class));

	return FutureUtils.toJava(blobKeyFuture);
}
 

@Test
@SuppressWarnings("unchecked")
public void testRetryOnError() throws Exception {
	final String testId = "Petit a petit, l'oiseau fait son nid";
	final String testEndpointAddress = "<test-address>";
	final UUID leaderId = UUID.randomUUID();

	// gateway that upon calls first responds with a failure, then with a success
	TestRegistrationGateway testGateway = mock(TestRegistrationGateway.class);

	when(testGateway.registrationCall(any(UUID.class), anyLong())).thenReturn(
			FutureUtils.completedExceptionally(new Exception("test exception")),
			CompletableFuture.completedFuture(new TestRegistrationSuccess(testId)));

	rpcService.registerGateway(testEndpointAddress, testGateway);

	TestRetryingRegistration registration = new TestRetryingRegistration(rpcService, testEndpointAddress, leaderId);

	long started = System.nanoTime();
	registration.startRegistration();

	CompletableFuture<Tuple2<TestRegistrationGateway, TestRegistrationSuccess>> future = registration.getFuture();
	Tuple2<TestRegistrationGateway, TestRegistrationSuccess> success =
			future.get(10, TimeUnit.SECONDS);

	long finished = System.nanoTime();
	long elapsedMillis = (finished - started) / 1000000;

	assertEquals(testId, success.f1.getCorrelationId());

	// validate that some retry-delay / back-off behavior happened
	assertTrue("retries did not properly back off",
			elapsedMillis >= TestRetryingRegistration.DELAY_ON_ERROR);
}
 

@Override
public CompletableFuture<StackTraceSampleResponse> requestStackTraceSample(
		ExecutionAttemptID executionAttemptID,
		int sampleId,
		int numSamples,
		Time delayBetweenSamples,
		int maxStackTraceDepth,
		Time timeout) {
	return FutureUtils.completedExceptionally(new UnsupportedOperationException());
}
 

@Override
public CompletableFuture<Acknowledge> cancelTask(ExecutionAttemptID executionAttemptID, Time timeout) {
	index++;

	if (index >= successfulOperations) {
		return FutureUtils.completedExceptionally(new IOException("Rpc call fails"));
	} else {
		return CompletableFuture.completedFuture(Acknowledge.get());
	}
}
 

@Test
public void testFailedJobSubmission() throws Exception {
	final String errorMessage = "test";
	DispatcherGateway mockGateway = new TestingDispatcherGateway.Builder()
		.setSubmitFunction(jobgraph -> FutureUtils.completedExceptionally(new Exception(errorMessage)))
		.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();

	JobGraph jobGraph = new JobGraph("testjob");
	try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
		objectOut.writeObject(jobGraph);
	}
	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());

	try {
		handler.handleRequest(new HandlerRequest<>(
				request,
				EmptyMessageParameters.getInstance(),
				Collections.emptyMap(),
				Collections.emptyMap(),
				Collections.singletonList(jobGraphFile.toFile())), mockGateway)
			.get();
	} catch (Exception e) {
		Throwable t = ExceptionUtils.stripExecutionException(e);
		Assert.assertEquals(errorMessage, t.getMessage());
	}
}
 

@Override
public CompletableFuture<Void> restart(final RestartCallback restarter, ScheduledExecutor executor) {
	if (isRestartTimestampsQueueFull()) {
		restartTimestampsDeque.remove();
	}
	restartTimestampsDeque.add(System.currentTimeMillis());
	return FutureUtils.scheduleWithDelay(restarter::triggerFullRecovery, delayInterval, executor);
}