org.apache.flink.runtime.webmonitor.RestfulGateway Java Examples

@Test
public void testShortUpdateInterval() throws InterruptedException {
	final long updateInterval = 1L;
	final AtomicInteger requestMetricQueryServiceGatewaysCounter = new AtomicInteger(0);
	final RestfulGateway restfulGateway = createRestfulGateway(requestMetricQueryServiceGatewaysCounter);

	final MetricFetcher fetcher = createMetricFetcher(updateInterval, restfulGateway);

	fetcher.update();

	final long start = System.currentTimeMillis();
	long difference = 0L;

	while (difference <= updateInterval) {
		Thread.sleep(2L * updateInterval);
		difference = System.currentTimeMillis() - start;
	}

	fetcher.update();

	assertThat(requestMetricQueryServiceGatewaysCounter.get(), is(2));
}
 

@Override
protected CompletableFuture<R> handleRequest(@Nonnull HandlerRequest<EmptyRequestBody, M> request, @Nonnull RestfulGateway gateway) throws RestHandlerException {
	JobID jobId = request.getPathParameter(JobIDPathParameter.class);

	CompletableFuture<AccessExecutionGraph> executionGraphFuture = executionGraphCache.getExecutionGraph(jobId, gateway);

	return executionGraphFuture.thenApplyAsync(
		executionGraph -> {
			try {
				return handleRequest(request, executionGraph);
			} catch (RestHandlerException rhe) {
				throw new CompletionException(rhe);
			}
		}, executor)
		.exceptionally(throwable -> {
			throwable = ExceptionUtils.stripCompletionException(throwable);
			if (throwable instanceof FlinkJobNotFoundException) {
				throw new CompletionException(
					new NotFoundException(String.format("Job %s not found", jobId), throwable));
			} else {
				throw new CompletionException(throwable);
			}
		});
}
 

public JobPlanHandler(
	GatewayRetriever<? extends RestfulGateway> leaderRetriever,
	Time timeout,
	Map<String, String> headers,
	MessageHeaders<EmptyRequestBody, JobPlanInfo, JobMessageParameters> messageHeaders,
	ExecutionGraphCache executionGraphCache,
	Executor executor) {

	super(
		leaderRetriever,
		timeout,
		headers,
		messageHeaders,
		executionGraphCache,
		executor);
}
 

@Override
protected CompletableFuture<EmptyResponseBody> handleRequest(
		@Nonnull final HandlerRequest<EmptyRequestBody, JarDeleteMessageParameters> request,
		@Nonnull final RestfulGateway gateway) throws RestHandlerException {

	final String jarId = request.getPathParameter(JarIdPathParameter.class);
	return CompletableFuture.supplyAsync(() -> {
		final Path jarToDelete = jarDir.resolve(jarId);
		if (!Files.exists(jarToDelete)) {
			throw new CompletionException(new RestHandlerException(
				String.format("File %s does not exist in %s.", jarId, jarDir),
				HttpResponseStatus.BAD_REQUEST));
		} else {
			try {
				Files.delete(jarToDelete);
				return EmptyResponseBody.getInstance();
			} catch (final IOException e) {
				throw new CompletionException(new RestHandlerException(
					String.format("Failed to delete jar %s.", jarToDelete),
					HttpResponseStatus.INTERNAL_SERVER_ERROR,
					e));
			}
		}
	}, executor);
}
 

@Before
public void setUp() throws Exception {
	MetricFetcher fetcher = new MetricFetcherImpl<RestfulGateway>(
		mock(GatewayRetriever.class),
		mock(MetricQueryServiceRetriever.class),
		Executors.directExecutor(),
		TestingUtils.TIMEOUT(),
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());
	store = fetcher.getMetricStore();

	Collection<MetricDump> metricDumps = getMetricDumps();
	for (MetricDump dump : metricDumps) {
		store.add(dump);
	}

	handler = getHandler(
		LEADER_RETRIEVER,
		TIMEOUT,
		TEST_HEADERS,
		EXECUTOR,
		fetcher);
	pathParameters = getPathParameters();
}
 

public JarPlanHandler(
		final GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		final Time timeout,
		final Map<String, String> responseHeaders,
		final MessageHeaders<JarPlanRequestBody, JobPlanInfo, JarPlanMessageParameters> messageHeaders,
		final Path jarDir,
		final Configuration configuration,
		final Executor executor) {
	this(
		leaderRetriever,
		timeout,
		responseHeaders,
		messageHeaders,
		jarDir,
		configuration,
		executor,
		jobGraph -> new JobPlanInfo(JsonPlanGenerator.generatePlan(jobGraph)));
}
 

public CheckpointStatisticDetailsHandler(
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Time timeout,
		Map<String, String> responseHeaders,
		MessageHeaders<EmptyRequestBody, CheckpointStatistics, CheckpointMessageParameters> messageHeaders,
		ExecutionGraphCache executionGraphCache,
		Executor executor,
		CheckpointStatsCache checkpointStatsCache) {
	super(
		leaderRetriever,
		timeout,
		responseHeaders,
		messageHeaders,
		executionGraphCache,
		executor,
		checkpointStatsCache);
}
 

public JobVertexDetailsHandler(
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Time timeout,
		Map<String, String> responseHeaders,
		MessageHeaders<EmptyRequestBody, JobVertexDetailsInfo, JobVertexMessageParameters> messageHeaders,
		ExecutionGraphCache executionGraphCache,
		Executor executor,
		MetricFetcher metricFetcher) {
	super(
		leaderRetriever,
		timeout,
		responseHeaders,
		messageHeaders,
		executionGraphCache,
		executor);
	this.metricFetcher = metricFetcher;
}
 

public MiniDispatcherRestEndpoint(
		RestServerEndpointConfiguration endpointConfiguration,
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Configuration clusterConfiguration,
		RestHandlerConfiguration restConfiguration,
		GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,
		TransientBlobService transientBlobService,
		ExecutorService executor,
		MetricFetcher metricFetcher,
		LeaderElectionService leaderElectionService,
		FatalErrorHandler fatalErrorHandler) throws IOException {
	super(
		endpointConfiguration,
		leaderRetriever,
		clusterConfiguration,
		restConfiguration,
		resourceManagerRetriever,
		transientBlobService,
		executor,
		metricFetcher,
		leaderElectionService,
		fatalErrorHandler);
}
 

public TaskCheckpointStatisticDetailsHandler(
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Time timeout,
		Map<String, String> responseHeaders,
		MessageHeaders<EmptyRequestBody, TaskCheckpointStatisticsWithSubtaskDetails, TaskCheckpointMessageParameters> messageHeaders,
		ExecutionGraphCache executionGraphCache,
		Executor executor,
		CheckpointStatsCache checkpointStatsCache) {
	super(
		leaderRetriever,
		timeout,
		responseHeaders,
		messageHeaders,
		executionGraphCache,
		executor,
		checkpointStatsCache);
}
 

public JobExceptionsHandler(
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Time timeout,
		Map<String, String> responseHeaders,
		MessageHeaders<EmptyRequestBody, JobExceptionsInfo, JobExceptionsMessageParameters> messageHeaders,
		ExecutionGraphCache executionGraphCache,
		Executor executor) {

	super(
		leaderRetriever,
		timeout,
		responseHeaders,
		messageHeaders,
		executionGraphCache,
		executor);
}
 

@Override
protected CompletableFuture<Acknowledge> triggerOperation(HandlerRequest<SavepointDisposalRequest, EmptyMessageParameters> request, RestfulGateway gateway) throws RestHandlerException {
	final String savepointPath = request.getRequestBody().getSavepointPath();
	if (savepointPath == null) {
		throw new RestHandlerException(
			String.format("Field %s must not be omitted or be null.",
				SavepointDisposalRequest.FIELD_NAME_SAVEPOINT_PATH),
			HttpResponseStatus.BAD_REQUEST);
	}
	return gateway.disposeSavepoint(savepointPath, RpcUtils.INF_TIMEOUT);
}
 

@Override
protected CompletableFuture<JobVertexBackPressureInfo> handleRequest(
		@Nonnull HandlerRequest<EmptyRequestBody, JobVertexMessageParameters> request,
		@Nonnull RestfulGateway gateway) throws RestHandlerException {
	final JobID jobId = request.getPathParameter(JobIDPathParameter.class);
	final JobVertexID jobVertexId = request.getPathParameter(JobVertexIdPathParameter.class);
	return gateway
		.requestOperatorBackPressureStats(jobId, jobVertexId)
		.thenApply(
			operatorBackPressureStats ->
				operatorBackPressureStats.getOperatorBackPressureStats().map(
					JobVertexBackPressureHandler::createJobVertexBackPressureInfo).orElse(
					JobVertexBackPressureInfo.deprecated()));
}
 

public ShutdownHandler(
		final GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		final Time timeout,
		final Map<String, String> responseHeaders,
		final MessageHeaders<EmptyRequestBody, EmptyResponseBody, EmptyMessageParameters> messageHeaders) {
	super(leaderRetriever, timeout, responseHeaders, messageHeaders);
}
 

public static JarListInfo listJars(JarListHandler handler, RestfulGateway restfulGateway) throws Exception {
	HandlerRequest<EmptyRequestBody, EmptyMessageParameters> listRequest = new HandlerRequest<>(
		EmptyRequestBody.getInstance(),
		EmptyMessageParameters.getInstance());
	return handler.handleRequest(listRequest, restfulGateway)
		.get();
}
 

@Override
public CompletableFuture<ClusterOverviewWithVersion> handleRequest(@Nonnull HandlerRequest<EmptyRequestBody, EmptyMessageParameters> request, @Nonnull RestfulGateway gateway) {
	CompletableFuture<ClusterOverview> overviewFuture = gateway.requestClusterOverview(timeout);

	return overviewFuture.thenApply(
		statusOverview -> ClusterOverviewWithVersion.fromStatusOverview(statusOverview, version, commitID));
}
 

@Override
protected AggregatingSubtasksMetricsHandler getHandler(GatewayRetriever<? extends RestfulGateway> leaderRetriever, Time timeout, Map<String, String> responseHeaders, Executor executor, MetricFetcher fetcher) {
	return new AggregatingSubtasksMetricsHandler(
		leaderRetriever,
		timeout,
		responseHeaders,
		executor,
		fetcher
	);
}
 

/**
 * Instantiates a new Abstract job vertex handler.
 *
 * @param leaderRetriever     the leader retriever
 * @param timeout             the timeout
 * @param responseHeaders     the response headers
 * @param messageHeaders      the message headers
 * @param executionGraphCache the execution graph cache
 * @param executor            the executor
 */
protected AbstractJobVertexHandler(
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Time timeout,
		Map<String, String> responseHeaders,
		MessageHeaders<EmptyRequestBody, R, M> messageHeaders,
		ExecutionGraphCache executionGraphCache,
		Executor executor) {

	super(leaderRetriever, timeout, responseHeaders, messageHeaders, executionGraphCache, executor);
}
 

public RescalingStatusHandler(
	GatewayRetriever<? extends RestfulGateway> leaderRetriever,
	Time timeout,
	Map<String, String> responseHeaders) {
	super(
		leaderRetriever,
		timeout,
		responseHeaders,
		RescalingStatusHeaders.getInstance());
}
 

@Override
protected AggregatingTaskManagersMetricsHandler getHandler(GatewayRetriever<? extends RestfulGateway> leaderRetriever, Time timeout, Map<String, String> responseHeaders, Executor executor, MetricFetcher fetcher) {
	return new AggregatingTaskManagersMetricsHandler(
		leaderRetriever,
		timeout,
		responseHeaders,
		executor,
		fetcher
	);
}
 

public JobsOverviewHandler(
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Time timeout,
		Map<String, String> responseHeaders,
		MessageHeaders<EmptyRequestBody, MultipleJobsDetails, EmptyMessageParameters> messageHeaders) {
	super(
		leaderRetriever,
		timeout,
		responseHeaders,
		messageHeaders);
}
 

public static void deleteJar(JarDeleteHandler handler, String jarName, RestfulGateway restfulGateway) throws Exception {
	JarDeleteMessageParameters deleteParameters = JarDeleteHeaders.getInstance().getUnresolvedMessageParameters();
	HandlerRequest<EmptyRequestBody, JarDeleteMessageParameters> deleteRequest = new HandlerRequest<>(
		EmptyRequestBody.getInstance(),
		deleteParameters,
		Collections.singletonMap(deleteParameters.jarIdPathParameter.getKey(), jarName),
		Collections.emptyMap(),
		Collections.emptyList());
	handler.handleRequest(deleteRequest, restfulGateway)
		.get();
}
 

public TaskManagerDetailsHandler(
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Time timeout,
		Map<String, String> responseHeaders,
		MessageHeaders<EmptyRequestBody, TaskManagerDetailsInfo, TaskManagerMessageParameters> messageHeaders,
		GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		MetricFetcher metricFetcher) {
	super(leaderRetriever, timeout, responseHeaders, messageHeaders, resourceManagerGatewayRetriever);

	this.metricFetcher = Preconditions.checkNotNull(metricFetcher);
	this.metricStore = metricFetcher.getMetricStore();
}
 

@Override
protected CompletableFuture<EmptyResponseBody> handleRequest(@Nonnull HandlerRequest<TestRequestBody, EmptyMessageParameters> request, @Nonnull RestfulGateway gateway) throws RestHandlerException {
	Collection<Path> uploadedFiles = request.getUploadedFiles().stream().map(File::toPath).collect(Collectors.toList());
	if (!uploadedFiles.isEmpty()) {
		throw new RestHandlerException("This handler should not have received file uploads.", HttpResponseStatus.INTERNAL_SERVER_ERROR);
	}
	this.lastReceivedRequest = request.getRequestBody();
	return CompletableFuture.completedFuture(EmptyResponseBody.getInstance());
}
 

public CheckpointConfigHandler(
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Time timeout,
		Map<String, String> responseHeaders,
		MessageHeaders<EmptyRequestBody, CheckpointConfigInfo, JobMessageParameters> messageHeaders,
		ExecutionGraphCache executionGraphCache,
		Executor executor) {
	super(
		leaderRetriever,
		timeout,
		responseHeaders,
		messageHeaders,
		executionGraphCache,
		executor);
}
 

@Nonnull
private MetricFetcher createMetricFetcher(long updateInterval, RestfulGateway restfulGateway) {
	return new MetricFetcherImpl<>(
		() -> CompletableFuture.completedFuture(restfulGateway),
		address -> null,
		Executors.directExecutor(),
		Time.seconds(10L),
		updateInterval);
}
 

@Override
protected CompletableFuture<JobPlanInfo> handleRequest(
		@Nonnull final HandlerRequest<JarPlanRequestBody, JarPlanMessageParameters> request,
		@Nonnull final RestfulGateway gateway) throws RestHandlerException {
	final JarHandlerContext context = JarHandlerContext.fromRequest(request, jarDir, log);

	return CompletableFuture.supplyAsync(() -> {
		final JobGraph jobGraph = context.toJobGraph(configuration, true);
		return planGenerator.apply(jobGraph);
	}, executor);
}
 

private RestfulGateway createRestfulGateway(AtomicInteger requestMetricQueryServiceGatewaysCounter) {
	return new TestingRestfulGateway.Builder()
		.setRequestMetricQueryServiceGatewaysSupplier(() -> {
			requestMetricQueryServiceGatewaysCounter.incrementAndGet();
			return new CompletableFuture<>();
		})
		.build();
}
 

/**
 * Instantiates a new Abstract job vertex handler.
 *
 * @param leaderRetriever     the leader retriever
 * @param timeout             the timeout
 * @param responseHeaders     the response headers
 * @param messageHeaders      the message headers
 * @param executionGraphCache the execution graph cache
 * @param executor            the executor
 */
public SubtaskExecutionAttemptAccumulatorsHandler(
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Time timeout,
		Map<String, String> responseHeaders,
		MessageHeaders<EmptyRequestBody, SubtaskExecutionAttemptAccumulatorsInfo, SubtaskAttemptMessageParameters> messageHeaders,
		ExecutionGraphCache executionGraphCache,
		Executor executor) {

	super(leaderRetriever, timeout, responseHeaders, messageHeaders, executionGraphCache, executor);
}
 

@Override
protected AggregatingJobsMetricsHandler getHandler(GatewayRetriever<? extends RestfulGateway> leaderRetriever, Time timeout, Map<String, String> responseHeaders, Executor executor, MetricFetcher fetcher) {
	return new AggregatingJobsMetricsHandler(
		leaderRetriever,
		timeout,
		responseHeaders,
		executor,
		fetcher
	);
}