org.apache.flink.runtime.messages.webmonitor.MultipleJobsDetails Java Examples

/**
 * This method replicates the JSON response that would be given by the JobsOverviewHandler when
 * listing both running and finished jobs.
 *
 * <p>Every job archive contains a joboverview.json file containing the same structure. Since jobs are archived on
 * their own however the list of finished jobs only contains a single job.
 *
 * <p>For the display in the HistoryServer WebFrontend we have to combine these overviews.
 */
private static void updateJobOverview(File webOverviewDir, File webDir) {
	try (JsonGenerator gen = jacksonFactory.createGenerator(HistoryServer.createOrGetFile(webDir, JobsOverviewHeaders.URL))) {
		File[] overviews = new File(webOverviewDir.getPath()).listFiles();
		if (overviews != null) {
			Collection<JobDetails> allJobs = new ArrayList<>(overviews.length);
			for (File overview : overviews) {
				MultipleJobsDetails subJobs = mapper.readValue(overview, MultipleJobsDetails.class);
				allJobs.addAll(subJobs.getJobs());
			}
			mapper.writeValue(gen, new MultipleJobsDetails(allJobs));
		}
	} catch (IOException ioe) {
		LOG.error("Failed to update job overview.", ioe);
	}
}
 

/**
 * This method replicates the JSON response that would be given by the JobsOverviewHandler when
 * listing both running and finished jobs.
 *
 * <p>Every job archive contains a joboverview.json file containing the same structure. Since jobs are archived on
 * their own however the list of finished jobs only contains a single job.
 *
 * <p>For the display in the HistoryServer WebFrontend we have to combine these overviews.
 */
private static void updateJobOverview(File webOverviewDir, File webDir) {
	try (JsonGenerator gen = jacksonFactory.createGenerator(HistoryServer.createOrGetFile(webDir, JobsOverviewHeaders.URL))) {
		File[] overviews = new File(webOverviewDir.getPath()).listFiles();
		if (overviews != null) {
			Collection<JobDetails> allJobs = new ArrayList<>(overviews.length);
			for (File overview : overviews) {
				MultipleJobsDetails subJobs = mapper.readValue(overview, MultipleJobsDetails.class);
				allJobs.addAll(subJobs.getJobs());
			}
			mapper.writeValue(gen, new MultipleJobsDetails(allJobs));
		}
	} catch (IOException ioe) {
		LOG.error("Failed to update job overview.", ioe);
	}
}
 

@Override
public CompletableFuture<MultipleJobsDetails> requestMultipleJobDetails(Time timeout) {
	List<CompletableFuture<Optional<JobDetails>>> individualOptionalJobDetails = queryJobMastersForInformation(
		(JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobDetails(timeout));

	CompletableFuture<Collection<Optional<JobDetails>>> optionalCombinedJobDetails = FutureUtils.combineAll(
		individualOptionalJobDetails);

	CompletableFuture<Collection<JobDetails>> combinedJobDetails = optionalCombinedJobDetails.thenApply(this::flattenOptionalCollection);

	final Collection<JobDetails> completedJobDetails = archivedExecutionGraphStore.getAvailableJobDetails();

	return combinedJobDetails.thenApply(
		(Collection<JobDetails> runningJobDetails) -> {
			final Collection<JobDetails> allJobDetails = new ArrayList<>(completedJobDetails.size() + runningJobDetails.size());

			allJobDetails.addAll(runningJobDetails);
			allJobDetails.addAll(completedJobDetails);

			return new MultipleJobsDetails(allJobDetails);
		});
}
 

@Override
public CompletableFuture<MultipleJobsDetails> requestMultipleJobDetails(Time timeout) {
	List<CompletableFuture<Optional<JobDetails>>> individualOptionalJobDetails = queryJobMastersForInformation(
		(JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobDetails(timeout));

	CompletableFuture<Collection<Optional<JobDetails>>> optionalCombinedJobDetails = FutureUtils.combineAll(
		individualOptionalJobDetails);

	CompletableFuture<Collection<JobDetails>> combinedJobDetails = optionalCombinedJobDetails.thenApply(this::flattenOptionalCollection);

	final Collection<JobDetails> completedJobDetails = archivedExecutionGraphStore.getAvailableJobDetails();

	return combinedJobDetails.thenApply(
		(Collection<JobDetails> runningJobDetails) -> {
			final Collection<JobDetails> allJobDetails = new ArrayList<>(completedJobDetails.size() + runningJobDetails.size());

			allJobDetails.addAll(runningJobDetails);
			allJobDetails.addAll(completedJobDetails);

			return new MultipleJobsDetails(allJobDetails);
		});
}
 

@Override
public CompletableFuture<MultipleJobsDetails> requestMultipleJobDetails(Time timeout) {
	List<CompletableFuture<Optional<JobDetails>>> individualOptionalJobDetails = queryJobMastersForInformation(
		(JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobDetails(timeout));

	CompletableFuture<Collection<Optional<JobDetails>>> optionalCombinedJobDetails = FutureUtils.combineAll(
		individualOptionalJobDetails);

	CompletableFuture<Collection<JobDetails>> combinedJobDetails = optionalCombinedJobDetails.thenApply(this::flattenOptionalCollection);

	final Collection<JobDetails> completedJobDetails = archivedExecutionGraphStore.getAvailableJobDetails();

	return combinedJobDetails.thenApply(
		(Collection<JobDetails> runningJobDetails) -> {
			final Collection<JobDetails> allJobDetails = new ArrayList<>(completedJobDetails.size() + runningJobDetails.size());

			allJobDetails.addAll(runningJobDetails);
			allJobDetails.addAll(completedJobDetails);

			return new MultipleJobsDetails(allJobDetails);
		});
}
 

/**
 * This method replicates the JSON response that would be given by the JobsOverviewHandler when
 * listing both running and finished jobs.
 *
 * <p>Every job archive contains a joboverview.json file containing the same structure. Since jobs are archived on
 * their own however the list of finished jobs only contains a single job.
 *
 * <p>For the display in the HistoryServer WebFrontend we have to combine these overviews.
 */
private static void updateJobOverview(File webOverviewDir, File webDir) {
	try (JsonGenerator gen = jacksonFactory.createGenerator(HistoryServer.createOrGetFile(webDir, JobsOverviewHeaders.URL))) {
		File[] overviews = new File(webOverviewDir.getPath()).listFiles();
		if (overviews != null) {
			Collection<JobDetails> allJobs = new ArrayList<>(overviews.length);
			for (File overview : overviews) {
				MultipleJobsDetails subJobs = mapper.readValue(overview, MultipleJobsDetails.class);
				allJobs.addAll(subJobs.getJobs());
			}
			mapper.writeValue(gen, new MultipleJobsDetails(allJobs));
		}
	} catch (IOException ioe) {
		LOG.error("Failed to update job overview.", ioe);
	}
}
 

/**
 * Lists the currently running and finished jobs on the cluster.
 *
 * @return future collection of running and finished jobs
 * @throws Exception if no connection to the cluster could be established
 */
public CompletableFuture<Collection<JobStatusMessage>> listJobs() throws Exception {
	final ActorGateway jobManager = getJobManagerGateway();

	Future<Object> response = jobManager.ask(new RequestJobDetails(true, false), timeout);
	CompletableFuture<Object> responseFuture = FutureUtils.toJava(response);

	return responseFuture.thenApply((responseMessage) -> {
		if (responseMessage instanceof MultipleJobsDetails) {
			MultipleJobsDetails details = (MultipleJobsDetails) responseMessage;

			final Collection<JobDetails> jobDetails = details.getJobs();
			Collection<JobStatusMessage> flattenedDetails = new ArrayList<>(jobDetails.size());
			jobDetails.forEach(detail -> flattenedDetails.add(new JobStatusMessage(detail.getJobId(), detail.getJobName(), detail.getStatus(), detail.getStartTime())));
			return flattenedDetails;
		} else {
			throw new CompletionException(
				new IllegalStateException("Unknown JobManager response of type " + responseMessage.getClass()));
		}
	});
}
 

public TestingRestfulGateway(
		String address,
		String hostname,
		Function<JobID, CompletableFuture<Acknowledge>> cancelJobFunction,
		Function<JobID, CompletableFuture<ArchivedExecutionGraph>> requestJobFunction,
		Function<JobID, CompletableFuture<JobResult>> requestJobResultFunction,
		Function<JobID, CompletableFuture<JobStatus>> requestJobStatusFunction,
		Supplier<CompletableFuture<MultipleJobsDetails>> requestMultipleJobDetailsSupplier,
		Supplier<CompletableFuture<ClusterOverview>> requestClusterOverviewSupplier,
		Supplier<CompletableFuture<Collection<String>>> requestMetricQueryServiceAddressesSupplier,
		Supplier<CompletableFuture<Collection<Tuple2<ResourceID, String>>>> requestTaskManagerMetricQueryServiceAddressesSupplier,
		BiFunction<JobID, JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction,
		BiFunction<JobID, String, CompletableFuture<String>> triggerSavepointFunction,
		BiFunction<JobID, String, CompletableFuture<String>> stopWithSavepointFunction,
		Supplier<CompletableFuture<Acknowledge>> clusterShutdownSupplier,
		TriFunction<JobID, OperatorID, SerializedValue<CoordinationRequest>, CompletableFuture<CoordinationResponse>> deliverCoordinationRequestToCoordinatorFunction) {
	this.address = address;
	this.hostname = hostname;
	this.cancelJobFunction = cancelJobFunction;
	this.requestJobFunction = requestJobFunction;
	this.requestJobResultFunction = requestJobResultFunction;
	this.requestJobStatusFunction = requestJobStatusFunction;
	this.requestMultipleJobDetailsSupplier = requestMultipleJobDetailsSupplier;
	this.requestClusterOverviewSupplier = requestClusterOverviewSupplier;
	this.requestMetricQueryServiceAddressesSupplier = requestMetricQueryServiceAddressesSupplier;
	this.requestTaskManagerMetricQueryServiceAddressesSupplier = requestTaskManagerMetricQueryServiceAddressesSupplier;
	this.requestOperatorBackPressureStatsFunction = requestOperatorBackPressureStatsFunction;
	this.triggerSavepointFunction = triggerSavepointFunction;
	this.stopWithSavepointFunction = stopWithSavepointFunction;
	this.clusterShutdownSupplier = clusterShutdownSupplier;
	this.deliverCoordinationRequestToCoordinatorFunction = deliverCoordinationRequestToCoordinatorFunction;
}
 

@Test
public void testMultipleJobDetails() {
	try {
		final Random rnd = new Random();
		GenericMessageTester.testMessageInstance(
				new MultipleJobsDetails(randomJobDetails(rnd)));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

public TestingDispatcherGateway(
		String address,
		String hostname,
		Function<JobID, CompletableFuture<Acknowledge>> cancelJobFunction,
		Function<JobID, CompletableFuture<ArchivedExecutionGraph>> requestJobFunction,
		Function<JobID, CompletableFuture<JobResult>> requestJobResultFunction,
		Function<JobID, CompletableFuture<JobStatus>> requestJobStatusFunction,
		Supplier<CompletableFuture<MultipleJobsDetails>> requestMultipleJobDetailsSupplier,
		Supplier<CompletableFuture<ClusterOverview>> requestClusterOverviewSupplier,
		Supplier<CompletableFuture<Collection<String>>> requestMetricQueryServiceAddressesSupplier,
		Supplier<CompletableFuture<Collection<Tuple2<ResourceID, String>>>> requestTaskManagerMetricQueryServiceGatewaysSupplier,
		BiFunction<JobID, JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction,
		BiFunction<JobID, String, CompletableFuture<String>> triggerSavepointFunction,
		BiFunction<JobID, String, CompletableFuture<String>> stopWithSavepointFunction,
		Function<JobGraph, CompletableFuture<Acknowledge>> submitFunction,
		Supplier<CompletableFuture<Collection<JobID>>> listFunction,
		int blobServerPort,
		DispatcherId fencingToken,
		Function<JobID, CompletableFuture<ArchivedExecutionGraph>> requestArchivedJobFunction) {
	super(
		address,
		hostname,
		cancelJobFunction,
		requestJobFunction,
		requestJobResultFunction,
		requestJobStatusFunction,
		requestMultipleJobDetailsSupplier,
		requestClusterOverviewSupplier,
		requestMetricQueryServiceAddressesSupplier,
		requestTaskManagerMetricQueryServiceGatewaysSupplier,
		requestOperatorBackPressureStatsFunction,
		triggerSavepointFunction,
		stopWithSavepointFunction);
	this.submitFunction = submitFunction;
	this.listFunction = listFunction;
	this.blobServerPort = blobServerPort;
	this.fencingToken = fencingToken;
	this.requestArchivedJobFunction = requestArchivedJobFunction;
}
 

public TestingRestfulGateway(
		String address,
		String hostname,
		Function<JobID, CompletableFuture<Acknowledge>> cancelJobFunction,
		Function<JobID, CompletableFuture<ArchivedExecutionGraph>> requestJobFunction,
		Function<JobID, CompletableFuture<JobResult>> requestJobResultFunction,
		Function<JobID, CompletableFuture<JobStatus>> requestJobStatusFunction,
		Supplier<CompletableFuture<MultipleJobsDetails>> requestMultipleJobDetailsSupplier,
		Supplier<CompletableFuture<ClusterOverview>> requestClusterOverviewSupplier,
		Supplier<CompletableFuture<Collection<String>>> requestMetricQueryServiceAddressesSupplier,
		Supplier<CompletableFuture<Collection<Tuple2<ResourceID, String>>>> requestTaskManagerMetricQueryServiceAddressesSupplier,
		BiFunction<JobID, JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction,
		BiFunction<JobID, String, CompletableFuture<String>> triggerSavepointFunction,
		BiFunction<JobID, String, CompletableFuture<String>> stopWithSavepointFunction) {
	this.address = address;
	this.hostname = hostname;
	this.cancelJobFunction = cancelJobFunction;
	this.requestJobFunction = requestJobFunction;
	this.requestJobResultFunction = requestJobResultFunction;
	this.requestJobStatusFunction = requestJobStatusFunction;
	this.requestMultipleJobDetailsSupplier = requestMultipleJobDetailsSupplier;
	this.requestClusterOverviewSupplier = requestClusterOverviewSupplier;
	this.requestMetricQueryServiceAddressesSupplier = requestMetricQueryServiceAddressesSupplier;
	this.requestTaskManagerMetricQueryServiceAddressesSupplier = requestTaskManagerMetricQueryServiceAddressesSupplier;
	this.requestOperatorBackPressureStatsFunction = requestOperatorBackPressureStatsFunction;
	this.triggerSavepointFunction = triggerSavepointFunction;
	this.stopWithSavepointFunction = stopWithSavepointFunction;
}
 

@Override
public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException {
	ResponseBody json = new MultipleJobsDetails(Collections.singleton(WebMonitorUtils.createDetailsForJob(graph)));
	String path = getMessageHeaders().getTargetRestEndpointURL()
		.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString());
	return Collections.singletonList(new ArchivedJson(path, json));
}
 

public JobsOverviewHandler(
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Time timeout,
		Map<String, String> responseHeaders,
		MessageHeaders<EmptyRequestBody, MultipleJobsDetails, EmptyMessageParameters> messageHeaders) {
	super(
		leaderRetriever,
		timeout,
		responseHeaders,
		messageHeaders);
}
 

public JobsOverviewHandler(
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Time timeout,
		Map<String, String> responseHeaders,
		MessageHeaders<EmptyRequestBody, MultipleJobsDetails, EmptyMessageParameters> messageHeaders) {
	super(
		leaderRetriever,
		timeout,
		responseHeaders,
		messageHeaders);
}
 

@Test
public void testHistoryServerIntegration() throws Exception {
	final int numJobs = 2;
	for (int x = 0; x < numJobs; x++) {
		runJob();
	}
	createLegacyArchive(jmDirectory.toPath());

	CountDownLatch numFinishedPolls = new CountDownLatch(1);

	Configuration historyServerConfig = new Configuration();
	historyServerConfig.setString(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_DIRS, jmDirectory.toURI().toString());
	historyServerConfig.setString(HistoryServerOptions.HISTORY_SERVER_WEB_DIR, hsDirectory.getAbsolutePath());

	historyServerConfig.setInteger(HistoryServerOptions.HISTORY_SERVER_WEB_PORT, 0);

	// the job is archived asynchronously after env.execute() returns
	File[] archives = jmDirectory.listFiles();
	while (archives == null || archives.length != numJobs + 1) {
		Thread.sleep(50);
		archives = jmDirectory.listFiles();
	}

	HistoryServer hs = new HistoryServer(historyServerConfig, numFinishedPolls);
	try {
		hs.start();
		String baseUrl = "http://localhost:" + hs.getWebPort();
		numFinishedPolls.await(10L, TimeUnit.SECONDS);

		ObjectMapper mapper = new ObjectMapper();
		String response = getFromHTTP(baseUrl + JobsOverviewHeaders.URL);
		MultipleJobsDetails overview = mapper.readValue(response, MultipleJobsDetails.class);

		Assert.assertEquals(numJobs + 1, overview.getJobs().size());
	} finally {
		hs.stop();
	}
}
 

@Override
public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException {
	ResponseBody json = new MultipleJobsDetails(Collections.singleton(WebMonitorUtils.createDetailsForJob(graph)));
	String path = getMessageHeaders().getTargetRestEndpointURL()
		.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString());
	return Collections.singletonList(new ArchivedJson(path, json));
}
 

@Test
public void testMultipleJobDetails() {
	try {
		final Random rnd = new Random();
		GenericMessageTester.testMessageInstance(
				new MultipleJobsDetails(randomJobDetails(rnd)));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

public TestingDispatcherGateway(
		String address,
		String hostname,
		Function<JobID, CompletableFuture<Acknowledge>> cancelJobFunction,
		Function<JobID, CompletableFuture<Acknowledge>> stopJobFunction,
		Function<JobID, CompletableFuture<ArchivedExecutionGraph>> requestJobFunction,
		Function<JobID, CompletableFuture<JobResult>> requestJobResultFunction,
		Function<JobID, CompletableFuture<JobStatus>> requestJobStatusFunction,
		Supplier<CompletableFuture<MultipleJobsDetails>> requestMultipleJobDetailsSupplier,
		Supplier<CompletableFuture<ClusterOverview>> requestClusterOverviewSupplier,
		Supplier<CompletableFuture<Collection<String>>> requestMetricQueryServicePathsSupplier,
		Supplier<CompletableFuture<Collection<Tuple2<ResourceID, String>>>> requestTaskManagerMetricQueryServicePathsSupplier,
		BiFunction<JobID, JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction,
		BiFunction<JobID, String, CompletableFuture<String>> triggerSavepointFunction,
		Function<JobGraph, CompletableFuture<Acknowledge>> submitFunction,
		Supplier<CompletableFuture<Collection<JobID>>> listFunction,
		int blobServerPort,
		DispatcherId fencingToken,
		Function<JobID, CompletableFuture<ArchivedExecutionGraph>> requestArchivedJobFunction) {
	super(
		address,
		hostname,
		cancelJobFunction,
		stopJobFunction,
		requestJobFunction,
		requestJobResultFunction,
		requestJobStatusFunction,
		requestMultipleJobDetailsSupplier,
		requestClusterOverviewSupplier,
		requestMetricQueryServicePathsSupplier,
		requestTaskManagerMetricQueryServicePathsSupplier,
		requestOperatorBackPressureStatsFunction,
		triggerSavepointFunction);
	this.submitFunction = submitFunction;
	this.listFunction = listFunction;
	this.blobServerPort = blobServerPort;
	this.fencingToken = fencingToken;
	this.requestArchivedJobFunction = requestArchivedJobFunction;
}
 

@Test
public void testHistoryServerIntegration() throws Exception {
	final int numJobs = 2;
	for (int x = 0; x < numJobs; x++) {
		runJob();
	}
	createLegacyArchive(jmDirectory.toPath());

	CountDownLatch numFinishedPolls = new CountDownLatch(1);

	Configuration historyServerConfig = new Configuration();
	historyServerConfig.setString(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_DIRS, jmDirectory.toURI().toString());
	historyServerConfig.setString(HistoryServerOptions.HISTORY_SERVER_WEB_DIR, hsDirectory.getAbsolutePath());

	historyServerConfig.setInteger(HistoryServerOptions.HISTORY_SERVER_WEB_PORT, 0);

	// the job is archived asynchronously after env.execute() returns
	File[] archives = jmDirectory.listFiles();
	while (archives == null || archives.length != numJobs + 1) {
		Thread.sleep(50);
		archives = jmDirectory.listFiles();
	}

	HistoryServer hs = new HistoryServer(historyServerConfig, numFinishedPolls);
	try {
		hs.start();
		String baseUrl = "http://localhost:" + hs.getWebPort();
		numFinishedPolls.await(10L, TimeUnit.SECONDS);

		ObjectMapper mapper = new ObjectMapper();
		String response = getFromHTTP(baseUrl + JobsOverviewHeaders.URL);
		MultipleJobsDetails overview = mapper.readValue(response, MultipleJobsDetails.class);

		Assert.assertEquals(numJobs + 1, overview.getJobs().size());
	} finally {
		hs.stop();
	}
}
 

public TestingRestfulGateway(
		String address,
		String hostname,
		Function<JobID, CompletableFuture<Acknowledge>> cancelJobFunction,
		Function<JobID, CompletableFuture<Acknowledge>> stopJobFunction,
		Function<JobID, CompletableFuture<ArchivedExecutionGraph>> requestJobFunction,
		Function<JobID, CompletableFuture<JobResult>> requestJobResultFunction,
		Function<JobID, CompletableFuture<JobStatus>> requestJobStatusFunction,
		Supplier<CompletableFuture<MultipleJobsDetails>> requestMultipleJobDetailsSupplier,
		Supplier<CompletableFuture<ClusterOverview>> requestClusterOverviewSupplier,
		Supplier<CompletableFuture<Collection<String>>> requestMetricQueryServicePathsSupplier,
		Supplier<CompletableFuture<Collection<Tuple2<ResourceID, String>>>> requestTaskManagerMetricQueryServicePathsSupplier,
		BiFunction<JobID, JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction,
		BiFunction<JobID, String, CompletableFuture<String>> triggerSavepointFunction) {
	this.address = address;
	this.hostname = hostname;
	this.cancelJobFunction = cancelJobFunction;
	this.stopJobFunction = stopJobFunction;
	this.requestJobFunction = requestJobFunction;
	this.requestJobResultFunction = requestJobResultFunction;
	this.requestJobStatusFunction = requestJobStatusFunction;
	this.requestMultipleJobDetailsSupplier = requestMultipleJobDetailsSupplier;
	this.requestClusterOverviewSupplier = requestClusterOverviewSupplier;
	this.requestMetricQueryServicePathsSupplier = requestMetricQueryServicePathsSupplier;
	this.requestTaskManagerMetricQueryServicePathsSupplier = requestTaskManagerMetricQueryServicePathsSupplier;
	this.requestOperatorBackPressureStatsFunction = requestOperatorBackPressureStatsFunction;
	this.triggerSavepointFunction = triggerSavepointFunction;
}
 

@Override
public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException {
	ResponseBody json = new MultipleJobsDetails(Collections.singleton(WebMonitorUtils.createDetailsForJob(graph)));
	String path = getMessageHeaders().getTargetRestEndpointURL()
		.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString());
	return Collections.singletonList(new ArchivedJson(path, json));
}
 

@Test
public void testMultipleJobDetails() {
	try {
		final Random rnd = new Random();
		GenericMessageTester.testMessageInstance(
				new MultipleJobsDetails(randomJobDetails(rnd)));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

public JobsOverviewHandler(
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Time timeout,
		Map<String, String> responseHeaders,
		MessageHeaders<EmptyRequestBody, MultipleJobsDetails, EmptyMessageParameters> messageHeaders) {
	super(
		leaderRetriever,
		timeout,
		responseHeaders,
		messageHeaders);
}
 

@Override
public Class<MultipleJobsDetails> getResponseClass() {
	return MultipleJobsDetails.class;
}
 

private static String convertLegacyJobOverview(String legacyOverview) throws IOException {
	JsonNode root = mapper.readTree(legacyOverview);
	JsonNode finishedJobs = root.get("finished");
	JsonNode job = finishedJobs.get(0);

	JobID jobId = JobID.fromHexString(job.get("jid").asText());
	String name = job.get("name").asText();
	JobStatus state = JobStatus.valueOf(job.get("state").asText());

	long startTime = job.get("start-time").asLong();
	long endTime = job.get("end-time").asLong();
	long duration = job.get("duration").asLong();
	long lastMod = job.get("last-modification").asLong();

	JsonNode tasks = job.get("tasks");
	int numTasks = tasks.get("total").asInt();
	JsonNode pendingNode = tasks.get("pending");
	// for flink version < 1.4 we have pending field,
	// when version >= 1.4 pending has been split into scheduled, deploying, and created.
	boolean versionLessThan14 = pendingNode != null;
	int created = 0;
	int scheduled;
	int deploying = 0;

	if (versionLessThan14) {
		// pending is a mix of CREATED/SCHEDULED/DEPLOYING
		// to maintain the correct number of task states we pick SCHEDULED
		scheduled = pendingNode.asInt();
	} else {
		created = tasks.get("created").asInt();
		scheduled = tasks.get("scheduled").asInt();
		deploying = tasks.get("deploying").asInt();
	}
	int running = tasks.get("running").asInt();
	int finished = tasks.get("finished").asInt();
	int canceling = tasks.get("canceling").asInt();
	int canceled = tasks.get("canceled").asInt();
	int failed = tasks.get("failed").asInt();

	int[] tasksPerState = new int[ExecutionState.values().length];
	tasksPerState[ExecutionState.CREATED.ordinal()] = created;
	tasksPerState[ExecutionState.SCHEDULED.ordinal()] = scheduled;
	tasksPerState[ExecutionState.DEPLOYING.ordinal()] = deploying;
	tasksPerState[ExecutionState.RUNNING.ordinal()] = running;
	tasksPerState[ExecutionState.FINISHED.ordinal()] = finished;
	tasksPerState[ExecutionState.CANCELING.ordinal()] = canceling;
	tasksPerState[ExecutionState.CANCELED.ordinal()] = canceled;
	tasksPerState[ExecutionState.FAILED.ordinal()] = failed;

	JobDetails jobDetails = new JobDetails(jobId, name, startTime, endTime, duration, state, lastMod, tasksPerState, numTasks);
	MultipleJobsDetails multipleJobsDetails = new MultipleJobsDetails(Collections.singleton(jobDetails));

	StringWriter sw = new StringWriter();
	mapper.writeValue(sw, multipleJobsDetails);
	return sw.toString();
}
 

private static String convertLegacyJobOverview(String legacyOverview) throws IOException {
	JsonNode root = mapper.readTree(legacyOverview);
	JsonNode finishedJobs = root.get("finished");
	JsonNode job = finishedJobs.get(0);

	JobID jobId = JobID.fromHexString(job.get("jid").asText());
	String name = job.get("name").asText();
	JobStatus state = JobStatus.valueOf(job.get("state").asText());

	long startTime = job.get("start-time").asLong();
	long endTime = job.get("end-time").asLong();
	long duration = job.get("duration").asLong();
	long lastMod = job.get("last-modification").asLong();

	JsonNode tasks = job.get("tasks");
	int numTasks = tasks.get("total").asInt();
	JsonNode pendingNode = tasks.get("pending");
	// for flink version < 1.4 we have pending field,
	// when version >= 1.4 pending has been split into scheduled, deploying, and created.
	boolean versionLessThan14 = pendingNode != null;
	int created = 0;
	int scheduled;
	int deploying = 0;

	if (versionLessThan14) {
		// pending is a mix of CREATED/SCHEDULED/DEPLOYING
		// to maintain the correct number of task states we pick SCHEDULED
		scheduled = pendingNode.asInt();
	} else {
		created = tasks.get("created").asInt();
		scheduled = tasks.get("scheduled").asInt();
		deploying = tasks.get("deploying").asInt();
	}
	int running = tasks.get("running").asInt();
	int finished = tasks.get("finished").asInt();
	int canceling = tasks.get("canceling").asInt();
	int canceled = tasks.get("canceled").asInt();
	int failed = tasks.get("failed").asInt();

	int[] tasksPerState = new int[ExecutionState.values().length];
	tasksPerState[ExecutionState.CREATED.ordinal()] = created;
	tasksPerState[ExecutionState.SCHEDULED.ordinal()] = scheduled;
	tasksPerState[ExecutionState.DEPLOYING.ordinal()] = deploying;
	tasksPerState[ExecutionState.RUNNING.ordinal()] = running;
	tasksPerState[ExecutionState.FINISHED.ordinal()] = finished;
	tasksPerState[ExecutionState.CANCELING.ordinal()] = canceling;
	tasksPerState[ExecutionState.CANCELED.ordinal()] = canceled;
	tasksPerState[ExecutionState.FAILED.ordinal()] = failed;

	JobDetails jobDetails = new JobDetails(jobId, name, startTime, endTime, duration, state, lastMod, tasksPerState, numTasks);
	MultipleJobsDetails multipleJobsDetails = new MultipleJobsDetails(Collections.singleton(jobDetails));

	StringWriter sw = new StringWriter();
	mapper.writeValue(sw, multipleJobsDetails);
	return sw.toString();
}
 

@Test
public void testUpdate() {
	final Time timeout = Time.seconds(10L);

	// ========= setup TaskManager =================================================================================

	JobID jobID = new JobID();
	ResourceID tmRID = ResourceID.generate();

	// ========= setup QueryServices ================================================================================

	final MetricQueryServiceGateway jmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], new byte[0], new byte[0], new byte[0], 0, 0, 0, 0)))
		.build();

	MetricDumpSerialization.MetricSerializationResult requestMetricsAnswer = createRequestDumpAnswer(tmRID, jobID);
	final MetricQueryServiceGateway tmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(requestMetricsAnswer))
		.build();

	// ========= setup JobManager ==================================================================================

	final TestingRestfulGateway restfulGateway = new TestingRestfulGateway.Builder()
		.setRequestMultipleJobDetailsSupplier(() -> CompletableFuture.completedFuture(new MultipleJobsDetails(Collections.emptyList())))
		.setRequestMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(jmQueryService.getAddress())))
		.setRequestTaskManagerMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(Tuple2.of(tmRID, tmQueryService.getAddress()))))
		.build();

	final GatewayRetriever<RestfulGateway> retriever = () -> CompletableFuture.completedFuture(restfulGateway);

	// ========= start MetricFetcher testing =======================================================================
	MetricFetcher fetcher = new MetricFetcherImpl<>(
		retriever,
		address -> CompletableFuture.completedFuture(tmQueryService),
		Executors.directExecutor(),
		timeout,
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());

	// verify that update fetches metrics and updates the store
	fetcher.update();
	MetricStore store = fetcher.getMetricStore();
	synchronized (store) {
		assertEquals("7", store.getJobManagerMetricStore().getMetric("abc.hist_min"));
		assertEquals("6", store.getJobManagerMetricStore().getMetric("abc.hist_max"));
		assertEquals("4.0", store.getJobManagerMetricStore().getMetric("abc.hist_mean"));
		assertEquals("0.5", store.getJobManagerMetricStore().getMetric("abc.hist_median"));
		assertEquals("5.0", store.getJobManagerMetricStore().getMetric("abc.hist_stddev"));
		assertEquals("0.75", store.getJobManagerMetricStore().getMetric("abc.hist_p75"));
		assertEquals("0.9", store.getJobManagerMetricStore().getMetric("abc.hist_p90"));
		assertEquals("0.95", store.getJobManagerMetricStore().getMetric("abc.hist_p95"));
		assertEquals("0.98", store.getJobManagerMetricStore().getMetric("abc.hist_p98"));
		assertEquals("0.99", store.getJobManagerMetricStore().getMetric("abc.hist_p99"));
		assertEquals("0.999", store.getJobManagerMetricStore().getMetric("abc.hist_p999"));

		assertEquals("x", store.getTaskManagerMetricStore(tmRID.toString()).metrics.get("abc.gauge"));
		assertEquals("5.0", store.getJobMetricStore(jobID.toString()).metrics.get("abc.jc"));
		assertEquals("2", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.abc.tc"));
		assertEquals("1", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.opname.abc.oc"));
	}
}
 

private static MultipleJobsDetails getJobsOverview(String baseUrl) throws Exception {
	Tuple2<Integer, String> response = getFromHTTP(baseUrl + JobsOverviewHeaders.URL);
	return OBJECT_MAPPER.readValue(response.f1, MultipleJobsDetails.class);
}
 

@Override
protected CompletableFuture<MultipleJobsDetails> handleRequest(@Nonnull HandlerRequest<EmptyRequestBody, EmptyMessageParameters> request, @Nonnull DispatcherGateway gateway) throws RestHandlerException {
	JobDetails running = new JobDetails(new JobID(), "job1", 0, 0, 0, JobStatus.RUNNING, 0, new int[9], 0);
	JobDetails finished = new JobDetails(new JobID(), "job2", 0, 0, 0, JobStatus.FINISHED, 0, new int[9], 0);
	return CompletableFuture.completedFuture(new MultipleJobsDetails(Arrays.asList(running, finished)));
}
 

public TestingDispatcherGateway(
		String address,
		String hostname,
		Function<JobID, CompletableFuture<Acknowledge>> cancelJobFunction,
		Function<JobID, CompletableFuture<ArchivedExecutionGraph>> requestJobFunction,
		Function<JobID, CompletableFuture<JobResult>> requestJobResultFunction,
		Function<JobID, CompletableFuture<JobStatus>> requestJobStatusFunction,
		Supplier<CompletableFuture<MultipleJobsDetails>> requestMultipleJobDetailsSupplier,
		Supplier<CompletableFuture<ClusterOverview>> requestClusterOverviewSupplier,
		Supplier<CompletableFuture<Collection<String>>> requestMetricQueryServiceAddressesSupplier,
		Supplier<CompletableFuture<Collection<Tuple2<ResourceID, String>>>> requestTaskManagerMetricQueryServiceGatewaysSupplier,
		BiFunction<JobID, JobVertexID, CompletableFuture<OperatorBackPressureStatsResponse>> requestOperatorBackPressureStatsFunction,
		BiFunction<JobID, String, CompletableFuture<String>> triggerSavepointFunction,
		BiFunction<JobID, String, CompletableFuture<String>> stopWithSavepointFunction,
		Function<JobGraph, CompletableFuture<Acknowledge>> submitFunction,
		Supplier<CompletableFuture<Collection<JobID>>> listFunction,
		int blobServerPort,
		DispatcherId fencingToken,
		Function<JobID, CompletableFuture<ArchivedExecutionGraph>> requestArchivedJobFunction,
		Supplier<CompletableFuture<Acknowledge>> clusterShutdownSupplier,
		Function<ApplicationStatus, CompletableFuture<Acknowledge>> clusterShutdownWithStatusFunction,
		TriFunction<JobID, OperatorID, SerializedValue<CoordinationRequest>, CompletableFuture<CoordinationResponse>> deliverCoordinationRequestToCoordinatorFunction) {
	super(
		address,
		hostname,
		cancelJobFunction,
		requestJobFunction,
		requestJobResultFunction,
		requestJobStatusFunction,
		requestMultipleJobDetailsSupplier,
		requestClusterOverviewSupplier,
		requestMetricQueryServiceAddressesSupplier,
		requestTaskManagerMetricQueryServiceGatewaysSupplier,
		requestOperatorBackPressureStatsFunction,
		triggerSavepointFunction,
		stopWithSavepointFunction,
		clusterShutdownSupplier,
		deliverCoordinationRequestToCoordinatorFunction);
	this.submitFunction = submitFunction;
	this.listFunction = listFunction;
	this.blobServerPort = blobServerPort;
	this.fencingToken = fencingToken;
	this.requestArchivedJobFunction = requestArchivedJobFunction;
	this.clusterShutdownWithStatusFunction = clusterShutdownWithStatusFunction;
}