org.apache.flink.runtime.rest.messages.JobsOverviewHeaders Java Examples

/**
 * This method replicates the JSON response that would be given by the JobsOverviewHandler when
 * listing both running and finished jobs.
 *
 * <p>Every job archive contains a joboverview.json file containing the same structure. Since jobs are archived on
 * their own however the list of finished jobs only contains a single job.
 *
 * <p>For the display in the HistoryServer WebFrontend we have to combine these overviews.
 */
private static void updateJobOverview(File webOverviewDir, File webDir) {
	try (JsonGenerator gen = jacksonFactory.createGenerator(HistoryServer.createOrGetFile(webDir, JobsOverviewHeaders.URL))) {
		File[] overviews = new File(webOverviewDir.getPath()).listFiles();
		if (overviews != null) {
			Collection<JobDetails> allJobs = new ArrayList<>(overviews.length);
			for (File overview : overviews) {
				MultipleJobsDetails subJobs = mapper.readValue(overview, MultipleJobsDetails.class);
				allJobs.addAll(subJobs.getJobs());
			}
			mapper.writeValue(gen, new MultipleJobsDetails(allJobs));
		}
	} catch (IOException ioe) {
		LOG.error("Failed to update job overview.", ioe);
	}
}
 

/**
 * This method replicates the JSON response that would be given by the JobsOverviewHandler when
 * listing both running and finished jobs.
 *
 * <p>Every job archive contains a joboverview.json file containing the same structure. Since jobs are archived on
 * their own however the list of finished jobs only contains a single job.
 *
 * <p>For the display in the HistoryServer WebFrontend we have to combine these overviews.
 */
private static void updateJobOverview(File webOverviewDir, File webDir) {
	try (JsonGenerator gen = jacksonFactory.createGenerator(HistoryServer.createOrGetFile(webDir, JobsOverviewHeaders.URL))) {
		File[] overviews = new File(webOverviewDir.getPath()).listFiles();
		if (overviews != null) {
			Collection<JobDetails> allJobs = new ArrayList<>(overviews.length);
			for (File overview : overviews) {
				MultipleJobsDetails subJobs = mapper.readValue(overview, MultipleJobsDetails.class);
				allJobs.addAll(subJobs.getJobs());
			}
			mapper.writeValue(gen, new MultipleJobsDetails(allJobs));
		}
	} catch (IOException ioe) {
		LOG.error("Failed to update job overview.", ioe);
	}
}
 

/**
 * This method replicates the JSON response that would be given by the JobsOverviewHandler when
 * listing both running and finished jobs.
 *
 * <p>Every job archive contains a joboverview.json file containing the same structure. Since jobs are archived on
 * their own however the list of finished jobs only contains a single job.
 *
 * <p>For the display in the HistoryServer WebFrontend we have to combine these overviews.
 */
private static void updateJobOverview(File webOverviewDir, File webDir) {
	try (JsonGenerator gen = jacksonFactory.createGenerator(HistoryServer.createOrGetFile(webDir, JobsOverviewHeaders.URL))) {
		File[] overviews = new File(webOverviewDir.getPath()).listFiles();
		if (overviews != null) {
			Collection<JobDetails> allJobs = new ArrayList<>(overviews.length);
			for (File overview : overviews) {
				MultipleJobsDetails subJobs = mapper.readValue(overview, MultipleJobsDetails.class);
				allJobs.addAll(subJobs.getJobs());
			}
			mapper.writeValue(gen, new MultipleJobsDetails(allJobs));
		}
	} catch (IOException ioe) {
		LOG.error("Failed to update job overview.", ioe);
	}
}
 

@Test
public void testHistoryServerIntegration() throws Exception {
	final int numJobs = 2;
	for (int x = 0; x < numJobs; x++) {
		runJob();
	}
	createLegacyArchive(jmDirectory.toPath());

	CountDownLatch numFinishedPolls = new CountDownLatch(1);

	Configuration historyServerConfig = new Configuration();
	historyServerConfig.setString(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_DIRS, jmDirectory.toURI().toString());
	historyServerConfig.setString(HistoryServerOptions.HISTORY_SERVER_WEB_DIR, hsDirectory.getAbsolutePath());

	historyServerConfig.setInteger(HistoryServerOptions.HISTORY_SERVER_WEB_PORT, 0);

	// the job is archived asynchronously after env.execute() returns
	File[] archives = jmDirectory.listFiles();
	while (archives == null || archives.length != numJobs + 1) {
		Thread.sleep(50);
		archives = jmDirectory.listFiles();
	}

	HistoryServer hs = new HistoryServer(historyServerConfig, numFinishedPolls);
	try {
		hs.start();
		String baseUrl = "http://localhost:" + hs.getWebPort();
		numFinishedPolls.await(10L, TimeUnit.SECONDS);

		ObjectMapper mapper = new ObjectMapper();
		String response = getFromHTTP(baseUrl + JobsOverviewHeaders.URL);
		MultipleJobsDetails overview = mapper.readValue(response, MultipleJobsDetails.class);

		Assert.assertEquals(numJobs + 1, overview.getJobs().size());
	} finally {
		hs.stop();
	}
}
 

@Override
public CompletableFuture<Collection<JobStatusMessage>> listJobs() {
	return sendRequest(JobsOverviewHeaders.getInstance())
		.thenApply(
			(multipleJobsDetails) -> multipleJobsDetails
				.getJobs()
				.stream()
				.map(detail -> new JobStatusMessage(
					detail.getJobId(),
					detail.getJobName(),
					detail.getStatus(),
					detail.getStartTime()))
				.collect(Collectors.toList()));
}
 

@Test
public void testHistoryServerIntegration() throws Exception {
	final int numJobs = 2;
	for (int x = 0; x < numJobs; x++) {
		runJob();
	}
	createLegacyArchive(jmDirectory.toPath());

	CountDownLatch numFinishedPolls = new CountDownLatch(1);

	Configuration historyServerConfig = new Configuration();
	historyServerConfig.setString(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_DIRS, jmDirectory.toURI().toString());
	historyServerConfig.setString(HistoryServerOptions.HISTORY_SERVER_WEB_DIR, hsDirectory.getAbsolutePath());

	historyServerConfig.setInteger(HistoryServerOptions.HISTORY_SERVER_WEB_PORT, 0);

	// the job is archived asynchronously after env.execute() returns
	File[] archives = jmDirectory.listFiles();
	while (archives == null || archives.length != numJobs + 1) {
		Thread.sleep(50);
		archives = jmDirectory.listFiles();
	}

	HistoryServer hs = new HistoryServer(historyServerConfig, numFinishedPolls);
	try {
		hs.start();
		String baseUrl = "http://localhost:" + hs.getWebPort();
		numFinishedPolls.await(10L, TimeUnit.SECONDS);

		ObjectMapper mapper = new ObjectMapper();
		String response = getFromHTTP(baseUrl + JobsOverviewHeaders.URL);
		MultipleJobsDetails overview = mapper.readValue(response, MultipleJobsDetails.class);

		Assert.assertEquals(numJobs + 1, overview.getJobs().size());
	} finally {
		hs.stop();
	}
}
 

@Override
public CompletableFuture<Collection<JobStatusMessage>> listJobs() {
	return sendRequest(JobsOverviewHeaders.getInstance())
		.thenApply(
			(multipleJobsDetails) -> multipleJobsDetails
				.getJobs()
				.stream()
				.map(detail -> new JobStatusMessage(
					detail.getJobId(),
					detail.getJobName(),
					detail.getStatus(),
					detail.getStartTime()))
				.collect(Collectors.toList()));
}
 

@Override
public CompletableFuture<Collection<JobStatusMessage>> listJobs() {
	return sendRequest(JobsOverviewHeaders.getInstance())
		.thenApply(
			(multipleJobsDetails) -> multipleJobsDetails
				.getJobs()
				.stream()
				.map(detail -> new JobStatusMessage(
					detail.getJobId(),
					detail.getJobName(),
					detail.getStatus(),
					detail.getStartTime()))
				.collect(Collectors.toList()));
}
 

private TestListJobsHandler() {
	super(JobsOverviewHeaders.getInstance());
}
 

private TestListJobsHandler() {
	super(JobsOverviewHeaders.getInstance());
}
 

private static MultipleJobsDetails getJobsOverview(String baseUrl) throws Exception {
	Tuple2<Integer, String> response = getFromHTTP(baseUrl + JobsOverviewHeaders.URL);
	return OBJECT_MAPPER.readValue(response.f1, MultipleJobsDetails.class);
}
 

private TestListJobsHandler() {
	super(JobsOverviewHeaders.getInstance());
}