org.apache.flink.runtime.messages.webmonitor.JobDetails Java Examples

/**
 * Tests that we obtain the correct collection of available job details.
 */
@Test
public void testAvailableJobDetails() throws IOException {
	final int numberExecutionGraphs = 10;
	final Collection<ArchivedExecutionGraph> executionGraphs = generateTerminalExecutionGraphs(numberExecutionGraphs);

	final Collection<JobDetails> jobDetails = generateJobDetails(executionGraphs);

	final File rootDir = temporaryFolder.newFolder();

	try (final FileArchivedExecutionGraphStore executionGraphStore = createDefaultExecutionGraphStore(rootDir)) {
		for (ArchivedExecutionGraph executionGraph : executionGraphs) {
			executionGraphStore.put(executionGraph);
		}

		assertThat(executionGraphStore.getAvailableJobDetails(), Matchers.containsInAnyOrder(jobDetails.toArray()));
	}
}
 

@Override
public CompletableFuture<MultipleJobsDetails> requestMultipleJobDetails(Time timeout) {
	List<CompletableFuture<Optional<JobDetails>>> individualOptionalJobDetails = queryJobMastersForInformation(
		(JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobDetails(timeout));

	CompletableFuture<Collection<Optional<JobDetails>>> optionalCombinedJobDetails = FutureUtils.combineAll(
		individualOptionalJobDetails);

	CompletableFuture<Collection<JobDetails>> combinedJobDetails = optionalCombinedJobDetails.thenApply(this::flattenOptionalCollection);

	final Collection<JobDetails> completedJobDetails = archivedExecutionGraphStore.getAvailableJobDetails();

	return combinedJobDetails.thenApply(
		(Collection<JobDetails> runningJobDetails) -> {
			final Collection<JobDetails> allJobDetails = new ArrayList<>(completedJobDetails.size() + runningJobDetails.size());

			allJobDetails.addAll(runningJobDetails);
			allJobDetails.addAll(completedJobDetails);

			return new MultipleJobsDetails(allJobDetails);
		});
}
 

private Collection<JobDetails> randomJobDetails(Random rnd) {
	final JobDetails[] details = new JobDetails[rnd.nextInt(10)];
	for (int k = 0; k < details.length; k++) {
		int[] numVerticesPerState = new int[ExecutionState.values().length];
		int numTotal = 0;

		for (int i = 0; i < numVerticesPerState.length; i++) {
			int count = rnd.nextInt(55);
			numVerticesPerState[i] = count;
			numTotal += count;
		}

		long time = rnd.nextLong();
		long endTime = rnd.nextBoolean() ? -1L : time + rnd.nextInt();
		long lastModified = endTime == -1 ? time + rnd.nextInt() : endTime;

		String name = new GenericMessageTester.StringInstantiator().instantiate(rnd);
		JobID jid = new JobID();
		JobStatus status = JobStatus.values()[rnd.nextInt(JobStatus.values().length)];

		details[k] = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
	}
	return Arrays.asList(details);
}
 

@Override
public CompletableFuture<JobStatus> requestJobStatus(JobID jobId, Time timeout) {

	final CompletableFuture<JobMasterGateway> jobMasterGatewayFuture = getJobMasterGatewayFuture(jobId);

	final CompletableFuture<JobStatus> jobStatusFuture = jobMasterGatewayFuture.thenCompose(
		(JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobStatus(timeout));

	return jobStatusFuture.exceptionally(
		(Throwable throwable) -> {
			final JobDetails jobDetails = archivedExecutionGraphStore.getAvailableJobDetails(jobId);

			// check whether it is a completed job
			if (jobDetails == null) {
				throw new CompletionException(ExceptionUtils.stripCompletionException(throwable));
			} else {
				return jobDetails.getStatus();
			}
		});
}
 

/**
 * This method replicates the JSON response that would be given by the JobsOverviewHandler when
 * listing both running and finished jobs.
 *
 * <p>Every job archive contains a joboverview.json file containing the same structure. Since jobs are archived on
 * their own however the list of finished jobs only contains a single job.
 *
 * <p>For the display in the HistoryServer WebFrontend we have to combine these overviews.
 */
private static void updateJobOverview(File webOverviewDir, File webDir) {
	try (JsonGenerator gen = jacksonFactory.createGenerator(HistoryServer.createOrGetFile(webDir, JobsOverviewHeaders.URL))) {
		File[] overviews = new File(webOverviewDir.getPath()).listFiles();
		if (overviews != null) {
			Collection<JobDetails> allJobs = new ArrayList<>(overviews.length);
			for (File overview : overviews) {
				MultipleJobsDetails subJobs = mapper.readValue(overview, MultipleJobsDetails.class);
				allJobs.addAll(subJobs.getJobs());
			}
			mapper.writeValue(gen, new MultipleJobsDetails(allJobs));
		}
	} catch (IOException ioe) {
		LOG.error("Failed to update job overview.", ioe);
	}
}
 

/**
 * Tests that we obtain the correct collection of available job details.
 */
@Test
public void testAvailableJobDetails() throws IOException {
	final int numberExecutionGraphs = 10;
	final Collection<ArchivedExecutionGraph> executionGraphs = generateTerminalExecutionGraphs(numberExecutionGraphs);

	final Collection<JobDetails> jobDetails = executionGraphs.stream().map(WebMonitorUtils::createDetailsForJob).collect(Collectors.toList());

	final File rootDir = temporaryFolder.newFolder();

	try (final FileArchivedExecutionGraphStore executionGraphStore = createDefaultExecutionGraphStore(rootDir)) {
		for (ArchivedExecutionGraph executionGraph : executionGraphs) {
			executionGraphStore.put(executionGraph);
		}

		assertThat(executionGraphStore.getAvailableJobDetails(), Matchers.containsInAnyOrder(jobDetails.toArray()));
	}
}
 

private Collection<JobDetails> randomJobDetails(Random rnd) {
	final JobDetails[] details = new JobDetails[rnd.nextInt(10)];
	for (int k = 0; k < details.length; k++) {
		int[] numVerticesPerState = new int[ExecutionState.values().length];
		int numTotal = 0;

		for (int i = 0; i < numVerticesPerState.length; i++) {
			int count = rnd.nextInt(55);
			numVerticesPerState[i] = count;
			numTotal += count;
		}

		long time = rnd.nextLong();
		long endTime = rnd.nextBoolean() ? -1L : time + rnd.nextInt();
		long lastModified = endTime == -1 ? time + rnd.nextInt() : endTime;

		String name = new GenericMessageTester.StringInstantiator().instantiate(rnd);
		JobID jid = new JobID();
		JobStatus status = JobStatus.values()[rnd.nextInt(JobStatus.values().length)];

		details[k] = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
	}
	return Arrays.asList(details);
}
 

/**
 * Lists the currently running and finished jobs on the cluster.
 *
 * @return future collection of running and finished jobs
 * @throws Exception if no connection to the cluster could be established
 */
public CompletableFuture<Collection<JobStatusMessage>> listJobs() throws Exception {
	final ActorGateway jobManager = getJobManagerGateway();

	Future<Object> response = jobManager.ask(new RequestJobDetails(true, false), timeout);
	CompletableFuture<Object> responseFuture = FutureUtils.toJava(response);

	return responseFuture.thenApply((responseMessage) -> {
		if (responseMessage instanceof MultipleJobsDetails) {
			MultipleJobsDetails details = (MultipleJobsDetails) responseMessage;

			final Collection<JobDetails> jobDetails = details.getJobs();
			Collection<JobStatusMessage> flattenedDetails = new ArrayList<>(jobDetails.size());
			jobDetails.forEach(detail -> flattenedDetails.add(new JobStatusMessage(detail.getJobId(), detail.getJobName(), detail.getStatus(), detail.getStartTime())));
			return flattenedDetails;
		} else {
			throw new CompletionException(
				new IllegalStateException("Unknown JobManager response of type " + responseMessage.getClass()));
		}
	});
}
 

@Override
public CompletableFuture<JobStatus> requestJobStatus(JobID jobId, Time timeout) {

	final CompletableFuture<JobMasterGateway> jobMasterGatewayFuture = getJobMasterGatewayFuture(jobId);

	final CompletableFuture<JobStatus> jobStatusFuture = jobMasterGatewayFuture.thenCompose(
		(JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobStatus(timeout));

	return jobStatusFuture.exceptionally(
		(Throwable throwable) -> {
			final JobDetails jobDetails = archivedExecutionGraphStore.getAvailableJobDetails(jobId);

			// check whether it is a completed job
			if (jobDetails == null) {
				throw new CompletionException(ExceptionUtils.stripCompletionException(throwable));
			} else {
				return jobDetails.getStatus();
			}
		});
}
 

@Override
public CompletableFuture<MultipleJobsDetails> requestMultipleJobDetails(Time timeout) {
	List<CompletableFuture<Optional<JobDetails>>> individualOptionalJobDetails = queryJobMastersForInformation(
		(JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobDetails(timeout));

	CompletableFuture<Collection<Optional<JobDetails>>> optionalCombinedJobDetails = FutureUtils.combineAll(
		individualOptionalJobDetails);

	CompletableFuture<Collection<JobDetails>> combinedJobDetails = optionalCombinedJobDetails.thenApply(this::flattenOptionalCollection);

	final Collection<JobDetails> completedJobDetails = archivedExecutionGraphStore.getAvailableJobDetails();

	return combinedJobDetails.thenApply(
		(Collection<JobDetails> runningJobDetails) -> {
			final Collection<JobDetails> allJobDetails = new ArrayList<>(completedJobDetails.size() + runningJobDetails.size());

			allJobDetails.addAll(runningJobDetails);
			allJobDetails.addAll(completedJobDetails);

			return new MultipleJobsDetails(allJobDetails);
		});
}
 

/**
 * This method replicates the JSON response that would be given by the JobsOverviewHandler when
 * listing both running and finished jobs.
 *
 * <p>Every job archive contains a joboverview.json file containing the same structure. Since jobs are archived on
 * their own however the list of finished jobs only contains a single job.
 *
 * <p>For the display in the HistoryServer WebFrontend we have to combine these overviews.
 */
private static void updateJobOverview(File webOverviewDir, File webDir) {
	try (JsonGenerator gen = jacksonFactory.createGenerator(HistoryServer.createOrGetFile(webDir, JobsOverviewHeaders.URL))) {
		File[] overviews = new File(webOverviewDir.getPath()).listFiles();
		if (overviews != null) {
			Collection<JobDetails> allJobs = new ArrayList<>(overviews.length);
			for (File overview : overviews) {
				MultipleJobsDetails subJobs = mapper.readValue(overview, MultipleJobsDetails.class);
				allJobs.addAll(subJobs.getJobs());
			}
			mapper.writeValue(gen, new MultipleJobsDetails(allJobs));
		}
	} catch (IOException ioe) {
		LOG.error("Failed to update job overview.", ioe);
	}
}
 

@Override
public CompletableFuture<MultipleJobsDetails> requestMultipleJobDetails(Time timeout) {
	List<CompletableFuture<Optional<JobDetails>>> individualOptionalJobDetails = queryJobMastersForInformation(
		(JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobDetails(timeout));

	CompletableFuture<Collection<Optional<JobDetails>>> optionalCombinedJobDetails = FutureUtils.combineAll(
		individualOptionalJobDetails);

	CompletableFuture<Collection<JobDetails>> combinedJobDetails = optionalCombinedJobDetails.thenApply(this::flattenOptionalCollection);

	final Collection<JobDetails> completedJobDetails = archivedExecutionGraphStore.getAvailableJobDetails();

	return combinedJobDetails.thenApply(
		(Collection<JobDetails> runningJobDetails) -> {
			final Collection<JobDetails> allJobDetails = new ArrayList<>(completedJobDetails.size() + runningJobDetails.size());

			allJobDetails.addAll(runningJobDetails);
			allJobDetails.addAll(completedJobDetails);

			return new MultipleJobsDetails(allJobDetails);
		});
}
 

@Override
public CompletableFuture<JobStatus> requestJobStatus(JobID jobId, Time timeout) {

	final CompletableFuture<JobMasterGateway> jobMasterGatewayFuture = getJobMasterGatewayFuture(jobId);

	final CompletableFuture<JobStatus> jobStatusFuture = jobMasterGatewayFuture.thenCompose(
		(JobMasterGateway jobMasterGateway) -> jobMasterGateway.requestJobStatus(timeout));

	return jobStatusFuture.exceptionally(
		(Throwable throwable) -> {
			final JobDetails jobDetails = archivedExecutionGraphStore.getAvailableJobDetails(jobId);

			// check whether it is a completed job
			if (jobDetails == null) {
				throw new CompletionException(ExceptionUtils.stripCompletionException(throwable));
			} else {
				return jobDetails.getStatus();
			}
		});
}
 

/**
 * This method replicates the JSON response that would be given by the JobsOverviewHandler when
 * listing both running and finished jobs.
 *
 * <p>Every job archive contains a joboverview.json file containing the same structure. Since jobs are archived on
 * their own however the list of finished jobs only contains a single job.
 *
 * <p>For the display in the HistoryServer WebFrontend we have to combine these overviews.
 */
private static void updateJobOverview(File webOverviewDir, File webDir) {
	try (JsonGenerator gen = jacksonFactory.createGenerator(HistoryServer.createOrGetFile(webDir, JobsOverviewHeaders.URL))) {
		File[] overviews = new File(webOverviewDir.getPath()).listFiles();
		if (overviews != null) {
			Collection<JobDetails> allJobs = new ArrayList<>(overviews.length);
			for (File overview : overviews) {
				MultipleJobsDetails subJobs = mapper.readValue(overview, MultipleJobsDetails.class);
				allJobs.addAll(subJobs.getJobs());
			}
			mapper.writeValue(gen, new MultipleJobsDetails(allJobs));
		}
	} catch (IOException ioe) {
		LOG.error("Failed to update job overview.", ioe);
	}
}
 

/**
 * Tests that we obtain the correct collection of available job details.
 */
@Test
public void testAvailableJobDetails() throws IOException {
	final int numberExecutionGraphs = 10;
	final Collection<ArchivedExecutionGraph> executionGraphs = generateTerminalExecutionGraphs(numberExecutionGraphs);

	final Collection<JobDetails> jobDetails = executionGraphs.stream().map(WebMonitorUtils::createDetailsForJob).collect(Collectors.toList());

	final File rootDir = temporaryFolder.newFolder();

	try (final FileArchivedExecutionGraphStore executionGraphStore = createDefaultExecutionGraphStore(rootDir)) {
		for (ArchivedExecutionGraph executionGraph : executionGraphs) {
			executionGraphStore.put(executionGraph);
		}

		assertThat(executionGraphStore.getAvailableJobDetails(), Matchers.containsInAnyOrder(jobDetails.toArray()));
	}
}
 

private Collection<JobDetails> randomJobDetails(Random rnd) {
	final JobDetails[] details = new JobDetails[rnd.nextInt(10)];
	for (int k = 0; k < details.length; k++) {
		int[] numVerticesPerState = new int[ExecutionState.values().length];
		int numTotal = 0;

		for (int i = 0; i < numVerticesPerState.length; i++) {
			int count = rnd.nextInt(55);
			numVerticesPerState[i] = count;
			numTotal += count;
		}

		long time = rnd.nextLong();
		long endTime = rnd.nextBoolean() ? -1L : time + rnd.nextInt();
		long lastModified = endTime == -1 ? time + rnd.nextInt() : endTime;

		String name = new GenericMessageTester.StringInstantiator().instantiate(rnd);
		JobID jid = new JobID();
		JobStatus status = JobStatus.values()[rnd.nextInt(JobStatus.values().length)];

		details[k] = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
	}
	return Arrays.asList(details);
}
 

@Nullable
@Override
public JobDetails getAvailableJobDetails(JobID jobId) {
	final ArchivedExecutionGraph archivedExecutionGraph = serializableExecutionGraphs.get(jobId);

	if (archivedExecutionGraph != null) {
		return WebMonitorUtils.createDetailsForJob(archivedExecutionGraph);
	} else {
		return null;
	}
}
 

@Test
public void testJobDetailsMessage() {
	try {
		final Random rnd = new Random();
		
		int[] numVerticesPerState = new int[ExecutionState.values().length];
		int numTotal = 0;

		for (int i = 0; i < numVerticesPerState.length; i++) {
			int count = rnd.nextInt(55);
			numVerticesPerState[i] = count;
			numTotal += count;
		}

		long time = rnd.nextLong();
		long endTime = rnd.nextBoolean() ? -1L : time + rnd.nextInt();
		long lastModified = endTime == -1 ? time + rnd.nextInt() : endTime;

		String name = GenericMessageTester.randomString(rnd);
		JobID jid = GenericMessageTester.randomJobId(rnd);
		JobStatus status = GenericMessageTester.randomJobStatus(rnd);
		
		JobDetails msg1 = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
		JobDetails msg2 = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
		
		GenericMessageTester.testMessageInstances(msg1, msg2);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

public static JobDetails createDetailsForJob(AccessExecutionGraph job) {
	JobStatus status = job.getState();

	long started = job.getStatusTimestamp(JobStatus.CREATED);
	long finished = status.isGloballyTerminalState() ? job.getStatusTimestamp(status) : -1L;
	long duration = (finished >= 0L ? finished : System.currentTimeMillis()) - started;

	int[] countsPerStatus = new int[ExecutionState.values().length];
	long lastChanged = 0;
	int numTotalTasks = 0;

	for (AccessExecutionJobVertex ejv : job.getVerticesTopologically()) {
		AccessExecutionVertex[] vertices = ejv.getTaskVertices();
		numTotalTasks += vertices.length;

		for (AccessExecutionVertex vertex : vertices) {
			ExecutionState state = vertex.getExecutionState();
			countsPerStatus[state.ordinal()]++;
			lastChanged = Math.max(lastChanged, vertex.getStateTimestamp(state));
		}
	}

	lastChanged = Math.max(lastChanged, finished);

	return new JobDetails(
		job.getJobID(),
		job.getJobName(),
		started,
		finished,
		duration,
		status,
		lastChanged,
		countsPerStatus,
		numTotalTasks);
}
 

public static JobDetails createDetailsForJob(AccessExecutionGraph job) {
	JobStatus status = job.getState();

	long started = job.getStatusTimestamp(JobStatus.CREATED);
	long finished = status.isGloballyTerminalState() ? job.getStatusTimestamp(status) : -1L;
	long duration = (finished >= 0L ? finished : System.currentTimeMillis()) - started;

	int[] countsPerStatus = new int[ExecutionState.values().length];
	long lastChanged = 0;
	int numTotalTasks = 0;

	for (AccessExecutionJobVertex ejv : job.getVerticesTopologically()) {
		AccessExecutionVertex[] vertices = ejv.getTaskVertices();
		numTotalTasks += vertices.length;

		for (AccessExecutionVertex vertex : vertices) {
			ExecutionState state = vertex.getExecutionState();
			countsPerStatus[state.ordinal()]++;
			lastChanged = Math.max(lastChanged, vertex.getStateTimestamp(state));
		}
	}

	lastChanged = Math.max(lastChanged, finished);

	return new JobDetails(
		job.getJobID(),
		job.getJobName(),
		started,
		finished,
		duration,
		status,
		lastChanged,
		countsPerStatus,
		numTotalTasks);
}
 

@Test
public void testJobDetailsMessage() {
	try {
		final Random rnd = new Random();
		
		int[] numVerticesPerState = new int[ExecutionState.values().length];
		int numTotal = 0;

		for (int i = 0; i < numVerticesPerState.length; i++) {
			int count = rnd.nextInt(55);
			numVerticesPerState[i] = count;
			numTotal += count;
		}

		long time = rnd.nextLong();
		long endTime = rnd.nextBoolean() ? -1L : time + rnd.nextInt();
		long lastModified = endTime == -1 ? time + rnd.nextInt() : endTime;

		String name = GenericMessageTester.randomString(rnd);
		JobID jid = GenericMessageTester.randomJobId(rnd);
		JobStatus status = GenericMessageTester.randomJobStatus(rnd);
		
		JobDetails msg1 = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
		JobDetails msg2 = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
		
		GenericMessageTester.testMessageInstances(msg1, msg2);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Nullable
@Override
public JobDetails getAvailableJobDetails(JobID jobId) {
	final ArchivedExecutionGraph archivedExecutionGraph = serializableExecutionGraphs.get(jobId);

	if (archivedExecutionGraph != null) {
		return WebMonitorUtils.createDetailsForJob(archivedExecutionGraph);
	} else {
		return null;
	}
}
 

@Override
public void put(ArchivedExecutionGraph archivedExecutionGraph) throws IOException {
	final JobStatus jobStatus = archivedExecutionGraph.getState();
	final JobID jobId = archivedExecutionGraph.getJobID();
	final String jobName = archivedExecutionGraph.getJobName();

	Preconditions.checkArgument(
		jobStatus.isGloballyTerminalState(),
		"The job " + jobName + '(' + jobId +
			") is not in a globally terminal state. Instead it is in state " + jobStatus + '.');

	switch (jobStatus) {
		case FINISHED:
			numFinishedJobs++;
			break;
		case CANCELED:
			numCanceledJobs++;
			break;
		case FAILED:
			numFailedJobs++;
			break;
		default:
			throw new IllegalStateException("The job " + jobName + '(' +
				jobId + ") should have been in a globally terminal state. " +
				"Instead it was in state " + jobStatus + '.');
	}

	// write the ArchivedExecutionGraph to disk
	storeArchivedExecutionGraph(archivedExecutionGraph);

	final JobDetails detailsForJob = WebMonitorUtils.createDetailsForJob(archivedExecutionGraph);

	jobDetailsCache.put(jobId, detailsForJob);
	archivedExecutionGraphCache.put(jobId, archivedExecutionGraph);
}
 

@Nullable
@Override
public JobDetails getAvailableJobDetails(JobID jobId) {
	final ArchivedExecutionGraph archivedExecutionGraph = serializableExecutionGraphs.get(jobId);

	if (archivedExecutionGraph != null) {
		return WebMonitorUtils.createDetailsForJob(archivedExecutionGraph);
	} else {
		return null;
	}
}
 

@Override
public void put(ArchivedExecutionGraph archivedExecutionGraph) throws IOException {
	final JobStatus jobStatus = archivedExecutionGraph.getState();
	final JobID jobId = archivedExecutionGraph.getJobID();
	final String jobName = archivedExecutionGraph.getJobName();

	Preconditions.checkArgument(
		jobStatus.isGloballyTerminalState(),
		"The job " + jobName + '(' + jobId +
			") is not in a globally terminal state. Instead it is in state " + jobStatus + '.');

	switch (jobStatus) {
		case FINISHED:
			numFinishedJobs++;
			break;
		case CANCELED:
			numCanceledJobs++;
			break;
		case FAILED:
			numFailedJobs++;
			break;
		default:
			throw new IllegalStateException("The job " + jobName + '(' +
				jobId + ") should have been in a globally terminal state. " +
				"Instead it was in state " + jobStatus + '.');
	}

	// write the ArchivedExecutionGraph to disk
	storeArchivedExecutionGraph(archivedExecutionGraph);

	final JobDetails detailsForJob = WebMonitorUtils.createDetailsForJob(archivedExecutionGraph);

	jobDetailsCache.put(jobId, detailsForJob);
	archivedExecutionGraphCache.put(jobId, archivedExecutionGraph);
}
 

public static JobDetails createDetailsForJob(AccessExecutionGraph job) {
	JobStatus status = job.getState();

	long started = job.getStatusTimestamp(JobStatus.CREATED);
	long finished = status.isGloballyTerminalState() ? job.getStatusTimestamp(status) : -1L;
	long duration = (finished >= 0L ? finished : System.currentTimeMillis()) - started;

	int[] countsPerStatus = new int[ExecutionState.values().length];
	long lastChanged = 0;
	int numTotalTasks = 0;

	for (AccessExecutionJobVertex ejv : job.getVerticesTopologically()) {
		AccessExecutionVertex[] vertices = ejv.getTaskVertices();
		numTotalTasks += vertices.length;

		for (AccessExecutionVertex vertex : vertices) {
			ExecutionState state = vertex.getExecutionState();
			countsPerStatus[state.ordinal()]++;
			lastChanged = Math.max(lastChanged, vertex.getStateTimestamp(state));
		}
	}

	lastChanged = Math.max(lastChanged, finished);

	return new JobDetails(
		job.getJobID(),
		job.getJobName(),
		started,
		finished,
		duration,
		status,
		lastChanged,
		countsPerStatus,
		numTotalTasks);
}
 

@Test
public void testJobDetailsMessage() {
	try {
		final Random rnd = new Random();
		
		int[] numVerticesPerState = new int[ExecutionState.values().length];
		int numTotal = 0;

		for (int i = 0; i < numVerticesPerState.length; i++) {
			int count = rnd.nextInt(55);
			numVerticesPerState[i] = count;
			numTotal += count;
		}

		long time = rnd.nextLong();
		long endTime = rnd.nextBoolean() ? -1L : time + rnd.nextInt();
		long lastModified = endTime == -1 ? time + rnd.nextInt() : endTime;

		String name = GenericMessageTester.randomString(rnd);
		JobID jid = GenericMessageTester.randomJobId(rnd);
		JobStatus status = GenericMessageTester.randomJobStatus(rnd);
		
		JobDetails msg1 = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
		JobDetails msg2 = new JobDetails(jid, name, time, endTime, endTime - time, status, lastModified, numVerticesPerState, numTotal);
		
		GenericMessageTester.testMessageInstances(msg1, msg2);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Override
public void put(ArchivedExecutionGraph archivedExecutionGraph) throws IOException {
	final JobStatus jobStatus = archivedExecutionGraph.getState();
	final JobID jobId = archivedExecutionGraph.getJobID();
	final String jobName = archivedExecutionGraph.getJobName();

	Preconditions.checkArgument(
		jobStatus.isGloballyTerminalState(),
		"The job " + jobName + '(' + jobId +
			") is not in a globally terminal state. Instead it is in state " + jobStatus + '.');

	switch (jobStatus) {
		case FINISHED:
			numFinishedJobs++;
			break;
		case CANCELED:
			numCanceledJobs++;
			break;
		case FAILED:
			numFailedJobs++;
			break;
		default:
			throw new IllegalStateException("The job " + jobName + '(' +
				jobId + ") should have been in a globally terminal state. " +
				"Instead it was in state " + jobStatus + '.');
	}

	// write the ArchivedExecutionGraph to disk
	storeArchivedExecutionGraph(archivedExecutionGraph);

	final JobDetails detailsForJob = WebMonitorUtils.createDetailsForJob(archivedExecutionGraph);

	jobDetailsCache.put(jobId, detailsForJob);
	archivedExecutionGraphCache.put(jobId, archivedExecutionGraph);
}
 

private Set<String> getIdsFromJobOverview(String baseUrl) throws Exception {
	return getJobsOverview(baseUrl).getJobs().stream()
		.map(JobDetails::getJobId)
		.map(JobID::toString)
		.collect(Collectors.toSet());
}
 

private static String convertLegacyJobOverview(String legacyOverview) throws IOException {
	JsonNode root = mapper.readTree(legacyOverview);
	JsonNode finishedJobs = root.get("finished");
	JsonNode job = finishedJobs.get(0);

	JobID jobId = JobID.fromHexString(job.get("jid").asText());
	String name = job.get("name").asText();
	JobStatus state = JobStatus.valueOf(job.get("state").asText());

	long startTime = job.get("start-time").asLong();
	long endTime = job.get("end-time").asLong();
	long duration = job.get("duration").asLong();
	long lastMod = job.get("last-modification").asLong();

	JsonNode tasks = job.get("tasks");
	int numTasks = tasks.get("total").asInt();
	JsonNode pendingNode = tasks.get("pending");
	// for flink version < 1.4 we have pending field,
	// when version >= 1.4 pending has been split into scheduled, deploying, and created.
	boolean versionLessThan14 = pendingNode != null;
	int created = 0;
	int scheduled;
	int deploying = 0;

	if (versionLessThan14) {
		// pending is a mix of CREATED/SCHEDULED/DEPLOYING
		// to maintain the correct number of task states we pick SCHEDULED
		scheduled = pendingNode.asInt();
	} else {
		created = tasks.get("created").asInt();
		scheduled = tasks.get("scheduled").asInt();
		deploying = tasks.get("deploying").asInt();
	}
	int running = tasks.get("running").asInt();
	int finished = tasks.get("finished").asInt();
	int canceling = tasks.get("canceling").asInt();
	int canceled = tasks.get("canceled").asInt();
	int failed = tasks.get("failed").asInt();

	int[] tasksPerState = new int[ExecutionState.values().length];
	tasksPerState[ExecutionState.CREATED.ordinal()] = created;
	tasksPerState[ExecutionState.SCHEDULED.ordinal()] = scheduled;
	tasksPerState[ExecutionState.DEPLOYING.ordinal()] = deploying;
	tasksPerState[ExecutionState.RUNNING.ordinal()] = running;
	tasksPerState[ExecutionState.FINISHED.ordinal()] = finished;
	tasksPerState[ExecutionState.CANCELING.ordinal()] = canceling;
	tasksPerState[ExecutionState.CANCELED.ordinal()] = canceled;
	tasksPerState[ExecutionState.FAILED.ordinal()] = failed;

	JobDetails jobDetails = new JobDetails(jobId, name, startTime, endTime, duration, state, lastMod, tasksPerState, numTasks);
	MultipleJobsDetails multipleJobsDetails = new MultipleJobsDetails(Collections.singleton(jobDetails));

	StringWriter sw = new StringWriter();
	mapper.writeValue(sw, multipleJobsDetails);
	return sw.toString();
}