org.apache.flink.runtime.checkpoint.AbstractCheckpointStats Java Examples

@Test
public void testCacheAddAndGet() throws Exception {
	AbstractCheckpointStats chk0 = createCheckpoint(0, CheckpointStatsStatus.COMPLETED);
	AbstractCheckpointStats chk1 = createCheckpoint(1, CheckpointStatsStatus.COMPLETED);
	AbstractCheckpointStats chk2 = createCheckpoint(2, CheckpointStatsStatus.IN_PROGRESS);

	CheckpointStatsCache cache = new CheckpointStatsCache(1);
	cache.tryAdd(chk0);
	assertEquals(chk0, cache.tryGet(0));

	cache.tryAdd(chk1);
	assertNull(cache.tryGet(0));
	assertEquals(chk1, cache.tryGet(1));

	cache.tryAdd(chk2);
	assertNull(cache.tryGet(2));
	assertNull(cache.tryGet(0));
	assertEquals(chk1, cache.tryGet(1));
}
 

@Override
public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException {
	CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot();
	if (stats == null) {
		return Collections.emptyList();
	}
	CheckpointStatsHistory history = stats.getHistory();
	List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size());
	for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) {
		for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) {
			ResponseBody json = createCheckpointDetails(checkpoint, subtaskStats);
			String path = getMessageHeaders().getTargetRestEndpointURL()
				.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
				.replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId()))
				.replace(':' + JobVertexIdPathParameter.KEY, subtaskStats.getJobVertexId().toString());
			archive.add(new ArchivedJson(path, json));
		}
	}
	return archive;
}
 

private static TaskCheckpointStatisticsWithSubtaskDetails createCheckpointDetails(AbstractCheckpointStats checkpointStats, TaskStateStats taskStatistics) {
	final TaskCheckpointStatisticsWithSubtaskDetails.Summary summary = createSummary(
		taskStatistics.getSummaryStats(),
		checkpointStats.getTriggerTimestamp());

	final List<SubtaskCheckpointStatistics> subtaskCheckpointStatistics = createSubtaskCheckpointStatistics(
		taskStatistics.getSubtaskStats(),
		checkpointStats.getTriggerTimestamp());

	return new TaskCheckpointStatisticsWithSubtaskDetails(
		checkpointStats.getCheckpointId(),
		checkpointStats.getStatus(),
		taskStatistics.getLatestAckTimestamp(),
		taskStatistics.getStateSize(),
		taskStatistics.getEndToEndDuration(checkpointStats.getTriggerTimestamp()),
		taskStatistics.getAlignmentBuffered(),
		taskStatistics.getNumberOfSubtasks(),
		taskStatistics.getNumberOfAcknowledgedSubtasks(),
		summary,
		subtaskCheckpointStatistics);
}
 

@Override
public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException {
	CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot();
	if (stats == null) {
		return Collections.emptyList();
	}
	CheckpointStatsHistory history = stats.getHistory();
	List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size());
	for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) {
		ResponseBody json = CheckpointStatistics.generateCheckpointStatistics(checkpoint, true);
		String path = getMessageHeaders().getTargetRestEndpointURL()
			.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
			.replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId()));
		archive.add(new ArchivedJson(path, json));
	}
	return archive;
}
 

@Override
protected R handleRequest(HandlerRequest<EmptyRequestBody, M> request, AccessExecutionGraph executionGraph) throws RestHandlerException {
	final long checkpointId = request.getPathParameter(CheckpointIdPathParameter.class);

	final CheckpointStatsSnapshot checkpointStatsSnapshot = executionGraph.getCheckpointStatsSnapshot();

	if (checkpointStatsSnapshot != null) {
		AbstractCheckpointStats checkpointStats = checkpointStatsSnapshot.getHistory().getCheckpointById(checkpointId);

		if (checkpointStats != null) {
			checkpointStatsCache.tryAdd(checkpointStats);
		} else {
			checkpointStats = checkpointStatsCache.tryGet(checkpointId);
		}

		if (checkpointStats != null) {
			return handleCheckpointRequest(request, checkpointStats);
		} else {
			throw new RestHandlerException("Could not find checkpointing statistics for checkpoint " + checkpointId + '.', HttpResponseStatus.NOT_FOUND);
		}
	} else {
		throw new RestHandlerException("Checkpointing was not enabled for job " + executionGraph.getJobID() + '.', HttpResponseStatus.NOT_FOUND);
	}
}
 

@Override
public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException {
	CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot();
	if (stats == null) {
		return Collections.emptyList();
	}
	CheckpointStatsHistory history = stats.getHistory();
	List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size());
	for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) {
		ResponseBody json = CheckpointStatistics.generateCheckpointStatistics(checkpoint, true);
		String path = getMessageHeaders().getTargetRestEndpointURL()
			.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
			.replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId()));
		archive.add(new ArchivedJson(path, json));
	}
	return archive;
}
 

@Test
public void testCacheAddAndGet() throws Exception {
	AbstractCheckpointStats chk0 = createCheckpoint(0, CheckpointStatsStatus.COMPLETED);
	AbstractCheckpointStats chk1 = createCheckpoint(1, CheckpointStatsStatus.COMPLETED);
	AbstractCheckpointStats chk2 = createCheckpoint(2, CheckpointStatsStatus.IN_PROGRESS);

	CheckpointStatsCache cache = new CheckpointStatsCache(1);
	cache.tryAdd(chk0);
	assertEquals(chk0, cache.tryGet(0));

	cache.tryAdd(chk1);
	assertNull(cache.tryGet(0));
	assertEquals(chk1, cache.tryGet(1));

	cache.tryAdd(chk2);
	assertNull(cache.tryGet(2));
	assertNull(cache.tryGet(0));
	assertEquals(chk1, cache.tryGet(1));
}
 

private static TaskCheckpointStatisticsWithSubtaskDetails createCheckpointDetails(AbstractCheckpointStats checkpointStats, TaskStateStats taskStatistics) {
	final TaskCheckpointStatisticsWithSubtaskDetails.Summary summary = createSummary(
		taskStatistics.getSummaryStats(),
		checkpointStats.getTriggerTimestamp());

	final List<SubtaskCheckpointStatistics> subtaskCheckpointStatistics = createSubtaskCheckpointStatistics(
		taskStatistics.getSubtaskStats(),
		checkpointStats.getTriggerTimestamp());

	return new TaskCheckpointStatisticsWithSubtaskDetails(
		checkpointStats.getCheckpointId(),
		checkpointStats.getStatus(),
		taskStatistics.getLatestAckTimestamp(),
		taskStatistics.getStateSize(),
		taskStatistics.getEndToEndDuration(checkpointStats.getTriggerTimestamp()),
		0,
		taskStatistics.getNumberOfSubtasks(),
		taskStatistics.getNumberOfAcknowledgedSubtasks(),
		summary,
		subtaskCheckpointStatistics);
}
 

@Override
public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException {
	CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot();
	if (stats == null) {
		return Collections.emptyList();
	}
	CheckpointStatsHistory history = stats.getHistory();
	List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size());
	for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) {
		ResponseBody json = CheckpointStatistics.generateCheckpointStatistics(checkpoint, true);
		String path = getMessageHeaders().getTargetRestEndpointURL()
			.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
			.replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId()));
		archive.add(new ArchivedJson(path, json));
	}
	return archive;
}
 

@Override
public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException {
	CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot();
	if (stats == null) {
		return Collections.emptyList();
	}
	CheckpointStatsHistory history = stats.getHistory();
	List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size());
	for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) {
		for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) {
			ResponseBody json = createCheckpointDetails(checkpoint, subtaskStats);
			String path = getMessageHeaders().getTargetRestEndpointURL()
				.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
				.replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId()))
				.replace(':' + JobVertexIdPathParameter.KEY, subtaskStats.getJobVertexId().toString());
			archive.add(new ArchivedJson(path, json));
		}
	}
	return archive;
}
 

private static TaskCheckpointStatisticsWithSubtaskDetails createCheckpointDetails(AbstractCheckpointStats checkpointStats, TaskStateStats taskStatistics) {
	final TaskCheckpointStatisticsWithSubtaskDetails.Summary summary = createSummary(
		taskStatistics.getSummaryStats(),
		checkpointStats.getTriggerTimestamp());

	final List<SubtaskCheckpointStatistics> subtaskCheckpointStatistics = createSubtaskCheckpointStatistics(
		taskStatistics.getSubtaskStats(),
		checkpointStats.getTriggerTimestamp());

	return new TaskCheckpointStatisticsWithSubtaskDetails(
		checkpointStats.getCheckpointId(),
		checkpointStats.getStatus(),
		taskStatistics.getLatestAckTimestamp(),
		taskStatistics.getStateSize(),
		taskStatistics.getEndToEndDuration(checkpointStats.getTriggerTimestamp()),
		taskStatistics.getAlignmentBuffered(),
		taskStatistics.getNumberOfSubtasks(),
		taskStatistics.getNumberOfAcknowledgedSubtasks(),
		summary,
		subtaskCheckpointStatistics);
}
 

@Override
public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException {
	CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot();
	if (stats == null) {
		return Collections.emptyList();
	}
	CheckpointStatsHistory history = stats.getHistory();
	List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size());
	for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) {
		for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) {
			ResponseBody json = createCheckpointDetails(checkpoint, subtaskStats);
			String path = getMessageHeaders().getTargetRestEndpointURL()
				.replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString())
				.replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId()))
				.replace(':' + JobVertexIdPathParameter.KEY, subtaskStats.getJobVertexId().toString());
			archive.add(new ArchivedJson(path, json));
		}
	}
	return archive;
}
 

@Test
public void testCacheAddAndGet() throws Exception {
	AbstractCheckpointStats chk0 = createCheckpoint(0, CheckpointStatsStatus.COMPLETED);
	AbstractCheckpointStats chk1 = createCheckpoint(1, CheckpointStatsStatus.COMPLETED);
	AbstractCheckpointStats chk2 = createCheckpoint(2, CheckpointStatsStatus.IN_PROGRESS);

	CheckpointStatsCache cache = new CheckpointStatsCache(1);
	cache.tryAdd(chk0);
	assertEquals(chk0, cache.tryGet(0));

	cache.tryAdd(chk1);
	assertNull(cache.tryGet(0));
	assertEquals(chk1, cache.tryGet(1));

	cache.tryAdd(chk2);
	assertNull(cache.tryGet(2));
	assertNull(cache.tryGet(0));
	assertEquals(chk1, cache.tryGet(1));
}
 

@Override
protected R handleRequest(HandlerRequest<EmptyRequestBody, M> request, AccessExecutionGraph executionGraph) throws RestHandlerException {
	final long checkpointId = request.getPathParameter(CheckpointIdPathParameter.class);

	final CheckpointStatsSnapshot checkpointStatsSnapshot = executionGraph.getCheckpointStatsSnapshot();

	if (checkpointStatsSnapshot != null) {
		AbstractCheckpointStats checkpointStats = checkpointStatsSnapshot.getHistory().getCheckpointById(checkpointId);

		if (checkpointStats != null) {
			checkpointStatsCache.tryAdd(checkpointStats);
		} else {
			checkpointStats = checkpointStatsCache.tryGet(checkpointId);
		}

		if (checkpointStats != null) {
			return handleCheckpointRequest(request, checkpointStats);
		} else {
			throw new RestHandlerException("Could not find checkpointing statistics for checkpoint " + checkpointId + '.', HttpResponseStatus.NOT_FOUND);
		}
	} else {
		throw new RestHandlerException("Checkpointing was not enabled for job " + executionGraph.getJobID() + '.', HttpResponseStatus.NOT_FOUND);
	}
}
 

@Override
protected R handleRequest(HandlerRequest<EmptyRequestBody, M> request, AccessExecutionGraph executionGraph) throws RestHandlerException {
	final long checkpointId = request.getPathParameter(CheckpointIdPathParameter.class);

	final CheckpointStatsSnapshot checkpointStatsSnapshot = executionGraph.getCheckpointStatsSnapshot();

	if (checkpointStatsSnapshot != null) {
		AbstractCheckpointStats checkpointStats = checkpointStatsSnapshot.getHistory().getCheckpointById(checkpointId);

		if (checkpointStats != null) {
			checkpointStatsCache.tryAdd(checkpointStats);
		} else {
			checkpointStats = checkpointStatsCache.tryGet(checkpointId);
		}

		if (checkpointStats != null) {
			return handleCheckpointRequest(request, checkpointStats);
		} else {
			throw new RestHandlerException("Could not find checkpointing statistics for checkpoint " + checkpointId + '.', HttpResponseStatus.NOT_FOUND);
		}
	} else {
		throw new RestHandlerException("Checkpointing was not enabled for job " + executionGraph.getJobID() + '.', HttpResponseStatus.NOT_FOUND);
	}
}
 

public CheckpointStatsCache(int maxNumEntries) {
	if (maxNumEntries > 0) {
		this.cache = CacheBuilder.<Long, AbstractCheckpointStats>newBuilder()
			.maximumSize(maxNumEntries)
			.build();
	} else {
		this.cache = null;
	}
}
 

@Test
public void testZeroSizeCache() throws Exception {
	AbstractCheckpointStats checkpoint = createCheckpoint(0, CheckpointStatsStatus.COMPLETED);

	CheckpointStatsCache cache = new CheckpointStatsCache(0);
	cache.tryAdd(checkpoint);
	assertNull(cache.tryGet(0L));
}
 

/**
 * Try to look up a checkpoint by it's ID in the cache.
 *
 * @param checkpointId ID of the checkpoint to look up.
 * @return The checkpoint or <code>null</code> if checkpoint not found.
 */
public AbstractCheckpointStats tryGet(long checkpointId) {
	if (cache != null) {
		return cache.getIfPresent(checkpointId);
	} else {
		return null;
	}
}
 

/**
 * Try to add the checkpoint to the cache.
 *
 * @param checkpoint Checkpoint to be added.
 */
public void tryAdd(AbstractCheckpointStats checkpoint) {
	// Don't add in progress checkpoints as they will be replaced by their
	// completed/failed version eventually.
	if (cache != null && checkpoint != null && !checkpoint.getStatus().isInProgress()) {
		cache.put(checkpoint.getCheckpointId(), checkpoint);
	}
}
 

public CheckpointStatsCache(int maxNumEntries) {
	if (maxNumEntries > 0) {
		this.cache = CacheBuilder.<Long, AbstractCheckpointStats>newBuilder()
			.maximumSize(maxNumEntries)
			.build();
	} else {
		this.cache = null;
	}
}
 

@Override
protected TaskCheckpointStatisticsWithSubtaskDetails handleCheckpointRequest(
		HandlerRequest<EmptyRequestBody, TaskCheckpointMessageParameters> request,
		AbstractCheckpointStats checkpointStats) throws RestHandlerException {

	final JobVertexID jobVertexId = request.getPathParameter(JobVertexIdPathParameter.class);

	final TaskStateStats taskStatistics = checkpointStats.getTaskStateStats(jobVertexId);

	if (taskStatistics == null) {
		throw new NotFoundException("There is no checkpoint statistics for task " + jobVertexId + '.');
	}

	return createCheckpointDetails(checkpointStats, taskStatistics);
}
 

@Override
protected TaskCheckpointStatisticsWithSubtaskDetails handleCheckpointRequest(
		HandlerRequest<EmptyRequestBody, TaskCheckpointMessageParameters> request,
		AbstractCheckpointStats checkpointStats) throws RestHandlerException {

	final JobVertexID jobVertexId = request.getPathParameter(JobVertexIdPathParameter.class);

	final TaskStateStats taskStatistics = checkpointStats.getTaskStateStats(jobVertexId);

	if (taskStatistics == null) {
		throw new NotFoundException("There is no checkpoint statistics for task " + jobVertexId + '.');
	}

	return createCheckpointDetails(checkpointStats, taskStatistics);
}
 

/**
 * Try to add the checkpoint to the cache.
 *
 * @param checkpoint Checkpoint to be added.
 */
public void tryAdd(AbstractCheckpointStats checkpoint) {
	// Don't add in progress checkpoints as they will be replaced by their
	// completed/failed version eventually.
	if (cache != null && checkpoint != null && !checkpoint.getStatus().isInProgress()) {
		cache.put(checkpoint.getCheckpointId(), checkpoint);
	}
}
 

/**
 * Try to look up a checkpoint by it's ID in the cache.
 *
 * @param checkpointId ID of the checkpoint to look up.
 * @return The checkpoint or <code>null</code> if checkpoint not found.
 */
public AbstractCheckpointStats tryGet(long checkpointId) {
	if (cache != null) {
		return cache.getIfPresent(checkpointId);
	} else {
		return null;
	}
}
 

@Test
public void testZeroSizeCache() throws Exception {
	AbstractCheckpointStats checkpoint = createCheckpoint(0, CheckpointStatsStatus.COMPLETED);

	CheckpointStatsCache cache = new CheckpointStatsCache(0);
	cache.tryAdd(checkpoint);
	assertNull(cache.tryGet(0L));
}
 

/**
 * Try to look up a checkpoint by it's ID in the cache.
 *
 * @param checkpointId ID of the checkpoint to look up.
 * @return The checkpoint or <code>null</code> if checkpoint not found.
 */
public AbstractCheckpointStats tryGet(long checkpointId) {
	if (cache != null) {
		return cache.getIfPresent(checkpointId);
	} else {
		return null;
	}
}
 

/**
 * Try to add the checkpoint to the cache.
 *
 * @param checkpoint Checkpoint to be added.
 */
public void tryAdd(AbstractCheckpointStats checkpoint) {
	// Don't add in progress checkpoints as they will be replaced by their
	// completed/failed version eventually.
	if (cache != null && checkpoint != null && !checkpoint.getStatus().isInProgress()) {
		cache.put(checkpoint.getCheckpointId(), checkpoint);
	}
}
 

public CheckpointStatsCache(int maxNumEntries) {
	if (maxNumEntries > 0) {
		this.cache = CacheBuilder.<Long, AbstractCheckpointStats>newBuilder()
			.maximumSize(maxNumEntries)
			.build();
	} else {
		this.cache = null;
	}
}
 

@Test
public void testZeroSizeCache() throws Exception {
	AbstractCheckpointStats checkpoint = createCheckpoint(0, CheckpointStatsStatus.COMPLETED);

	CheckpointStatsCache cache = new CheckpointStatsCache(0);
	cache.tryAdd(checkpoint);
	assertNull(cache.tryGet(0L));
}
 

@Override
protected TaskCheckpointStatisticsWithSubtaskDetails handleCheckpointRequest(
		HandlerRequest<EmptyRequestBody, TaskCheckpointMessageParameters> request,
		AbstractCheckpointStats checkpointStats) throws RestHandlerException {

	final JobVertexID jobVertexId = request.getPathParameter(JobVertexIdPathParameter.class);

	final TaskStateStats taskStatistics = checkpointStats.getTaskStateStats(jobVertexId);

	if (taskStatistics == null) {
		throw new NotFoundException("There is no checkpoint statistics for task " + jobVertexId + '.');
	}

	return createCheckpointDetails(checkpointStats, taskStatistics);
}