Java Code Examples for org.apache.flink.runtime.rest.handler.legacy.metrics.MetricStore#ComponentMetricStore

@Nonnull
@Override
Collection<? extends MetricStore.ComponentMetricStore> getStores(MetricStore store, HandlerRequest<EmptyRequestBody, AggregateTaskManagerMetricsParameters> request) {
	List<ResourceID> taskmanagers = request.getQueryParameter(TaskManagersFilterQueryParameter.class);
	if (taskmanagers.isEmpty()) {
		return store.getTaskManagers().values();
	} else {
		Collection<MetricStore.TaskManagerMetricStore> taskmanagerStores = new ArrayList<>(taskmanagers.size());
		for (ResourceID taskmanager : taskmanagers) {
			MetricStore.TaskManagerMetricStore taskManagerMetricStore = store.getTaskManagerMetricStore(taskmanager.getResourceIdString());
			if (taskManagerMetricStore != null) {
				taskmanagerStores.add(taskManagerMetricStore);
			}
		}
		return taskmanagerStores;
	}
}
 

@Nonnull
@Override
Collection<? extends MetricStore.ComponentMetricStore> getStores(MetricStore store, HandlerRequest<EmptyRequestBody, AggregatedSubtaskMetricsParameters> request) {
	JobID jobID = request.getPathParameter(JobIDPathParameter.class);
	JobVertexID taskID = request.getPathParameter(JobVertexIdPathParameter.class);

	Collection<String> subtaskRanges = request.getQueryParameter(SubtasksFilterQueryParameter.class);
	if (subtaskRanges.isEmpty()) {
		MetricStore.TaskMetricStore taskMetricStore = store.getTaskMetricStore(jobID.toString(), taskID.toString());
		if (taskMetricStore != null) {
			return taskMetricStore.getAllSubtaskMetricStores();
		} else {
			return Collections.emptyList();
		}
	} else {
		Iterable<Integer> subtasks = getIntegerRangeFromString(subtaskRanges);
		Collection<MetricStore.ComponentMetricStore> subtaskStores = new ArrayList<>(8);
		for (int subtask : subtasks) {
			MetricStore.ComponentMetricStore subtaskMetricStore = store.getSubtaskMetricStore(jobID.toString(), taskID.toString(), subtask);
			if (subtaskMetricStore != null) {
				subtaskStores.add(subtaskMetricStore);
			}
		}
		return subtaskStores;
	}
}
 

@Nonnull
@Override
Collection<? extends MetricStore.ComponentMetricStore> getStores(MetricStore store, HandlerRequest<EmptyRequestBody, AggregatedSubtaskMetricsParameters> request) {
	JobID jobID = request.getPathParameter(JobIDPathParameter.class);
	JobVertexID taskID = request.getPathParameter(JobVertexIdPathParameter.class);

	Collection<String> subtaskRanges = request.getQueryParameter(SubtasksFilterQueryParameter.class);
	if (subtaskRanges.isEmpty()) {
		MetricStore.TaskMetricStore taskMetricStore = store.getTaskMetricStore(jobID.toString(), taskID.toString());
		if (taskMetricStore != null) {
			return taskMetricStore.getAllSubtaskMetricStores();
		} else {
			return Collections.emptyList();
		}
	} else {
		Iterable<Integer> subtasks = getIntegerRangeFromString(subtaskRanges);
		Collection<MetricStore.ComponentMetricStore> subtaskStores = new ArrayList<>(8);
		for (int subtask : subtasks) {
			MetricStore.ComponentMetricStore subtaskMetricStore = store.getSubtaskMetricStore(jobID.toString(), taskID.toString(), subtask);
			if (subtaskMetricStore != null) {
				subtaskStores.add(subtaskMetricStore);
			}
		}
		return subtaskStores;
	}
}
 

@Override
protected MetricStore.ComponentMetricStore getComponentMetricStore(
		HandlerRequest<EmptyRequestBody, JobVertexMetricsMessageParameters> request,
		MetricStore metricStore) {

	final JobID jobId = request.getPathParameter(JobIDPathParameter.class);
	final JobVertexID vertexId = request.getPathParameter(JobVertexIdPathParameter.class);

	return metricStore.getTaskMetricStore(jobId.toString(), vertexId.toString());
}
 

@Nullable
@Override
protected MetricStore.ComponentMetricStore getComponentMetricStore(
	HandlerRequest<EmptyRequestBody,
		TestMessageParameters> request,
	MetricStore metricStore) {
	return returnComponentMetricStore ? metricStore.getJobManager() : null;
}
 

private static List<Metric> getRequestedMetrics(
		MetricStore.ComponentMetricStore componentMetricStore,
		Set<String> requestedMetrics) throws RestHandlerException {

	final List<Metric> metrics = new ArrayList<>(requestedMetrics.size());
	for (final String requestedMetric : requestedMetrics) {
		final String value = componentMetricStore.getMetric(requestedMetric, null);
		if (value != null) {
			metrics.add(new Metric(requestedMetric, value));
		}
	}
	return metrics;
}
 

private static List<Metric> getAvailableMetrics(MetricStore.ComponentMetricStore componentMetricStore) {
	return componentMetricStore.metrics
		.keySet()
		.stream()
		.map(Metric::new)
		.collect(Collectors.toList());
}
 

private static List<Metric> getRequestedMetrics(
		MetricStore.ComponentMetricStore componentMetricStore,
		Set<String> requestedMetrics) throws RestHandlerException {

	final List<Metric> metrics = new ArrayList<>(requestedMetrics.size());
	for (final String requestedMetric : requestedMetrics) {
		final String value = componentMetricStore.getMetric(requestedMetric, null);
		if (value != null) {
			metrics.add(new Metric(requestedMetric, value));
		}
	}
	return metrics;
}
 

private static List<Metric> getAvailableMetrics(MetricStore.ComponentMetricStore componentMetricStore) {
	return componentMetricStore.metrics
		.keySet()
		.stream()
		.map(Metric::new)
		.collect(Collectors.toList());
}
 

@Nullable
@Override
protected MetricStore.ComponentMetricStore getComponentMetricStore(
		final HandlerRequest<EmptyRequestBody, TaskManagerMetricsMessageParameters> request,
		final MetricStore metricStore) {
	final ResourceID taskManagerId = request.getPathParameter(TaskManagerIdPathParameter.class);
	return metricStore.getTaskManagerMetricStore(taskManagerId.toString());
}
 

@Nullable
@Override
protected MetricStore.ComponentMetricStore getComponentMetricStore(
		final HandlerRequest<EmptyRequestBody, JobManagerMetricsMessageParameters> request,
		final MetricStore metricStore) {
	return metricStore.getJobManagerMetricStore();
}
 

@Nullable
@Override
protected MetricStore.ComponentMetricStore getComponentMetricStore(
		final HandlerRequest<EmptyRequestBody, JobManagerMetricsMessageParameters> request,
		final MetricStore metricStore) {
	return metricStore.getJobManagerMetricStore();
}
 

/**
 * Extracts and aggregates all requested metrics from the given metric stores, and maps the result to a JSON string.
 *
 * @param stores available metrics
 * @param requestedMetrics ids of requested metrics
 * @param requestedAggregationsFactories requested aggregations
 * @return JSON string containing the requested metrics
 */
private AggregatedMetricsResponseBody getAggregatedMetricValues(
		Collection<? extends MetricStore.ComponentMetricStore> stores,
		List<String> requestedMetrics,
		MetricAccumulatorFactory requestedAggregationsFactories) {

	Collection<AggregatedMetric> aggregatedMetrics = new ArrayList<>(requestedMetrics.size());
	for (String requestedMetric : requestedMetrics) {
		final Collection<Double> values = new ArrayList<>(stores.size());
		try {
			for (MetricStore.ComponentMetricStore store : stores) {
				String stringValue = store.metrics.get(requestedMetric);
				if (stringValue != null) {
					values.add(Double.valueOf(stringValue));
				}
			}
		} catch (NumberFormatException nfe) {
			log.warn("The metric {} is not numeric and can't be aggregated.", requestedMetric, nfe);
			// metric is not numeric so we can't perform aggregations => ignore it
			continue;
		}
		if (!values.isEmpty()) {

			Iterator<Double> valuesIterator = values.iterator();
			MetricAccumulator acc = requestedAggregationsFactories.get(requestedMetric, valuesIterator.next());
			valuesIterator.forEachRemaining(acc::add);

			aggregatedMetrics.add(acc.get());
		} else {
			return new AggregatedMetricsResponseBody(Collections.emptyList());
		}
	}
	return new AggregatedMetricsResponseBody(aggregatedMetrics);
}
 

@Nullable
@Override
protected MetricStore.ComponentMetricStore getComponentMetricStore(
		final HandlerRequest<EmptyRequestBody, TaskManagerMetricsMessageParameters> request,
		final MetricStore metricStore) {
	final ResourceID taskManagerId = request.getPathParameter(TaskManagerIdPathParameter.class);
	return metricStore.getTaskManagerMetricStore(taskManagerId.toString());
}
 

/**
 * Returns the {@link MetricStore.ComponentMetricStore} that should be queried for metrics.
 */
@Nullable
protected abstract MetricStore.ComponentMetricStore getComponentMetricStore(
	HandlerRequest<EmptyRequestBody, M> request,
	MetricStore metricStore);
 

/**
 * Adds the IO metrics for the given attempt to this object. If the {@link AccessExecution} is in
 * a terminal state the contained {@link IOMetrics} object is added. Otherwise the given {@link MetricFetcher} is
 * used to retrieve the required metrics.
 *
 * @param attempt Attempt whose IO metrics should be added
 * @param fetcher MetricFetcher to retrieve metrics for running jobs
 * @param jobID JobID to which the attempt belongs
 * @param taskID TaskID to which the attempt belongs
 */
public void addIOMetrics(AccessExecution attempt, @Nullable MetricFetcher fetcher, String jobID, String taskID) {
	if (attempt.getState().isTerminal()) {
		IOMetrics ioMetrics = attempt.getIOMetrics();
		if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph
			this.numBytesInLocal += ioMetrics.getNumBytesInLocal();
			this.numBytesInRemote += ioMetrics.getNumBytesInRemote();
			this.numBytesOut += ioMetrics.getNumBytesOut();
			this.numRecordsIn += ioMetrics.getNumRecordsIn();
			this.numRecordsOut += ioMetrics.getNumRecordsOut();
		}
	} else { // execAttempt is still running, use MetricQueryService instead
		if (fetcher != null) {
			fetcher.update();
			MetricStore.ComponentMetricStore metrics = fetcher.getMetricStore()
				.getSubtaskMetricStore(jobID, taskID, attempt.getParallelSubtaskIndex());
			if (metrics != null) {
				/**
				 * We want to keep track of missing metrics to be able to make a difference between 0 as a value
				 * and a missing value.
				 * In case a metric is missing for a parallel instance of a task, we set the complete flag as
				 * false.
				 */
				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_LOCAL) == null){
					this.numBytesInLocalComplete = false;
				}
				else {
					this.numBytesInLocal += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_LOCAL));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_REMOTE) == null){
					this.numBytesInRemoteComplete = false;
				}
				else {
					this.numBytesInRemote += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_REMOTE));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT) == null){
					this.numBytesOutComplete = false;
				}
				else {
					this.numBytesOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN) == null){
					this.numRecordsInComplete = false;
				}
				else {
					this.numRecordsIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT) == null){
					this.numRecordsOutComplete = false;
				}
				else {
					this.numRecordsOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT));
				}
			}
			else {
				this.numBytesInLocalComplete = false;
				this.numBytesInRemoteComplete = false;
				this.numBytesOutComplete = false;
				this.numRecordsInComplete = false;
				this.numRecordsOutComplete = false;
			}
		}
	}
}
 

/**
 * Adds the IO metrics for the given attempt to this object. If the {@link AccessExecution} is in
 * a terminal state the contained {@link IOMetrics} object is added. Otherwise the given {@link MetricFetcher} is
 * used to retrieve the required metrics.
 *
 * @param attempt Attempt whose IO metrics should be added
 * @param fetcher MetricFetcher to retrieve metrics for running jobs
 * @param jobID JobID to which the attempt belongs
 * @param taskID TaskID to which the attempt belongs
 */
public void addIOMetrics(AccessExecution attempt, @Nullable MetricFetcher fetcher, String jobID, String taskID) {
	if (attempt.getState().isTerminal()) {
		IOMetrics ioMetrics = attempt.getIOMetrics();
		if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph
			this.numBytesIn += ioMetrics.getNumBytesIn();
			this.numBytesOut += ioMetrics.getNumBytesOut();
			this.numRecordsIn += ioMetrics.getNumRecordsIn();
			this.numRecordsOut += ioMetrics.getNumRecordsOut();
		}
	} else { // execAttempt is still running, use MetricQueryService instead
		if (fetcher != null) {
			fetcher.update();
			MetricStore.ComponentMetricStore metrics = fetcher.getMetricStore()
				.getSubtaskMetricStore(jobID, taskID, attempt.getParallelSubtaskIndex());
			if (metrics != null) {
				/**
				 * We want to keep track of missing metrics to be able to make a difference between 0 as a value
				 * and a missing value.
				 * In case a metric is missing for a parallel instance of a task, we set the complete flag as
				 * false.
				 */
				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_IN) == null){
					this.numBytesInComplete = false;
				}
				else {
					this.numBytesIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT) == null){
					this.numBytesOutComplete = false;
				}
				else {
					this.numBytesOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN) == null){
					this.numRecordsInComplete = false;
				}
				else {
					this.numRecordsIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT) == null){
					this.numRecordsOutComplete = false;
				}
				else {
					this.numRecordsOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT));
				}
			}
			else {
				this.numBytesInComplete = false;
				this.numBytesOutComplete = false;
				this.numRecordsInComplete = false;
				this.numRecordsOutComplete = false;
			}
		}
	}
}
 

@Nonnull
abstract Collection<? extends MetricStore.ComponentMetricStore> getStores(MetricStore store, HandlerRequest<EmptyRequestBody, P> request);
 

@Nonnull
abstract Collection<? extends MetricStore.ComponentMetricStore> getStores(MetricStore store, HandlerRequest<EmptyRequestBody, P> request);
 

@Nonnull
abstract Collection<? extends MetricStore.ComponentMetricStore> getStores(MetricStore store, HandlerRequest<EmptyRequestBody, P> request);