Java Code Examples for org.apache.flink.runtime.executiongraph.IOMetrics#getNumBytesOut()

The following examples show how to use org.apache.flink.runtime.executiongraph.IOMetrics#getNumBytesOut() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MutableIOMetrics.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Adds the IO metrics for the given attempt to this object. If the {@link AccessExecution} is in
 * a terminal state the contained {@link IOMetrics} object is added. Otherwise the given {@link MetricFetcher} is
 * used to retrieve the required metrics.
 *
 * @param attempt Attempt whose IO metrics should be added
 * @param fetcher MetricFetcher to retrieve metrics for running jobs
 * @param jobID JobID to which the attempt belongs
 * @param taskID TaskID to which the attempt belongs
 */
public void addIOMetrics(AccessExecution attempt, @Nullable MetricFetcher fetcher, String jobID, String taskID) {
	if (attempt.getState().isTerminal()) {
		IOMetrics ioMetrics = attempt.getIOMetrics();
		if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph
			this.numBytesInLocal += ioMetrics.getNumBytesInLocal();
			this.numBytesInRemote += ioMetrics.getNumBytesInRemote();
			this.numBytesOut += ioMetrics.getNumBytesOut();
			this.numRecordsIn += ioMetrics.getNumRecordsIn();
			this.numRecordsOut += ioMetrics.getNumRecordsOut();
		}
	} else { // execAttempt is still running, use MetricQueryService instead
		if (fetcher != null) {
			fetcher.update();
			MetricStore.ComponentMetricStore metrics = fetcher.getMetricStore()
				.getSubtaskMetricStore(jobID, taskID, attempt.getParallelSubtaskIndex());
			if (metrics != null) {
				/**
				 * We want to keep track of missing metrics to be able to make a difference between 0 as a value
				 * and a missing value.
				 * In case a metric is missing for a parallel instance of a task, we set the complete flag as
				 * false.
				 */
				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_LOCAL) == null){
					this.numBytesInLocalComplete = false;
				}
				else {
					this.numBytesInLocal += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_LOCAL));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_REMOTE) == null){
					this.numBytesInRemoteComplete = false;
				}
				else {
					this.numBytesInRemote += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_REMOTE));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT) == null){
					this.numBytesOutComplete = false;
				}
				else {
					this.numBytesOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN) == null){
					this.numRecordsInComplete = false;
				}
				else {
					this.numRecordsIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT) == null){
					this.numRecordsOutComplete = false;
				}
				else {
					this.numRecordsOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT));
				}
			}
			else {
				this.numBytesInLocalComplete = false;
				this.numBytesInRemoteComplete = false;
				this.numBytesOutComplete = false;
				this.numRecordsInComplete = false;
				this.numRecordsOutComplete = false;
			}
		}
	}
}
Example 2
Source File: MutableIOMetrics.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Adds the IO metrics for the given attempt to this object. If the {@link AccessExecution} is in
 * a terminal state the contained {@link IOMetrics} object is added. Otherwise the given {@link MetricFetcher} is
 * used to retrieve the required metrics.
 *
 * @param attempt Attempt whose IO metrics should be added
 * @param fetcher MetricFetcher to retrieve metrics for running jobs
 * @param jobID JobID to which the attempt belongs
 * @param taskID TaskID to which the attempt belongs
 */
public void addIOMetrics(AccessExecution attempt, @Nullable MetricFetcher fetcher, String jobID, String taskID) {
	if (attempt.getState().isTerminal()) {
		IOMetrics ioMetrics = attempt.getIOMetrics();
		if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph
			this.numBytesIn += ioMetrics.getNumBytesIn();
			this.numBytesOut += ioMetrics.getNumBytesOut();
			this.numRecordsIn += ioMetrics.getNumRecordsIn();
			this.numRecordsOut += ioMetrics.getNumRecordsOut();
		}
	} else { // execAttempt is still running, use MetricQueryService instead
		if (fetcher != null) {
			fetcher.update();
			MetricStore.ComponentMetricStore metrics = fetcher.getMetricStore()
				.getSubtaskMetricStore(jobID, taskID, attempt.getParallelSubtaskIndex());
			if (metrics != null) {
				/**
				 * We want to keep track of missing metrics to be able to make a difference between 0 as a value
				 * and a missing value.
				 * In case a metric is missing for a parallel instance of a task, we set the complete flag as
				 * false.
				 */
				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_IN) == null){
					this.numBytesInComplete = false;
				}
				else {
					this.numBytesIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT) == null){
					this.numBytesOutComplete = false;
				}
				else {
					this.numBytesOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN) == null){
					this.numRecordsInComplete = false;
				}
				else {
					this.numRecordsIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT) == null){
					this.numRecordsOutComplete = false;
				}
				else {
					this.numRecordsOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT));
				}
			}
			else {
				this.numBytesInComplete = false;
				this.numBytesOutComplete = false;
				this.numRecordsInComplete = false;
				this.numRecordsOutComplete = false;
			}
		}
	}
}
Example 3
Source File: MutableIOMetrics.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Adds the IO metrics for the given attempt to this object. If the {@link AccessExecution} is in
 * a terminal state the contained {@link IOMetrics} object is added. Otherwise the given {@link MetricFetcher} is
 * used to retrieve the required metrics.
 *
 * @param attempt Attempt whose IO metrics should be added
 * @param fetcher MetricFetcher to retrieve metrics for running jobs
 * @param jobID JobID to which the attempt belongs
 * @param taskID TaskID to which the attempt belongs
 */
public void addIOMetrics(AccessExecution attempt, @Nullable MetricFetcher fetcher, String jobID, String taskID) {
	if (attempt.getState().isTerminal()) {
		IOMetrics ioMetrics = attempt.getIOMetrics();
		if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph
			this.numBytesIn += ioMetrics.getNumBytesIn();
			this.numBytesOut += ioMetrics.getNumBytesOut();
			this.numRecordsIn += ioMetrics.getNumRecordsIn();
			this.numRecordsOut += ioMetrics.getNumRecordsOut();
		}
	} else { // execAttempt is still running, use MetricQueryService instead
		if (fetcher != null) {
			fetcher.update();
			MetricStore.ComponentMetricStore metrics = fetcher.getMetricStore()
				.getSubtaskMetricStore(jobID, taskID, attempt.getParallelSubtaskIndex());
			if (metrics != null) {
				/**
				 * We want to keep track of missing metrics to be able to make a difference between 0 as a value
				 * and a missing value.
				 * In case a metric is missing for a parallel instance of a task, we set the complete flag as
				 * false.
				 */
				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_IN) == null){
					this.numBytesInComplete = false;
				}
				else {
					this.numBytesIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT) == null){
					this.numBytesOutComplete = false;
				}
				else {
					this.numBytesOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN) == null){
					this.numRecordsInComplete = false;
				}
				else {
					this.numRecordsIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT) == null){
					this.numRecordsOutComplete = false;
				}
				else {
					this.numRecordsOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT));
				}
			}
			else {
				this.numBytesInComplete = false;
				this.numBytesOutComplete = false;
				this.numRecordsInComplete = false;
				this.numRecordsOutComplete = false;
			}
		}
	}
}