org.apache.flink.runtime.executiongraph.IOMetrics Java Examples

/**
 * Creates a new task execution state update, with an attached exception.
 * This constructor may never throw an exception.
 *
 * @param jobID
 *        the ID of the job the task belongs to
 * @param executionId
 *        the ID of the task execution whose state is to be reported
 * @param executionState
 *        the execution state to be reported
 * @param error
 *        an optional error
 * @param accumulators
 *        The flink and user-defined accumulators which may be null.
 */
public TaskExecutionState(JobID jobID, ExecutionAttemptID executionId,
		ExecutionState executionState, Throwable error,
		AccumulatorSnapshot accumulators, IOMetrics ioMetrics) {

	if (jobID == null || executionId == null || executionState == null) {
		throw new NullPointerException();
	}

	this.jobID = jobID;
	this.executionId = executionId;
	this.executionState = executionState;
	if (error != null) {
		this.throwable = new SerializedThrowable(error);
	} else {
		this.throwable = null;
	}
	this.accumulators = accumulators;
	this.ioMetrics = ioMetrics;
}
 

/**
 * Creates a new task execution state update, with an attached exception.
 * This constructor may never throw an exception.
 *
 * @param jobID
 *        the ID of the job the task belongs to
 * @param executionId
 *        the ID of the task execution whose state is to be reported
 * @param executionState
 *        the execution state to be reported
 * @param error
 *        an optional error
 * @param accumulators
 *        The flink and user-defined accumulators which may be null.
 */
public TaskExecutionState(JobID jobID, ExecutionAttemptID executionId,
		ExecutionState executionState, Throwable error,
		AccumulatorSnapshot accumulators, IOMetrics ioMetrics) {

	if (jobID == null || executionId == null || executionState == null) {
		throw new NullPointerException();
	}

	this.jobID = jobID;
	this.executionId = executionId;
	this.executionState = executionState;
	if (error != null) {
		this.throwable = new SerializedThrowable(error);
	} else {
		this.throwable = null;
	}
	this.accumulators = accumulators;
	this.ioMetrics = ioMetrics;
}
 

/**
 * Creates a new task execution state update, with an attached exception.
 * This constructor may never throw an exception.
 *
 * @param jobID
 *        the ID of the job the task belongs to
 * @param executionId
 *        the ID of the task execution whose state is to be reported
 * @param executionState
 *        the execution state to be reported
 * @param error
 *        an optional error
 * @param accumulators
 *        The flink and user-defined accumulators which may be null.
 */
public TaskExecutionState(JobID jobID, ExecutionAttemptID executionId,
		ExecutionState executionState, Throwable error,
		AccumulatorSnapshot accumulators, IOMetrics ioMetrics) {

	if (jobID == null || executionId == null || executionState == null) {
		throw new NullPointerException();
	}

	this.jobID = jobID;
	this.executionId = executionId;
	this.executionState = executionState;
	if (error != null) {
		this.throwable = new SerializedThrowable(error);
	} else {
		this.throwable = null;
	}
	this.accumulators = accumulators;
	this.ioMetrics = ioMetrics;
}
 

@Test
public void testTaskIOMetricGroup() {
	TaskMetricGroup task = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
	TaskIOMetricGroup taskIO = task.getIOMetricGroup();

	// test counter forwarding
	assertNotNull(taskIO.getNumRecordsInCounter());
	assertNotNull(taskIO.getNumRecordsOutCounter());

	Counter c1 = new SimpleCounter();
	c1.inc(32L);
	Counter c2 = new SimpleCounter();
	c2.inc(64L);

	taskIO.reuseRecordsInputCounter(c1);
	taskIO.reuseRecordsOutputCounter(c2);
	assertEquals(32L, taskIO.getNumRecordsInCounter().getCount());
	assertEquals(64L, taskIO.getNumRecordsOutCounter().getCount());

	// test IOMetrics instantiation
	taskIO.getNumBytesInCounter().inc(100L);
	taskIO.getNumBytesOutCounter().inc(250L);
	taskIO.getNumBuffersOutCounter().inc(3L);
	taskIO.getIdleTimeMsPerSecond().markEvent(2L);

	IOMetrics io = taskIO.createSnapshot();
	assertEquals(32L, io.getNumRecordsIn());
	assertEquals(64L, io.getNumRecordsOut());
	assertEquals(100L, io.getNumBytesIn());
	assertEquals(250L, io.getNumBytesOut());
	assertEquals(3L, taskIO.getNumBuffersOutCounter().getCount());
	assertEquals(2L, taskIO.getIdleTimeMsPerSecond().getCount());
}
 

@Test
public void testTaskIOMetricGroup() {
	TaskMetricGroup task = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
	TaskIOMetricGroup taskIO = task.getIOMetricGroup();

	// test counter forwarding
	assertNotNull(taskIO.getNumRecordsInCounter());
	assertNotNull(taskIO.getNumRecordsOutCounter());

	Counter c1 = new SimpleCounter();
	c1.inc(32L);
	Counter c2 = new SimpleCounter();
	c2.inc(64L);

	taskIO.reuseRecordsInputCounter(c1);
	taskIO.reuseRecordsOutputCounter(c2);
	assertEquals(32L, taskIO.getNumRecordsInCounter().getCount());
	assertEquals(64L, taskIO.getNumRecordsOutCounter().getCount());

	// test IOMetrics instantiation
	taskIO.getNumBytesInLocalCounter().inc(100L);
	taskIO.getNumBytesInRemoteCounter().inc(150L);
	taskIO.getNumBytesOutCounter().inc(250L);
	taskIO.getNumBuffersInLocalCounter().inc(1L);
	taskIO.getNumBuffersInRemoteCounter().inc(2L);
	taskIO.getNumBuffersOutCounter().inc(3L);

	IOMetrics io = taskIO.createSnapshot();
	assertEquals(32L, io.getNumRecordsIn());
	assertEquals(64L, io.getNumRecordsOut());
	assertEquals(100L, io.getNumBytesInLocal());
	assertEquals(150L, io.getNumBytesInRemote());
	assertEquals(250L, io.getNumBytesOut());
	assertEquals(1L, taskIO.getNumBuffersInLocalCounter().getCount());
	assertEquals(2L, taskIO.getNumBuffersInRemoteCounter().getCount());
	assertEquals(3L, taskIO.getNumBuffersOutCounter().getCount());
}
 

@Test
public void testTaskIOMetricGroup() {
	TaskMetricGroup task = UnregisteredMetricGroups.createUnregisteredTaskMetricGroup();
	TaskIOMetricGroup taskIO = task.getIOMetricGroup();

	// test counter forwarding
	assertNotNull(taskIO.getNumRecordsInCounter());
	assertNotNull(taskIO.getNumRecordsOutCounter());

	Counter c1 = new SimpleCounter();
	c1.inc(32L);
	Counter c2 = new SimpleCounter();
	c2.inc(64L);

	taskIO.reuseRecordsInputCounter(c1);
	taskIO.reuseRecordsOutputCounter(c2);
	assertEquals(32L, taskIO.getNumRecordsInCounter().getCount());
	assertEquals(64L, taskIO.getNumRecordsOutCounter().getCount());

	// test IOMetrics instantiation
	taskIO.getNumBytesInCounter().inc(100L);
	taskIO.getNumBytesOutCounter().inc(250L);
	taskIO.getNumBuffersOutCounter().inc(3L);

	IOMetrics io = taskIO.createSnapshot();
	assertEquals(32L, io.getNumRecordsIn());
	assertEquals(64L, io.getNumRecordsOut());
	assertEquals(100L, io.getNumBytesIn());
	assertEquals(250L, io.getNumBytesOut());
	assertEquals(3L, taskIO.getNumBuffersOutCounter().getCount());
}
 

public IOMetrics createSnapshot() {
	return new IOMetrics(numRecordsInRate, numRecordsOutRate, numBytesInRate, numBytesOutRate);
}
 

/**
 * Adds the IO metrics for the given attempt to this object. If the {@link AccessExecution} is in
 * a terminal state the contained {@link IOMetrics} object is added. Otherwise the given {@link MetricFetcher} is
 * used to retrieve the required metrics.
 *
 * @param attempt Attempt whose IO metrics should be added
 * @param fetcher MetricFetcher to retrieve metrics for running jobs
 * @param jobID JobID to which the attempt belongs
 * @param taskID TaskID to which the attempt belongs
 */
public void addIOMetrics(AccessExecution attempt, @Nullable MetricFetcher fetcher, String jobID, String taskID) {
	if (attempt.getState().isTerminal()) {
		IOMetrics ioMetrics = attempt.getIOMetrics();
		if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph
			this.numBytesIn += ioMetrics.getNumBytesIn();
			this.numBytesOut += ioMetrics.getNumBytesOut();
			this.numRecordsIn += ioMetrics.getNumRecordsIn();
			this.numRecordsOut += ioMetrics.getNumRecordsOut();
		}
	} else { // execAttempt is still running, use MetricQueryService instead
		if (fetcher != null) {
			fetcher.update();
			MetricStore.ComponentMetricStore metrics = fetcher.getMetricStore()
				.getSubtaskMetricStore(jobID, taskID, attempt.getParallelSubtaskIndex());
			if (metrics != null) {
				/**
				 * We want to keep track of missing metrics to be able to make a difference between 0 as a value
				 * and a missing value.
				 * In case a metric is missing for a parallel instance of a task, we set the complete flag as
				 * false.
				 */
				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_IN) == null){
					this.numBytesInComplete = false;
				}
				else {
					this.numBytesIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT) == null){
					this.numBytesOutComplete = false;
				}
				else {
					this.numBytesOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN) == null){
					this.numRecordsInComplete = false;
				}
				else {
					this.numRecordsIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT) == null){
					this.numRecordsOutComplete = false;
				}
				else {
					this.numRecordsOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT));
				}
			}
			else {
				this.numBytesInComplete = false;
				this.numBytesOutComplete = false;
				this.numRecordsInComplete = false;
				this.numRecordsOutComplete = false;
			}
		}
	}
}
 

public IOMetrics createSnapshot() {
	return new IOMetrics(numRecordsInRate, numRecordsOutRate, numBytesInRate, numBytesOutRate);
}
 

public IOMetrics getIOMetrics() {
	return ioMetrics;
}
 

public ArchivedExecutionBuilder setIOMetrics(IOMetrics ioMetrics) {
	this.ioMetrics = ioMetrics;
	return this;
}
 

/**
 * Adds the IO metrics for the given attempt to this object. If the {@link AccessExecution} is in
 * a terminal state the contained {@link IOMetrics} object is added. Otherwise the given {@link MetricFetcher} is
 * used to retrieve the required metrics.
 *
 * @param attempt Attempt whose IO metrics should be added
 * @param fetcher MetricFetcher to retrieve metrics for running jobs
 * @param jobID JobID to which the attempt belongs
 * @param taskID TaskID to which the attempt belongs
 */
public void addIOMetrics(AccessExecution attempt, @Nullable MetricFetcher fetcher, String jobID, String taskID) {
	if (attempt.getState().isTerminal()) {
		IOMetrics ioMetrics = attempt.getIOMetrics();
		if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph
			this.numBytesIn += ioMetrics.getNumBytesIn();
			this.numBytesOut += ioMetrics.getNumBytesOut();
			this.numRecordsIn += ioMetrics.getNumRecordsIn();
			this.numRecordsOut += ioMetrics.getNumRecordsOut();
		}
	} else { // execAttempt is still running, use MetricQueryService instead
		if (fetcher != null) {
			fetcher.update();
			MetricStore.ComponentMetricStore metrics = fetcher.getMetricStore()
				.getSubtaskMetricStore(jobID, taskID, attempt.getParallelSubtaskIndex());
			if (metrics != null) {
				/**
				 * We want to keep track of missing metrics to be able to make a difference between 0 as a value
				 * and a missing value.
				 * In case a metric is missing for a parallel instance of a task, we set the complete flag as
				 * false.
				 */
				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_IN) == null){
					this.numBytesInComplete = false;
				}
				else {
					this.numBytesIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT) == null){
					this.numBytesOutComplete = false;
				}
				else {
					this.numBytesOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN) == null){
					this.numRecordsInComplete = false;
				}
				else {
					this.numRecordsIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT) == null){
					this.numRecordsOutComplete = false;
				}
				else {
					this.numRecordsOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT));
				}
			}
			else {
				this.numBytesInComplete = false;
				this.numBytesOutComplete = false;
				this.numRecordsInComplete = false;
				this.numRecordsOutComplete = false;
			}
		}
	}
}
 

public IOMetrics getIOMetrics() {
	return ioMetrics;
}
 

public ArchivedExecutionBuilder setIOMetrics(IOMetrics ioMetrics) {
	this.ioMetrics = ioMetrics;
	return this;
}
 

public static void compareIoMetrics(IOMetrics expectedMetrics, JsonNode writtenMetrics) {
	assertEquals(expectedMetrics.getNumBytesInTotal(), writtenMetrics.get("read-bytes").asLong());
	assertEquals(expectedMetrics.getNumBytesOut(), writtenMetrics.get("write-bytes").asLong());
	assertEquals(expectedMetrics.getNumRecordsIn(), writtenMetrics.get("read-records").asLong());
	assertEquals(expectedMetrics.getNumRecordsOut(), writtenMetrics.get("write-records").asLong());
}
 

/**
 * Adds the IO metrics for the given attempt to this object. If the {@link AccessExecution} is in
 * a terminal state the contained {@link IOMetrics} object is added. Otherwise the given {@link MetricFetcher} is
 * used to retrieve the required metrics.
 *
 * @param attempt Attempt whose IO metrics should be added
 * @param fetcher MetricFetcher to retrieve metrics for running jobs
 * @param jobID JobID to which the attempt belongs
 * @param taskID TaskID to which the attempt belongs
 */
public void addIOMetrics(AccessExecution attempt, @Nullable MetricFetcher fetcher, String jobID, String taskID) {
	if (attempt.getState().isTerminal()) {
		IOMetrics ioMetrics = attempt.getIOMetrics();
		if (ioMetrics != null) { // execAttempt is already finished, use final metrics stored in ExecutionGraph
			this.numBytesInLocal += ioMetrics.getNumBytesInLocal();
			this.numBytesInRemote += ioMetrics.getNumBytesInRemote();
			this.numBytesOut += ioMetrics.getNumBytesOut();
			this.numRecordsIn += ioMetrics.getNumRecordsIn();
			this.numRecordsOut += ioMetrics.getNumRecordsOut();
		}
	} else { // execAttempt is still running, use MetricQueryService instead
		if (fetcher != null) {
			fetcher.update();
			MetricStore.ComponentMetricStore metrics = fetcher.getMetricStore()
				.getSubtaskMetricStore(jobID, taskID, attempt.getParallelSubtaskIndex());
			if (metrics != null) {
				/**
				 * We want to keep track of missing metrics to be able to make a difference between 0 as a value
				 * and a missing value.
				 * In case a metric is missing for a parallel instance of a task, we set the complete flag as
				 * false.
				 */
				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_LOCAL) == null){
					this.numBytesInLocalComplete = false;
				}
				else {
					this.numBytesInLocal += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_LOCAL));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_REMOTE) == null){
					this.numBytesInRemoteComplete = false;
				}
				else {
					this.numBytesInRemote += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_IN_REMOTE));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT) == null){
					this.numBytesOutComplete = false;
				}
				else {
					this.numBytesOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_BYTES_OUT));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN) == null){
					this.numRecordsInComplete = false;
				}
				else {
					this.numRecordsIn += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_IN));
				}

				if (metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT) == null){
					this.numRecordsOutComplete = false;
				}
				else {
					this.numRecordsOut += Long.valueOf(metrics.getMetric(MetricNames.IO_NUM_RECORDS_OUT));
				}
			}
			else {
				this.numBytesInLocalComplete = false;
				this.numBytesInRemoteComplete = false;
				this.numBytesOutComplete = false;
				this.numRecordsInComplete = false;
				this.numRecordsOutComplete = false;
			}
		}
	}
}
 

public IOMetrics createSnapshot() {
	return new IOMetrics(numRecordsInRate, numRecordsOutRate, numBytesInRateLocal, numBytesInRateRemote, numBytesOutRate);
}
 

public IOMetrics getIOMetrics() {
	return ioMetrics;
}