Java Code Examples for org.apache.flink.metrics.MetricGroup#gauge()

private static void instantiateCPUMetrics(MetricGroup metrics, SystemResourcesCounter usageCounter) {
	metrics.<Double, Gauge<Double>>gauge("Usage", usageCounter::getCpuUsage);
	metrics.<Double, Gauge<Double>>gauge("Idle", usageCounter::getCpuIdle);
	metrics.<Double, Gauge<Double>>gauge("Sys", usageCounter::getCpuSys);
	metrics.<Double, Gauge<Double>>gauge("User", usageCounter::getCpuUser);
	metrics.<Double, Gauge<Double>>gauge("IOWait", usageCounter::getIOWait);
	metrics.<Double, Gauge<Double>>gauge("Nice", usageCounter::getCpuNice);
	metrics.<Double, Gauge<Double>>gauge("Irq", usageCounter::getCpuIrq);
	metrics.<Double, Gauge<Double>>gauge("SoftIrq", usageCounter::getCpuSoftIrq);

	metrics.<Double, Gauge<Double>>gauge("Load1min", usageCounter::getCpuLoad1);
	metrics.<Double, Gauge<Double>>gauge("Load5min", usageCounter::getCpuLoad5);
	metrics.<Double, Gauge<Double>>gauge("Load15min", usageCounter::getCpuLoad15);

	for (int i = 0; i < usageCounter.getProcessorsCount(); i++) {
		final int processor = i;
		metrics.<Double, Gauge<Double>>gauge(
			String.format("UsageCPU%d", processor),
			() -> usageCounter.getCpuUsagePerProcessor(processor));
	}
}
 

/**
 * For each partition, register a new metric group to expose current offsets and committed offsets.
 * Per-partition metric groups can be scoped by user variables {@link KafkaConsumerMetricConstants#OFFSETS_BY_TOPIC_METRICS_GROUP}
 * and {@link KafkaConsumerMetricConstants#OFFSETS_BY_PARTITION_METRICS_GROUP}.
 *
 * <p>Note: this method also registers gauges for deprecated offset metrics, to maintain backwards compatibility.
 *
 * @param consumerMetricGroup The consumer metric group
 * @param partitionOffsetStates The partition offset state holders, whose values will be used to update metrics
 */
private void registerOffsetMetrics(
		MetricGroup consumerMetricGroup,
		List<KafkaTopicPartitionState<KPH>> partitionOffsetStates) {

	for (KafkaTopicPartitionState<KPH> ktp : partitionOffsetStates) {
		MetricGroup topicPartitionGroup = consumerMetricGroup
			.addGroup(OFFSETS_BY_TOPIC_METRICS_GROUP, ktp.getTopic())
			.addGroup(OFFSETS_BY_PARTITION_METRICS_GROUP, Integer.toString(ktp.getPartition()));

		topicPartitionGroup.gauge(CURRENT_OFFSETS_METRICS_GAUGE, new OffsetGauge(ktp, OffsetGaugeType.CURRENT_OFFSET));
		topicPartitionGroup.gauge(COMMITTED_OFFSETS_METRICS_GAUGE, new OffsetGauge(ktp, OffsetGaugeType.COMMITTED_OFFSET));

		legacyCurrentOffsetsMetricGroup.gauge(getLegacyOffsetsMetricsGaugeName(ktp), new OffsetGauge(ktp, OffsetGaugeType.CURRENT_OFFSET));
		legacyCommittedOffsetsMetricGroup.gauge(getLegacyOffsetsMetricsGaugeName(ktp), new OffsetGauge(ktp, OffsetGaugeType.COMMITTED_OFFSET));
	}
}
 

/**
 * register reader group metrics
 *
 */
private void registerMetrics() {
    Preconditions.checkState(readerGroup != null, "Reader Group is not created");
    MetricGroup pravegaReaderMetricGroup = getRuntimeContext().getMetricGroup().addGroup(PRAVEGA_READER_METRICS_GROUP);
    MetricGroup readerGroupMetricGroup = pravegaReaderMetricGroup.addGroup(READER_GROUP_METRICS_GROUP);
    readerGroupMetricGroup.gauge(UNREAD_BYTES_METRICS_GAUGE, new UnreadBytesGauge(readerGroup));
    readerGroupMetricGroup.gauge(READER_GROUP_NAME_METRICS_GAUGE, new ReaderGroupNameGauge(readerGroup));
    readerGroupMetricGroup.gauge(SCOPE_NAME_METRICS_GAUGE, new ScopeNameGauge(readerGroup));
    readerGroupMetricGroup.gauge(ONLINE_READERS_METRICS_GAUGE, new OnlineReadersGauge(readerGroup));
    readerGroupMetricGroup.gauge(STREAM_NAMES_METRICS_GAUGE, new StreamNamesGauge(readerGroup));

    Set<String> streamNames = readerGroup.getStreamNames();
    for (String scopedStream: streamNames) {
        String[] streamInfo = scopedStream.split("/", 2);
        Preconditions.checkArgument(streamInfo.length == 2, "not a fully qualified stream expected: scopeName/streamName");
        MetricGroup streamMetricGroup = readerGroupMetricGroup
                .addGroup(STREAM_METRICS_GROUP + "." + streamInfo[0]+ "_"+ streamInfo[1]);
        streamMetricGroup.gauge(SEGMENT_POSITIONS_METRICS_GAUGE,
                new SegmentPositionsGauge(readerGroup, streamInfo[0], streamInfo[1]));
    }
}
 

/**
 * For each partition, register a new metric group to expose current offsets and committed offsets.
 * Per-partition metric groups can be scoped by user variables {@link KafkaConsumerMetricConstants#OFFSETS_BY_TOPIC_METRICS_GROUP}
 * and {@link KafkaConsumerMetricConstants#OFFSETS_BY_PARTITION_METRICS_GROUP}.
 *
 * <p>Note: this method also registers gauges for deprecated offset metrics, to maintain backwards compatibility.
 *
 * @param consumerMetricGroup The consumer metric group
 * @param partitionOffsetStates The partition offset state holders, whose values will be used to update metrics
 */
private void registerOffsetMetrics(
		MetricGroup consumerMetricGroup,
		List<KafkaTopicPartitionState<T, KPH>> partitionOffsetStates) {

	for (KafkaTopicPartitionState<T, KPH> ktp : partitionOffsetStates) {
		MetricGroup topicPartitionGroup = consumerMetricGroup
			.addGroup(OFFSETS_BY_TOPIC_METRICS_GROUP, ktp.getTopic())
			.addGroup(OFFSETS_BY_PARTITION_METRICS_GROUP, Integer.toString(ktp.getPartition()));

		topicPartitionGroup.gauge(CURRENT_OFFSETS_METRICS_GAUGE, new OffsetGauge(ktp, OffsetGaugeType.CURRENT_OFFSET));
		topicPartitionGroup.gauge(COMMITTED_OFFSETS_METRICS_GAUGE, new OffsetGauge(ktp, OffsetGaugeType.COMMITTED_OFFSET));

		legacyCurrentOffsetsMetricGroup.gauge(getLegacyOffsetsMetricsGaugeName(ktp), new OffsetGauge(ktp, OffsetGaugeType.CURRENT_OFFSET));
		legacyCommittedOffsetsMetricGroup.gauge(getLegacyOffsetsMetricsGaugeName(ktp), new OffsetGauge(ktp, OffsetGaugeType.COMMITTED_OFFSET));
	}
}
 

/**
 * Registers a metric group associated with the shard id of the provided {@link KinesisStreamShardState shardState}.
 *
 * @return a {@link ShardMetricsReporter} that can be used to update metric values
 */
private static ShardMetricsReporter registerShardMetrics(MetricGroup metricGroup, KinesisStreamShardState shardState) {
	ShardMetricsReporter shardMetrics = new ShardMetricsReporter();

	MetricGroup streamShardMetricGroup = metricGroup
		.addGroup(
			KinesisConsumerMetricConstants.STREAM_METRICS_GROUP,
			shardState.getStreamShardHandle().getStreamName())
		.addGroup(
			KinesisConsumerMetricConstants.SHARD_METRICS_GROUP,
			shardState.getStreamShardHandle().getShard().getShardId());

	streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.MILLIS_BEHIND_LATEST_GAUGE, shardMetrics::getMillisBehindLatest);
	streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.MAX_RECORDS_PER_FETCH, shardMetrics::getMaxNumberOfRecordsPerFetch);
	streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.NUM_AGGREGATED_RECORDS_PER_FETCH, shardMetrics::getNumberOfAggregatedRecords);
	streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.NUM_DEAGGREGATED_RECORDS_PER_FETCH, shardMetrics::getNumberOfDeaggregatedRecords);
	streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.AVG_RECORD_SIZE_BYTES, shardMetrics::getAverageRecordSizeBytes);
	streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.BYTES_PER_READ, shardMetrics::getBytesPerRead);
	streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.RUNTIME_LOOP_NANOS, shardMetrics::getRunLoopTimeNanos);
	streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.LOOP_FREQUENCY_HZ, shardMetrics::getLoopFrequencyHz);
	streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.SLEEP_TIME_MILLIS, shardMetrics::getSleepTimeMillis);
	return shardMetrics;
}
 

private static void instantiateCPUMetrics(MetricGroup metrics) {
	try {
		final com.sun.management.OperatingSystemMXBean mxBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();

		metrics.<Double, Gauge<Double>>gauge("Load", mxBean::getProcessCpuLoad);
		metrics.<Long, Gauge<Long>>gauge("Time", mxBean::getProcessCpuTime);
	} catch (Exception e) {
		LOG.warn("Cannot access com.sun.management.OperatingSystemMXBean.getProcessCpuLoad()" +
			" - CPU load metrics will not be available.", e);
	}
}
 

private static void registerShuffleMetrics(
		String groupName,
		MetricGroup metricGroup,
		NetworkBufferPool networkBufferPool) {
	MetricGroup networkGroup = metricGroup.addGroup(groupName);
	networkGroup.<Integer, Gauge<Integer>>gauge(METRIC_TOTAL_MEMORY_SEGMENT,
		networkBufferPool::getTotalNumberOfMemorySegments);
	networkGroup.<Integer, Gauge<Integer>>gauge(METRIC_AVAILABLE_MEMORY_SEGMENT,
		networkBufferPool::getNumberOfAvailableMemorySegments);
}
 

private static void instantiateCPUMetrics(MetricGroup metrics) {
	try {
		final com.sun.management.OperatingSystemMXBean mxBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();

		metrics.<Double, Gauge<Double>>gauge("Load", mxBean::getProcessCpuLoad);
		metrics.<Long, Gauge<Long>>gauge("Time", mxBean::getProcessCpuTime);
	} catch (Exception e) {
		LOG.warn("Cannot access com.sun.management.OperatingSystemMXBean.getProcessCpuLoad()" +
			" - CPU load metrics will not be available.", e);
	}
}
 

/**
 * Register gauges to pull native metrics for the column family.
 * @param columnFamilyName group name for the new gauges
 * @param handle native handle to the column family
 */
void registerColumnFamily(String columnFamilyName, ColumnFamilyHandle handle) {
	MetricGroup group = metricGroup.addGroup(columnFamilyName);

	for (String property : options.getProperties()) {
		RocksDBNativeMetricView gauge = new RocksDBNativeMetricView(handle, property);
		group.gauge(property, gauge);
	}
}
 

private static void instantiateNetworkMetrics(MetricGroup metrics, SystemResourcesCounter usageCounter) {
	for (int i = 0; i < usageCounter.getNetworkInterfaceNames().length; i++) {
		MetricGroup interfaceGroup = metrics.addGroup(usageCounter.getNetworkInterfaceNames()[i]);

		final int interfaceNo = i;
		interfaceGroup.<Long, Gauge<Long>>gauge("ReceiveRate", () -> usageCounter.getReceiveRatePerInterface(interfaceNo));
		interfaceGroup.<Long, Gauge<Long>>gauge("SendRate", () -> usageCounter.getSendRatePerInterface(interfaceNo));
	}
}
 

WatermarkSyncCallback(ProcessingTimeService timerService, long interval) {
	this.timerService = checkNotNull(timerService);
	this.interval = interval;
	MetricGroup shardMetricsGroup = consumerMetricGroup.addGroup("subtaskId",
		String.valueOf(indexOfThisConsumerSubtask));
	shardMetricsGroup.gauge("localWatermark", () -> nextWatermark);
	shardMetricsGroup.gauge("globalWatermark", () -> lastGlobalWatermark);
}
 

/**
 * Initialize Buffer Metrics for a task.
 */
public void initializeBufferMetrics(Task task) {
	final MetricGroup buffers = addGroup("buffers");
	buffers.gauge("inputQueueLength", new InputBuffersGauge(task));
	buffers.gauge("outputQueueLength", new OutputBuffersGauge(task));
	buffers.gauge("inPoolUsage", new InputBufferPoolUsageGauge(task));
	buffers.gauge("outPoolUsage", new OutputBufferPoolUsageGauge(task));
}
 

/**
 * Register the exposed metrics.
 *
 * @param metricGroup Metric group to use for the metrics.
 */
private void registerMetrics(MetricGroup metricGroup) {
	metricGroup.gauge(NUMBER_OF_CHECKPOINTS_METRIC, new CheckpointsCounter());
	metricGroup.gauge(NUMBER_OF_IN_PROGRESS_CHECKPOINTS_METRIC, new InProgressCheckpointsCounter());
	metricGroup.gauge(NUMBER_OF_COMPLETED_CHECKPOINTS_METRIC, new CompletedCheckpointsCounter());
	metricGroup.gauge(NUMBER_OF_FAILED_CHECKPOINTS_METRIC, new FailedCheckpointsCounter());
	metricGroup.gauge(LATEST_RESTORED_CHECKPOINT_TIMESTAMP_METRIC, new LatestRestoredCheckpointTimestampGauge());
	metricGroup.gauge(LATEST_COMPLETED_CHECKPOINT_SIZE_METRIC, new LatestCompletedCheckpointSizeGauge());
	metricGroup.gauge(LATEST_COMPLETED_CHECKPOINT_DURATION_METRIC, new LatestCompletedCheckpointDurationGauge());
	metricGroup.gauge(LATEST_COMPLETED_CHECKPOINT_EXTERNAL_PATH_METRIC, new LatestCompletedCheckpointExternalPathGauge());
}
 

/**
 * Register the exposed metrics.
 *
 * @param metricGroup Metric group to use for the metrics.
 */
private void registerMetrics(MetricGroup metricGroup) {
	metricGroup.gauge(NUMBER_OF_CHECKPOINTS_METRIC, new CheckpointsCounter());
	metricGroup.gauge(NUMBER_OF_IN_PROGRESS_CHECKPOINTS_METRIC, new InProgressCheckpointsCounter());
	metricGroup.gauge(NUMBER_OF_COMPLETED_CHECKPOINTS_METRIC, new CompletedCheckpointsCounter());
	metricGroup.gauge(NUMBER_OF_FAILED_CHECKPOINTS_METRIC, new FailedCheckpointsCounter());
	metricGroup.gauge(LATEST_RESTORED_CHECKPOINT_TIMESTAMP_METRIC, new LatestRestoredCheckpointTimestampGauge());
	metricGroup.gauge(LATEST_COMPLETED_CHECKPOINT_SIZE_METRIC, new LatestCompletedCheckpointSizeGauge());
	metricGroup.gauge(LATEST_COMPLETED_CHECKPOINT_DURATION_METRIC, new LatestCompletedCheckpointDurationGauge());
	metricGroup.gauge(LATEST_COMPLETED_CHECKPOINT_ALIGNMENT_BUFFERED_METRIC, new LatestCompletedCheckpointAlignmentBufferedGauge());
	metricGroup.gauge(LATEST_COMPLETED_CHECKPOINT_EXTERNAL_PATH_METRIC, new LatestCompletedCheckpointExternalPathGauge());
}
 

private static void instantiateMemoryMetrics(MetricGroup metrics, GlobalMemory memory) {
	metrics.<Long, Gauge<Long>>gauge("Available", memory::getAvailable);
	metrics.<Long, Gauge<Long>>gauge("Total", memory::getTotal);
}
 

private void registerDispatcherMetrics(MetricGroup jobManagerMetricGroup) {
	jobManagerMetricGroup.gauge(MetricNames.NUM_RUNNING_JOBS,
		() -> (long) jobManagerRunnerFutures.size());
}
 

private static void instantiateMemoryMetrics(MetricGroup metrics, GlobalMemory memory) {
	metrics.<Long, Gauge<Long>>gauge("Available", memory::getAvailable);
	metrics.<Long, Gauge<Long>>gauge("Total", memory::getTotal);
}
 

private void registerDispatcherMetrics(MetricGroup jobManagerMetricGroup) {
	jobManagerMetricGroup.gauge(MetricNames.NUM_RUNNING_JOBS,
		() -> (long) jobManagerRunnerFutures.size());
}
 

private static void instantiateClassLoaderMetrics(MetricGroup metrics) {
	final ClassLoadingMXBean mxBean = ManagementFactory.getClassLoadingMXBean();
	metrics.<Long, Gauge<Long>>gauge("ClassesLoaded", mxBean::getTotalLoadedClassCount);
	metrics.<Long, Gauge<Long>>gauge("ClassesUnloaded", mxBean::getUnloadedClassCount);
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	// check and pass the configuration properties
	KinesisProducerConfiguration producerConfig = KinesisConfigUtil.getValidatedProducerConfiguration(configProps);

	producer = getKinesisProducer(producerConfig);

	final MetricGroup kinesisMectricGroup = getRuntimeContext().getMetricGroup().addGroup(KINESIS_PRODUCER_METRIC_GROUP);
	this.backpressureCycles = kinesisMectricGroup.counter(METRIC_BACKPRESSURE_CYCLES);
	kinesisMectricGroup.gauge(METRIC_OUTSTANDING_RECORDS_COUNT, producer::getOutstandingRecordsCount);

	backpressureLatch = new TimeoutLatch();
	callback = new FutureCallback<UserRecordResult>() {
		@Override
		public void onSuccess(UserRecordResult result) {
			backpressureLatch.trigger();
			if (!result.isSuccessful()) {
				if (failOnError) {
					// only remember the first thrown exception
					if (thrownException == null) {
						thrownException = new RuntimeException("Record was not sent successful");
					}
				} else {
					LOG.warn("Record was not sent successful");
				}
			}
		}

		@Override
		public void onFailure(Throwable t) {
			backpressureLatch.trigger();
			if (failOnError) {
				thrownException = t;
			} else {
				LOG.warn("An exception occurred while processing a record", t);
			}
		}
	};

	if (this.customPartitioner != null) {
		this.customPartitioner.initialize(getRuntimeContext().getIndexOfThisSubtask(), getRuntimeContext().getNumberOfParallelSubtasks());
	}

	LOG.info("Started Kinesis producer instance for region '{}'", producerConfig.getRegion());
}