Java Code Examples for org.apache.flink.metrics.MetricGroup#addGroup()

public SumAndCount(String name, MetricGroup metricGroup) {
	MetricGroup group = metricGroup.addGroup(name);
	count = group.counter("count");
	group.gauge("sum", new Gauge<Double>() {
		@Override
		public Double getValue() {
			return sum;
		}
	});
	group.gauge("avg", new Gauge<Double>() {
		@Override
		public Double getValue() {
			if (System.currentTimeMillis() - currentAvgTime > AVG_INTERVAL) {
				return 0.0;
			}
			return currentAvg;
		}
	});
}
 

public static TaskManagerMetricGroup instantiateTaskManagerMetricGroup(
		MetricRegistry metricRegistry,
		TaskManagerLocation taskManagerLocation,
		NetworkEnvironment network,
		Optional<Time> systemResourceProbeInterval) {
	final TaskManagerMetricGroup taskManagerMetricGroup = new TaskManagerMetricGroup(
		metricRegistry,
		taskManagerLocation.getHostname(),
		taskManagerLocation.getResourceID().toString());

	MetricGroup statusGroup = taskManagerMetricGroup.addGroup(METRIC_GROUP_STATUS_NAME);

	// Initialize the TM metrics
	instantiateStatusMetrics(statusGroup);

	MetricGroup networkGroup = statusGroup
		.addGroup("Network");
	instantiateNetworkMetrics(networkGroup, network);

	if (systemResourceProbeInterval.isPresent()) {
		instantiateSystemMetrics(taskManagerMetricGroup, systemResourceProbeInterval.get());
	}
	return taskManagerMetricGroup;
}
 

/**
 * Registers legacy network metric groups before shuffle service refactoring.
 *
 * <p>Registers legacy metric groups if shuffle service implementation is original default one.
 *
 * @deprecated should be removed in future
 */
@SuppressWarnings("DeprecatedIsStillUsed")
@Deprecated
public static void registerLegacyNetworkMetrics(
		boolean isDetailedMetrics,
		MetricGroup metricGroup,
		ResultPartitionWriter[] producedPartitions,
		InputGate[] inputGates) {
	checkNotNull(metricGroup);
	checkNotNull(producedPartitions);
	checkNotNull(inputGates);

	// add metrics for buffers
	final MetricGroup buffersGroup = metricGroup.addGroup(METRIC_GROUP_BUFFERS_DEPRECATED);

	// similar to MetricUtils.instantiateNetworkMetrics() but inside this IOMetricGroup (metricGroup)
	final MetricGroup networkGroup = metricGroup.addGroup(METRIC_GROUP_NETWORK_DEPRECATED);
	final MetricGroup outputGroup = networkGroup.addGroup(METRIC_GROUP_OUTPUT);
	final MetricGroup inputGroup = networkGroup.addGroup(METRIC_GROUP_INPUT);

	ResultPartition[] resultPartitions = Arrays.copyOf(producedPartitions, producedPartitions.length, ResultPartition[].class);
	registerOutputMetrics(isDetailedMetrics, outputGroup, buffersGroup, resultPartitions);

	SingleInputGate[] singleInputGates = Arrays.copyOf(inputGates, inputGates.length, SingleInputGate[].class);
	registerInputMetrics(isDetailedMetrics, inputGroup, buffersGroup, singleInputGates);
}
 

/**
 * Verifies that when attempting to create a group with the name of an existing one the existing one will be returned instead.
 */
@Test
public void testDuplicateGroupName() throws Exception {
	Configuration config = new Configuration();

	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(config));

	MetricGroup root = new TaskManagerMetricGroup(registry, "host", "id");

	MetricGroup group1 = root.addGroup("group");
	MetricGroup group2 = root.addGroup("group");
	MetricGroup group3 = root.addGroup("group");
	Assert.assertTrue(group1 == group2 && group2 == group3);

	registry.shutdown().get();
}
 

public static void instantiateSystemMetrics(MetricGroup metricGroup, Time probeInterval) {
	try {
		MetricGroup system = metricGroup.addGroup("System");

		SystemResourcesCounter systemResourcesCounter = new SystemResourcesCounter(probeInterval);
		systemResourcesCounter.start();

		SystemInfo systemInfo = new SystemInfo();
		HardwareAbstractionLayer hardwareAbstractionLayer = systemInfo.getHardware();

		instantiateMemoryMetrics(system.addGroup("Memory"), hardwareAbstractionLayer.getMemory());
		instantiateSwapMetrics(system.addGroup("Swap"), hardwareAbstractionLayer.getMemory());
		instantiateCPUMetrics(system.addGroup("CPU"), systemResourcesCounter);
		instantiateNetworkMetrics(system.addGroup("Network"), systemResourcesCounter);
	}
	catch (NoClassDefFoundError ex) {
		LOG.warn(
			"Failed to initialize system resource metrics because of missing class definitions." +
			" Did you forget to explicitly add the oshi-core optional dependency?",
			ex);
	}
}
 

/**
 * Registers legacy network metric groups before shuffle service refactoring.
 *
 * <p>Registers legacy metric groups if shuffle service implementation is original default one.
 *
 * @deprecated should be removed in future
 */
@SuppressWarnings("DeprecatedIsStillUsed")
@Deprecated
public static void registerLegacyNetworkMetrics(
		boolean isDetailedMetrics,
		boolean isCreditBased,
		MetricGroup metricGroup,
		ResultPartitionWriter[] producedPartitions,
		InputGate[] inputGates) {
	checkNotNull(metricGroup);
	checkNotNull(producedPartitions);
	checkNotNull(inputGates);

	// add metrics for buffers
	final MetricGroup buffersGroup = metricGroup.addGroup(METRIC_GROUP_BUFFERS_DEPRECATED);

	// similar to MetricUtils.instantiateNetworkMetrics() but inside this IOMetricGroup (metricGroup)
	final MetricGroup networkGroup = metricGroup.addGroup(METRIC_GROUP_NETWORK_DEPRECATED);
	final MetricGroup outputGroup = networkGroup.addGroup(METRIC_GROUP_OUTPUT);
	final MetricGroup inputGroup = networkGroup.addGroup(METRIC_GROUP_INPUT);

	ResultPartition[] resultPartitions = Arrays.copyOf(producedPartitions, producedPartitions.length, ResultPartition[].class);
	registerOutputMetrics(isDetailedMetrics, outputGroup, buffersGroup, resultPartitions);

	SingleInputGate[] singleInputGates = Arrays.copyOf(inputGates, inputGates.length, SingleInputGate[].class);
	registerInputMetrics(isDetailedMetrics, isCreditBased, inputGroup, buffersGroup, singleInputGates);
}
 

private static void registerShuffleMetrics(
		String groupName,
		MetricGroup metricGroup,
		NetworkBufferPool networkBufferPool) {
	MetricGroup networkGroup = metricGroup.addGroup(groupName);
	networkGroup.<Integer, Gauge<Integer>>gauge(METRIC_TOTAL_MEMORY_SEGMENT,
		networkBufferPool::getTotalNumberOfMemorySegments);
	networkGroup.<Integer, Gauge<Integer>>gauge(METRIC_AVAILABLE_MEMORY_SEGMENT,
		networkBufferPool::getNumberOfAvailableMemorySegments);
}
 

public static void registerQueueLengthMetrics(MetricGroup parent, SingleInputGate[] gates) {
	for (int i = 0; i < gates.length; i++) {
		InputGateMetrics metrics = new InputGateMetrics(gates[i]);

		MetricGroup group = parent.addGroup(i);
		group.gauge("totalQueueLen", metrics.getTotalQueueLenGauge());
		group.gauge("minQueueLen", metrics.getMinQueueLenGauge());
		group.gauge("maxQueueLen", metrics.getMaxQueueLenGauge());
		group.gauge("avgQueueLen", metrics.getAvgQueueLenGauge());
	}
}
 

@Override
MetricGroup createSourceMetricGroups(
		MetricGroup base,
		LatencyMarker marker,
		OperatorID operatorId,
		int operatorSubtaskIndex) {
	return base
		.addGroup("source_id", String.valueOf(marker.getOperatorId()));
}
 

private static void instantiateGarbageCollectorMetrics(MetricGroup metrics) {
	List<GarbageCollectorMXBean> garbageCollectors = ManagementFactory.getGarbageCollectorMXBeans();

	for (final GarbageCollectorMXBean garbageCollector: garbageCollectors) {
		MetricGroup gcGroup = metrics.addGroup(garbageCollector.getName());

		gcGroup.<Long, Gauge<Long>>gauge("Count", garbageCollector::getCollectionCount);
		gcGroup.<Long, Gauge<Long>>gauge("Time", garbageCollector::getCollectionTime);
	}
}
 

public static void registerQueueLengthMetrics(MetricGroup parent, ResultPartition[] partitions) {
	for (int i = 0; i < partitions.length; i++) {
		ResultPartitionMetrics metrics = new ResultPartitionMetrics(partitions[i]);

		MetricGroup group = parent.addGroup(i);
		group.gauge("totalQueueLen", metrics.getTotalQueueLenGauge());
		group.gauge("minQueueLen", metrics.getMinQueueLenGauge());
		group.gauge("maxQueueLen", metrics.getMaxQueueLenGauge());
		group.gauge("avgQueueLen", metrics.getAvgQueueLenGauge());
	}
}
 

private static void instantiateNetworkMetrics(MetricGroup metrics, SystemResourcesCounter usageCounter) {
	for (int i = 0; i < usageCounter.getNetworkInterfaceNames().length; i++) {
		MetricGroup interfaceGroup = metrics.addGroup(usageCounter.getNetworkInterfaceNames()[i]);

		final int interfaceNo = i;
		interfaceGroup.<Long, Gauge<Long>>gauge("ReceiveRate", () -> usageCounter.getReceiveRatePerInterface(interfaceNo));
		interfaceGroup.<Long, Gauge<Long>>gauge("SendRate", () -> usageCounter.getSendRatePerInterface(interfaceNo));
	}
}
 

public static void instantiateStatusMetrics(
		MetricGroup metricGroup) {
	MetricGroup jvm = metricGroup.addGroup("JVM");

	instantiateClassLoaderMetrics(jvm.addGroup("ClassLoader"));
	instantiateGarbageCollectorMetrics(jvm.addGroup("GarbageCollector"));
	instantiateMemoryMetrics(jvm.addGroup("Memory"));
	instantiateThreadMetrics(jvm.addGroup("Threads"));
	instantiateCPUMetrics(jvm.addGroup("CPU"));
}
 

@Test
public void testGetAllVariablesDoesNotDeadlock() throws InterruptedException {
	final TestMetricRegistry registry = new TestMetricRegistry();

	final MetricGroup parent = new GenericMetricGroup(registry, UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), "parent");
	final MetricGroup child = parent.addGroup("child");

	final Thread parentRegisteringThread = new Thread(() -> parent.counter("parent_counter"));
	final Thread childRegisteringThread = new Thread(() -> child.counter("child_counter"));

	final BlockerSync parentSync = new BlockerSync();
	final BlockerSync childSync = new BlockerSync();

	try {
		// start both threads and have them block in the registry, so they acquire the lock of their respective group
		registry.setOnRegistrationAction(childSync::blockNonInterruptible);
		childRegisteringThread.start();
		childSync.awaitBlocker();

		registry.setOnRegistrationAction(parentSync::blockNonInterruptible);
		parentRegisteringThread.start();
		parentSync.awaitBlocker();

		// the parent thread remains blocked to simulate the child thread holding some lock in the registry/reporter
		// the child thread continues execution and calls getAllVariables()
		// in the past this would block indefinitely since the method acquires the locks of all parent groups
		childSync.releaseBlocker();
		// wait with a timeout to ensure the finally block is executed _at some point_, un-blocking the parent
		childRegisteringThread.join(1000 * 10);

		parentSync.releaseBlocker();
		parentRegisteringThread.join();
	} finally {
		parentSync.releaseBlocker();
		childSync.releaseBlocker();
		parentRegisteringThread.join();
		childRegisteringThread.join();
	}
}
 

private static void instantiateGarbageCollectorMetrics(MetricGroup metrics) {
	List<GarbageCollectorMXBean> garbageCollectors = ManagementFactory.getGarbageCollectorMXBeans();

	for (final GarbageCollectorMXBean garbageCollector: garbageCollectors) {
		MetricGroup gcGroup = metrics.addGroup(garbageCollector.getName());

		gcGroup.<Long, Gauge<Long>>gauge("Count", garbageCollector::getCollectionCount);
		gcGroup.<Long, Gauge<Long>>gauge("Time", garbageCollector::getCollectionTime);
	}
}
 

@Test
public void testGetAllVariablesDoesNotDeadlock() throws InterruptedException {
	final TestMetricRegistry registry = new TestMetricRegistry();

	final MetricGroup parent = new GenericMetricGroup(registry, UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), "parent");
	final MetricGroup child = parent.addGroup("child");

	final Thread parentRegisteringThread = new Thread(() -> parent.counter("parent_counter"));
	final Thread childRegisteringThread = new Thread(() -> child.counter("child_counter"));

	final BlockerSync parentSync = new BlockerSync();
	final BlockerSync childSync = new BlockerSync();

	try {
		// start both threads and have them block in the registry, so they acquire the lock of their respective group
		registry.setOnRegistrationAction(childSync::blockNonInterruptible);
		childRegisteringThread.start();
		childSync.awaitBlocker();

		registry.setOnRegistrationAction(parentSync::blockNonInterruptible);
		parentRegisteringThread.start();
		parentSync.awaitBlocker();

		// the parent thread remains blocked to simulate the child thread holding some lock in the registry/reporter
		// the child thread continues execution and calls getAllVariables()
		// in the past this would block indefinitely since the method acquires the locks of all parent groups
		childSync.releaseBlocker();
		// wait with a timeout to ensure the finally block is executed _at some point_, un-blocking the parent
		childRegisteringThread.join(1000 * 10);

		parentSync.releaseBlocker();
		parentRegisteringThread.join();
	} finally {
		parentSync.releaseBlocker();
		childSync.releaseBlocker();
		parentRegisteringThread.join();
		childRegisteringThread.join();
	}
}
 

@Override
public ShuffleIOOwnerContext createShuffleIOOwnerContext(
		String ownerName,
		ExecutionAttemptID executionAttemptID,
		MetricGroup parentGroup) {
	MetricGroup nettyGroup = createShuffleIOOwnerMetricGroup(checkNotNull(parentGroup));
	return new ShuffleIOOwnerContext(
		checkNotNull(ownerName),
		checkNotNull(executionAttemptID),
		parentGroup,
		nettyGroup.addGroup(METRIC_GROUP_OUTPUT),
		nettyGroup.addGroup(METRIC_GROUP_INPUT));
}
 

@Test
public void testGetAllVariablesDoesNotDeadlock() throws InterruptedException {
	final TestMetricRegistry registry = new TestMetricRegistry();

	final MetricGroup parent = new GenericMetricGroup(registry, UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), "parent");
	final MetricGroup child = parent.addGroup("child");

	final Thread parentRegisteringThread = new Thread(() -> parent.counter("parent_counter"));
	final Thread childRegisteringThread = new Thread(() -> child.counter("child_counter"));

	final BlockerSync parentSync = new BlockerSync();
	final BlockerSync childSync = new BlockerSync();

	try {
		// start both threads and have them block in the registry, so they acquire the lock of their respective group
		registry.setOnRegistrationAction(childSync::blockNonInterruptible);
		childRegisteringThread.start();
		childSync.awaitBlocker();

		registry.setOnRegistrationAction(parentSync::blockNonInterruptible);
		parentRegisteringThread.start();
		parentSync.awaitBlocker();

		// the parent thread remains blocked to simulate the child thread holding some lock in the registry/reporter
		// the child thread continues execution and calls getAllVariables()
		// in the past this would block indefinitely since the method acquires the locks of all parent groups
		childSync.releaseBlocker();
		// wait with a timeout to ensure the finally block is executed _at some point_, un-blocking the parent
		childRegisteringThread.join(1000 * 10);

		parentSync.releaseBlocker();
		parentRegisteringThread.join();
	} finally {
		parentSync.releaseBlocker();
		childSync.releaseBlocker();
		parentRegisteringThread.join();
		childRegisteringThread.join();
	}
}
 

protected AbstractFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> seedPartitionsWithInitialOffsets,
		SerializedValue<WatermarkStrategy<T>> watermarkStrategy,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval,
		ClassLoader userCodeClassLoader,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	this.sourceContext = checkNotNull(sourceContext);
	this.watermarkOutput = new SourceContextWatermarkOutputAdapter<>(sourceContext);
	this.watermarkOutputMultiplexer = new WatermarkOutputMultiplexer(watermarkOutput);
	this.checkpointLock = sourceContext.getCheckpointLock();
	this.userCodeClassLoader = checkNotNull(userCodeClassLoader);

	this.useMetrics = useMetrics;
	this.consumerMetricGroup = checkNotNull(consumerMetricGroup);
	this.legacyCurrentOffsetsMetricGroup = consumerMetricGroup.addGroup(LEGACY_CURRENT_OFFSETS_METRICS_GROUP);
	this.legacyCommittedOffsetsMetricGroup = consumerMetricGroup.addGroup(LEGACY_COMMITTED_OFFSETS_METRICS_GROUP);

	this.watermarkStrategy = watermarkStrategy;

	if (watermarkStrategy == null) {
		timestampWatermarkMode = NO_TIMESTAMPS_WATERMARKS;
	} else {
		timestampWatermarkMode = WITH_WATERMARK_GENERATOR;
	}

	this.unassignedPartitionsQueue = new ClosableBlockingQueue<>();

	// initialize subscribed partition states with seed partitions
	this.subscribedPartitionStates = createPartitionStateHolders(
			seedPartitionsWithInitialOffsets,
			timestampWatermarkMode,
			watermarkStrategy,
			userCodeClassLoader);

	// check that all seed partition states have a defined offset
	for (KafkaTopicPartitionState<?, ?> partitionState : subscribedPartitionStates) {
		if (!partitionState.isOffsetDefined()) {
			throw new IllegalArgumentException("The fetcher was assigned seed partitions with undefined initial offsets.");
		}
	}

	// all seed partitions are not assigned yet, so should be added to the unassigned partitions queue
	for (KafkaTopicPartitionState<T, KPH> partition : subscribedPartitionStates) {
		unassignedPartitionsQueue.add(partition);
	}

	// register metrics for the initial seed partitions
	if (useMetrics) {
		registerOffsetMetrics(consumerMetricGroup, subscribedPartitionStates);
	}

	// if we have periodic watermarks, kick off the interval scheduler
	if (timestampWatermarkMode == WITH_WATERMARK_GENERATOR && autoWatermarkInterval > 0) {
		PeriodicWatermarkEmitter<T, KPH> periodicEmitter = new PeriodicWatermarkEmitter<>(
				checkpointLock,
				subscribedPartitionStates,
				watermarkOutputMultiplexer,
				processingTimeProvider,
				autoWatermarkInterval);

		periodicEmitter.start();
	}
}
 

protected AbstractFetcher(
		SourceContext<T> sourceContext,
		Map<KafkaTopicPartition, Long> seedPartitionsWithInitialOffsets,
		SerializedValue<AssignerWithPeriodicWatermarks<T>> watermarksPeriodic,
		SerializedValue<AssignerWithPunctuatedWatermarks<T>> watermarksPunctuated,
		ProcessingTimeService processingTimeProvider,
		long autoWatermarkInterval,
		ClassLoader userCodeClassLoader,
		MetricGroup consumerMetricGroup,
		boolean useMetrics) throws Exception {
	this.sourceContext = checkNotNull(sourceContext);
	this.checkpointLock = sourceContext.getCheckpointLock();
	this.userCodeClassLoader = checkNotNull(userCodeClassLoader);

	this.useMetrics = useMetrics;
	this.consumerMetricGroup = checkNotNull(consumerMetricGroup);
	this.legacyCurrentOffsetsMetricGroup = consumerMetricGroup.addGroup(LEGACY_CURRENT_OFFSETS_METRICS_GROUP);
	this.legacyCommittedOffsetsMetricGroup = consumerMetricGroup.addGroup(LEGACY_COMMITTED_OFFSETS_METRICS_GROUP);

	// figure out what we watermark mode we will be using
	this.watermarksPeriodic = watermarksPeriodic;
	this.watermarksPunctuated = watermarksPunctuated;

	if (watermarksPeriodic == null) {
		if (watermarksPunctuated == null) {
			// simple case, no watermarks involved
			timestampWatermarkMode = NO_TIMESTAMPS_WATERMARKS;
		} else {
			timestampWatermarkMode = PUNCTUATED_WATERMARKS;
		}
	} else {
		if (watermarksPunctuated == null) {
			timestampWatermarkMode = PERIODIC_WATERMARKS;
		} else {
			throw new IllegalArgumentException("Cannot have both periodic and punctuated watermarks");
		}
	}

	this.unassignedPartitionsQueue = new ClosableBlockingQueue<>();

	// initialize subscribed partition states with seed partitions
	this.subscribedPartitionStates = createPartitionStateHolders(
			seedPartitionsWithInitialOffsets,
			timestampWatermarkMode,
			watermarksPeriodic,
			watermarksPunctuated,
			userCodeClassLoader);

	// check that all seed partition states have a defined offset
	for (KafkaTopicPartitionState partitionState : subscribedPartitionStates) {
		if (!partitionState.isOffsetDefined()) {
			throw new IllegalArgumentException("The fetcher was assigned seed partitions with undefined initial offsets.");
		}
	}

	// all seed partitions are not assigned yet, so should be added to the unassigned partitions queue
	for (KafkaTopicPartitionState<KPH> partition : subscribedPartitionStates) {
		unassignedPartitionsQueue.add(partition);
	}

	// register metrics for the initial seed partitions
	if (useMetrics) {
		registerOffsetMetrics(consumerMetricGroup, subscribedPartitionStates);
	}

	// if we have periodic watermarks, kick off the interval scheduler
	if (timestampWatermarkMode == PERIODIC_WATERMARKS) {
		@SuppressWarnings("unchecked")
		PeriodicWatermarkEmitter periodicEmitter = new PeriodicWatermarkEmitter(
				subscribedPartitionStates,
				sourceContext,
				processingTimeProvider,
				autoWatermarkInterval);

		periodicEmitter.start();
	}
}