Java Code Examples for org.apache.flink.metrics.MetricGroup#counter()

public SumAndCount(String name, MetricGroup metricGroup) {
	MetricGroup group = metricGroup.addGroup(name);
	count = group.counter("count");
	group.gauge("sum", new Gauge<Double>() {
		@Override
		public Double getValue() {
			return sum;
		}
	});
	group.gauge("avg", new Gauge<Double>() {
		@Override
		public Double getValue() {
			if (System.currentTimeMillis() - currentAvgTime > AVG_INTERVAL) {
				return 0.0;
			}
			return currentAvg;
		}
	});
}
 

@Test
public void testScopeCachingForMultipleReporters() throws Exception {
	Configuration config = new Configuration();
	config.setString(MetricOptions.SCOPE_NAMING_TM, "A.B.C.D");
	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter1.class.getName());
	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "-");
	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter2.class.getName());
	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "!");

	MetricRegistryImpl testRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(config));
	try {
		MetricGroup tmGroup = new TaskManagerMetricGroup(testRegistry, "host", "id");
		tmGroup.counter("1");
		assertEquals("Reporters were not properly instantiated", 2, testRegistry.getReporters().size());
		for (MetricReporter reporter : testRegistry.getReporters()) {
			ScopeCheckingTestReporter typedReporter = (ScopeCheckingTestReporter) reporter;
			if (typedReporter.failureCause != null) {
				throw typedReporter.failureCause;
			}
		}
	} finally {
		testRegistry.shutdown().get();
	}
}
 

@Test
public void testLogicalScopeCachingForMultipleReporters() throws Exception {
	Configuration config = new Configuration();
	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, LogicalScopeReporter1.class.getName());
	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, LogicalScopeReporter2.class.getName());

	MetricRegistryImpl testRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(config));
	try {
		MetricGroup tmGroup = new TaskManagerMetricGroup(testRegistry, "host", "id")
			.addGroup("B")
			.addGroup("C");
		tmGroup.counter("1");
		assertEquals("Reporters were not properly instantiated", 2, testRegistry.getReporters().size());
		for (MetricReporter reporter : testRegistry.getReporters()) {
			ScopeCheckingTestReporter typedReporter = (ScopeCheckingTestReporter) reporter;
			if (typedReporter.failureCause != null) {
				throw typedReporter.failureCause;
			}
		}
	} finally {
		testRegistry.shutdown().get();
	}
}
 

@Test
public void testLogicalScopeCachingForMultipleReporters() throws Exception {
	MetricRegistryImpl testRegistry = new MetricRegistryImpl(
		MetricRegistryConfiguration.defaultMetricRegistryConfiguration(),
		Arrays.asList(
			ReporterSetup.forReporter("test1", new LogicalScopeReporter1()),
			ReporterSetup.forReporter("test2", new LogicalScopeReporter2())));
	try {
		MetricGroup tmGroup = new TaskManagerMetricGroup(testRegistry, "host", "id")
			.addGroup("B")
			.addGroup("C");
		tmGroup.counter("1");
		assertEquals("Reporters were not properly instantiated", 2, testRegistry.getReporters().size());
		for (MetricReporter reporter : testRegistry.getReporters()) {
			ScopeCheckingTestReporter typedReporter = (ScopeCheckingTestReporter) reporter;
			if (typedReporter.failureCause != null) {
				throw typedReporter.failureCause;
			}
		}
	} finally {
		testRegistry.shutdown().get();
	}
}
 

@Test
public void testLogicalScopeCachingForMultipleReporters() throws Exception {
	MetricRegistryImpl testRegistry = new MetricRegistryImpl(
		MetricRegistryConfiguration.defaultMetricRegistryConfiguration(),
		Arrays.asList(
			ReporterSetup.forReporter("test1", new LogicalScopeReporter1()),
			ReporterSetup.forReporter("test2", new LogicalScopeReporter2())));
	try {
		MetricGroup tmGroup = new TaskManagerMetricGroup(testRegistry, "host", "id")
			.addGroup("B")
			.addGroup("C");
		tmGroup.counter("1");
		assertEquals("Reporters were not properly instantiated", 2, testRegistry.getReporters().size());
		for (MetricReporter reporter : testRegistry.getReporters()) {
			ScopeCheckingTestReporter typedReporter = (ScopeCheckingTestReporter) reporter;
			if (typedReporter.failureCause != null) {
				throw typedReporter.failureCause;
			}
		}
	} finally {
		testRegistry.shutdown().get();
	}
}
 

@Test
public void testGetAllVariablesDoesNotDeadlock() throws InterruptedException {
	final TestMetricRegistry registry = new TestMetricRegistry();

	final MetricGroup parent = new GenericMetricGroup(registry, UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), "parent");
	final MetricGroup child = parent.addGroup("child");

	final Thread parentRegisteringThread = new Thread(() -> parent.counter("parent_counter"));
	final Thread childRegisteringThread = new Thread(() -> child.counter("child_counter"));

	final BlockerSync parentSync = new BlockerSync();
	final BlockerSync childSync = new BlockerSync();

	try {
		// start both threads and have them block in the registry, so they acquire the lock of their respective group
		registry.setOnRegistrationAction(childSync::blockNonInterruptible);
		childRegisteringThread.start();
		childSync.awaitBlocker();

		registry.setOnRegistrationAction(parentSync::blockNonInterruptible);
		parentRegisteringThread.start();
		parentSync.awaitBlocker();

		// the parent thread remains blocked to simulate the child thread holding some lock in the registry/reporter
		// the child thread continues execution and calls getAllVariables()
		// in the past this would block indefinitely since the method acquires the locks of all parent groups
		childSync.releaseBlocker();
		// wait with a timeout to ensure the finally block is executed _at some point_, un-blocking the parent
		childRegisteringThread.join(1000 * 10);

		parentSync.releaseBlocker();
		parentRegisteringThread.join();
	} finally {
		parentSync.releaseBlocker();
		childSync.releaseBlocker();
		parentRegisteringThread.join();
		childRegisteringThread.join();
	}
}
 

@Test
public void testGetAllVariablesDoesNotDeadlock() throws InterruptedException {
	final TestMetricRegistry registry = new TestMetricRegistry();

	final MetricGroup parent = new GenericMetricGroup(registry, UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), "parent");
	final MetricGroup child = parent.addGroup("child");

	final Thread parentRegisteringThread = new Thread(() -> parent.counter("parent_counter"));
	final Thread childRegisteringThread = new Thread(() -> child.counter("child_counter"));

	final BlockerSync parentSync = new BlockerSync();
	final BlockerSync childSync = new BlockerSync();

	try {
		// start both threads and have them block in the registry, so they acquire the lock of their respective group
		registry.setOnRegistrationAction(childSync::blockNonInterruptible);
		childRegisteringThread.start();
		childSync.awaitBlocker();

		registry.setOnRegistrationAction(parentSync::blockNonInterruptible);
		parentRegisteringThread.start();
		parentSync.awaitBlocker();

		// the parent thread remains blocked to simulate the child thread holding some lock in the registry/reporter
		// the child thread continues execution and calls getAllVariables()
		// in the past this would block indefinitely since the method acquires the locks of all parent groups
		childSync.releaseBlocker();
		// wait with a timeout to ensure the finally block is executed _at some point_, un-blocking the parent
		childRegisteringThread.join(1000 * 10);

		parentSync.releaseBlocker();
		parentRegisteringThread.join();
	} finally {
		parentSync.releaseBlocker();
		childSync.releaseBlocker();
		parentRegisteringThread.join();
		childRegisteringThread.join();
	}
}
 

@Override
public void open() throws Exception {
  super.open();
  final int indexOfThisSubtask = getRuntimeContext().getIndexOfThisSubtask();
  final SubtaskFeedbackKey<V> key = this.key.withSubTaskIndex(indexOfThisSubtask);

  FeedbackChannelBroker broker = FeedbackChannelBroker.get();
  this.channel = broker.getChannel(key);

  // metrics
  MetricGroup metrics = getRuntimeContext().getMetricGroup();
  SimpleCounter produced = metrics.counter("produced", new SimpleCounter());
  metrics.meter("producedRate", new MeterView(produced, 60));
  this.totalProduced = produced;
}
 

default Counter createOrGet(Counter numRecordsOut, RuntimeContext runtimeContext) {
    if (numRecordsOut == null) {
        MetricGroup metricGroup = runtimeContext.getMetricGroup().addGroup(metricGroupName());
        numRecordsOut = metricGroup.counter(MetricNames.IO_NUM_RECORDS_OUT);
        metricGroup.meter(MetricNames.IO_NUM_RECORDS_OUT_RATE, new MeterView(numRecordsOut, 60));
    }
    return numRecordsOut;
}
 

@Test
public void testScopeCachingForMultipleReporters() throws Exception {
	Configuration config = new Configuration();
	config.setString(MetricOptions.SCOPE_NAMING_TM, "A.B.C.D");

	MetricConfig metricConfig1 = new MetricConfig();
	metricConfig1.setProperty(ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "-");

	MetricConfig metricConfig2 = new MetricConfig();
	metricConfig2.setProperty(ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "!");

	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter1.class.getName());
	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test1." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "-");
	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter2.class.getName());
	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test2." + ConfigConstants.METRICS_REPORTER_SCOPE_DELIMITER, "!");

	MetricRegistryImpl testRegistry = new MetricRegistryImpl(
		MetricRegistryConfiguration.fromConfiguration(config),
		Arrays.asList(
			ReporterSetup.forReporter("test1", metricConfig1, new TestReporter1()),
			ReporterSetup.forReporter("test2", metricConfig2, new TestReporter2())));
	try {
		MetricGroup tmGroup = new TaskManagerMetricGroup(testRegistry, "host", "id");
		tmGroup.counter("1");
		assertEquals("Reporters were not properly instantiated", 2, testRegistry.getReporters().size());
		for (MetricReporter reporter : testRegistry.getReporters()) {
			ScopeCheckingTestReporter typedReporter = (ScopeCheckingTestReporter) reporter;
			if (typedReporter.failureCause != null) {
				throw typedReporter.failureCause;
			}
		}
	} finally {
		testRegistry.shutdown().get();
	}
}
 

@Test
public void testGetAllVariablesDoesNotDeadlock() throws InterruptedException {
	final TestMetricRegistry registry = new TestMetricRegistry();

	final MetricGroup parent = new GenericMetricGroup(registry, UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), "parent");
	final MetricGroup child = parent.addGroup("child");

	final Thread parentRegisteringThread = new Thread(() -> parent.counter("parent_counter"));
	final Thread childRegisteringThread = new Thread(() -> child.counter("child_counter"));

	final BlockerSync parentSync = new BlockerSync();
	final BlockerSync childSync = new BlockerSync();

	try {
		// start both threads and have them block in the registry, so they acquire the lock of their respective group
		registry.setOnRegistrationAction(childSync::blockNonInterruptible);
		childRegisteringThread.start();
		childSync.awaitBlocker();

		registry.setOnRegistrationAction(parentSync::blockNonInterruptible);
		parentRegisteringThread.start();
		parentSync.awaitBlocker();

		// the parent thread remains blocked to simulate the child thread holding some lock in the registry/reporter
		// the child thread continues execution and calls getAllVariables()
		// in the past this would block indefinitely since the method acquires the locks of all parent groups
		childSync.releaseBlocker();
		// wait with a timeout to ensure the finally block is executed _at some point_, un-blocking the parent
		childRegisteringThread.join(1000 * 10);

		parentSync.releaseBlocker();
		parentRegisteringThread.join();
	} finally {
		parentSync.releaseBlocker();
		childSync.releaseBlocker();
		parentRegisteringThread.join();
		childRegisteringThread.join();
	}
}
 

@Override
public void open() throws Exception {
  super.open();
  final int indexOfThisSubtask = getRuntimeContext().getIndexOfThisSubtask();
  final SubtaskFeedbackKey<V> key = this.key.withSubTaskIndex(indexOfThisSubtask);

  FeedbackChannelBroker broker = FeedbackChannelBroker.get();
  this.channel = broker.getChannel(key);

  // metrics
  MetricGroup metrics = getRuntimeContext().getMetricGroup();
  SimpleCounter produced = metrics.counter("produced", new SimpleCounter());
  metrics.meter("producedRate", new MeterView(produced, 60));
  this.totalProduced = produced;
}
 

/**
 * Verifies that group methods instantiate the correct metric with the given name.
 */
@Test
public void testMetricInstantiation() throws Exception {
	MetricRegistryImpl registry = new MetricRegistryImpl(
		MetricRegistryConfiguration.defaultMetricRegistryConfiguration(),
		Collections.singletonList(ReporterSetup.forReporter("test", new TestReporter1())));

	MetricGroup root = new TaskManagerMetricGroup(registry, "host", "id");

	Counter counter = root.counter("counter");
	assertEquals(counter, TestReporter1.lastPassedMetric);
	assertEquals("counter", TestReporter1.lastPassedName);

	Gauge<Object> gauge = root.gauge("gauge", new Gauge<Object>() {
		@Override
		public Object getValue() {
			return null;
		}
	});

	Assert.assertEquals(gauge, TestReporter1.lastPassedMetric);
	assertEquals("gauge", TestReporter1.lastPassedName);

	Histogram histogram = root.histogram("histogram", new Histogram() {
		@Override
		public void update(long value) {

		}

		@Override
		public long getCount() {
			return 0;
		}

		@Override
		public HistogramStatistics getStatistics() {
			return null;
		}
	});

	Assert.assertEquals(histogram, TestReporter1.lastPassedMetric);
	assertEquals("histogram", TestReporter1.lastPassedName);
	registry.shutdown().get();
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	schema.open(() -> getRuntimeContext().getMetricGroup().addGroup("user"));

	// check and pass the configuration properties
	KinesisProducerConfiguration producerConfig = KinesisConfigUtil.getValidatedProducerConfiguration(configProps);

	producer = getKinesisProducer(producerConfig);

	final MetricGroup kinesisMectricGroup = getRuntimeContext().getMetricGroup().addGroup(KINESIS_PRODUCER_METRIC_GROUP);
	this.backpressureCycles = kinesisMectricGroup.counter(METRIC_BACKPRESSURE_CYCLES);
	kinesisMectricGroup.gauge(METRIC_OUTSTANDING_RECORDS_COUNT, producer::getOutstandingRecordsCount);

	backpressureLatch = new TimeoutLatch();
	callback = new FutureCallback<UserRecordResult>() {
		@Override
		public void onSuccess(UserRecordResult result) {
			backpressureLatch.trigger();
			if (!result.isSuccessful()) {
				if (failOnError) {
					// only remember the first thrown exception
					if (thrownException == null) {
						thrownException = new RuntimeException("Record was not sent successful");
					}
				} else {
					LOG.warn("Record was not sent successful");
				}
			}
		}

		@Override
		public void onFailure(Throwable t) {
			backpressureLatch.trigger();
			if (failOnError) {
				thrownException = t;
			} else {
				LOG.warn("An exception occurred while processing a record", t);
			}
		}
	};

	if (this.customPartitioner != null) {
		this.customPartitioner.initialize(getRuntimeContext().getIndexOfThisSubtask(), getRuntimeContext().getNumberOfParallelSubtasks());
	}

	LOG.info("Started Kinesis producer instance for region '{}'", producerConfig.getRegion());
}
 

private static SimpleCounter metered(MetricGroup metrics, String name) {
  SimpleCounter counter = metrics.counter(name, new SimpleCounter());
  metrics.meter(name + "Rate", new MeterView(counter, 60));
  return counter;
}
 

@Override
public void registerMetrics(MetricGroup metricGroup) {
	metricGroup.counter("task_failures", numTaskFailures);
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	// check and pass the configuration properties
	KinesisProducerConfiguration producerConfig = KinesisConfigUtil.getValidatedProducerConfiguration(configProps);

	producer = getKinesisProducer(producerConfig);

	final MetricGroup kinesisMectricGroup = getRuntimeContext().getMetricGroup().addGroup(KINESIS_PRODUCER_METRIC_GROUP);
	this.backpressureCycles = kinesisMectricGroup.counter(METRIC_BACKPRESSURE_CYCLES);
	kinesisMectricGroup.gauge(METRIC_OUTSTANDING_RECORDS_COUNT, producer::getOutstandingRecordsCount);

	backpressureLatch = new TimeoutLatch();
	callback = new FutureCallback<UserRecordResult>() {
		@Override
		public void onSuccess(UserRecordResult result) {
			backpressureLatch.trigger();
			if (!result.isSuccessful()) {
				if (failOnError) {
					// only remember the first thrown exception
					if (thrownException == null) {
						thrownException = new RuntimeException("Record was not sent successful");
					}
				} else {
					LOG.warn("Record was not sent successful");
				}
			}
		}

		@Override
		public void onFailure(Throwable t) {
			backpressureLatch.trigger();
			if (failOnError) {
				thrownException = t;
			} else {
				LOG.warn("An exception occurred while processing a record", t);
			}
		}
	};

	if (this.customPartitioner != null) {
		this.customPartitioner.initialize(getRuntimeContext().getIndexOfThisSubtask(), getRuntimeContext().getNumberOfParallelSubtasks());
	}

	LOG.info("Started Kinesis producer instance for region '{}'", producerConfig.getRegion());
}
 

/**
 * Verifies that group methods instantiate the correct metric with the given name.
 */
@Test
public void testMetricInstantiation() throws Exception {
	Configuration config = new Configuration();
	config.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, TestReporter1.class.getName());

	MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.fromConfiguration(config));

	MetricGroup root = new TaskManagerMetricGroup(registry, "host", "id");

	Counter counter = root.counter("counter");
	assertEquals(counter, TestReporter1.lastPassedMetric);
	assertEquals("counter", TestReporter1.lastPassedName);

	Gauge<Object> gauge = root.gauge("gauge", new Gauge<Object>() {
		@Override
		public Object getValue() {
			return null;
		}
	});

	Assert.assertEquals(gauge, TestReporter1.lastPassedMetric);
	assertEquals("gauge", TestReporter1.lastPassedName);

	Histogram histogram = root.histogram("histogram", new Histogram() {
		@Override
		public void update(long value) {

		}

		@Override
		public long getCount() {
			return 0;
		}

		@Override
		public HistogramStatistics getStatistics() {
			return null;
		}
	});

	Assert.assertEquals(histogram, TestReporter1.lastPassedMetric);
	assertEquals("histogram", TestReporter1.lastPassedName);
	registry.shutdown().get();
}
 

@Override
public void registerMetrics(MetricGroup metricGroup) {
	metricGroup.counter("task_failures", numTaskFailures);
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	// check and pass the configuration properties
	KinesisProducerConfiguration producerConfig = KinesisConfigUtil.getValidatedProducerConfiguration(configProps);

	producer = getKinesisProducer(producerConfig);

	final MetricGroup kinesisMectricGroup = getRuntimeContext().getMetricGroup().addGroup(KINESIS_PRODUCER_METRIC_GROUP);
	this.backpressureCycles = kinesisMectricGroup.counter(METRIC_BACKPRESSURE_CYCLES);
	kinesisMectricGroup.gauge(METRIC_OUTSTANDING_RECORDS_COUNT, producer::getOutstandingRecordsCount);

	backpressureLatch = new TimeoutLatch();
	callback = new FutureCallback<UserRecordResult>() {
		@Override
		public void onSuccess(UserRecordResult result) {
			backpressureLatch.trigger();
			if (!result.isSuccessful()) {
				if (failOnError) {
					// only remember the first thrown exception
					if (thrownException == null) {
						thrownException = new RuntimeException("Record was not sent successful");
					}
				} else {
					LOG.warn("Record was not sent successful");
				}
			}
		}

		@Override
		public void onFailure(Throwable t) {
			backpressureLatch.trigger();
			if (failOnError) {
				thrownException = t;
			} else {
				LOG.warn("An exception occurred while processing a record", t);
			}
		}
	};

	if (this.customPartitioner != null) {
		this.customPartitioner.initialize(getRuntimeContext().getIndexOfThisSubtask(), getRuntimeContext().getNumberOfParallelSubtasks());
	}

	LOG.info("Started Kinesis producer instance for region '{}'", producerConfig.getRegion());
}