org.apache.beam.sdk.metrics.GaugeResult Java Examples
The following examples show how to use
org.apache.beam.sdk.metrics.GaugeResult.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FlinkMetricContainer.java From flink with Apache License 2.0 | 6 votes |
private void updateGauge(Iterable<MetricResult<GaugeResult>> gauges) { for (MetricResult<GaugeResult> metricResult : gauges) { if (!isUserMetric(metricResult)) { continue; } // get identifier String flinkMetricIdentifier = getFlinkMetricIdentifierString(metricResult.getKey()); GaugeResult update = metricResult.getAttempted(); // update flink metric FlinkGauge gauge = flinkGaugeCache.get(flinkMetricIdentifier); if (gauge == null) { MetricGroup metricGroup = registerMetricGroup(metricResult.getKey(), baseMetricGroup); gauge = metricGroup.gauge( metricResult.getKey().metricName().getName(), new FlinkGauge(update)); flinkGaugeCache.put(flinkMetricIdentifier, gauge); } else { gauge.update(update); } } }
Example #2
Source File: DataflowMetrics.java From beam with Apache License 2.0 | 6 votes |
@Override public MetricQueryResults queryMetrics(@Nullable MetricsFilter filter) { List<MetricUpdate> metricUpdates; ImmutableList<MetricResult<Long>> counters = ImmutableList.of(); ImmutableList<MetricResult<DistributionResult>> distributions = ImmutableList.of(); ImmutableList<MetricResult<GaugeResult>> gauges = ImmutableList.of(); JobMetrics jobMetrics; try { jobMetrics = getJobMetrics(); } catch (IOException e) { LOG.warn("Unable to query job metrics.\n"); return MetricQueryResults.create(counters, distributions, gauges); } metricUpdates = firstNonNull(jobMetrics.getMetrics(), Collections.emptyList()); return populateMetricQueryResults(metricUpdates, filter); }
Example #3
Source File: FlinkMetricContainer.java From beam with Apache License 2.0 | 6 votes |
private void updateGauge(Iterable<MetricResult<GaugeResult>> gauges) { for (MetricResult<GaugeResult> metricResult : gauges) { String flinkMetricName = getFlinkMetricNameString(metricResult.getKey()); GaugeResult update = metricResult.getAttempted(); // update flink metric FlinkGauge gauge = flinkGaugeCache.get(flinkMetricName); if (gauge == null) { gauge = runtimeContext.getMetricGroup().gauge(flinkMetricName, new FlinkGauge(update)); flinkGaugeCache.put(flinkMetricName, gauge); } else { gauge.update(update); } } }
Example #4
Source File: DirectMetrics.java From beam with Apache License 2.0 | 6 votes |
@Override public MetricQueryResults queryMetrics(@Nullable MetricsFilter filter) { ImmutableList.Builder<MetricResult<Long>> counterResults = ImmutableList.builder(); for (Entry<MetricKey, DirectMetric<Long, Long>> counter : counters.entries()) { maybeExtractResult(filter, counterResults, counter); } ImmutableList.Builder<MetricResult<DistributionResult>> distributionResults = ImmutableList.builder(); for (Entry<MetricKey, DirectMetric<DistributionData, DistributionResult>> distribution : distributions.entries()) { maybeExtractResult(filter, distributionResults, distribution); } ImmutableList.Builder<MetricResult<GaugeResult>> gaugeResults = ImmutableList.builder(); for (Entry<MetricKey, DirectMetric<GaugeData, GaugeResult>> gauge : gauges.entries()) { maybeExtractResult(filter, gaugeResults, gauge); } return MetricQueryResults.create( counterResults.build(), distributionResults.build(), gaugeResults.build()); }
Example #5
Source File: FailedRunningPipelineResults.java From beam with Apache License 2.0 | 6 votes |
@Override public MetricResults metrics() { return new MetricResults() { @Override public MetricQueryResults queryMetrics(@Nullable MetricsFilter filter) { return new MetricQueryResults() { @Override public Iterable<MetricResult<Long>> getCounters() { return Collections.emptyList(); } @Override public Iterable<MetricResult<DistributionResult>> getDistributions() { return Collections.emptyList(); } @Override public Iterable<MetricResult<GaugeResult>> getGauges() { return Collections.emptyList(); } }; } }; }
Example #6
Source File: PortableMetrics.java From beam with Apache License 2.0 | 5 votes |
private static Iterable<MetricResult<GaugeResult>> extractGaugeMetricsFromJobMetrics( List<MetricsApi.MonitoringInfo> monitoringInfoList) { return monitoringInfoList.stream() .filter(item -> LATEST_INT64_TYPE.equals(item.getType())) .filter(item -> item.getLabelsMap().get(NAMESPACE_LABEL) != null) .map(PortableMetrics::convertGaugeMonitoringInfoToGauge) .collect(Collectors.toList()); }
Example #7
Source File: MetricsGraphiteSink.java From beam with Apache License 2.0 | 5 votes |
@Override public void writeMetrics(MetricQueryResults metricQueryResults) throws Exception { final long metricTimestamp = System.currentTimeMillis() / 1000L; Socket socket = new Socket(InetAddress.getByName(address), port); BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), charset)); StringBuilder messagePayload = new StringBuilder(); Iterable<MetricResult<Long>> counters = metricQueryResults.getCounters(); Iterable<MetricResult<GaugeResult>> gauges = metricQueryResults.getGauges(); Iterable<MetricResult<DistributionResult>> distributions = metricQueryResults.getDistributions(); for (MetricResult<Long> counter : counters) { messagePayload.append(new CounterMetricMessage(counter, "value", metricTimestamp).toString()); } for (MetricResult<GaugeResult> gauge : gauges) { messagePayload.append(new GaugeMetricMessage(gauge, "value").toString()); } for (MetricResult<DistributionResult> distribution : distributions) { messagePayload.append( new DistributionMetricMessage(distribution, "min", metricTimestamp).toString()); messagePayload.append( new DistributionMetricMessage(distribution, "max", metricTimestamp).toString()); messagePayload.append( new DistributionMetricMessage(distribution, "count", metricTimestamp).toString()); messagePayload.append( new DistributionMetricMessage(distribution, "sum", metricTimestamp).toString()); messagePayload.append( new DistributionMetricMessage(distribution, "mean", metricTimestamp).toString()); } writer.write(messagePayload.toString()); writer.flush(); writer.close(); socket.close(); }
Example #8
Source File: SamzaMetricsContainer.java From beam with Apache License 2.0 | 5 votes |
@Override public void accept(MetricResult<GaugeResult> metricResult) { final String metricName = getMetricName(metricResult); @SuppressWarnings("unchecked") Gauge<Long> gauge = (Gauge<Long>) getSamzaMetricFor(metricName); if (gauge == null) { gauge = metricsRegistry.newGauge(BEAM_METRICS_GROUP, metricName, 0L); } gauge.set(metricResult.getAttempted().getValue()); }
Example #9
Source File: CustomMetricQueryResults.java From beam with Apache License 2.0 | 5 votes |
@Override public List<MetricResult<GaugeResult>> getGauges() { return makeResults( "s3", "n3", GaugeResult.create(100L, new Instant(345862800L)), GaugeResult.create(120L, new Instant(345862800L))); }
Example #10
Source File: MetricsHelper.java From dbeam with Apache License 2.0 | 5 votes |
public static Map<String, Long> getMetrics(final PipelineResult result) { final MetricQueryResults metricQueryResults = result.metrics().queryMetrics(MetricsFilter.builder().build()); final Map<String, Long> gauges = StreamSupport.stream(metricQueryResults.getGauges().spliterator(), false) .collect( Collectors.groupingBy( MetricResult::getName, Collectors.reducing( GaugeResult.empty(), GET_COMMITTED_GAUGE, BinaryOperator.maxBy(Comparator.comparing(GaugeResult::getTimestamp))))) .entrySet() .stream() .collect(Collectors.toMap(e -> e.getKey().getName(), e -> e.getValue().getValue())); final Map<String, Long> counters = StreamSupport.stream(metricQueryResults.getCounters().spliterator(), false) .collect( Collectors.groupingBy( m -> m.getName().getName(), Collectors.summingLong(GET_COMMITTED_COUNTER))); Map<String, Long> ret = new HashMap<>(); ret.putAll(gauges); ret.putAll(counters); addCalculatedMetrics(counters, ret); return Collections.unmodifiableMap(ret); }
Example #11
Source File: JetMetricResults.java From beam with Apache License 2.0 | 5 votes |
private QueryResults( Iterable<MetricResult<Long>> counters, Iterable<MetricResult<DistributionResult>> distributions, Iterable<MetricResult<GaugeResult>> gauges) { this.counters = counters; this.distributions = distributions; this.gauges = gauges; }
Example #12
Source File: PortableMetrics.java From beam with Apache License 2.0 | 5 votes |
private static MetricResult<GaugeResult> convertGaugeMonitoringInfoToGauge( MetricsApi.MonitoringInfo monitoringInfo) { Map<String, String> labelsMap = monitoringInfo.getLabelsMap(); MetricKey key = MetricKey.create( labelsMap.get(STEP_NAME_LABEL), MetricName.named(labelsMap.get(NAMESPACE_LABEL), labelsMap.get(METRIC_NAME_LABEL))); GaugeData data = decodeInt64Gauge(monitoringInfo.getPayload()); GaugeResult result = GaugeResult.create(data.value(), data.timestamp()); return MetricResult.create(key, false, result); }
Example #13
Source File: PortableMetrics.java From beam with Apache License 2.0 | 5 votes |
private static PortableMetrics convertMonitoringInfosToMetricResults( JobApi.MetricResults jobMetrics) { List<MetricsApi.MonitoringInfo> monitoringInfoList = new ArrayList<>(); monitoringInfoList.addAll(jobMetrics.getAttemptedList()); monitoringInfoList.addAll(jobMetrics.getCommittedList()); Iterable<MetricResult<Long>> countersFromJobMetrics = extractCountersFromJobMetrics(monitoringInfoList); Iterable<MetricResult<DistributionResult>> distributionsFromMetrics = extractDistributionMetricsFromJobMetrics(monitoringInfoList); Iterable<MetricResult<GaugeResult>> gaugesFromMetrics = extractGaugeMetricsFromJobMetrics(monitoringInfoList); return new PortableMetrics(countersFromJobMetrics, distributionsFromMetrics, gaugesFromMetrics); }
Example #14
Source File: PortableMetrics.java From beam with Apache License 2.0 | 5 votes |
private PortableMetrics( Iterable<MetricResult<Long>> counters, Iterable<MetricResult<DistributionResult>> distributions, Iterable<MetricResult<GaugeResult>> gauges) { this.counters = counters; this.distributions = distributions; this.gauges = gauges; }
Example #15
Source File: MetricsContainerStepMapTest.java From beam with Apache License 2.0 | 5 votes |
private void assertGauge( String name, MetricQueryResults metricQueryResults, String step, GaugeResult expected, boolean isCommitted) { assertThat( metricQueryResults.getGauges(), hasItem(metricsResult(NAMESPACE, name, step, expected, isCommitted))); }
Example #16
Source File: MetricsContainerStepMapTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void testGaugeCommittedUnsupportedInAttemptedAccumulatedMetricResults() { MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap(); attemptedMetrics.update(STEP1, metricsContainer); MetricResults metricResults = asAttemptedOnlyMetricResults(attemptedMetrics); MetricQueryResults step1res = metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build()); thrown.expect(UnsupportedOperationException.class); thrown.expectMessage("This runner does not currently support committed metrics results."); assertGauge(GAUGE_NAME, step1res, STEP1, GaugeResult.empty(), true); }
Example #17
Source File: DefaultMetricResults.java From beam with Apache License 2.0 | 5 votes |
public DefaultMetricResults( Iterable<MetricResult<Long>> counters, Iterable<MetricResult<DistributionResult>> distributions, Iterable<MetricResult<GaugeResult>> gauges) { this.counters = counters; this.distributions = distributions; this.gauges = gauges; }
Example #18
Source File: JetMetricResults.java From beam with Apache License 2.0 | 4 votes |
@Override public Iterable<MetricResult<GaugeResult>> getGauges() { return gauges; }
Example #19
Source File: DirectMetrics.java From beam with Apache License 2.0 | 4 votes |
@Override public GaugeResult extract(GaugeData data) { return data.extractResult(); }
Example #20
Source File: MetricsGraphiteSink.java From beam with Apache License 2.0 | 4 votes |
private GaugeMetricMessage(MetricResult<GaugeResult> gauge, String valueType) { this.valueType = valueType; this.gauge = gauge; }
Example #21
Source File: DirectMetricsTest.java From beam with Apache License 2.0 | 4 votes |
@SuppressWarnings("unchecked") @Test public void testApplyCommittedNoFilter() { metrics.commitLogical( bundle1, MetricUpdates.create( ImmutableList.of( MetricUpdate.create(MetricKey.create("step1", NAME1), 5L), MetricUpdate.create(MetricKey.create("step1", NAME2), 8L)), ImmutableList.of( MetricUpdate.create( MetricKey.create("step1", NAME1), DistributionData.create(8, 2, 3, 5))), ImmutableList.of( MetricUpdate.create(MetricKey.create("step1", NAME4), GaugeData.create(15L))))); metrics.commitLogical( bundle1, MetricUpdates.create( ImmutableList.of( MetricUpdate.create(MetricKey.create("step2", NAME1), 7L), MetricUpdate.create(MetricKey.create("step1", NAME2), 4L)), ImmutableList.of( MetricUpdate.create( MetricKey.create("step1", NAME1), DistributionData.create(4, 1, 4, 4))), ImmutableList.of( MetricUpdate.create(MetricKey.create("step1", NAME4), GaugeData.create(27L))))); MetricQueryResults results = metrics.allMetrics(); assertThat( results.getCounters(), containsInAnyOrder( attemptedMetricsResult("ns1", "name1", "step1", 0L), attemptedMetricsResult("ns1", "name2", "step1", 0L), attemptedMetricsResult("ns1", "name1", "step2", 0L))); assertThat( results.getCounters(), containsInAnyOrder( committedMetricsResult("ns1", "name1", "step1", 5L), committedMetricsResult("ns1", "name2", "step1", 12L), committedMetricsResult("ns1", "name1", "step2", 7L))); assertThat( results.getDistributions(), contains( attemptedMetricsResult("ns1", "name1", "step1", DistributionResult.IDENTITY_ELEMENT))); assertThat( results.getDistributions(), contains( committedMetricsResult( "ns1", "name1", "step1", DistributionResult.create(12, 3, 3, 5)))); assertThat( results.getGauges(), contains(attemptedMetricsResult("ns2", "name2", "step1", GaugeResult.empty()))); assertThat( results.getGauges(), contains( committedMetricsResult( "ns2", "name2", "step1", GaugeResult.create(27L, Instant.now())))); }
Example #22
Source File: FlinkMetricContainer.java From flink with Apache License 2.0 | 4 votes |
FlinkGauge(GaugeResult data) { this.data = data; }
Example #23
Source File: DataflowMetrics.java From beam with Apache License 2.0 | 4 votes |
public Iterable<MetricResult<GaugeResult>> getGaugeResults() { return gaugeResults.build(); }
Example #24
Source File: FlinkMetricContainer.java From beam with Apache License 2.0 | 4 votes |
FlinkGauge(GaugeResult data) { this.data = data; }
Example #25
Source File: JetMetricResults.java From beam with Apache License 2.0 | 4 votes |
private MetricResult<GaugeResult> toUpdateResult(Map.Entry<MetricKey, GaugeData> entry) { MetricKey key = entry.getKey(); GaugeResult gaugeResult = entry.getValue().extractResult(); return MetricResult.create(key, gaugeResult, gaugeResult); }
Example #26
Source File: MetricsContainerStepMapTest.java From beam with Apache License 2.0 | 4 votes |
@Test public void testReset() { MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap(); attemptedMetrics.update(STEP1, metricsContainer); attemptedMetrics.update(STEP2, metricsContainer); attemptedMetrics.update(STEP2, metricsContainer); MetricResults metricResults = asAttemptedOnlyMetricResults(attemptedMetrics); MetricQueryResults allres = metricResults.allMetrics(); assertCounter(COUNTER_NAME, allres, STEP1, VALUE, false); assertDistribution( DISTRIBUTION_NAME, allres, STEP1, DistributionResult.create(VALUE * 3, 2, VALUE, VALUE * 2), false); assertGauge(GAUGE_NAME, allres, STEP1, GaugeResult.create(VALUE, Instant.now()), false); assertCounter(COUNTER_NAME, allres, STEP2, VALUE * 2, false); assertDistribution( DISTRIBUTION_NAME, allres, STEP2, DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2), false); assertGauge(GAUGE_NAME, allres, STEP2, GaugeResult.create(VALUE, Instant.now()), false); attemptedMetrics.reset(); metricResults = asAttemptedOnlyMetricResults(attemptedMetrics); allres = metricResults.allMetrics(); // Check that the metrics container for STEP1 is reset assertCounter(COUNTER_NAME, allres, STEP1, 0L, false); assertDistribution( DISTRIBUTION_NAME, allres, STEP1, DistributionResult.IDENTITY_ELEMENT, false); assertGauge(GAUGE_NAME, allres, STEP1, GaugeResult.empty(), false); // Check that the metrics container for STEP2 is reset assertCounter(COUNTER_NAME, allres, STEP2, 0L, false); assertDistribution( DISTRIBUTION_NAME, allres, STEP2, DistributionResult.IDENTITY_ELEMENT, false); assertGauge(GAUGE_NAME, allres, STEP2, GaugeResult.empty(), false); }
Example #27
Source File: MetricsContainerStepMapTest.java From beam with Apache License 2.0 | 4 votes |
@Test public void testAttemptedAndCommittedAccumulatedMetricResults() { MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap(); attemptedMetrics.update(STEP1, metricsContainer); attemptedMetrics.update(STEP1, metricsContainer); attemptedMetrics.update(STEP2, metricsContainer); attemptedMetrics.update(STEP2, metricsContainer); attemptedMetrics.update(STEP2, metricsContainer); MetricsContainerStepMap committedMetrics = new MetricsContainerStepMap(); committedMetrics.update(STEP1, metricsContainer); committedMetrics.update(STEP2, metricsContainer); committedMetrics.update(STEP2, metricsContainer); MetricResults metricResults = asMetricResults(attemptedMetrics, committedMetrics); MetricQueryResults step1res = metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build()); assertIterableSize(step1res.getCounters(), 1); assertIterableSize(step1res.getDistributions(), 1); assertIterableSize(step1res.getGauges(), 1); assertCounter(COUNTER_NAME, step1res, STEP1, VALUE * 2, false); assertDistribution( DISTRIBUTION_NAME, step1res, STEP1, DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2), false); assertGauge(GAUGE_NAME, step1res, STEP1, GaugeResult.create(VALUE, Instant.now()), false); assertCounter(COUNTER_NAME, step1res, STEP1, VALUE, true); assertDistribution( DISTRIBUTION_NAME, step1res, STEP1, DistributionResult.create(VALUE * 3, 2, VALUE, VALUE * 2), true); assertGauge(GAUGE_NAME, step1res, STEP1, GaugeResult.create(VALUE, Instant.now()), true); MetricQueryResults step2res = metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP2).build()); assertIterableSize(step2res.getCounters(), 1); assertIterableSize(step2res.getDistributions(), 1); assertIterableSize(step2res.getGauges(), 1); assertCounter(COUNTER_NAME, step2res, STEP2, VALUE * 3, false); assertDistribution( DISTRIBUTION_NAME, step2res, STEP2, DistributionResult.create(VALUE * 9, 6, VALUE, VALUE * 2), false); assertGauge(GAUGE_NAME, step2res, STEP2, GaugeResult.create(VALUE, Instant.now()), false); assertCounter(COUNTER_NAME, step2res, STEP2, VALUE * 2, true); assertDistribution( DISTRIBUTION_NAME, step2res, STEP2, DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2), true); assertGauge(GAUGE_NAME, step2res, STEP2, GaugeResult.create(VALUE, Instant.now()), true); MetricQueryResults allres = metricResults.queryMetrics(MetricsFilter.builder().build()); assertIterableSize(allres.getCounters(), 2); assertIterableSize(allres.getDistributions(), 2); assertIterableSize(allres.getGauges(), 2); }
Example #28
Source File: MetricsContainerStepMapTest.java From beam with Apache License 2.0 | 4 votes |
@Test public void testAttemptedAccumulatedMetricResults() { MetricsContainerStepMap attemptedMetrics = new MetricsContainerStepMap(); attemptedMetrics.update(STEP1, metricsContainer); attemptedMetrics.update(STEP2, metricsContainer); attemptedMetrics.update(STEP2, metricsContainer); MetricResults metricResults = asAttemptedOnlyMetricResults(attemptedMetrics); MetricQueryResults step1res = metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP1).build()); assertIterableSize(step1res.getCounters(), 1); assertIterableSize(step1res.getDistributions(), 1); assertIterableSize(step1res.getGauges(), 1); assertCounter(COUNTER_NAME, step1res, STEP1, VALUE, false); assertDistribution( DISTRIBUTION_NAME, step1res, STEP1, DistributionResult.create(VALUE * 3, 2, VALUE, VALUE * 2), false); assertGauge(GAUGE_NAME, step1res, STEP1, GaugeResult.create(VALUE, Instant.now()), false); MetricQueryResults step2res = metricResults.queryMetrics(MetricsFilter.builder().addStep(STEP2).build()); assertIterableSize(step2res.getCounters(), 1); assertIterableSize(step2res.getDistributions(), 1); assertIterableSize(step2res.getGauges(), 1); assertCounter(COUNTER_NAME, step2res, STEP2, VALUE * 2, false); assertDistribution( DISTRIBUTION_NAME, step2res, STEP2, DistributionResult.create(VALUE * 6, 4, VALUE, VALUE * 2), false); assertGauge(GAUGE_NAME, step2res, STEP2, GaugeResult.create(VALUE, Instant.now()), false); MetricQueryResults allres = metricResults.allMetrics(); assertIterableSize(allres.getCounters(), 2); assertIterableSize(allres.getDistributions(), 2); assertIterableSize(allres.getGauges(), 2); }
Example #29
Source File: GaugeData.java From beam with Apache License 2.0 | 4 votes |
@Override public GaugeResult extractResult() { return GaugeResult.empty(); }
Example #30
Source File: GaugeData.java From beam with Apache License 2.0 | 4 votes |
public GaugeResult extractResult() { return GaugeResult.create(value(), timestamp()); }