org.apache.flink.runtime.metrics.dump.MetricDumpSerialization Java Examples
The following examples show how to use
org.apache.flink.runtime.metrics.dump.MetricDumpSerialization.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MetricRegistryImplTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testMetricQueryServiceSetup() throws Exception { MetricRegistryImpl metricRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.defaultMetricRegistryConfiguration()); Assert.assertNull(metricRegistry.getMetricQueryServiceGatewayRpcAddress()); metricRegistry.startQueryService(new TestingRpcService(), new ResourceID("mqs")); MetricQueryServiceGateway metricQueryServiceGateway = metricRegistry.getMetricQueryServiceGateway(); Assert.assertNotNull(metricQueryServiceGateway); metricRegistry.register(new SimpleCounter(), "counter", UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup()); boolean metricsSuccessfullyQueried = false; for (int x = 0; x < 10; x++) { MetricDumpSerialization.MetricSerializationResult metricSerializationResult = metricQueryServiceGateway.queryMetrics(Time.seconds(5)) .get(5, TimeUnit.SECONDS); if (metricSerializationResult.numCounters == 1) { metricsSuccessfullyQueried = true; } else { Thread.sleep(50); } } Assert.assertTrue("metrics query did not return expected result", metricsSuccessfullyQueried); }
Example #2
Source File: MetricFetcherImpl.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Query the metrics from the given QueryServiceGateway. * * @param queryServiceGateway to query for metrics */ private void queryMetrics(final MetricQueryServiceGateway queryServiceGateway) { LOG.debug("Query metrics for {}.", queryServiceGateway.getAddress()); queryServiceGateway .queryMetrics(timeout) .whenCompleteAsync( (MetricDumpSerialization.MetricSerializationResult result, Throwable t) -> { if (t != null) { LOG.debug("Fetching metrics failed.", t); } else { metrics.addAll(deserializer.deserialize(result)); } }, executor); }
Example #3
Source File: MetricFetcherImpl.java From flink with Apache License 2.0 | 6 votes |
/** * Query the metrics from the given QueryServiceGateway. * * @param queryServiceGateway to query for metrics */ private void queryMetrics(final MetricQueryServiceGateway queryServiceGateway) { LOG.debug("Query metrics for {}.", queryServiceGateway.getAddress()); queryServiceGateway .queryMetrics(timeout) .whenCompleteAsync( (MetricDumpSerialization.MetricSerializationResult result, Throwable t) -> { if (t != null) { LOG.debug("Fetching metrics failed.", t); } else { metrics.addAll(deserializer.deserialize(result)); } }, executor); }
Example #4
Source File: MetricRegistryImplTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testMetricQueryServiceSetup() throws Exception { MetricRegistryImpl metricRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.defaultMetricRegistryConfiguration()); Assert.assertNull(metricRegistry.getMetricQueryServiceGatewayRpcAddress()); metricRegistry.startQueryService(new TestingRpcService(), new ResourceID("mqs")); MetricQueryServiceGateway metricQueryServiceGateway = metricRegistry.getMetricQueryServiceGateway(); Assert.assertNotNull(metricQueryServiceGateway); metricRegistry.register(new SimpleCounter(), "counter", UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup()); boolean metricsSuccessfullyQueried = false; for (int x = 0; x < 10; x++) { MetricDumpSerialization.MetricSerializationResult metricSerializationResult = metricQueryServiceGateway.queryMetrics(Time.seconds(5)) .get(5, TimeUnit.SECONDS); if (metricSerializationResult.numCounters == 1) { metricsSuccessfullyQueried = true; } else { Thread.sleep(50); } } Assert.assertTrue("metrics query did not return expected result", metricsSuccessfullyQueried); }
Example #5
Source File: MetricFetcherImpl.java From flink with Apache License 2.0 | 6 votes |
/** * Query the metrics from the given QueryServiceGateway. * * @param queryServiceGateway to query for metrics */ private void queryMetrics(final MetricQueryServiceGateway queryServiceGateway) { LOG.debug("Query metrics for {}.", queryServiceGateway.getAddress()); queryServiceGateway .queryMetrics(timeout) .whenCompleteAsync( (MetricDumpSerialization.MetricSerializationResult result, Throwable t) -> { if (t != null) { LOG.debug("Fetching metrics failed.", t); } else { metrics.addAll(deserializer.deserialize(result)); } }, executor); }
Example #6
Source File: AkkaQueryServiceGateway.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public CompletableFuture<MetricDumpSerialization.MetricSerializationResult> queryMetrics(Time timeout) { return FutureUtils.toJava( Patterns.ask(queryServiceActorRef, MetricQueryService.getCreateDump(), timeout.toMilliseconds()) .mapTo(ClassTag$.MODULE$.apply(MetricDumpSerialization.MetricSerializationResult.class)) ); }
Example #7
Source File: MetricFetcherTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testUpdate() { final Time timeout = Time.seconds(10L); // ========= setup TaskManager ================================================================================= JobID jobID = new JobID(); ResourceID tmRID = ResourceID.generate(); // ========= setup JobManager ================================================================================== final String jmMetricQueryServicePath = "/jm/" + MetricQueryService.METRIC_QUERY_SERVICE_NAME; final String tmMetricQueryServicePath = "/tm/" + MetricQueryService.METRIC_QUERY_SERVICE_NAME + "_" + tmRID.getResourceIdString(); final TestingRestfulGateway restfulGateway = new TestingRestfulGateway.Builder() .setRequestMultipleJobDetailsSupplier(() -> CompletableFuture.completedFuture(new MultipleJobsDetails(Collections.emptyList()))) .setRequestMetricQueryServicePathsSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(jmMetricQueryServicePath))) .setRequestTaskManagerMetricQueryServicePathsSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(Tuple2.of(tmRID, tmMetricQueryServicePath)))) .build(); final GatewayRetriever<RestfulGateway> retriever = () -> CompletableFuture.completedFuture(restfulGateway); // ========= setup QueryServices ================================================================================ final MetricQueryServiceGateway jmQueryService = new TestingMetricQueryServiceGateway.Builder() .setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], new byte[0], new byte[0], new byte[0], 0, 0, 0, 0))) .build(); MetricDumpSerialization.MetricSerializationResult requestMetricsAnswer = createRequestDumpAnswer(tmRID, jobID); final MetricQueryServiceGateway tmQueryService = new TestingMetricQueryServiceGateway.Builder() .setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(requestMetricsAnswer)) .build(); final MetricQueryServiceRetriever queryServiceRetriever = (path) -> { if (path.equals(jmMetricQueryServicePath)) { return CompletableFuture.completedFuture(jmQueryService); } else if (path.equals(tmMetricQueryServicePath)) { return CompletableFuture.completedFuture(tmQueryService); } else { throw new IllegalArgumentException("Unexpected argument."); } }; // ========= start MetricFetcher testing ======================================================================= MetricFetcher fetcher = new MetricFetcherImpl<>( retriever, queryServiceRetriever, Executors.directExecutor(), timeout, MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue()); // verify that update fetches metrics and updates the store fetcher.update(); MetricStore store = fetcher.getMetricStore(); synchronized (store) { assertEquals("7", store.getJobManagerMetricStore().getMetric("abc.hist_min")); assertEquals("6", store.getJobManagerMetricStore().getMetric("abc.hist_max")); assertEquals("4.0", store.getJobManagerMetricStore().getMetric("abc.hist_mean")); assertEquals("0.5", store.getJobManagerMetricStore().getMetric("abc.hist_median")); assertEquals("5.0", store.getJobManagerMetricStore().getMetric("abc.hist_stddev")); assertEquals("0.75", store.getJobManagerMetricStore().getMetric("abc.hist_p75")); assertEquals("0.9", store.getJobManagerMetricStore().getMetric("abc.hist_p90")); assertEquals("0.95", store.getJobManagerMetricStore().getMetric("abc.hist_p95")); assertEquals("0.98", store.getJobManagerMetricStore().getMetric("abc.hist_p98")); assertEquals("0.99", store.getJobManagerMetricStore().getMetric("abc.hist_p99")); assertEquals("0.999", store.getJobManagerMetricStore().getMetric("abc.hist_p999")); assertEquals("x", store.getTaskManagerMetricStore(tmRID.toString()).metrics.get("abc.gauge")); assertEquals("5.0", store.getJobMetricStore(jobID.toString()).metrics.get("abc.jc")); assertEquals("2", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.abc.tc")); assertEquals("1", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.opname.abc.oc")); } }
Example #8
Source File: MetricFetcherTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
private static MetricDumpSerialization.MetricSerializationResult createRequestDumpAnswer(ResourceID tmRID, JobID jobID) { Map<Counter, Tuple2<QueryScopeInfo, String>> counters = new HashMap<>(); Map<Gauge<?>, Tuple2<QueryScopeInfo, String>> gauges = new HashMap<>(); Map<Histogram, Tuple2<QueryScopeInfo, String>> histograms = new HashMap<>(); Map<Meter, Tuple2<QueryScopeInfo, String>> meters = new HashMap<>(); SimpleCounter c1 = new SimpleCounter(); SimpleCounter c2 = new SimpleCounter(); c1.inc(1); c2.inc(2); counters.put(c1, new Tuple2<>(new QueryScopeInfo.OperatorQueryScopeInfo(jobID.toString(), "taskid", 2, "opname", "abc"), "oc")); counters.put(c2, new Tuple2<>(new QueryScopeInfo.TaskQueryScopeInfo(jobID.toString(), "taskid", 2, "abc"), "tc")); meters.put(new Meter() { @Override public void markEvent() { } @Override public void markEvent(long n) { } @Override public double getRate() { return 5; } @Override public long getCount() { return 10; } }, new Tuple2<>(new QueryScopeInfo.JobQueryScopeInfo(jobID.toString(), "abc"), "jc")); gauges.put(new Gauge<String>() { @Override public String getValue() { return "x"; } }, new Tuple2<>(new QueryScopeInfo.TaskManagerQueryScopeInfo(tmRID.toString(), "abc"), "gauge")); histograms.put(new TestHistogram(), new Tuple2<>(new QueryScopeInfo.JobManagerQueryScopeInfo("abc"), "hist")); MetricDumpSerialization.MetricDumpSerializer serializer = new MetricDumpSerialization.MetricDumpSerializer(); MetricDumpSerialization.MetricSerializationResult dump = serializer.serialize(counters, gauges, histograms, meters); serializer.close(); return dump; }
Example #9
Source File: TestingMetricQueryServiceGateway.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
public Builder setQueryMetricsSupplier(Supplier<CompletableFuture<MetricDumpSerialization.MetricSerializationResult>> queryMetricsSupplier) { this.queryMetricsSupplier = queryMetricsSupplier; return this; }
Example #10
Source File: TestingMetricQueryServiceGateway.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Override public CompletableFuture<MetricDumpSerialization.MetricSerializationResult> queryMetrics(Time timeout) { return queryMetricsSupplier.get(); }
Example #11
Source File: MetricFetcherTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testUpdate() { final Time timeout = Time.seconds(10L); // ========= setup TaskManager ================================================================================= JobID jobID = new JobID(); ResourceID tmRID = ResourceID.generate(); // ========= setup QueryServices ================================================================================ final MetricQueryServiceGateway jmQueryService = new TestingMetricQueryServiceGateway.Builder() .setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], new byte[0], new byte[0], new byte[0], 0, 0, 0, 0))) .build(); MetricDumpSerialization.MetricSerializationResult requestMetricsAnswer = createRequestDumpAnswer(tmRID, jobID); final MetricQueryServiceGateway tmQueryService = new TestingMetricQueryServiceGateway.Builder() .setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(requestMetricsAnswer)) .build(); // ========= setup JobManager ================================================================================== final TestingRestfulGateway restfulGateway = new TestingRestfulGateway.Builder() .setRequestMultipleJobDetailsSupplier(() -> CompletableFuture.completedFuture(new MultipleJobsDetails(Collections.emptyList()))) .setRequestMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(jmQueryService.getAddress()))) .setRequestTaskManagerMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(Tuple2.of(tmRID, tmQueryService.getAddress())))) .build(); final GatewayRetriever<RestfulGateway> retriever = () -> CompletableFuture.completedFuture(restfulGateway); // ========= start MetricFetcher testing ======================================================================= MetricFetcher fetcher = new MetricFetcherImpl<>( retriever, address -> CompletableFuture.completedFuture(tmQueryService), Executors.directExecutor(), timeout, MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue()); // verify that update fetches metrics and updates the store fetcher.update(); MetricStore store = fetcher.getMetricStore(); synchronized (store) { assertEquals("7", store.getJobManagerMetricStore().getMetric("abc.hist_min")); assertEquals("6", store.getJobManagerMetricStore().getMetric("abc.hist_max")); assertEquals("4.0", store.getJobManagerMetricStore().getMetric("abc.hist_mean")); assertEquals("0.5", store.getJobManagerMetricStore().getMetric("abc.hist_median")); assertEquals("5.0", store.getJobManagerMetricStore().getMetric("abc.hist_stddev")); assertEquals("0.75", store.getJobManagerMetricStore().getMetric("abc.hist_p75")); assertEquals("0.9", store.getJobManagerMetricStore().getMetric("abc.hist_p90")); assertEquals("0.95", store.getJobManagerMetricStore().getMetric("abc.hist_p95")); assertEquals("0.98", store.getJobManagerMetricStore().getMetric("abc.hist_p98")); assertEquals("0.99", store.getJobManagerMetricStore().getMetric("abc.hist_p99")); assertEquals("0.999", store.getJobManagerMetricStore().getMetric("abc.hist_p999")); assertEquals("x", store.getTaskManagerMetricStore(tmRID.toString()).metrics.get("abc.gauge")); assertEquals("5.0", store.getJobMetricStore(jobID.toString()).metrics.get("abc.jc")); assertEquals("2", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.abc.tc")); assertEquals("1", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.opname.abc.oc")); } }
Example #12
Source File: MetricFetcherTest.java From flink with Apache License 2.0 | 4 votes |
private static MetricDumpSerialization.MetricSerializationResult createRequestDumpAnswer(ResourceID tmRID, JobID jobID) { Map<Counter, Tuple2<QueryScopeInfo, String>> counters = new HashMap<>(); Map<Gauge<?>, Tuple2<QueryScopeInfo, String>> gauges = new HashMap<>(); Map<Histogram, Tuple2<QueryScopeInfo, String>> histograms = new HashMap<>(); Map<Meter, Tuple2<QueryScopeInfo, String>> meters = new HashMap<>(); SimpleCounter c1 = new SimpleCounter(); SimpleCounter c2 = new SimpleCounter(); c1.inc(1); c2.inc(2); counters.put(c1, new Tuple2<>(new QueryScopeInfo.OperatorQueryScopeInfo(jobID.toString(), "taskid", 2, "opname", "abc"), "oc")); counters.put(c2, new Tuple2<>(new QueryScopeInfo.TaskQueryScopeInfo(jobID.toString(), "taskid", 2, "abc"), "tc")); meters.put(new Meter() { @Override public void markEvent() { } @Override public void markEvent(long n) { } @Override public double getRate() { return 5; } @Override public long getCount() { return 10; } }, new Tuple2<>(new QueryScopeInfo.JobQueryScopeInfo(jobID.toString(), "abc"), "jc")); gauges.put(new Gauge<String>() { @Override public String getValue() { return "x"; } }, new Tuple2<>(new QueryScopeInfo.TaskManagerQueryScopeInfo(tmRID.toString(), "abc"), "gauge")); histograms.put(new TestHistogram(), new Tuple2<>(new QueryScopeInfo.JobManagerQueryScopeInfo("abc"), "hist")); MetricDumpSerialization.MetricDumpSerializer serializer = new MetricDumpSerialization.MetricDumpSerializer(); MetricDumpSerialization.MetricSerializationResult dump = serializer.serialize(counters, gauges, histograms, meters); serializer.close(); return dump; }
Example #13
Source File: TestingMetricQueryServiceGateway.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
public TestingMetricQueryServiceGateway(@Nonnull Supplier<CompletableFuture<MetricDumpSerialization.MetricSerializationResult>> queryMetricsSupplier, @Nonnull String address) { this.queryMetricsSupplier = queryMetricsSupplier; this.address = address; }
Example #14
Source File: MetricFetcherTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testUpdate() { final Time timeout = Time.seconds(10L); // ========= setup TaskManager ================================================================================= JobID jobID = new JobID(); ResourceID tmRID = ResourceID.generate(); // ========= setup QueryServices ================================================================================ final MetricQueryServiceGateway jmQueryService = new TestingMetricQueryServiceGateway.Builder() .setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], new byte[0], new byte[0], new byte[0], 0, 0, 0, 0))) .build(); MetricDumpSerialization.MetricSerializationResult requestMetricsAnswer = createRequestDumpAnswer(tmRID, jobID); final MetricQueryServiceGateway tmQueryService = new TestingMetricQueryServiceGateway.Builder() .setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(requestMetricsAnswer)) .build(); // ========= setup JobManager ================================================================================== final TestingRestfulGateway restfulGateway = new TestingRestfulGateway.Builder() .setRequestMultipleJobDetailsSupplier(() -> CompletableFuture.completedFuture(new MultipleJobsDetails(Collections.emptyList()))) .setRequestMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(jmQueryService.getAddress()))) .setRequestTaskManagerMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(Tuple2.of(tmRID, tmQueryService.getAddress())))) .build(); final GatewayRetriever<RestfulGateway> retriever = () -> CompletableFuture.completedFuture(restfulGateway); // ========= start MetricFetcher testing ======================================================================= MetricFetcher fetcher = new MetricFetcherImpl<>( retriever, address -> CompletableFuture.completedFuture(tmQueryService), Executors.directExecutor(), timeout, MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue()); // verify that update fetches metrics and updates the store fetcher.update(); MetricStore store = fetcher.getMetricStore(); synchronized (store) { assertEquals("7", store.getJobManagerMetricStore().getMetric("abc.hist_min")); assertEquals("6", store.getJobManagerMetricStore().getMetric("abc.hist_max")); assertEquals("4.0", store.getJobManagerMetricStore().getMetric("abc.hist_mean")); assertEquals("0.5", store.getJobManagerMetricStore().getMetric("abc.hist_median")); assertEquals("5.0", store.getJobManagerMetricStore().getMetric("abc.hist_stddev")); assertEquals("0.75", store.getJobManagerMetricStore().getMetric("abc.hist_p75")); assertEquals("0.9", store.getJobManagerMetricStore().getMetric("abc.hist_p90")); assertEquals("0.95", store.getJobManagerMetricStore().getMetric("abc.hist_p95")); assertEquals("0.98", store.getJobManagerMetricStore().getMetric("abc.hist_p98")); assertEquals("0.99", store.getJobManagerMetricStore().getMetric("abc.hist_p99")); assertEquals("0.999", store.getJobManagerMetricStore().getMetric("abc.hist_p999")); assertEquals("x", store.getTaskManagerMetricStore(tmRID.toString()).metrics.get("abc.gauge")); assertEquals("5.0", store.getJobMetricStore(jobID.toString()).metrics.get("abc.jc")); assertEquals("2", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.abc.tc")); assertEquals("1", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.opname.abc.oc")); } }
Example #15
Source File: MetricFetcherTest.java From flink with Apache License 2.0 | 4 votes |
private static MetricDumpSerialization.MetricSerializationResult createRequestDumpAnswer(ResourceID tmRID, JobID jobID) { Map<Counter, Tuple2<QueryScopeInfo, String>> counters = new HashMap<>(); Map<Gauge<?>, Tuple2<QueryScopeInfo, String>> gauges = new HashMap<>(); Map<Histogram, Tuple2<QueryScopeInfo, String>> histograms = new HashMap<>(); Map<Meter, Tuple2<QueryScopeInfo, String>> meters = new HashMap<>(); SimpleCounter c1 = new SimpleCounter(); SimpleCounter c2 = new SimpleCounter(); c1.inc(1); c2.inc(2); counters.put(c1, new Tuple2<>(new QueryScopeInfo.OperatorQueryScopeInfo(jobID.toString(), "taskid", 2, "opname", "abc"), "oc")); counters.put(c2, new Tuple2<>(new QueryScopeInfo.TaskQueryScopeInfo(jobID.toString(), "taskid", 2, "abc"), "tc")); meters.put(new Meter() { @Override public void markEvent() { } @Override public void markEvent(long n) { } @Override public double getRate() { return 5; } @Override public long getCount() { return 10; } }, new Tuple2<>(new QueryScopeInfo.JobQueryScopeInfo(jobID.toString(), "abc"), "jc")); gauges.put(new Gauge<String>() { @Override public String getValue() { return "x"; } }, new Tuple2<>(new QueryScopeInfo.TaskManagerQueryScopeInfo(tmRID.toString(), "abc"), "gauge")); histograms.put(new TestHistogram(), new Tuple2<>(new QueryScopeInfo.JobManagerQueryScopeInfo("abc"), "hist")); MetricDumpSerialization.MetricDumpSerializer serializer = new MetricDumpSerialization.MetricDumpSerializer(); MetricDumpSerialization.MetricSerializationResult dump = serializer.serialize(counters, gauges, histograms, meters); serializer.close(); return dump; }
Example #16
Source File: MetricQueryServiceGateway.java From flink with Apache License 2.0 | votes |
CompletableFuture<MetricDumpSerialization.MetricSerializationResult> queryMetrics(Time timeout);
Example #17
Source File: MetricQueryServiceGateway.java From flink with Apache License 2.0 | votes |
CompletableFuture<MetricDumpSerialization.MetricSerializationResult> queryMetrics(Time timeout);
Example #18
Source File: MetricQueryServiceGateway.java From Flink-CEPplus with Apache License 2.0 | votes |
CompletableFuture<MetricDumpSerialization.MetricSerializationResult> queryMetrics(Time timeout);