Java Code Examples for org.apache.flink.runtime.metrics.dump.MetricDumpSerialization#MetricSerializationResult

The following examples show how to use org.apache.flink.runtime.metrics.dump.MetricDumpSerialization#MetricSerializationResult . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MetricRegistryImplTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testMetricQueryServiceSetup() throws Exception {
	MetricRegistryImpl metricRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.defaultMetricRegistryConfiguration());

	Assert.assertNull(metricRegistry.getMetricQueryServiceGatewayRpcAddress());

	metricRegistry.startQueryService(new TestingRpcService(), new ResourceID("mqs"));

	MetricQueryServiceGateway metricQueryServiceGateway = metricRegistry.getMetricQueryServiceGateway();
	Assert.assertNotNull(metricQueryServiceGateway);

	metricRegistry.register(new SimpleCounter(), "counter", UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup());

	boolean metricsSuccessfullyQueried = false;
	for (int x = 0; x < 10; x++) {
		MetricDumpSerialization.MetricSerializationResult metricSerializationResult = metricQueryServiceGateway.queryMetrics(Time.seconds(5))
			.get(5, TimeUnit.SECONDS);

		if (metricSerializationResult.numCounters == 1) {
			metricsSuccessfullyQueried = true;
		} else {
			Thread.sleep(50);
		}
	}
	Assert.assertTrue("metrics query did not return expected result", metricsSuccessfullyQueried);
}
Example 2
Source File: MetricRegistryImplTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testMetricQueryServiceSetup() throws Exception {
	MetricRegistryImpl metricRegistry = new MetricRegistryImpl(MetricRegistryConfiguration.defaultMetricRegistryConfiguration());

	Assert.assertNull(metricRegistry.getMetricQueryServiceGatewayRpcAddress());

	metricRegistry.startQueryService(new TestingRpcService(), new ResourceID("mqs"));

	MetricQueryServiceGateway metricQueryServiceGateway = metricRegistry.getMetricQueryServiceGateway();
	Assert.assertNotNull(metricQueryServiceGateway);

	metricRegistry.register(new SimpleCounter(), "counter", UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup());

	boolean metricsSuccessfullyQueried = false;
	for (int x = 0; x < 10; x++) {
		MetricDumpSerialization.MetricSerializationResult metricSerializationResult = metricQueryServiceGateway.queryMetrics(Time.seconds(5))
			.get(5, TimeUnit.SECONDS);

		if (metricSerializationResult.numCounters == 1) {
			metricsSuccessfullyQueried = true;
		} else {
			Thread.sleep(50);
		}
	}
	Assert.assertTrue("metrics query did not return expected result", metricsSuccessfullyQueried);
}
Example 3
Source File: AkkaQueryServiceGateway.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public CompletableFuture<MetricDumpSerialization.MetricSerializationResult> queryMetrics(Time timeout) {
	return FutureUtils.toJava(
		Patterns.ask(queryServiceActorRef, MetricQueryService.getCreateDump(), timeout.toMilliseconds())
			.mapTo(ClassTag$.MODULE$.apply(MetricDumpSerialization.MetricSerializationResult.class))
	);
}
Example 4
Source File: TestingMetricQueryServiceGateway.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public TestingMetricQueryServiceGateway(@Nonnull Supplier<CompletableFuture<MetricDumpSerialization.MetricSerializationResult>> queryMetricsSupplier, @Nonnull String address) {
	this.queryMetricsSupplier = queryMetricsSupplier;
	this.address = address;
}
Example 5
Source File: TestingMetricQueryServiceGateway.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Override
public CompletableFuture<MetricDumpSerialization.MetricSerializationResult> queryMetrics(Time timeout) {
	return queryMetricsSupplier.get();
}
Example 6
Source File: TestingMetricQueryServiceGateway.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public Builder setQueryMetricsSupplier(Supplier<CompletableFuture<MetricDumpSerialization.MetricSerializationResult>> queryMetricsSupplier) {
	this.queryMetricsSupplier = queryMetricsSupplier;
	return this;
}
Example 7
Source File: MetricFetcherTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testUpdate() {
	final Time timeout = Time.seconds(10L);

	// ========= setup TaskManager =================================================================================
	JobID jobID = new JobID();
	ResourceID tmRID = ResourceID.generate();

	// ========= setup JobManager ==================================================================================

	final String jmMetricQueryServicePath = "/jm/" + MetricQueryService.METRIC_QUERY_SERVICE_NAME;
	final String tmMetricQueryServicePath = "/tm/" + MetricQueryService.METRIC_QUERY_SERVICE_NAME + "_" + tmRID.getResourceIdString();

	final TestingRestfulGateway restfulGateway = new TestingRestfulGateway.Builder()
		.setRequestMultipleJobDetailsSupplier(() -> CompletableFuture.completedFuture(new MultipleJobsDetails(Collections.emptyList())))
		.setRequestMetricQueryServicePathsSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(jmMetricQueryServicePath)))
		.setRequestTaskManagerMetricQueryServicePathsSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(Tuple2.of(tmRID, tmMetricQueryServicePath))))
		.build();

	final GatewayRetriever<RestfulGateway> retriever = () -> CompletableFuture.completedFuture(restfulGateway);

	// ========= setup QueryServices ================================================================================
	final MetricQueryServiceGateway jmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], new byte[0], new byte[0], new byte[0], 0, 0, 0, 0)))
		.build();

	MetricDumpSerialization.MetricSerializationResult requestMetricsAnswer = createRequestDumpAnswer(tmRID, jobID);
	final MetricQueryServiceGateway tmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(requestMetricsAnswer))
		.build();

	final MetricQueryServiceRetriever queryServiceRetriever = (path) -> {
		if (path.equals(jmMetricQueryServicePath))  {
			return CompletableFuture.completedFuture(jmQueryService);
		} else if (path.equals(tmMetricQueryServicePath)) {
			return CompletableFuture.completedFuture(tmQueryService);
		} else {
			throw new IllegalArgumentException("Unexpected argument.");
		}
	};

	// ========= start MetricFetcher testing =======================================================================
	MetricFetcher fetcher = new MetricFetcherImpl<>(
		retriever,
		queryServiceRetriever,
		Executors.directExecutor(),
		timeout,
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());

	// verify that update fetches metrics and updates the store
	fetcher.update();
	MetricStore store = fetcher.getMetricStore();
	synchronized (store) {
		assertEquals("7", store.getJobManagerMetricStore().getMetric("abc.hist_min"));
		assertEquals("6", store.getJobManagerMetricStore().getMetric("abc.hist_max"));
		assertEquals("4.0", store.getJobManagerMetricStore().getMetric("abc.hist_mean"));
		assertEquals("0.5", store.getJobManagerMetricStore().getMetric("abc.hist_median"));
		assertEquals("5.0", store.getJobManagerMetricStore().getMetric("abc.hist_stddev"));
		assertEquals("0.75", store.getJobManagerMetricStore().getMetric("abc.hist_p75"));
		assertEquals("0.9", store.getJobManagerMetricStore().getMetric("abc.hist_p90"));
		assertEquals("0.95", store.getJobManagerMetricStore().getMetric("abc.hist_p95"));
		assertEquals("0.98", store.getJobManagerMetricStore().getMetric("abc.hist_p98"));
		assertEquals("0.99", store.getJobManagerMetricStore().getMetric("abc.hist_p99"));
		assertEquals("0.999", store.getJobManagerMetricStore().getMetric("abc.hist_p999"));

		assertEquals("x", store.getTaskManagerMetricStore(tmRID.toString()).metrics.get("abc.gauge"));
		assertEquals("5.0", store.getJobMetricStore(jobID.toString()).metrics.get("abc.jc"));
		assertEquals("2", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.abc.tc"));
		assertEquals("1", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.opname.abc.oc"));
	}
}
Example 8
Source File: MetricFetcherTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
private static MetricDumpSerialization.MetricSerializationResult createRequestDumpAnswer(ResourceID tmRID, JobID jobID) {
	Map<Counter, Tuple2<QueryScopeInfo, String>> counters = new HashMap<>();
	Map<Gauge<?>, Tuple2<QueryScopeInfo, String>> gauges = new HashMap<>();
	Map<Histogram, Tuple2<QueryScopeInfo, String>> histograms = new HashMap<>();
	Map<Meter, Tuple2<QueryScopeInfo, String>> meters = new HashMap<>();

	SimpleCounter c1 = new SimpleCounter();
	SimpleCounter c2 = new SimpleCounter();

	c1.inc(1);
	c2.inc(2);

	counters.put(c1, new Tuple2<>(new QueryScopeInfo.OperatorQueryScopeInfo(jobID.toString(), "taskid", 2, "opname", "abc"), "oc"));
	counters.put(c2, new Tuple2<>(new QueryScopeInfo.TaskQueryScopeInfo(jobID.toString(), "taskid", 2, "abc"), "tc"));
	meters.put(new Meter() {
		@Override
		public void markEvent() {
		}

		@Override
		public void markEvent(long n) {
		}

		@Override
		public double getRate() {
			return 5;
		}

		@Override
		public long getCount() {
			return 10;
		}
	}, new Tuple2<>(new QueryScopeInfo.JobQueryScopeInfo(jobID.toString(), "abc"), "jc"));
	gauges.put(new Gauge<String>() {
		@Override
		public String getValue() {
			return "x";
		}
	}, new Tuple2<>(new QueryScopeInfo.TaskManagerQueryScopeInfo(tmRID.toString(), "abc"), "gauge"));
	histograms.put(new TestHistogram(), new Tuple2<>(new QueryScopeInfo.JobManagerQueryScopeInfo("abc"), "hist"));

	MetricDumpSerialization.MetricDumpSerializer serializer = new MetricDumpSerialization.MetricDumpSerializer();
	MetricDumpSerialization.MetricSerializationResult dump = serializer.serialize(counters, gauges, histograms, meters);
	serializer.close();

	return dump;
}
Example 9
Source File: MetricFetcherTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testUpdate() {
	final Time timeout = Time.seconds(10L);

	// ========= setup TaskManager =================================================================================

	JobID jobID = new JobID();
	ResourceID tmRID = ResourceID.generate();

	// ========= setup QueryServices ================================================================================

	final MetricQueryServiceGateway jmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], new byte[0], new byte[0], new byte[0], 0, 0, 0, 0)))
		.build();

	MetricDumpSerialization.MetricSerializationResult requestMetricsAnswer = createRequestDumpAnswer(tmRID, jobID);
	final MetricQueryServiceGateway tmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(requestMetricsAnswer))
		.build();

	// ========= setup JobManager ==================================================================================

	final TestingRestfulGateway restfulGateway = new TestingRestfulGateway.Builder()
		.setRequestMultipleJobDetailsSupplier(() -> CompletableFuture.completedFuture(new MultipleJobsDetails(Collections.emptyList())))
		.setRequestMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(jmQueryService.getAddress())))
		.setRequestTaskManagerMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(Tuple2.of(tmRID, tmQueryService.getAddress()))))
		.build();

	final GatewayRetriever<RestfulGateway> retriever = () -> CompletableFuture.completedFuture(restfulGateway);

	// ========= start MetricFetcher testing =======================================================================
	MetricFetcher fetcher = new MetricFetcherImpl<>(
		retriever,
		address -> CompletableFuture.completedFuture(tmQueryService),
		Executors.directExecutor(),
		timeout,
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());

	// verify that update fetches metrics and updates the store
	fetcher.update();
	MetricStore store = fetcher.getMetricStore();
	synchronized (store) {
		assertEquals("7", store.getJobManagerMetricStore().getMetric("abc.hist_min"));
		assertEquals("6", store.getJobManagerMetricStore().getMetric("abc.hist_max"));
		assertEquals("4.0", store.getJobManagerMetricStore().getMetric("abc.hist_mean"));
		assertEquals("0.5", store.getJobManagerMetricStore().getMetric("abc.hist_median"));
		assertEquals("5.0", store.getJobManagerMetricStore().getMetric("abc.hist_stddev"));
		assertEquals("0.75", store.getJobManagerMetricStore().getMetric("abc.hist_p75"));
		assertEquals("0.9", store.getJobManagerMetricStore().getMetric("abc.hist_p90"));
		assertEquals("0.95", store.getJobManagerMetricStore().getMetric("abc.hist_p95"));
		assertEquals("0.98", store.getJobManagerMetricStore().getMetric("abc.hist_p98"));
		assertEquals("0.99", store.getJobManagerMetricStore().getMetric("abc.hist_p99"));
		assertEquals("0.999", store.getJobManagerMetricStore().getMetric("abc.hist_p999"));

		assertEquals("x", store.getTaskManagerMetricStore(tmRID.toString()).metrics.get("abc.gauge"));
		assertEquals("5.0", store.getJobMetricStore(jobID.toString()).metrics.get("abc.jc"));
		assertEquals("2", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.abc.tc"));
		assertEquals("1", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.opname.abc.oc"));
	}
}
Example 10
Source File: MetricFetcherTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private static MetricDumpSerialization.MetricSerializationResult createRequestDumpAnswer(ResourceID tmRID, JobID jobID) {
	Map<Counter, Tuple2<QueryScopeInfo, String>> counters = new HashMap<>();
	Map<Gauge<?>, Tuple2<QueryScopeInfo, String>> gauges = new HashMap<>();
	Map<Histogram, Tuple2<QueryScopeInfo, String>> histograms = new HashMap<>();
	Map<Meter, Tuple2<QueryScopeInfo, String>> meters = new HashMap<>();

	SimpleCounter c1 = new SimpleCounter();
	SimpleCounter c2 = new SimpleCounter();

	c1.inc(1);
	c2.inc(2);

	counters.put(c1, new Tuple2<>(new QueryScopeInfo.OperatorQueryScopeInfo(jobID.toString(), "taskid", 2, "opname", "abc"), "oc"));
	counters.put(c2, new Tuple2<>(new QueryScopeInfo.TaskQueryScopeInfo(jobID.toString(), "taskid", 2, "abc"), "tc"));
	meters.put(new Meter() {
		@Override
		public void markEvent() {
		}

		@Override
		public void markEvent(long n) {
		}

		@Override
		public double getRate() {
			return 5;
		}

		@Override
		public long getCount() {
			return 10;
		}
	}, new Tuple2<>(new QueryScopeInfo.JobQueryScopeInfo(jobID.toString(), "abc"), "jc"));
	gauges.put(new Gauge<String>() {
		@Override
		public String getValue() {
			return "x";
		}
	}, new Tuple2<>(new QueryScopeInfo.TaskManagerQueryScopeInfo(tmRID.toString(), "abc"), "gauge"));
	histograms.put(new TestHistogram(), new Tuple2<>(new QueryScopeInfo.JobManagerQueryScopeInfo("abc"), "hist"));

	MetricDumpSerialization.MetricDumpSerializer serializer = new MetricDumpSerialization.MetricDumpSerializer();
	MetricDumpSerialization.MetricSerializationResult dump = serializer.serialize(counters, gauges, histograms, meters);
	serializer.close();

	return dump;
}
Example 11
Source File: MetricFetcherTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testUpdate() {
	final Time timeout = Time.seconds(10L);

	// ========= setup TaskManager =================================================================================

	JobID jobID = new JobID();
	ResourceID tmRID = ResourceID.generate();

	// ========= setup QueryServices ================================================================================

	final MetricQueryServiceGateway jmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], new byte[0], new byte[0], new byte[0], 0, 0, 0, 0)))
		.build();

	MetricDumpSerialization.MetricSerializationResult requestMetricsAnswer = createRequestDumpAnswer(tmRID, jobID);
	final MetricQueryServiceGateway tmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(requestMetricsAnswer))
		.build();

	// ========= setup JobManager ==================================================================================

	final TestingRestfulGateway restfulGateway = new TestingRestfulGateway.Builder()
		.setRequestMultipleJobDetailsSupplier(() -> CompletableFuture.completedFuture(new MultipleJobsDetails(Collections.emptyList())))
		.setRequestMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(jmQueryService.getAddress())))
		.setRequestTaskManagerMetricQueryServiceGatewaysSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(Tuple2.of(tmRID, tmQueryService.getAddress()))))
		.build();

	final GatewayRetriever<RestfulGateway> retriever = () -> CompletableFuture.completedFuture(restfulGateway);

	// ========= start MetricFetcher testing =======================================================================
	MetricFetcher fetcher = new MetricFetcherImpl<>(
		retriever,
		address -> CompletableFuture.completedFuture(tmQueryService),
		Executors.directExecutor(),
		timeout,
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());

	// verify that update fetches metrics and updates the store
	fetcher.update();
	MetricStore store = fetcher.getMetricStore();
	synchronized (store) {
		assertEquals("7", store.getJobManagerMetricStore().getMetric("abc.hist_min"));
		assertEquals("6", store.getJobManagerMetricStore().getMetric("abc.hist_max"));
		assertEquals("4.0", store.getJobManagerMetricStore().getMetric("abc.hist_mean"));
		assertEquals("0.5", store.getJobManagerMetricStore().getMetric("abc.hist_median"));
		assertEquals("5.0", store.getJobManagerMetricStore().getMetric("abc.hist_stddev"));
		assertEquals("0.75", store.getJobManagerMetricStore().getMetric("abc.hist_p75"));
		assertEquals("0.9", store.getJobManagerMetricStore().getMetric("abc.hist_p90"));
		assertEquals("0.95", store.getJobManagerMetricStore().getMetric("abc.hist_p95"));
		assertEquals("0.98", store.getJobManagerMetricStore().getMetric("abc.hist_p98"));
		assertEquals("0.99", store.getJobManagerMetricStore().getMetric("abc.hist_p99"));
		assertEquals("0.999", store.getJobManagerMetricStore().getMetric("abc.hist_p999"));

		assertEquals("x", store.getTaskManagerMetricStore(tmRID.toString()).metrics.get("abc.gauge"));
		assertEquals("5.0", store.getJobMetricStore(jobID.toString()).metrics.get("abc.jc"));
		assertEquals("2", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.abc.tc"));
		assertEquals("1", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.opname.abc.oc"));
	}
}
Example 12
Source File: MetricFetcherTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
private static MetricDumpSerialization.MetricSerializationResult createRequestDumpAnswer(ResourceID tmRID, JobID jobID) {
	Map<Counter, Tuple2<QueryScopeInfo, String>> counters = new HashMap<>();
	Map<Gauge<?>, Tuple2<QueryScopeInfo, String>> gauges = new HashMap<>();
	Map<Histogram, Tuple2<QueryScopeInfo, String>> histograms = new HashMap<>();
	Map<Meter, Tuple2<QueryScopeInfo, String>> meters = new HashMap<>();

	SimpleCounter c1 = new SimpleCounter();
	SimpleCounter c2 = new SimpleCounter();

	c1.inc(1);
	c2.inc(2);

	counters.put(c1, new Tuple2<>(new QueryScopeInfo.OperatorQueryScopeInfo(jobID.toString(), "taskid", 2, "opname", "abc"), "oc"));
	counters.put(c2, new Tuple2<>(new QueryScopeInfo.TaskQueryScopeInfo(jobID.toString(), "taskid", 2, "abc"), "tc"));
	meters.put(new Meter() {
		@Override
		public void markEvent() {
		}

		@Override
		public void markEvent(long n) {
		}

		@Override
		public double getRate() {
			return 5;
		}

		@Override
		public long getCount() {
			return 10;
		}
	}, new Tuple2<>(new QueryScopeInfo.JobQueryScopeInfo(jobID.toString(), "abc"), "jc"));
	gauges.put(new Gauge<String>() {
		@Override
		public String getValue() {
			return "x";
		}
	}, new Tuple2<>(new QueryScopeInfo.TaskManagerQueryScopeInfo(tmRID.toString(), "abc"), "gauge"));
	histograms.put(new TestHistogram(), new Tuple2<>(new QueryScopeInfo.JobManagerQueryScopeInfo("abc"), "hist"));

	MetricDumpSerialization.MetricDumpSerializer serializer = new MetricDumpSerialization.MetricDumpSerializer();
	MetricDumpSerialization.MetricSerializationResult dump = serializer.serialize(counters, gauges, histograms, meters);
	serializer.close();

	return dump;
}
Example 13
Source File: MetricQueryServiceGateway.java    From Flink-CEPplus with Apache License 2.0 votes vote down vote up 
CompletableFuture<MetricDumpSerialization.MetricSerializationResult> queryMetrics(Time timeout);
Example 14
Source File: MetricQueryServiceGateway.java    From flink with Apache License 2.0 votes vote down vote up 
CompletableFuture<MetricDumpSerialization.MetricSerializationResult> queryMetrics(Time timeout);
Example 15
Source File: MetricQueryServiceGateway.java    From flink with Apache License 2.0 votes vote down vote up 
CompletableFuture<MetricDumpSerialization.MetricSerializationResult> queryMetrics(Time timeout);