org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceRetriever Java Examples

The following examples show how to use org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceRetriever. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AggregatingMetricsHandlerTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() throws Exception {
	MetricFetcher fetcher = new MetricFetcherImpl<RestfulGateway>(
		mock(GatewayRetriever.class),
		mock(MetricQueryServiceRetriever.class),
		Executors.directExecutor(),
		TestingUtils.TIMEOUT(),
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());
	store = fetcher.getMetricStore();

	Collection<MetricDump> metricDumps = getMetricDumps();
	for (MetricDump dump : metricDumps) {
		store.add(dump);
	}

	handler = getHandler(
		LEADER_RETRIEVER,
		TIMEOUT,
		TEST_HEADERS,
		EXECUTOR,
		fetcher);
	pathParameters = getPathParameters();
}
Example #2
Source File: MetricFetcherImpl.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Nonnull
public static <T extends RestfulGateway> MetricFetcherImpl<T> fromConfiguration(
		final Configuration configuration,
		final MetricQueryServiceRetriever metricQueryServiceRetriever,
		final GatewayRetriever<T> dispatcherGatewayRetriever,
		final ExecutorService executor) {
	final Time timeout = Time.milliseconds(configuration.getLong(WebOptions.TIMEOUT));
	final long updateInterval = configuration.getLong(MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL);

	return new MetricFetcherImpl<>(
		dispatcherGatewayRetriever,
		metricQueryServiceRetriever,
		executor,
		timeout,
		updateInterval);
}
Example #3
Source File: AggregatingMetricsHandlerTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() throws Exception {
	MetricFetcher fetcher = new MetricFetcherImpl<RestfulGateway>(
		mock(GatewayRetriever.class),
		mock(MetricQueryServiceRetriever.class),
		Executors.directExecutor(),
		TestingUtils.TIMEOUT(),
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());
	store = fetcher.getMetricStore();

	Collection<MetricDump> metricDumps = getMetricDumps();
	for (MetricDump dump : metricDumps) {
		store.add(dump);
	}

	handler = getHandler(
		LEADER_RETRIEVER,
		TIMEOUT,
		TEST_HEADERS,
		EXECUTOR,
		fetcher);
	pathParameters = getPathParameters();
}
Example #4
Source File: MiniCluster.java    From flink with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
protected Collection<? extends DispatcherResourceManagerComponent<?>> createDispatcherResourceManagerComponents(
		Configuration configuration,
		RpcServiceFactory rpcServiceFactory,
		HighAvailabilityServices haServices,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		MetricQueryServiceRetriever metricQueryServiceRetriever,
		FatalErrorHandler fatalErrorHandler) throws Exception {
	SessionDispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createDispatcherResourceManagerComponentFactory();
	return Collections.singleton(
		dispatcherResourceManagerComponentFactory.create(
			configuration,
			rpcServiceFactory.createRpcService(),
			haServices,
			blobServer,
			heartbeatServices,
			metricRegistry,
			new MemoryArchivedExecutionGraphStore(),
			metricQueryServiceRetriever,
			fatalErrorHandler));
}
Example #5
Source File: MiniCluster.java    From flink with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
protected Collection<? extends DispatcherResourceManagerComponent> createDispatcherResourceManagerComponents(
		Configuration configuration,
		RpcServiceFactory rpcServiceFactory,
		HighAvailabilityServices haServices,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		MetricQueryServiceRetriever metricQueryServiceRetriever,
		FatalErrorHandler fatalErrorHandler) throws Exception {
	DispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createDispatcherResourceManagerComponentFactory();
	return Collections.singleton(
		dispatcherResourceManagerComponentFactory.create(
			configuration,
			ioExecutor,
			rpcServiceFactory.createRpcService(),
			haServices,
			blobServer,
			heartbeatServices,
			metricRegistry,
			new MemoryArchivedExecutionGraphStore(),
			metricQueryServiceRetriever,
			fatalErrorHandler));
}
Example #6
Source File: MiniCluster.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
protected Collection<? extends DispatcherResourceManagerComponent<?>> createDispatcherResourceManagerComponents(
		Configuration configuration,
		RpcServiceFactory rpcServiceFactory,
		HighAvailabilityServices haServices,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		MetricQueryServiceRetriever metricQueryServiceRetriever,
		FatalErrorHandler fatalErrorHandler) throws Exception {
	SessionDispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createDispatcherResourceManagerComponentFactory();
	return Collections.singleton(
		dispatcherResourceManagerComponentFactory.create(
			configuration,
			rpcServiceFactory.createRpcService(),
			haServices,
			blobServer,
			heartbeatServices,
			metricRegistry,
			new MemoryArchivedExecutionGraphStore(),
			metricQueryServiceRetriever,
			fatalErrorHandler));
}
Example #7
Source File: MiniCluster.java    From flink with Apache License 2.0 6 votes vote down vote up 
@GuardedBy("lock")
private void setupDispatcherResourceManagerComponents(Configuration configuration, RpcServiceFactory dispatcherResourceManagreComponentRpcServiceFactory, MetricQueryServiceRetriever metricQueryServiceRetriever) throws Exception {
	dispatcherResourceManagerComponents.addAll(createDispatcherResourceManagerComponents(
		configuration,
		dispatcherResourceManagreComponentRpcServiceFactory,
		haServices,
		blobServer,
		heartbeatServices,
		metricRegistry,
		metricQueryServiceRetriever,
		new ShutDownFatalErrorHandler()
	));

	final Collection<CompletableFuture<ApplicationStatus>> shutDownFutures = new ArrayList<>(dispatcherResourceManagerComponents.size());

	for (DispatcherResourceManagerComponent dispatcherResourceManagerComponent : dispatcherResourceManagerComponents) {
		final CompletableFuture<ApplicationStatus> shutDownFuture = dispatcherResourceManagerComponent.getShutDownFuture();
		FutureUtils.assertNoException(shutDownFuture.thenRun(dispatcherResourceManagerComponent::closeAsync));
		shutDownFutures.add(shutDownFuture);
	}

	FutureUtils.assertNoException(FutureUtils.completeAll(shutDownFutures).thenRun(this::closeAsync));
}
Example #8
Source File: AggregatingMetricsHandlerTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() throws Exception {
	MetricFetcher fetcher = new MetricFetcherImpl<RestfulGateway>(
		mock(GatewayRetriever.class),
		mock(MetricQueryServiceRetriever.class),
		Executors.directExecutor(),
		TestingUtils.TIMEOUT(),
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());
	store = fetcher.getMetricStore();

	Collection<MetricDump> metricDumps = getMetricDumps();
	for (MetricDump dump : metricDumps) {
		store.add(dump);
	}

	handler = getHandler(
		LEADER_RETRIEVER,
		TIMEOUT,
		TEST_HEADERS,
		EXECUTOR,
		fetcher);
	pathParameters = getPathParameters();
}
Example #9
Source File: DispatcherResourceManagerComponentFactory.java    From flink with Apache License 2.0 5 votes vote down vote up 
DispatcherResourceManagerComponent<T> create(
Configuration configuration,
RpcService rpcService,
HighAvailabilityServices highAvailabilityServices,
BlobServer blobServer,
HeartbeatServices heartbeatServices,
MetricRegistry metricRegistry,
ArchivedExecutionGraphStore archivedExecutionGraphStore,
MetricQueryServiceRetriever metricQueryServiceRetriever,
FatalErrorHandler fatalErrorHandler) throws Exception;
Example #10
Source File: TestingMiniCluster.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected Collection<? extends DispatcherResourceManagerComponent> createDispatcherResourceManagerComponents(
		Configuration configuration,
		RpcServiceFactory rpcServiceFactory,
		HighAvailabilityServices haServices,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		MetricQueryServiceRetriever metricQueryServiceRetriever,
		FatalErrorHandler fatalErrorHandler) throws Exception {
	DispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createDispatcherResourceManagerComponentFactory();

	final List<DispatcherResourceManagerComponent> result = new ArrayList<>(numberDispatcherResourceManagerComponents);

	for (int i = 0; i < numberDispatcherResourceManagerComponents; i++) {
		result.add(
			dispatcherResourceManagerComponentFactory.create(
				configuration,
				getIOExecutor(),
				rpcServiceFactory.createRpcService(),
				haServices,
				blobServer,
				heartbeatServices,
				metricRegistry,
				new MemoryArchivedExecutionGraphStore(),
				metricQueryServiceRetriever,
				fatalErrorHandler));
	}

	return result;
}
Example #11
Source File: DispatcherResourceManagerComponentFactory.java    From flink with Apache License 2.0 5 votes vote down vote up 
DispatcherResourceManagerComponent create(
Configuration configuration,
Executor ioExecutor,
RpcService rpcService,
HighAvailabilityServices highAvailabilityServices,
BlobServer blobServer,
HeartbeatServices heartbeatServices,
MetricRegistry metricRegistry,
ArchivedExecutionGraphStore archivedExecutionGraphStore,
MetricQueryServiceRetriever metricQueryServiceRetriever,
FatalErrorHandler fatalErrorHandler) throws Exception;
Example #12
Source File: MetricFetcherImpl.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Nonnull
public static <T extends RestfulGateway> MetricFetcherImpl<T> fromConfiguration(
		final Configuration configuration,
		final MetricQueryServiceRetriever metricQueryServiceGatewayRetriever, final GatewayRetriever<T> dispatcherGatewayRetriever,
		final ExecutorService executor) {
	final Time timeout = Time.milliseconds(configuration.getLong(WebOptions.TIMEOUT));
	final long updateInterval = configuration.getLong(MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL);

	return new MetricFetcherImpl<>(
		dispatcherGatewayRetriever,
		metricQueryServiceGatewayRetriever,
		executor,
		timeout,
		updateInterval);
}
Example #13
Source File: MetricFetcherImpl.java    From flink with Apache License 2.0 5 votes vote down vote up 
public MetricFetcherImpl(
		GatewayRetriever<T> retriever,
		MetricQueryServiceRetriever queryServiceRetriever,
		Executor executor,
		Time timeout,
		long updateInterval) {
	this.retriever = Preconditions.checkNotNull(retriever);
	this.queryServiceRetriever = Preconditions.checkNotNull(queryServiceRetriever);
	this.executor = Preconditions.checkNotNull(executor);
	this.timeout = Preconditions.checkNotNull(timeout);

	Preconditions.checkArgument(updateInterval > 0, "The update interval must be larger than 0.");
	this.updateInterval = updateInterval;
}
Example #14
Source File: TestingMiniCluster.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
protected Collection<? extends DispatcherResourceManagerComponent<?>> createDispatcherResourceManagerComponents(
		Configuration configuration,
		RpcServiceFactory rpcServiceFactory,
		HighAvailabilityServices haServices,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		MetricQueryServiceRetriever metricQueryServiceRetriever,
		FatalErrorHandler fatalErrorHandler) throws Exception {
	SessionDispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createTestingDispatcherResourceManagerComponentFactory();

	final List<DispatcherResourceManagerComponent<?>> result = new ArrayList<>(numberDispatcherResourceManagerComponents);

	for (int i = 0; i < numberDispatcherResourceManagerComponents; i++) {
		result.add(
			dispatcherResourceManagerComponentFactory.create(
				configuration,
				rpcServiceFactory.createRpcService(),
				haServices,
				blobServer,
				heartbeatServices,
				metricRegistry,
				new MemoryArchivedExecutionGraphStore(),
				metricQueryServiceRetriever,
				fatalErrorHandler));
	}

	return result;
}
Example #15
Source File: MetricFetcherImpl.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Nonnull
public static <T extends RestfulGateway> MetricFetcherImpl<T> fromConfiguration(
		final Configuration configuration,
		final MetricQueryServiceRetriever metricQueryServiceGatewayRetriever, final GatewayRetriever<T> dispatcherGatewayRetriever,
		final ExecutorService executor) {
	final Time timeout = Time.milliseconds(configuration.getLong(WebOptions.TIMEOUT));
	final long updateInterval = configuration.getLong(MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL);

	return new MetricFetcherImpl<>(
		dispatcherGatewayRetriever,
		metricQueryServiceGatewayRetriever,
		executor,
		timeout,
		updateInterval);
}
Example #16
Source File: MetricFetcherImpl.java    From flink with Apache License 2.0 5 votes vote down vote up 
public MetricFetcherImpl(
		GatewayRetriever<T> retriever,
		MetricQueryServiceRetriever queryServiceRetriever,
		Executor executor,
		Time timeout,
		long updateInterval) {
	this.retriever = Preconditions.checkNotNull(retriever);
	this.queryServiceRetriever = Preconditions.checkNotNull(queryServiceRetriever);
	this.executor = Preconditions.checkNotNull(executor);
	this.timeout = Preconditions.checkNotNull(timeout);

	Preconditions.checkArgument(updateInterval > 0, "The update interval must be larger than 0.");
	this.updateInterval = updateInterval;
}
Example #17
Source File: TestingMiniCluster.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
protected Collection<? extends DispatcherResourceManagerComponent<?>> createDispatcherResourceManagerComponents(
		Configuration configuration,
		RpcServiceFactory rpcServiceFactory,
		HighAvailabilityServices haServices,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		MetricQueryServiceRetriever metricQueryServiceRetriever,
		FatalErrorHandler fatalErrorHandler) throws Exception {
	SessionDispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createTestingDispatcherResourceManagerComponentFactory();

	final List<DispatcherResourceManagerComponent<?>> result = new ArrayList<>(numberDispatcherResourceManagerComponents);

	for (int i = 0; i < numberDispatcherResourceManagerComponents; i++) {
		result.add(
			dispatcherResourceManagerComponentFactory.create(
				configuration,
				rpcServiceFactory.createRpcService(),
				haServices,
				blobServer,
				heartbeatServices,
				metricRegistry,
				new MemoryArchivedExecutionGraphStore(),
				metricQueryServiceRetriever,
				fatalErrorHandler));
	}

	return result;
}
Example #18
Source File: DispatcherResourceManagerComponentFactory.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
DispatcherResourceManagerComponent<T> create(
Configuration configuration,
RpcService rpcService,
HighAvailabilityServices highAvailabilityServices,
BlobServer blobServer,
HeartbeatServices heartbeatServices,
MetricRegistry metricRegistry,
ArchivedExecutionGraphStore archivedExecutionGraphStore,
MetricQueryServiceRetriever metricQueryServiceRetriever,
FatalErrorHandler fatalErrorHandler) throws Exception;
Example #19
Source File: MetricFetcherImpl.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public MetricFetcherImpl(
		GatewayRetriever<T> retriever,
		MetricQueryServiceRetriever queryServiceRetriever,
		Executor executor,
		Time timeout,
		long updateInterval) {
	this.retriever = Preconditions.checkNotNull(retriever);
	this.queryServiceRetriever = Preconditions.checkNotNull(queryServiceRetriever);
	this.executor = Preconditions.checkNotNull(executor);
	this.timeout = Preconditions.checkNotNull(timeout);

	Preconditions.checkArgument(updateInterval > 0, "The update interval must be larger than 0.");
	this.updateInterval = updateInterval;
}
Example #20
Source File: MetricFetcherTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test
public void testUpdate() {
	final Time timeout = Time.seconds(10L);

	// ========= setup TaskManager =================================================================================
	JobID jobID = new JobID();
	ResourceID tmRID = ResourceID.generate();

	// ========= setup JobManager ==================================================================================

	final String jmMetricQueryServicePath = "/jm/" + MetricQueryService.METRIC_QUERY_SERVICE_NAME;
	final String tmMetricQueryServicePath = "/tm/" + MetricQueryService.METRIC_QUERY_SERVICE_NAME + "_" + tmRID.getResourceIdString();

	final TestingRestfulGateway restfulGateway = new TestingRestfulGateway.Builder()
		.setRequestMultipleJobDetailsSupplier(() -> CompletableFuture.completedFuture(new MultipleJobsDetails(Collections.emptyList())))
		.setRequestMetricQueryServicePathsSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(jmMetricQueryServicePath)))
		.setRequestTaskManagerMetricQueryServicePathsSupplier(() -> CompletableFuture.completedFuture(Collections.singleton(Tuple2.of(tmRID, tmMetricQueryServicePath))))
		.build();

	final GatewayRetriever<RestfulGateway> retriever = () -> CompletableFuture.completedFuture(restfulGateway);

	// ========= setup QueryServices ================================================================================
	final MetricQueryServiceGateway jmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(new MetricDumpSerialization.MetricSerializationResult(new byte[0], new byte[0], new byte[0], new byte[0], 0, 0, 0, 0)))
		.build();

	MetricDumpSerialization.MetricSerializationResult requestMetricsAnswer = createRequestDumpAnswer(tmRID, jobID);
	final MetricQueryServiceGateway tmQueryService = new TestingMetricQueryServiceGateway.Builder()
		.setQueryMetricsSupplier(() -> CompletableFuture.completedFuture(requestMetricsAnswer))
		.build();

	final MetricQueryServiceRetriever queryServiceRetriever = (path) -> {
		if (path.equals(jmMetricQueryServicePath))  {
			return CompletableFuture.completedFuture(jmQueryService);
		} else if (path.equals(tmMetricQueryServicePath)) {
			return CompletableFuture.completedFuture(tmQueryService);
		} else {
			throw new IllegalArgumentException("Unexpected argument.");
		}
	};

	// ========= start MetricFetcher testing =======================================================================
	MetricFetcher fetcher = new MetricFetcherImpl<>(
		retriever,
		queryServiceRetriever,
		Executors.directExecutor(),
		timeout,
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());

	// verify that update fetches metrics and updates the store
	fetcher.update();
	MetricStore store = fetcher.getMetricStore();
	synchronized (store) {
		assertEquals("7", store.getJobManagerMetricStore().getMetric("abc.hist_min"));
		assertEquals("6", store.getJobManagerMetricStore().getMetric("abc.hist_max"));
		assertEquals("4.0", store.getJobManagerMetricStore().getMetric("abc.hist_mean"));
		assertEquals("0.5", store.getJobManagerMetricStore().getMetric("abc.hist_median"));
		assertEquals("5.0", store.getJobManagerMetricStore().getMetric("abc.hist_stddev"));
		assertEquals("0.75", store.getJobManagerMetricStore().getMetric("abc.hist_p75"));
		assertEquals("0.9", store.getJobManagerMetricStore().getMetric("abc.hist_p90"));
		assertEquals("0.95", store.getJobManagerMetricStore().getMetric("abc.hist_p95"));
		assertEquals("0.98", store.getJobManagerMetricStore().getMetric("abc.hist_p98"));
		assertEquals("0.99", store.getJobManagerMetricStore().getMetric("abc.hist_p99"));
		assertEquals("0.999", store.getJobManagerMetricStore().getMetric("abc.hist_p999"));

		assertEquals("x", store.getTaskManagerMetricStore(tmRID.toString()).metrics.get("abc.gauge"));
		assertEquals("5.0", store.getJobMetricStore(jobID.toString()).metrics.get("abc.jc"));
		assertEquals("2", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.abc.tc"));
		assertEquals("1", store.getTaskMetricStore(jobID.toString(), "taskid").metrics.get("2.opname.abc.oc"));
	}
}