org.apache.flink.runtime.rest.RestClient Java Examples
The following examples show how to use
org.apache.flink.runtime.rest.RestClient.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RestClusterClientTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Nonnull private RestClient createRestClient() throws ConfigurationException { return new RestClient(RestClientConfiguration.fromConfiguration(restConfig), executor) { @Override public <M extends MessageHeaders<R, P, U>, U extends MessageParameters, R extends RequestBody, P extends ResponseBody> CompletableFuture<P> sendRequest( final String targetAddress, final int targetPort, final M messageHeaders, final U messageParameters, final R request, final Collection<FileUpload> files) throws IOException { if (failHttpRequest.test(messageHeaders, messageParameters, request)) { return FutureUtils.completedExceptionally(new IOException("expected")); } else { return super.sendRequest(targetAddress, targetPort, messageHeaders, messageParameters, request, files); } } }; }
Example #2
Source File: MetricsAvailabilityITCase.java From flink with Apache License 2.0 | 6 votes |
private static Collection<ResourceID> getTaskManagerIds(final RestClient restClient) throws Exception { final TaskManagersHeaders headers = TaskManagersHeaders.getInstance(); final TaskManagersInfo response = fetchMetric(() -> restClient.sendRequest( HOST, PORT, headers, EmptyMessageParameters.getInstance(), EmptyRequestBody.getInstance()), taskManagersInfo -> !taskManagersInfo.getTaskManagerInfos().isEmpty()); return response.getTaskManagerInfos().stream() .map(TaskManagerInfo::getResourceId) .collect(Collectors.toList()); }
Example #3
Source File: MetricsAvailabilityITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private static Collection<ResourceID> getTaskManagerIds(final RestClient restClient) throws Exception { final TaskManagersHeaders headers = TaskManagersHeaders.getInstance(); final TaskManagersInfo response = fetchMetric(() -> restClient.sendRequest( HOST, PORT, headers, EmptyMessageParameters.getInstance(), EmptyRequestBody.getInstance()), taskManagersInfo -> !taskManagersInfo.getTaskManagerInfos().isEmpty()); return response.getTaskManagerInfos().stream() .map(TaskManagerInfo::getResourceId) .collect(Collectors.toList()); }
Example #4
Source File: RestClusterClientTest.java From flink with Apache License 2.0 | 6 votes |
@Nonnull private RestClient createRestClient() throws ConfigurationException { return new RestClient(RestClientConfiguration.fromConfiguration(restConfig), executor) { @Override public <M extends MessageHeaders<R, P, U>, U extends MessageParameters, R extends RequestBody, P extends ResponseBody> CompletableFuture<P> sendRequest( final String targetAddress, final int targetPort, final M messageHeaders, final U messageParameters, final R request, final Collection<FileUpload> files) throws IOException { if (failHttpRequest.test(messageHeaders, messageParameters, request)) { return FutureUtils.completedExceptionally(new IOException("expected")); } else { return super.sendRequest(targetAddress, targetPort, messageHeaders, messageParameters, request, files); } } }; }
Example #5
Source File: RestClusterClient.java From flink with Apache License 2.0 | 6 votes |
private RestClusterClient( Configuration configuration, @Nullable RestClient restClient, T clusterId, WaitStrategy waitStrategy, ClientHighAvailabilityServices clientHAServices) throws Exception { this.configuration = checkNotNull(configuration); this.restClusterClientConfiguration = RestClusterClientConfiguration.fromConfiguration(configuration); if (restClient != null) { this.restClient = restClient; } else { this.restClient = new RestClient(restClusterClientConfiguration.getRestClientConfiguration(), executorService); } this.waitStrategy = checkNotNull(waitStrategy); this.clusterId = checkNotNull(clusterId); this.clientHAServices = checkNotNull(clientHAServices); this.webMonitorRetrievalService = clientHAServices.getClusterRestEndpointLeaderRetriever(); this.retryExecutorService = Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-RestClusterClient-Retry")); startLeaderRetrievers(); }
Example #6
Source File: RestClusterClientTest.java From flink with Apache License 2.0 | 6 votes |
@Nonnull private RestClient createRestClient() throws ConfigurationException { return new RestClient(RestClientConfiguration.fromConfiguration(restConfig), executor) { @Override public <M extends MessageHeaders<R, P, U>, U extends MessageParameters, R extends RequestBody, P extends ResponseBody> CompletableFuture<P> sendRequest( final String targetAddress, final int targetPort, final M messageHeaders, final U messageParameters, final R request, final Collection<FileUpload> files) throws IOException { if (failHttpRequest.test(messageHeaders, messageParameters, request)) { return FutureUtils.completedExceptionally(new IOException("expected")); } else { return super.sendRequest(targetAddress, targetPort, messageHeaders, messageParameters, request, files); } } }; }
Example #7
Source File: MetricsAvailabilityITCase.java From flink with Apache License 2.0 | 6 votes |
private static Collection<ResourceID> getTaskManagerIds(final RestClient restClient) throws Exception { final TaskManagersHeaders headers = TaskManagersHeaders.getInstance(); final TaskManagersInfo response = fetchMetric(() -> restClient.sendRequest( HOST, PORT, headers, EmptyMessageParameters.getInstance(), EmptyRequestBody.getInstance()), taskManagersInfo -> !taskManagersInfo.getTaskManagerInfos().isEmpty()); return response.getTaskManagerInfos().stream() .map(TaskManagerInfo::getResourceId) .collect(Collectors.toList()); }
Example #8
Source File: MetricsAvailabilityITCase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testReporter() throws Exception { dist.startFlinkCluster(); final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(new Configuration()), scheduledExecutorService); checkJobManagerMetricAvailability(restClient); final Collection<ResourceID> taskManagerIds = getTaskManagerIds(restClient); for (final ResourceID taskManagerId : taskManagerIds) { checkTaskManagerMetricAvailability(restClient, taskManagerId); } }
Example #9
Source File: YARNSessionCapacitySchedulerITCase.java From flink with Apache License 2.0 | 5 votes |
@BeforeClass public static void setup() throws Exception { YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.set("yarn.scheduler.capacity.root.queues", "default,qa-team"); YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.default.capacity", 40); YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.qa-team.capacity", 60); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-capacityscheduler"); startYARNWithConfig(YARN_CONFIGURATION); restClientExecutor = Executors.newSingleThreadExecutor(); restClient = new RestClient(RestClientConfiguration.fromConfiguration(new Configuration()), restClientExecutor); }
Example #10
Source File: MetricsAvailabilityITCase.java From flink with Apache License 2.0 | 5 votes |
private static void checkTaskManagerMetricAvailability(final RestClient restClient, final ResourceID taskManagerId) throws Exception { final TaskManagerMetricsHeaders headers = TaskManagerMetricsHeaders.getInstance(); final TaskManagerMetricsMessageParameters parameters = headers.getUnresolvedMessageParameters(); parameters.taskManagerIdParameter.resolve(taskManagerId); parameters.metricsFilterParameter.resolve(Collections.singletonList("Status.Network.TotalMemorySegments")); fetchMetric(() -> restClient.sendRequest(HOST, PORT, headers, parameters, EmptyRequestBody.getInstance()), getMetricNamePredicate("Status.Network.TotalMemorySegments")); }
Example #11
Source File: MetricsAvailabilityITCase.java From flink with Apache License 2.0 | 5 votes |
private static void checkJobManagerMetricAvailability(final RestClient restClient) throws Exception { final JobManagerMetricsHeaders headers = JobManagerMetricsHeaders.getInstance(); final JobManagerMetricsMessageParameters parameters = headers.getUnresolvedMessageParameters(); parameters.metricsFilterParameter.resolve(Collections.singletonList("numRegisteredTaskManagers")); fetchMetric(() -> restClient.sendRequest(HOST, PORT, headers, parameters, EmptyRequestBody.getInstance()), getMetricNamePredicate("numRegisteredTaskManagers")); }
Example #12
Source File: MetricsAvailabilityITCase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testReporter() throws Exception { try (ClusterController ignored = dist.startCluster(1)) { final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(new Configuration()), scheduledExecutorService); checkJobManagerMetricAvailability(restClient); final Collection<ResourceID> taskManagerIds = getTaskManagerIds(restClient); for (final ResourceID taskManagerId : taskManagerIds) { checkTaskManagerMetricAvailability(restClient, taskManagerId); } } }
Example #13
Source File: RestClusterClientSavepointTriggerTest.java From flink with Apache License 2.0 | 5 votes |
private RestClusterClient<StandaloneClusterId> createRestClusterClient(final int port) throws Exception { final Configuration clientConfig = new Configuration(REST_CONFIG); clientConfig.setInteger(RestOptions.PORT, port); return new RestClusterClient<>( clientConfig, new RestClient(RestClientConfiguration.fromConfiguration(REST_CONFIG), executor), StandaloneClusterId.getInstance(), (attempt) -> 0); }
Example #14
Source File: RestClusterClient.java From flink with Apache License 2.0 | 5 votes |
@VisibleForTesting RestClusterClient( Configuration configuration, @Nullable RestClient restClient, T clusterId, WaitStrategy waitStrategy) throws Exception { this( configuration, restClient, clusterId, waitStrategy, HighAvailabilityServicesUtils.createClientHAService(configuration)); }
Example #15
Source File: YARNSessionCapacitySchedulerITCase.java From flink with Apache License 2.0 | 5 votes |
@BeforeClass public static void setup() throws Exception { YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.set("yarn.scheduler.capacity.root.queues", "default,qa-team"); YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.default.capacity", 40); YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.qa-team.capacity", 60); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-capacityscheduler"); startYARNWithConfig(YARN_CONFIGURATION); restClientExecutor = Executors.newSingleThreadExecutor(); restClient = new RestClient(RestClientConfiguration.fromConfiguration(new Configuration()), restClientExecutor); }
Example #16
Source File: MetricsAvailabilityITCase.java From flink with Apache License 2.0 | 5 votes |
private static void checkTaskManagerMetricAvailability(final RestClient restClient, final ResourceID taskManagerId) throws Exception { final TaskManagerMetricsHeaders headers = TaskManagerMetricsHeaders.getInstance(); final TaskManagerMetricsMessageParameters parameters = headers.getUnresolvedMessageParameters(); parameters.taskManagerIdParameter.resolve(taskManagerId); parameters.metricsFilterParameter.resolve(Collections.singletonList("Status.Network.TotalMemorySegments")); fetchMetric(() -> restClient.sendRequest(HOST, PORT, headers, parameters, EmptyRequestBody.getInstance()), getMetricNamePredicate("Status.Network.TotalMemorySegments")); }
Example #17
Source File: MetricsAvailabilityITCase.java From flink with Apache License 2.0 | 5 votes |
private static void checkJobManagerMetricAvailability(final RestClient restClient) throws Exception { final JobManagerMetricsHeaders headers = JobManagerMetricsHeaders.getInstance(); final JobManagerMetricsMessageParameters parameters = headers.getUnresolvedMessageParameters(); parameters.metricsFilterParameter.resolve(Collections.singletonList("numRegisteredTaskManagers")); fetchMetric(() -> restClient.sendRequest(HOST, PORT, headers, parameters, EmptyRequestBody.getInstance()), getMetricNamePredicate("numRegisteredTaskManagers")); }
Example #18
Source File: RestClusterClientSavepointTriggerTest.java From flink with Apache License 2.0 | 5 votes |
private RestClusterClient<StandaloneClusterId> createRestClusterClient(final int port) throws Exception { final Configuration clientConfig = new Configuration(REST_CONFIG); clientConfig.setInteger(RestOptions.PORT, port); return new RestClusterClient<>( clientConfig, new RestClient(RestClientConfiguration.fromConfiguration(REST_CONFIG), executor), StandaloneClusterId.getInstance(), (attempt) -> 0, null); }
Example #19
Source File: RestClusterClient.java From flink with Apache License 2.0 | 5 votes |
@VisibleForTesting RestClusterClient( Configuration configuration, @Nullable RestClient restClient, T clusterId, WaitStrategy waitStrategy, @Nullable LeaderRetrievalService webMonitorRetrievalService) throws Exception { super(configuration); this.restClusterClientConfiguration = RestClusterClientConfiguration.fromConfiguration(configuration); if (restClient != null) { this.restClient = restClient; } else { this.restClient = new RestClient(restClusterClientConfiguration.getRestClientConfiguration(), executorService); } this.waitStrategy = Preconditions.checkNotNull(waitStrategy); this.clusterId = Preconditions.checkNotNull(clusterId); if (webMonitorRetrievalService == null) { this.webMonitorRetrievalService = highAvailabilityServices.getWebMonitorLeaderRetriever(); } else { this.webMonitorRetrievalService = webMonitorRetrievalService; } this.retryExecutorService = Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-RestClusterClient-Retry")); startLeaderRetrievers(); }
Example #20
Source File: YARNSessionCapacitySchedulerITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@BeforeClass public static void setup() throws Exception { YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.set("yarn.scheduler.capacity.root.queues", "default,qa-team"); YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.default.capacity", 40); YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.qa-team.capacity", 60); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-capacityscheduler"); startYARNWithConfig(YARN_CONFIGURATION); restClientExecutor = Executors.newSingleThreadExecutor(); restClient = new RestClient(RestClientConfiguration.fromConfiguration(new Configuration()), restClientExecutor); }
Example #21
Source File: MetricsAvailabilityITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private static void checkTaskManagerMetricAvailability(final RestClient restClient, final ResourceID taskManagerId) throws Exception { final TaskManagerMetricsHeaders headers = TaskManagerMetricsHeaders.getInstance(); final TaskManagerMetricsMessageParameters parameters = headers.getUnresolvedMessageParameters(); parameters.taskManagerIdParameter.resolve(taskManagerId); parameters.metricsFilterParameter.resolve(Collections.singletonList("Status.Network.TotalMemorySegments")); fetchMetric(() -> restClient.sendRequest(HOST, PORT, headers, parameters, EmptyRequestBody.getInstance()), getMetricNamePredicate("Status.Network.TotalMemorySegments")); }
Example #22
Source File: MetricsAvailabilityITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private static void checkJobManagerMetricAvailability(final RestClient restClient) throws Exception { final JobManagerMetricsHeaders headers = JobManagerMetricsHeaders.getInstance(); final JobManagerMetricsMessageParameters parameters = headers.getUnresolvedMessageParameters(); parameters.metricsFilterParameter.resolve(Collections.singletonList("numRegisteredTaskManagers")); fetchMetric(() -> restClient.sendRequest(HOST, PORT, headers, parameters, EmptyRequestBody.getInstance()), getMetricNamePredicate("numRegisteredTaskManagers")); }
Example #23
Source File: MetricsAvailabilityITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testReporter() throws Exception { dist.startFlinkCluster(); final RestClient restClient = new RestClient(RestClientConfiguration.fromConfiguration(new Configuration()), scheduledExecutorService); checkJobManagerMetricAvailability(restClient); final Collection<ResourceID> taskManagerIds = getTaskManagerIds(restClient); for (final ResourceID taskManagerId : taskManagerIds) { checkTaskManagerMetricAvailability(restClient, taskManagerId); } }
Example #24
Source File: RestClusterClientSavepointTriggerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private RestClusterClient<StandaloneClusterId> createRestClusterClient(final int port) throws Exception { final Configuration clientConfig = new Configuration(REST_CONFIG); clientConfig.setInteger(RestOptions.PORT, port); return new RestClusterClient<>( clientConfig, new RestClient(RestClientConfiguration.fromConfiguration(REST_CONFIG), executor), StandaloneClusterId.getInstance(), (attempt) -> 0, null); }
Example #25
Source File: RestClusterClient.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@VisibleForTesting RestClusterClient( Configuration configuration, @Nullable RestClient restClient, T clusterId, WaitStrategy waitStrategy, @Nullable LeaderRetrievalService webMonitorRetrievalService) throws Exception { super(configuration); this.restClusterClientConfiguration = RestClusterClientConfiguration.fromConfiguration(configuration); if (restClient != null) { this.restClient = restClient; } else { this.restClient = new RestClient(restClusterClientConfiguration.getRestClientConfiguration(), executorService); } this.waitStrategy = Preconditions.checkNotNull(waitStrategy); this.clusterId = Preconditions.checkNotNull(clusterId); if (webMonitorRetrievalService == null) { this.webMonitorRetrievalService = highAvailabilityServices.getWebMonitorLeaderRetriever(); } else { this.webMonitorRetrievalService = webMonitorRetrievalService; } this.dispatcherRetrievalService = highAvailabilityServices.getDispatcherLeaderRetriever(); this.retryExecutorService = Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-RestClusterClient-Retry")); startLeaderRetrievers(); }
Example #26
Source File: JarRunHandlerTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testRunJar() throws Exception { Path uploadDir = TMP.newFolder().toPath(); Path actualUploadDir = uploadDir.resolve("flink-web-upload"); Files.createDirectory(actualUploadDir); Path emptyJar = actualUploadDir.resolve("empty.jar"); Files.createFile(emptyJar); Configuration config = new Configuration(); config.setString(WebOptions.UPLOAD_DIR, uploadDir.toString()); MiniClusterResource clusterResource = new MiniClusterResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(config) .setNumberTaskManagers(1) .setNumberSlotsPerTaskManager(1) .build()); clusterResource.before(); try { Configuration clientConfig = clusterResource.getClientConfiguration(); RestClient client = new RestClient(RestClientConfiguration.fromConfiguration(clientConfig), TestingUtils.defaultExecutor()); try { JarRunHeaders headers = JarRunHeaders.getInstance(); JarRunMessageParameters parameters = headers.getUnresolvedMessageParameters(); parameters.jarIdPathParameter.resolve(emptyJar.getFileName().toString()); String host = clientConfig.getString(RestOptions.ADDRESS); int port = clientConfig.getInteger(RestOptions.PORT); try { client.sendRequest(host, port, headers, parameters, new JarRunRequestBody()) .get(); } catch (Exception e) { Optional<RestClientException> expected = ExceptionUtils.findThrowable(e, RestClientException.class); if (expected.isPresent()) { // implies the job was actually submitted assertTrue(expected.get().getMessage().contains("ProgramInvocationException")); // original cause is preserved in stack trace assertThat(expected.get().getMessage(), containsString("ZipException")); // implies the jar was registered for the job graph (otherwise the jar name would not occur in the exception) // implies the jar was uploaded (otherwise the file would not be found at all) assertTrue(expected.get().getMessage().contains("empty.jar'. zip file is empty")); } else { throw e; } } } finally { client.shutdown(Time.milliseconds(10)); } } finally { clusterResource.after(); } }
Example #27
Source File: BatchFineGrainedRecoveryITCase.java From flink with Apache License 2.0 | 4 votes |
private RestClient createRestClient() throws ConfigurationException { return new RestClient( RestClientConfiguration.fromConfiguration(new UnmodifiableConfiguration(new Configuration())), executorService); }
Example #28
Source File: JarRunHandlerTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testRunJar() throws Exception { Path uploadDir = TMP.newFolder().toPath(); Path actualUploadDir = uploadDir.resolve("flink-web-upload"); Files.createDirectory(actualUploadDir); Path emptyJar = actualUploadDir.resolve("empty.jar"); Files.createFile(emptyJar); Configuration config = new Configuration(); config.setString(WebOptions.UPLOAD_DIR, uploadDir.toString()); MiniClusterResource clusterResource = new MiniClusterResource( new MiniClusterResourceConfiguration.Builder() .setConfiguration(config) .setNumberTaskManagers(1) .setNumberSlotsPerTaskManager(1) .build()); clusterResource.before(); try { Configuration clientConfig = clusterResource.getClientConfiguration(); RestClient client = new RestClient(RestClientConfiguration.fromConfiguration(clientConfig), TestingUtils.defaultExecutor()); try { JarRunHeaders headers = JarRunHeaders.getInstance(); JarRunMessageParameters parameters = headers.getUnresolvedMessageParameters(); parameters.jarIdPathParameter.resolve(emptyJar.getFileName().toString()); String host = clientConfig.getString(RestOptions.ADDRESS); int port = clientConfig.getInteger(RestOptions.PORT); try { client.sendRequest(host, port, headers, parameters, new JarRunRequestBody()) .get(); } catch (Exception e) { Optional<RestClientException> expected = ExceptionUtils.findThrowable(e, RestClientException.class); if (expected.isPresent()) { // implies the job was actually submitted assertTrue(expected.get().getMessage().contains("ProgramInvocationException")); // original cause is preserved in stack trace assertThat(expected.get().getMessage(), containsString("ZipException: zip file is empty")); // implies the jar was registered for the job graph (otherwise the jar name would not occur in the exception) // implies the jar was uploaded (otherwise the file would not be found at all) assertTrue(expected.get().getMessage().contains("empty.jar")); } else { throw e; } } } finally { client.shutdown(Time.milliseconds(10)); } } finally { clusterResource.after(); } }
Example #29
Source File: BatchFineGrainedRecoveryITCase.java From flink with Apache License 2.0 | 4 votes |
private RestClient createRestClient() throws ConfigurationException { return new RestClient( RestClientConfiguration.fromConfiguration(new UnmodifiableConfiguration(new Configuration())), executorService); }