org.apache.flink.client.deployment.ClusterDescriptor Java Examples

/**
 * Retrieves the {@link ClusterClient} from the given {@link CustomCommandLine} and runs the given
 * {@link ClusterAction} against it.
 *
 * @param activeCommandLine to create the {@link ClusterDescriptor} from
 * @param commandLine containing the parsed command line options
 * @param clusterAction the cluster action to run against the retrieved {@link ClusterClient}.
 * @param <ClusterID> type of the cluster id
 * @throws FlinkException if something goes wrong
 */
private <ClusterID> void runClusterAction(CustomCommandLine activeCommandLine, CommandLine commandLine, ClusterAction<ClusterID> clusterAction) throws FlinkException {
	final Configuration executorConfig = activeCommandLine.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ClusterID> clusterClientFactory = clusterClientServiceLoader.getClusterClientFactory(executorConfig);

	final ClusterID clusterId = clusterClientFactory.getClusterId(executorConfig);
	if (clusterId == null) {
		throw new FlinkException("No cluster id was specified. Please specify a cluster to which you would like to connect.");
	}

	try (final ClusterDescriptor<ClusterID> clusterDescriptor = clusterClientFactory.createClusterDescriptor(executorConfig)) {
		try (final ClusterClient<ClusterID> clusterClient = clusterDescriptor.retrieve(clusterId).getClusterClient()) {
			clusterAction.runAction(clusterClient);
		}
	}
}
 

@Override
public CompletableFuture<JobClient> execute(@Nonnull final Pipeline pipeline, @Nonnull final Configuration configuration) throws Exception {
	final JobGraph jobGraph = PipelineExecutorUtils.getJobGraph(pipeline, configuration);

	try (final ClusterDescriptor<ClusterID> clusterDescriptor = clusterClientFactory.createClusterDescriptor(configuration)) {
		final ClusterID clusterID = clusterClientFactory.getClusterId(configuration);
		checkState(clusterID != null);

		final ClusterClientProvider<ClusterID> clusterClientProvider = clusterDescriptor.retrieve(clusterID);
		ClusterClient<ClusterID> clusterClient = clusterClientProvider.getClusterClient();
		return clusterClient
				.submitJob(jobGraph)
				.thenApplyAsync(jobID -> (JobClient) new ClusterClientJobClientAdapter<>(
						clusterClientProvider,
						jobID))
				.whenComplete((ignored1, ignored2) -> clusterClient.close());
	}
}
 

@Override
public CompletableFuture<JobClient> execute(@Nonnull final Pipeline pipeline, @Nonnull final Configuration configuration) throws Exception {
	final JobGraph jobGraph = PipelineExecutorUtils.getJobGraph(pipeline, configuration);

	try (final ClusterDescriptor<ClusterID> clusterDescriptor = clusterClientFactory.createClusterDescriptor(configuration)) {
		final ExecutionConfigAccessor configAccessor = ExecutionConfigAccessor.fromConfiguration(configuration);

		final ClusterSpecification clusterSpecification = clusterClientFactory.getClusterSpecification(configuration);

		final ClusterClientProvider<ClusterID> clusterClientProvider = clusterDescriptor
				.deployJobCluster(clusterSpecification, jobGraph, configAccessor.getDetachedMode());
		LOG.info("Job has been submitted with JobID " + jobGraph.getJobID());

		return CompletableFuture.completedFuture(
				new ClusterClientJobClientAdapter<>(clusterClientProvider, jobGraph.getJobID()));
	}
}
 

/**
 * Tests that the configuration is properly passed via the DefaultCLI to the
 * created ClusterDescriptor.
 */
@Test
public void testConfigurationPassing() throws Exception {
	final Configuration configuration = getConfiguration();

	final String localhost = "localhost";
	final int port = 1234;

	configuration.setString(JobManagerOptions.ADDRESS, localhost);
	configuration.setInteger(JobManagerOptions.PORT, port);

	@SuppressWarnings("unchecked")
	final AbstractCustomCommandLine<StandaloneClusterId> defaultCLI =
		(AbstractCustomCommandLine<StandaloneClusterId>) getCli(configuration);

	final String[] args = {};

	CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);

	final ClusterDescriptor<StandaloneClusterId> clusterDescriptor =
		defaultCLI.createClusterDescriptor(commandLine);

	final ClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine));

	final LeaderConnectionInfo clusterConnectionInfo = clusterClient.getClusterConnectionInfo();

	assertThat(clusterConnectionInfo.getHostname(), Matchers.equalTo(localhost));
	assertThat(clusterConnectionInfo.getPort(), Matchers.equalTo(port));
}
 

private ClusterClient<?> getClusterClient(AbstractCustomCommandLine defaultCLI, CommandLine commandLine) throws FlinkException {
	final ClusterClientServiceLoader serviceLoader = new DefaultClusterClientServiceLoader();
	final Configuration executorConfig = defaultCLI.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<StandaloneClusterId> clusterFactory = serviceLoader.getClusterClientFactory(executorConfig);
	checkState(clusterFactory != null);

	final ClusterDescriptor<StandaloneClusterId> clusterDescriptor = clusterFactory.createClusterDescriptor(executorConfig);
	return clusterDescriptor.retrieve(clusterFactory.getClusterId(executorConfig)).getClusterClient();
}
 

/**
 * Tests that command line options override the configuration settings.
 */
@Test
public void testRESTManualConfigurationOverride() throws Exception {
	final String configuredHostname = "localhost";
	final int configuredPort = 1234;
	final Configuration configuration = new Configuration();

	configuration.setString(JobManagerOptions.ADDRESS, configuredHostname);
	configuration.setInteger(JobManagerOptions.PORT, configuredPort);
	configuration.setString(RestOptions.ADDRESS, configuredHostname);
	configuration.setInteger(RestOptions.PORT, configuredPort);

	final DefaultCLI defaultCLI = new DefaultCLI(configuration);

	final String manualHostname = "123.123.123.123";
	final int manualPort = 4321;
	final String[] args = {"-m", manualHostname + ':' + manualPort};

	CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);

	final ClusterClientServiceLoader serviceLoader = new DefaultClusterClientServiceLoader();
	final Configuration executorConfig = defaultCLI.applyCommandLineOptionsToConfiguration(commandLine);

	final ClusterClientFactory<StandaloneClusterId> clusterFactory = serviceLoader.getClusterClientFactory(executorConfig);
	checkState(clusterFactory != null);

	final ClusterDescriptor<StandaloneClusterId> clusterDescriptor = clusterFactory.createClusterDescriptor(executorConfig);
	final RestClusterClient<?> clusterClient = (RestClusterClient<?>) clusterDescriptor
			.retrieve(clusterFactory.getClusterId(executorConfig))
			.getClusterClient();

	URL webMonitorBaseUrl = clusterClient.getWebMonitorBaseUrl().get();
	assertThat(webMonitorBaseUrl.getHost(), equalTo(manualHostname));
	assertThat(webMonitorBaseUrl.getPort(), equalTo(manualPort));
}
 

public <ClusterID> void run(
		final Configuration configuration,
		final ApplicationConfiguration applicationConfiguration) throws Exception {
	checkNotNull(configuration);
	checkNotNull(applicationConfiguration);

	LOG.info("Submitting application in 'Application Mode'.");

	final ClusterClientFactory<ClusterID> clientFactory = clientServiceLoader.getClusterClientFactory(configuration);
	try (final ClusterDescriptor<ClusterID> clusterDescriptor = clientFactory.createClusterDescriptor(configuration)) {
		final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(configuration);

		clusterDescriptor.deployApplicationCluster(clusterSpecification, applicationConfiguration);
	}
}
 

/**
 * Tests that command line options override the configuration settings.
 */
@Test
public void testManualConfigurationOverride() throws Exception {
	final String localhost = "localhost";
	final int port = 1234;
	final Configuration configuration = getConfiguration();

	configuration.setString(JobManagerOptions.ADDRESS, localhost);
	configuration.setInteger(JobManagerOptions.PORT, port);

	@SuppressWarnings("unchecked")
	final AbstractCustomCommandLine<StandaloneClusterId> defaultCLI =
		(AbstractCustomCommandLine<StandaloneClusterId>) getCli(configuration);

	final String manualHostname = "123.123.123.123";
	final int manualPort = 4321;
	final String[] args = {"-m", manualHostname + ':' + manualPort};

	CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);

	final ClusterDescriptor<StandaloneClusterId> clusterDescriptor =
		defaultCLI.createClusterDescriptor(commandLine);

	final ClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine));

	final LeaderConnectionInfo clusterConnectionInfo = clusterClient.getClusterConnectionInfo();

	assertThat(clusterConnectionInfo.getHostname(), Matchers.equalTo(manualHostname));
	assertThat(clusterConnectionInfo.getPort(), Matchers.equalTo(manualPort));
}
 

private <T> void deployJobOnNewCluster(
		ClusterDescriptor<T> clusterDescriptor,
		JobGraph jobGraph,
		Result<T> result,
		ClassLoader classLoader) throws Exception {
	ClusterClient<T> clusterClient = null;
	try {
		// deploy job cluster with job attached
		clusterClient = clusterDescriptor.deployJobCluster(context.getClusterSpec(), jobGraph, false);
		// save information about the new cluster
		result.setClusterInformation(clusterClient.getClusterId(), clusterClient.getWebInterfaceURL());
		// get result
		if (awaitJobResult) {
			// we need to hard cast for now
			final JobExecutionResult jobResult = ((RestClusterClient<T>) clusterClient)
					.requestJobResult(jobGraph.getJobID())
					.get()
					.toJobExecutionResult(context.getClassLoader()); // throws exception if job fails
			executionResultBucket.add(jobResult);
		}
	} finally {
		try {
			if (clusterClient != null) {
				clusterClient.shutdown();
			}
		} catch (Exception e) {
			// ignore
		}
	}
}
 

/**
 * Tests that command line options override the configuration settings.
 */
@Test
public void testManualConfigurationOverride() throws Exception {
	final String localhost = "localhost";
	final int port = 1234;
	final Configuration configuration = getConfiguration();

	configuration.setString(JobManagerOptions.ADDRESS, localhost);
	configuration.setInteger(JobManagerOptions.PORT, port);

	@SuppressWarnings("unchecked")
	final AbstractCustomCommandLine<StandaloneClusterId> defaultCLI =
		(AbstractCustomCommandLine<StandaloneClusterId>) getCli(configuration);

	final String manualHostname = "123.123.123.123";
	final int manualPort = 4321;
	final String[] args = {"-m", manualHostname + ':' + manualPort};

	CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);

	final ClusterDescriptor<StandaloneClusterId> clusterDescriptor =
		defaultCLI.createClusterDescriptor(commandLine);

	final ClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine));

	final LeaderConnectionInfo clusterConnectionInfo = clusterClient.getClusterConnectionInfo();

	assertThat(clusterConnectionInfo.getHostname(), Matchers.equalTo(manualHostname));
	assertThat(clusterConnectionInfo.getPort(), Matchers.equalTo(manualPort));
}
 

/**
 * Tests that the configuration is properly passed via the DefaultCLI to the
 * created ClusterDescriptor.
 */
@Test
public void testConfigurationPassing() throws Exception {
	final Configuration configuration = getConfiguration();

	final String localhost = "localhost";
	final int port = 1234;

	configuration.setString(JobManagerOptions.ADDRESS, localhost);
	configuration.setInteger(JobManagerOptions.PORT, port);

	@SuppressWarnings("unchecked")
	final AbstractCustomCommandLine<StandaloneClusterId> defaultCLI =
		(AbstractCustomCommandLine<StandaloneClusterId>) getCli(configuration);

	final String[] args = {};

	CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);

	final ClusterDescriptor<StandaloneClusterId> clusterDescriptor =
		defaultCLI.createClusterDescriptor(commandLine);

	final ClusterClient<?> clusterClient = clusterDescriptor.retrieve(defaultCLI.getClusterId(commandLine));

	final LeaderConnectionInfo clusterConnectionInfo = clusterClient.getClusterConnectionInfo();

	assertThat(clusterConnectionInfo.getHostname(), Matchers.equalTo(localhost));
	assertThat(clusterConnectionInfo.getPort(), Matchers.equalTo(port));
}
 

private <T> void deployJobOnNewCluster(
		ClusterDescriptor<T> clusterDescriptor,
		JobGraph jobGraph,
		Result<T> result,
		ClassLoader classLoader) throws Exception {
	ClusterClient<T> clusterClient = null;
	try {
		// deploy job cluster with job attached
		clusterClient = clusterDescriptor.deployJobCluster(context.getClusterSpec(), jobGraph, false);
		// save information about the new cluster
		result.setClusterInformation(clusterClient.getClusterId(), clusterClient.getWebInterfaceURL());
		// get result
		if (awaitJobResult) {
			// we need to hard cast for now
			final JobExecutionResult jobResult = ((RestClusterClient<T>) clusterClient)
					.requestJobResult(jobGraph.getJobID())
					.get()
					.toJobExecutionResult(context.getClassLoader()); // throws exception if job fails
			executionResultBucket.add(jobResult);
		}
	} finally {
		try {
			if (clusterClient != null) {
				clusterClient.shutdown();
			}
		} catch (Exception e) {
			// ignore
		}
	}
}
 

@Override
public ClusterDescriptor<T> createClusterDescriptor(CommandLine commandLine) {
	return new DummyClusterDescriptor<>(clusterClient);
}
 

public ClusterDescriptor<T> createClusterDescriptor() throws Exception {
	return activeCommandLine.createClusterDescriptor(commandLine);
}
 

public ClusterDescriptor<ClusterID> createClusterDescriptor() {
	return clusterClientFactory.createClusterDescriptor(flinkConfig);
}
 

public ClusterDescriptor<T> createClusterDescriptor() throws Exception {
	return activeCommandLine.createClusterDescriptor(commandLine);
}
 

@Override
public ClusterDescriptor<T> createClusterDescriptor(CommandLine commandLine) {
	return new DummyClusterDescriptor<>(clusterClient);
}
 

@Override
public ClusterDescriptor<ClusterID> createClusterDescriptor(Configuration configuration) {
	return new DummyClusterDescriptor<>(checkNotNull(clusterClient));
}
 

/**
 * Create a {@link ClusterDescriptor} from the given configuration, configuration directory
 * and the command line.
 *
 * @param commandLine containing command line options relevant for the ClusterDescriptor
 * @return ClusterDescriptor
 * @throws FlinkException if the ClusterDescriptor could not be created
 */
ClusterDescriptor<T> createClusterDescriptor(CommandLine commandLine) throws FlinkException;
 

/**
 * Create a {@link ClusterDescriptor} from the given configuration, configuration directory
 * and the command line.
 *
 * @param commandLine containing command line options relevant for the ClusterDescriptor
 * @return ClusterDescriptor
 * @throws FlinkException if the ClusterDescriptor could not be created
 */
ClusterDescriptor<T> createClusterDescriptor(CommandLine commandLine) throws FlinkException;