org.apache.flink.runtime.entrypoint.ClusterEntrypoint Java Examples

The following examples show how to use org.apache.flink.runtime.entrypoint.ClusterEntrypoint. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MiniDispatcherTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the {@link MiniDispatcher} recovers the single job with which it
 * was started.
 */
@Test
public void testSingleJobRecovery() throws Exception {
	final MiniDispatcher miniDispatcher = createMiniDispatcher(ClusterEntrypoint.ExecutionMode.DETACHED);

	miniDispatcher.start();

	try {
		// wait until the Dispatcher is the leader
		dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get();

		final JobGraph actualJobGraph = jobGraphFuture.get();

		assertThat(actualJobGraph.getJobID(), is(jobGraph.getJobID()));
	} finally {
		RpcUtils.terminateRpcEndpoint(miniDispatcher, timeout);
	}
}
 
Example #2
Source File: DispatcherProcess.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Entrypoint of the DispatcherProcessEntryPoint.
 *
 * <p>Other arguments are parsed to a {@link Configuration} and passed to the Dispatcher,
 * for instance: <code>--high-availability ZOOKEEPER --high-availability.zookeeper.quorum
 * "xyz:123:456"</code>.
 */
public static void main(String[] args) {
	try {
		ParameterTool params = ParameterTool.fromArgs(args);
		Configuration config = params.getConfiguration();
		LOG.info("Configuration: {}.", config);

		config.setInteger(JobManagerOptions.PORT, 0);
		config.setString(RestOptions.BIND_PORT, "0");

		final StandaloneSessionClusterEntrypoint clusterEntrypoint = new StandaloneSessionClusterEntrypoint(config);

		ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint);
	}
	catch (Throwable t) {
		LOG.error("Failed to start Dispatcher process", t);
		System.exit(1);
	}
}
 
Example #3
Source File: MiniDispatcherTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private MiniDispatcher createMiniDispatcher(ClusterEntrypoint.ExecutionMode executionMode) throws Exception {
	return new MiniDispatcher(
		rpcService,
		DispatcherId.generate(),
		new DispatcherServices(
			configuration,
			highAvailabilityServices,
			() -> CompletableFuture.completedFuture(resourceManagerGateway),
			blobServer,
			heartbeatServices,
			archivedExecutionGraphStore,
			testingFatalErrorHandlerResource.getFatalErrorHandler(),
			VoidHistoryServerArchivist.INSTANCE,
			null,
			UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(),
			highAvailabilityServices.getJobGraphStore(),
			testingJobManagerRunnerFactory),
		new DefaultDispatcherBootstrap(Collections.singletonList(jobGraph)),
		executionMode);
}
 
Example #4
Source File: MiniDispatcherTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that in detached mode, the {@link MiniDispatcher} will complete the future that
 * signals job termination.
 */
@Test
public void testTerminationAfterJobCompletion() throws Exception {
	final MiniDispatcher miniDispatcher = createMiniDispatcher(ClusterEntrypoint.ExecutionMode.DETACHED);

	miniDispatcher.start();

	try {
		// wait until we have submitted the job
		final TestingJobManagerRunner testingJobManagerRunner = testingJobManagerRunnerFactory.takeCreatedJobManagerRunner();

		testingJobManagerRunner.completeResultFuture(archivedExecutionGraph);

		// wait until we terminate
		miniDispatcher.getShutDownFuture().get();
	} finally {
		RpcUtils.terminateRpcEndpoint(miniDispatcher, timeout);
	}
}
 
Example #5
Source File: MiniDispatcher.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<JobResult> requestJobResult(JobID jobId, Time timeout) {
	final CompletableFuture<JobResult> jobResultFuture = super.requestJobResult(jobId, timeout);

	if (executionMode == ClusterEntrypoint.ExecutionMode.NORMAL) {
		// terminate the MiniDispatcher once we served the first JobResult successfully
		jobResultFuture.thenAccept((JobResult result) -> {
			ApplicationStatus status = result.getSerializedThrowable().isPresent() ?
					ApplicationStatus.FAILED : ApplicationStatus.SUCCEEDED;

			LOG.debug("Shutting down per-job cluster because someone retrieved the job result.");
			shutDownFuture.complete(status);
		});
	} else {
		LOG.debug("Not shutting down per-job cluster after someone retrieved the job result.");
	}

	return jobResultFuture;
}
 
Example #6
Source File: YarnJobClusterEntrypoint.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, YarnJobClusterEntrypoint.class.getSimpleName(), args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	Map<String, String> env = System.getenv();

	final String workingDirectory = env.get(ApplicationConstants.Environment.PWD.key());
	Preconditions.checkArgument(
		workingDirectory != null,
		"Working directory variable (%s) not set",
		ApplicationConstants.Environment.PWD.key());

	try {
		YarnEntrypointUtils.logYarnEnvironmentInformation(env, LOG);
	} catch (IOException e) {
		LOG.warn("Could not log YARN environment information.", e);
	}

	Configuration configuration = YarnEntrypointUtils.loadConfiguration(workingDirectory, env);

	YarnJobClusterEntrypoint yarnJobClusterEntrypoint = new YarnJobClusterEntrypoint(configuration);

	ClusterEntrypoint.runClusterEntrypoint(yarnJobClusterEntrypoint);
}
 
Example #7
Source File: YarnSessionClusterEntrypoint.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, YarnSessionClusterEntrypoint.class.getSimpleName(), args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	Map<String, String> env = System.getenv();

	final String workingDirectory = env.get(ApplicationConstants.Environment.PWD.key());
	Preconditions.checkArgument(
		workingDirectory != null,
		"Working directory variable (%s) not set",
		ApplicationConstants.Environment.PWD.key());

	try {
		YarnEntrypointUtils.logYarnEnvironmentInformation(env, LOG);
	} catch (IOException e) {
		LOG.warn("Could not log YARN environment information.", e);
	}

	Configuration configuration = YarnEntrypointUtils.loadConfiguration(workingDirectory, env);

	YarnSessionClusterEntrypoint yarnSessionClusterEntrypoint = new YarnSessionClusterEntrypoint(configuration);

	ClusterEntrypoint.runClusterEntrypoint(yarnSessionClusterEntrypoint);
}
 
Example #8
Source File: MesosJobClusterEntrypoint.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, MesosJobClusterEntrypoint.class.getSimpleName(), args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	// load configuration incl. dynamic properties
	CommandLineParser parser = new PosixParser();
	CommandLine cmd;
	try {
		cmd = parser.parse(ALL_OPTIONS, args);
	}
	catch (Exception e){
		LOG.error("Could not parse the command-line options.", e);
		System.exit(STARTUP_FAILURE_RETURN_CODE);
		return;
	}

	Configuration dynamicProperties = BootstrapTools.parseDynamicProperties(cmd);
	Configuration configuration = MesosUtils.loadConfiguration(dynamicProperties, LOG);

	MesosJobClusterEntrypoint clusterEntrypoint = new MesosJobClusterEntrypoint(configuration);

	ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint);
}
 
Example #9
Source File: MesosSessionClusterEntrypoint.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, MesosSessionClusterEntrypoint.class.getSimpleName(), args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	// load configuration incl. dynamic properties
	CommandLineParser parser = new PosixParser();
	CommandLine cmd;
	try {
		cmd = parser.parse(ALL_OPTIONS, args);
	}
	catch (Exception e){
		LOG.error("Could not parse the command-line options.", e);
		System.exit(STARTUP_FAILURE_RETURN_CODE);
		return;
	}

	Configuration dynamicProperties = BootstrapTools.parseDynamicProperties(cmd);
	Configuration configuration = MesosUtils.loadConfiguration(dynamicProperties, LOG);

	MesosSessionClusterEntrypoint clusterEntrypoint = new MesosSessionClusterEntrypoint(configuration);

	ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint);
}
 
Example #10
Source File: YarnJobDescriptor.java    From sylph with Apache License 2.0 6 votes vote down vote up
public ClusterClient<ApplicationId> deploy(JobGraph jobGraph, boolean detached)
        throws Exception
{
    // this is required because the slots are allocated lazily
    jobGraph.setAllowQueuedScheduling(true);
    //
    ApplicationReport report = startAppMaster(jobGraph);

    Configuration flinkConfiguration = getFlinkConfiguration();

    ClusterEntrypoint.ExecutionMode executionMode = detached ? ClusterEntrypoint.ExecutionMode.DETACHED : ClusterEntrypoint.ExecutionMode.NORMAL;
    flinkConfiguration.setString(ClusterEntrypoint.EXECUTION_MODE, executionMode.toString());
    String host = report.getHost();
    int port = report.getRpcPort();
    flinkConfiguration.setString(JobManagerOptions.ADDRESS, host);
    flinkConfiguration.setInteger(JobManagerOptions.PORT, port);
    flinkConfiguration.setString(RestOptions.ADDRESS, host);
    flinkConfiguration.setInteger(RestOptions.PORT, port);
    return new RestClusterClient<>(flinkConfiguration, report.getApplicationId());
}
 
Example #11
Source File: DispatcherProcess.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Entrypoint of the DispatcherProcessEntryPoint.
 *
 * <p>Other arguments are parsed to a {@link Configuration} and passed to the Dispatcher,
 * for instance: <code>--high-availability ZOOKEEPER --high-availability.zookeeper.quorum
 * "xyz:123:456"</code>.
 */
public static void main(String[] args) {
	try {
		ParameterTool params = ParameterTool.fromArgs(args);
		Configuration config = params.getConfiguration();
		LOG.info("Configuration: {}.", config);

		config.setInteger(JobManagerOptions.PORT, 0);
		config.setString(RestOptions.BIND_PORT, "0");

		final StandaloneSessionClusterEntrypoint clusterEntrypoint = new StandaloneSessionClusterEntrypoint(config);

		ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint);
	}
	catch (Throwable t) {
		LOG.error("Failed to start Dispatcher process", t);
		System.exit(1);
	}
}
 
Example #12
Source File: MiniDispatcherTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private MiniDispatcher createMiniDispatcher(ClusterEntrypoint.ExecutionMode executionMode) throws Exception {
	return new MiniDispatcher(
		rpcService,
		UUID.randomUUID().toString(),
		configuration,
		highAvailabilityServices,
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		heartbeatServices,
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(),
		null,
		archivedExecutionGraphStore,
		testingJobManagerRunnerFactory,
		testingFatalErrorHandler,
		VoidHistoryServerArchivist.INSTANCE,
		jobGraph,
		executionMode);
}
 
Example #13
Source File: MiniDispatcherTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that in detached mode, the {@link MiniDispatcher} will complete the future that
 * signals job termination.
 */
@Test
public void testTerminationAfterJobCompletion() throws Exception {
	final MiniDispatcher miniDispatcher = createMiniDispatcher(ClusterEntrypoint.ExecutionMode.DETACHED);

	miniDispatcher.start();

	try {
		// wait until the Dispatcher is the leader
		dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get();

		// wait until we have submitted the job
		jobGraphFuture.get();

		resultFuture.complete(archivedExecutionGraph);

		// wait until we terminate
		miniDispatcher.getJobTerminationFuture().get();
	} finally {
		RpcUtils.terminateRpcEndpoint(miniDispatcher, timeout);
	}
}
 
Example #14
Source File: MesosSessionClusterEntrypoint.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, MesosSessionClusterEntrypoint.class.getSimpleName(), args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	// load configuration incl. dynamic properties
	CommandLineParser parser = new PosixParser();
	CommandLine cmd;
	try {
		cmd = parser.parse(ALL_OPTIONS, args);
	}
	catch (Exception e){
		LOG.error("Could not parse the command-line options.", e);
		System.exit(STARTUP_FAILURE_RETURN_CODE);
		return;
	}

	Configuration dynamicProperties = BootstrapTools.parseDynamicProperties(cmd);
	Configuration configuration = MesosEntrypointUtils.loadConfiguration(dynamicProperties, LOG);

	MesosSessionClusterEntrypoint clusterEntrypoint = new MesosSessionClusterEntrypoint(configuration, dynamicProperties);

	ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint);
}
 
Example #15
Source File: MiniDispatcher.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<JobResult> requestJobResult(JobID jobId, Time timeout) {
	final CompletableFuture<JobResult> jobResultFuture = super.requestJobResult(jobId, timeout);

	if (executionMode == ClusterEntrypoint.ExecutionMode.NORMAL) {
		// terminate the MiniDispatcher once we served the first JobResult successfully
		jobResultFuture.thenAccept((JobResult result) -> {
			ApplicationStatus status = result.getSerializedThrowable().isPresent() ?
					ApplicationStatus.FAILED : ApplicationStatus.SUCCEEDED;

			jobTerminationFuture.complete(status);
		});
	}

	return jobResultFuture;
}
 
Example #16
Source File: MesosJobClusterEntrypoint.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, MesosJobClusterEntrypoint.class.getSimpleName(), args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	// load configuration incl. dynamic properties
	CommandLineParser parser = new PosixParser();
	CommandLine cmd;
	try {
		cmd = parser.parse(ALL_OPTIONS, args);
	}
	catch (Exception e){
		LOG.error("Could not parse the command-line options.", e);
		System.exit(STARTUP_FAILURE_RETURN_CODE);
		return;
	}

	Configuration dynamicProperties = BootstrapTools.parseDynamicProperties(cmd);
	Configuration configuration = MesosEntrypointUtils.loadConfiguration(dynamicProperties, LOG);

	MesosJobClusterEntrypoint clusterEntrypoint = new MesosJobClusterEntrypoint(configuration, dynamicProperties);

	ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint);
}
 
Example #17
Source File: MiniDispatcherTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that in detached mode, the {@link MiniDispatcher} will complete the future that
 * signals job termination.
 */
@Test
public void testTerminationAfterJobCompletion() throws Exception {
	final MiniDispatcher miniDispatcher = createMiniDispatcher(ClusterEntrypoint.ExecutionMode.DETACHED);

	miniDispatcher.start();

	try {
		// wait until the Dispatcher is the leader
		dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get();

		// wait until we have submitted the job
		jobGraphFuture.get();

		resultFuture.complete(archivedExecutionGraph);

		// wait until we terminate
		miniDispatcher.getJobTerminationFuture().get();
	} finally {
		RpcUtils.terminateRpcEndpoint(miniDispatcher, timeout);
	}
}
 
Example #18
Source File: MiniDispatcherTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the {@link MiniDispatcher} recovers the single job with which it
 * was started.
 */
@Test
public void testSingleJobRecovery() throws Exception {
	final MiniDispatcher miniDispatcher = createMiniDispatcher(ClusterEntrypoint.ExecutionMode.DETACHED);

	miniDispatcher.start();

	try {
		// wait until the Dispatcher is the leader
		dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get();

		final JobGraph actualJobGraph = jobGraphFuture.get();

		assertThat(actualJobGraph.getJobID(), is(jobGraph.getJobID()));
	} finally {
		RpcUtils.terminateRpcEndpoint(miniDispatcher, timeout);
	}
}
 
Example #19
Source File: MiniDispatcherTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Nonnull
private MiniDispatcher createMiniDispatcher(ClusterEntrypoint.ExecutionMode executionMode) throws Exception {
	return new MiniDispatcher(
		rpcService,
		UUID.randomUUID().toString(),
		configuration,
		highAvailabilityServices,
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		heartbeatServices,
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(),
		null,
		archivedExecutionGraphStore,
		testingJobManagerRunnerFactory,
		testingFatalErrorHandler,
		VoidHistoryServerArchivist.INSTANCE,
		jobGraph,
		executionMode);
}
 
Example #20
Source File: MesosJobClusterEntrypoint.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, MesosJobClusterEntrypoint.class.getSimpleName(), args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	// load configuration incl. dynamic properties
	CommandLineParser parser = new PosixParser();
	CommandLine cmd;
	try {
		cmd = parser.parse(ALL_OPTIONS, args);
	}
	catch (Exception e){
		LOG.error("Could not parse the command-line options.", e);
		System.exit(STARTUP_FAILURE_RETURN_CODE);
		return;
	}

	Configuration dynamicProperties = BootstrapTools.parseDynamicProperties(cmd);
	Configuration configuration = MesosEntrypointUtils.loadConfiguration(dynamicProperties, LOG);

	MesosJobClusterEntrypoint clusterEntrypoint = new MesosJobClusterEntrypoint(configuration, dynamicProperties);

	ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint);
}
 
Example #21
Source File: DispatcherProcess.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Entrypoint of the DispatcherProcessEntryPoint.
 *
 * <p>Other arguments are parsed to a {@link Configuration} and passed to the Dispatcher,
 * for instance: <code>--high-availability ZOOKEEPER --high-availability.zookeeper.quorum
 * "xyz:123:456"</code>.
 */
public static void main(String[] args) {
	try {
		ParameterTool params = ParameterTool.fromArgs(args);
		Configuration config = params.getConfiguration();
		LOG.info("Configuration: {}.", config);

		config.setInteger(JobManagerOptions.PORT, 0);
		config.setString(RestOptions.BIND_PORT, "0");

		final StandaloneSessionClusterEntrypoint clusterEntrypoint = new StandaloneSessionClusterEntrypoint(config);

		ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint);
	}
	catch (Throwable t) {
		LOG.error("Failed to start Dispatcher process", t);
		System.exit(1);
	}
}
 
Example #22
Source File: MiniDispatcher.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<JobResult> requestJobResult(JobID jobId, Time timeout) {
	final CompletableFuture<JobResult> jobResultFuture = super.requestJobResult(jobId, timeout);

	if (executionMode == ClusterEntrypoint.ExecutionMode.NORMAL) {
		// terminate the MiniDispatcher once we served the first JobResult successfully
		jobResultFuture.thenAccept((JobResult result) -> {
			ApplicationStatus status = result.getSerializedThrowable().isPresent() ?
					ApplicationStatus.FAILED : ApplicationStatus.SUCCEEDED;

			jobTerminationFuture.complete(status);
		});
	}

	return jobResultFuture;
}
 
Example #23
Source File: StatefulFunctionsClusterEntryPoint.java    From flink-statefun with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
static void setDefaultExecutionModeIfNotConfigured(Configuration configuration) {
  if (isNoExecutionModeConfigured(configuration)) {
    // In contrast to other places, the default for standalone job clusters is
    // ExecutionMode.DETACHED
    configuration.setString(ClusterEntrypoint.EXECUTION_MODE, ExecutionMode.DETACHED.toString());
  }
}
 
Example #24
Source File: StandaloneJobClusterEntryPoint.java    From flink with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
static void setDefaultExecutionModeIfNotConfigured(Configuration configuration) {
	if (isNoExecutionModeConfigured(configuration)) {
		// In contrast to other places, the default for standalone job clusters is ExecutionMode.DETACHED
		configuration.setString(ClusterEntrypoint.EXECUTION_MODE, ExecutionMode.DETACHED.toString());
	}
}
 
Example #25
Source File: KubernetesSessionClusterEntrypoint.java    From flink with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, KubernetesSessionClusterEntrypoint.class.getSimpleName(), args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	final ClusterEntrypoint entrypoint = new KubernetesSessionClusterEntrypoint(
		KubernetesEntrypointUtils.loadConfiguration());
	ClusterEntrypoint.runClusterEntrypoint(entrypoint);
}
 
Example #26
Source File: MiniDispatcher.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
protected void jobReachedGloballyTerminalState(ArchivedExecutionGraph archivedExecutionGraph) {
	super.jobReachedGloballyTerminalState(archivedExecutionGraph);

	if (executionMode == ClusterEntrypoint.ExecutionMode.DETACHED) {
		// shut down since we don't have to wait for the execution result retrieval
		jobTerminationFuture.complete(ApplicationStatus.fromJobStatus(archivedExecutionGraph.getState()));
	}
}
 
Example #27
Source File: JobDispatcherFactory.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public MiniDispatcher createDispatcher(
		Configuration configuration,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		JobManagerMetricGroup jobManagerMetricGroup,
		@Nullable String metricQueryServicePath,
		ArchivedExecutionGraphStore archivedExecutionGraphStore,
		FatalErrorHandler fatalErrorHandler,
		HistoryServerArchivist historyServerArchivist) throws Exception {
	final JobGraph jobGraph = jobGraphRetriever.retrieveJobGraph(configuration);

	final String executionModeValue = configuration.getString(EXECUTION_MODE);

	final ClusterEntrypoint.ExecutionMode executionMode = ClusterEntrypoint.ExecutionMode.valueOf(executionModeValue);

	return new MiniDispatcher(
		rpcService,
		getEndpointId(),
		configuration,
		highAvailabilityServices,
		resourceManagerGatewayRetriever,
		blobServer,
		heartbeatServices,
		jobManagerMetricGroup,
		metricQueryServicePath,
		archivedExecutionGraphStore,
		DefaultJobManagerRunnerFactory.INSTANCE,
		fatalErrorHandler,
		historyServerArchivist,
		jobGraph,
		executionMode);
}
 
Example #28
Source File: YarnJobClusterEntrypoint.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
	// startup checks and logging
	EnvironmentInformation.logEnvironmentInfo(LOG, YarnJobClusterEntrypoint.class.getSimpleName(), args);
	SignalHandler.register(LOG);
	JvmShutdownSafeguard.installAsShutdownHook(LOG);

	Map<String, String> env = System.getenv();

	final String workingDirectory = env.get(ApplicationConstants.Environment.PWD.key());
	Preconditions.checkArgument(
		workingDirectory != null,
		"Working directory variable (%s) not set",
		ApplicationConstants.Environment.PWD.key());

	try {
		YarnEntrypointUtils.logYarnEnvironmentInformation(env, LOG);
	} catch (IOException e) {
		LOG.warn("Could not log YARN environment information.", e);
	}

	Configuration configuration = YarnEntrypointUtils.loadConfiguration(workingDirectory, env, LOG);

	YarnJobClusterEntrypoint yarnJobClusterEntrypoint = new YarnJobClusterEntrypoint(
		configuration,
		workingDirectory);

	ClusterEntrypoint.runClusterEntrypoint(yarnJobClusterEntrypoint);
}
 
Example #29
Source File: MiniDispatcher.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected void jobReachedGloballyTerminalState(ArchivedExecutionGraph archivedExecutionGraph) {
	super.jobReachedGloballyTerminalState(archivedExecutionGraph);

	if (executionMode == ClusterEntrypoint.ExecutionMode.DETACHED) {
		// shut down since we don't have to wait for the execution result retrieval
		shutDownFuture.complete(ApplicationStatus.fromJobStatus(archivedExecutionGraph.getState()));
	}
}
 
Example #30
Source File: MiniDispatcherTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the {@link MiniDispatcher} recovers the single job with which it
 * was started.
 */
@Test
public void testSingleJobRecovery() throws Exception {
	final MiniDispatcher miniDispatcher = createMiniDispatcher(ClusterEntrypoint.ExecutionMode.DETACHED);

	miniDispatcher.start();

	try {
		final TestingJobManagerRunner testingJobManagerRunner = testingJobManagerRunnerFactory.takeCreatedJobManagerRunner();

		assertThat(testingJobManagerRunner.getJobID(), is(jobGraph.getJobID()));
	} finally {
		RpcUtils.terminateRpcEndpoint(miniDispatcher, timeout);
	}
}