org.apache.flink.runtime.jobmaster.JobMaster Java Examples

The following examples show how to use org.apache.flink.runtime.jobmaster.JobMaster. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DefaultJobMasterServiceFactory.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public JobMaster createJobMasterService(JobGraph jobGraph, OnCompletionActions jobCompletionActions, ClassLoader userCodeClassloader) throws Exception {
	return new JobMaster(
		rpcService,
		jobMasterConfiguration,
		ResourceID.generate(),
		jobGraph,
		haServices,
		slotPoolFactory,
		schedulerFactory,
		jobManagerSharedServices,
		heartbeatServices,
		jobManagerJobMetricGroupFactory,
		jobCompletionActions,
		fatalErrorHandler,
		userCodeClassloader);
}
Example #2
Source File: StandaloneUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Creates a {@link StandaloneLeaderRetrievalService} form the given configuration and the
 * JobManager name. The host and port for the remote Akka URL are retrieved from the provided
 * configuration. Instead of using the standard JobManager Akka name, the provided one is used
 * for the remote Akka URL.
 *
 * @param configuration Configuration instance containing hte host and port information
 * @param resolveInitialHostName If true, resolves the hostname of the StandaloneLeaderRetrievalService
 * @param jobManagerName Name of the JobManager actor
 * @return StandaloneLeaderRetrievalService
 * @throws ConfigurationException if the job manager address cannot be retrieved from the configuration
 * @throws UnknownHostException if the job manager address cannot be resolved
 */
public static StandaloneLeaderRetrievalService createLeaderRetrievalService(
		Configuration configuration,
		boolean resolveInitialHostName,
		String jobManagerName)
	throws ConfigurationException, UnknownHostException {
	Tuple2<String, Integer> hostnamePort = HighAvailabilityServicesUtils.getJobManagerAddress(configuration);

	String jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
		hostnamePort.f0,
		hostnamePort.f1,
		jobManagerName != null ? jobManagerName : JobMaster.JOB_MANAGER_NAME,
		resolveInitialHostName ? AddressResolution.TRY_ADDRESS_RESOLUTION : AddressResolution.NO_ADDRESS_RESOLUTION,
		configuration);

	return new StandaloneLeaderRetrievalService(jobManagerAkkaUrl);
}
Example #3
Source File: LeaderRetrievalServiceHostnameResolutionTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testUnresolvableHostname2() throws Exception {

	try {
		Configuration config = new Configuration();

		config.setString(JobManagerOptions.ADDRESS, nonExistingHostname);
		config.setInteger(JobManagerOptions.PORT, 17234);

		StandaloneUtils.createLeaderRetrievalService(
			config,
			true,
			JobMaster.JOB_MANAGER_NAME);
		fail("This should fail with an UnknownHostException");
	}
	catch (UnknownHostException e) {
		// that is what we want!
	}
}
Example #4
Source File: StandaloneUtils.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Creates a {@link StandaloneLeaderRetrievalService} form the given configuration and the
 * JobManager name. The host and port for the remote Akka URL are retrieved from the provided
 * configuration. Instead of using the standard JobManager Akka name, the provided one is used
 * for the remote Akka URL.
 *
 * @param configuration Configuration instance containing hte host and port information
 * @param resolveInitialHostName If true, resolves the hostname of the StandaloneLeaderRetrievalService
 * @param jobManagerName Name of the JobManager actor
 * @return StandaloneLeaderRetrievalService
 * @throws ConfigurationException if the job manager address cannot be retrieved from the configuration
 * @throws UnknownHostException if the job manager address cannot be resolved
 */
public static StandaloneLeaderRetrievalService createLeaderRetrievalService(
		Configuration configuration,
		boolean resolveInitialHostName,
		String jobManagerName)
	throws ConfigurationException, UnknownHostException {
	Tuple2<String, Integer> hostnamePort = HighAvailabilityServicesUtils.getJobManagerAddress(configuration);

	String jobManagerAkkaUrl = AkkaRpcServiceUtils.getRpcUrl(
		hostnamePort.f0,
		hostnamePort.f1,
		jobManagerName != null ? jobManagerName : JobMaster.JOB_MANAGER_NAME,
		resolveInitialHostName ? AddressResolution.TRY_ADDRESS_RESOLUTION : AddressResolution.NO_ADDRESS_RESOLUTION,
		configuration);

	return new StandaloneLeaderRetrievalService(jobManagerAkkaUrl);
}
Example #5
Source File: LeaderRetrievalServiceHostnameResolutionTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testUnresolvableHostname2() throws Exception {

	try {
		Configuration config = new Configuration();

		config.setString(JobManagerOptions.ADDRESS, nonExistingHostname);
		config.setInteger(JobManagerOptions.PORT, 17234);

		StandaloneUtils.createLeaderRetrievalService(
			config,
			true,
			JobMaster.JOB_MANAGER_NAME);
		fail("This should fail with an UnknownHostException");
	}
	catch (UnknownHostException e) {
		// that is what we want!
	}
}
Example #6
Source File: JobMasterBuilder.java    From flink with Apache License 2.0 6 votes vote down vote up 
public JobMaster createJobMaster() throws Exception {
	final JobMasterConfiguration jobMasterConfiguration = JobMasterConfiguration.fromConfiguration(configuration);

	return new JobMaster(
		rpcService,
		jobMasterConfiguration,
		jmResourceId,
		jobGraph,
		highAvailabilityServices,
		slotPoolFactory != null ? slotPoolFactory : DefaultSlotPoolFactory.fromConfiguration(configuration),
		schedulerFactory != null ? schedulerFactory : DefaultSchedulerFactory.fromConfiguration(configuration),
		jobManagerSharedServices,
		heartbeatServices,
		UnregisteredJobManagerJobMetricGroupFactory.INSTANCE,
		onCompletionActions,
		fatalErrorHandler,
		JobMasterBuilder.class.getClassLoader(),
		SchedulerNGFactoryFactory.createSchedulerNGFactory(configuration),
		shuffleMaster,
		partitionTrackerFactory);
}
Example #7
Source File: LeaderRetrievalServiceHostnameResolutionTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testUnresolvableHostname1() throws UnknownHostException, ConfigurationException {
	Configuration config = new Configuration();

	config.setString(JobManagerOptions.ADDRESS, nonExistingHostname);
	config.setInteger(JobManagerOptions.PORT, 17234);

	StandaloneUtils.createLeaderRetrievalService(
		config,
		false,
		JobMaster.JOB_MANAGER_NAME);
}
Example #8
Source File: DefaultJobMasterServiceFactory.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public JobMaster createJobMasterService(
		JobGraph jobGraph,
		OnCompletionActions jobCompletionActions,
		ClassLoader userCodeClassloader) throws Exception {

	return new JobMaster(
		rpcService,
		jobMasterConfiguration,
		ResourceID.generate(),
		jobGraph,
		haServices,
		slotPoolFactory,
		schedulerFactory,
		jobManagerSharedServices,
		heartbeatServices,
		jobManagerJobMetricGroupFactory,
		jobCompletionActions,
		fatalErrorHandler,
		userCodeClassloader,
		schedulerNGFactory,
		shuffleMaster,
		lookup -> new PartitionTrackerImpl(
			jobGraph.getJobID(),
			shuffleMaster,
			lookup
		));
}
Example #9
Source File: LeaderRetrievalServiceHostnameResolutionTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testUnresolvableHostname1() throws UnknownHostException, ConfigurationException {
	Configuration config = new Configuration();

	config.setString(JobManagerOptions.ADDRESS, nonExistingHostname);
	config.setInteger(JobManagerOptions.PORT, 17234);

	StandaloneUtils.createLeaderRetrievalService(
		config,
		false,
		JobMaster.JOB_MANAGER_NAME);
}
Example #10
Source File: DefaultJobMasterServiceFactory.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public JobMaster createJobMasterService(
		JobGraph jobGraph,
		OnCompletionActions jobCompletionActions,
		ClassLoader userCodeClassloader) throws Exception {

	return new JobMaster(
		rpcService,
		jobMasterConfiguration,
		ResourceID.generate(),
		jobGraph,
		haServices,
		slotPoolFactory,
		schedulerFactory,
		jobManagerSharedServices,
		heartbeatServices,
		jobManagerJobMetricGroupFactory,
		jobCompletionActions,
		fatalErrorHandler,
		userCodeClassloader,
		schedulerNGFactory,
		shuffleMaster,
		lookup -> new JobMasterPartitionTrackerImpl(
			jobGraph.getJobID(),
			shuffleMaster,
			lookup
		));
}
Example #11
Source File: HighAvailabilityServicesUtils.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = LeaderRetrievalUtils.getRecoveryMode(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				JobMaster.JOB_MANAGER_NAME,
				addressResolution,
				configuration);
			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				ResourceManager.RESOURCE_MANAGER_NAME,
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				Dispatcher.DISPATCHER_NAME,
				addressResolution,
				configuration);

			final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
				"%s must be set",
				RestOptions.ADDRESS.key());
			final int port = configuration.getInteger(RestOptions.PORT);
			final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
			final String protocol = enableSSL ? "https://" : "http://";

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				jobManagerRpcUrl,
				String.format("%s%s:%s", protocol, address, port));
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}
Example #12
Source File: HighAvailabilityServicesUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
public static HighAvailabilityServices createHighAvailabilityServices(
	Configuration configuration,
	Executor executor,
	AddressResolution addressResolution) throws Exception {

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			final Tuple2<String, Integer> hostnamePort = getJobManagerAddress(configuration);

			final String jobManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				JobMaster.JOB_MANAGER_NAME,
				addressResolution,
				configuration);
			final String resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				ResourceManager.RESOURCE_MANAGER_NAME,
				addressResolution,
				configuration);
			final String dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl(
				hostnamePort.f0,
				hostnamePort.f1,
				Dispatcher.DISPATCHER_NAME,
				addressResolution,
				configuration);

			final String address = checkNotNull(configuration.getString(RestOptions.ADDRESS),
				"%s must be set",
				RestOptions.ADDRESS.key());
			final int port = configuration.getInteger(RestOptions.PORT);
			final boolean enableSSL = SSLUtils.isRestSSLEnabled(configuration);
			final String protocol = enableSSL ? "https://" : "http://";

			return new StandaloneHaServices(
				resourceManagerRpcUrl,
				dispatcherRpcUrl,
				jobManagerRpcUrl,
				String.format("%s%s:%s", protocol, address, port));
		case ZOOKEEPER:
			BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(configuration);

			return new ZooKeeperHaServices(
				ZooKeeperUtils.startCuratorFramework(configuration),
				executor,
				configuration,
				blobStoreService);

		case FACTORY_CLASS:
			return createCustomHAServices(configuration, executor);

		default:
			throw new Exception("Recovery mode " + highAvailabilityMode + " is not supported.");
	}
}