org.apache.flink.runtime.rpc.RpcService Java Examples

The following examples show how to use org.apache.flink.runtime.rpc.RpcService. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MiniCluster.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Factory method to instantiate the RPC service.
 *
 * @param akkaRpcServiceConfig
 *            The default RPC timeout for asynchronous "ask" requests.
 * @param remoteEnabled
 *            True, if the RPC service should be reachable from other (remote) RPC services.
 * @param bindAddress
 *            The address to bind the RPC service to. Only relevant when "remoteEnabled" is true.
 *
 * @return The instantiated RPC service
 */
protected RpcService createRpcService(
		AkkaRpcServiceConfiguration akkaRpcServiceConfig,
		boolean remoteEnabled,
		String bindAddress) {

	final Config akkaConfig;

	if (remoteEnabled) {
		akkaConfig = AkkaUtils.getAkkaConfig(akkaRpcServiceConfig.getConfiguration(), bindAddress, 0);
	} else {
		akkaConfig = AkkaUtils.getAkkaConfig(akkaRpcServiceConfig.getConfiguration());
	}

	final Config effectiveAkkaConfig = AkkaUtils.testDispatcherConfig().withFallback(akkaConfig);

	final ActorSystem actorSystem = AkkaUtils.createActorSystem(effectiveAkkaConfig);

	return new AkkaRpcService(actorSystem, akkaRpcServiceConfig);
}
 
Example #2
Source File: TaskManagerRunnerConfigurationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskManagerRpcServiceShouldBindToConfiguredTaskManagerHostname() throws Exception {
	final String taskmanagerHost = "testhostname";
	final Configuration config = createFlinkConfigWithPredefinedTaskManagerHostname(taskmanagerHost);
	final HighAvailabilityServices highAvailabilityServices = createHighAvailabilityServices(config);

	RpcService taskManagerRpcService = null;
	try {
		taskManagerRpcService = TaskManagerRunner.createRpcService(config, highAvailabilityServices);

		assertThat(taskManagerRpcService.getPort(), is(greaterThanOrEqualTo(0)));
		assertThat(taskManagerRpcService.getAddress(), is(equalTo(taskmanagerHost)));
	} finally {
		maybeCloseRpcService(taskManagerRpcService);
		highAvailabilityServices.closeAndCleanupAllData();
	}
}
 
Example #3
Source File: AkkaRpcActorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that actors are properly terminated when the AkkaRpcService is shut down.
 */
@Test
public void testActorTerminationWhenServiceShutdown() throws Exception {
	final ActorSystem rpcActorSystem = AkkaUtils.createDefaultActorSystem();
	final RpcService rpcService = new AkkaRpcService(
		rpcActorSystem, AkkaRpcServiceConfiguration.defaultConfiguration());

	try {
		SimpleRpcEndpoint rpcEndpoint = new SimpleRpcEndpoint(rpcService, SimpleRpcEndpoint.class.getSimpleName());

		rpcEndpoint.start();

		CompletableFuture<Void> terminationFuture = rpcEndpoint.getTerminationFuture();

		rpcService.stopService();

		terminationFuture.get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);
	} finally {
		rpcActorSystem.terminate();
		Await.ready(rpcActorSystem.whenTerminated(), FutureUtils.toFiniteDuration(timeout));
	}
}
 
Example #4
Source File: TestingResourceManager.java    From flink with Apache License 2.0 6 votes vote down vote up
public TestingResourceManager(
		RpcService rpcService,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		ResourceManagerPartitionTrackerFactory clusterPartitionTrackerFactory,
		JobLeaderIdService jobLeaderIdService,
		FatalErrorHandler fatalErrorHandler,
		ResourceManagerMetricGroup resourceManagerMetricGroup) {
	super(
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		clusterPartitionTrackerFactory,
		jobLeaderIdService,
		new ClusterInformation("localhost", 1234),
		fatalErrorHandler,
		resourceManagerMetricGroup,
		RpcUtils.INF_TIMEOUT);
}
 
Example #5
Source File: JobLeaderService.java    From flink with Apache License 2.0 6 votes vote down vote up
JobManagerRetryingRegistration(
		Logger log,
		RpcService rpcService,
		String targetName,
		Class<JobMasterGateway> targetType,
		String targetAddress,
		JobMasterId jobMasterId,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		String taskManagerRpcAddress,
		TaskManagerLocation taskManagerLocation) {
	super(
		log,
		rpcService,
		targetName,
		targetType,
		targetAddress,
		jobMasterId,
		retryingRegistrationConfiguration);

	this.taskManagerRpcAddress = taskManagerRpcAddress;
	this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
}
 
Example #6
Source File: TaskManagerRunnerStartupTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static void startTaskManager(
	Configuration configuration,
	RpcService rpcService,
	HighAvailabilityServices highAvailabilityServices
) throws Exception {

	TaskManagerRunner.startTaskManager(
		configuration,
		ResourceID.generate(),
		rpcService,
		highAvailabilityServices,
		new TestingHeartbeatServices(),
		NoOpMetricRegistry.INSTANCE,
		new BlobCacheService(
			configuration,
			new VoidBlobStore(),
			null),
		false,
		ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
		error -> {});
}
 
Example #7
Source File: MiniCluster.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private CompletionStage<Void> terminateRpcServices() {
	synchronized (lock) {
		final int numRpcServices = 1 + rpcServices.size();

		final Collection<CompletableFuture<?>> rpcTerminationFutures = new ArrayList<>(numRpcServices);

		rpcTerminationFutures.add(commonRpcService.stopService());

		for (RpcService rpcService : rpcServices) {
			rpcTerminationFutures.add(rpcService.stopService());
		}

		commonRpcService = null;
		rpcServices.clear();

		return FutureUtils.completeAll(rpcTerminationFutures);
	}
}
 
Example #8
Source File: TestingJobManagerRunnerFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public JobManagerRunner createJobManagerRunner(
		JobGraph jobGraph,
		Configuration configuration,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		JobManagerSharedServices jobManagerSharedServices,
		JobManagerJobMetricGroupFactory jobManagerJobMetricGroupFactory,
		FatalErrorHandler fatalErrorHandler) throws Exception {
	final Supplier<Exception> exceptionSupplier = failJobMasterCreationWith.get();

	if (exceptionSupplier != null) {
		throw exceptionSupplier.get();
	} else {
		jobGraphFuture.complete(jobGraph);

		final JobManagerRunner mock = mock(JobManagerRunner.class);
		when(mock.getResultFuture()).thenReturn(resultFuture);
		when(mock.closeAsync()).thenReturn(terminationFuture);
		when(mock.getJobGraph()).thenReturn(jobGraph);

		return mock;
	}
}
 
Example #9
Source File: TaskSubmissionTestEnvironment.java    From flink with Apache License 2.0 6 votes vote down vote up
static void registerJobMasterConnection(
		JobTable jobTable,
		JobID jobId,
		RpcService testingRpcService,
		JobMasterGateway jobMasterGateway,
		TaskManagerActions taskManagerActions,
		Time timeout,
		MainThreadExecutable mainThreadExecutable) {
	mainThreadExecutable.runAsync(() -> {
		final JobTable.Job job = jobTable.getOrCreateJob(jobId, () -> TestingJobServices.newBuilder().build());
		job.connect(
			ResourceID.generate(),
			jobMasterGateway,
			taskManagerActions,
			new TestCheckpointResponder(),
			new TestGlobalAggregateManager(),
			new RpcResultPartitionConsumableNotifier(jobMasterGateway, testingRpcService.getExecutor(), timeout),
			TestingPartitionProducerStateChecker.newBuilder()
				.setPartitionProducerStateFunction((jobID, intermediateDataSetID, resultPartitionID) -> CompletableFuture.completedFuture(ExecutionState.RUNNING))
				.build());
	});
}
 
Example #10
Source File: TestingTaskExecutor.java    From flink with Apache License 2.0 6 votes vote down vote up
public TestingTaskExecutor(
		RpcService rpcService,
		TaskManagerConfiguration taskManagerConfiguration,
		HighAvailabilityServices haServices,
		TaskManagerServices taskExecutorServices,
		HeartbeatServices heartbeatServices,
		TaskManagerMetricGroup taskManagerMetricGroup,
		@Nullable String metricQueryServiceAddress,
		BlobCacheService blobCacheService,
		FatalErrorHandler fatalErrorHandler,
		PartitionTable<JobID> partitionTable) {
	super(
		rpcService,
		taskManagerConfiguration,
		haServices,
		taskExecutorServices,
		heartbeatServices,
		taskManagerMetricGroup,
		metricQueryServiceAddress,
		blobCacheService,
		fatalErrorHandler,
		partitionTable);
}
 
Example #11
Source File: TaskExecutorToResourceManagerConnection.java    From flink with Apache License 2.0 6 votes vote down vote up
ResourceManagerRegistration(
		Logger log,
		RpcService rpcService,
		String targetAddress,
		ResourceManagerId resourceManagerId,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		String taskExecutorAddress,
		ResourceID resourceID,
		int dataPort,
		HardwareDescription hardwareDescription) {

	super(log, rpcService, "ResourceManager", ResourceManagerGateway.class, targetAddress, resourceManagerId, retryingRegistrationConfiguration);
	this.taskExecutorAddress = checkNotNull(taskExecutorAddress);
	this.resourceID = checkNotNull(resourceID);
	this.dataPort = dataPort;
	this.hardwareDescription = checkNotNull(hardwareDescription);
}
 
Example #12
Source File: RetryingRegistration.java    From flink with Apache License 2.0 6 votes vote down vote up
public RetryingRegistration(
		Logger log,
		RpcService rpcService,
		String targetName,
		Class<G> targetType,
		String targetAddress,
		F fencingToken,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration) {

	this.log = checkNotNull(log);
	this.rpcService = checkNotNull(rpcService);
	this.targetName = checkNotNull(targetName);
	this.targetType = checkNotNull(targetType);
	this.targetAddress = checkNotNull(targetAddress);
	this.fencingToken = checkNotNull(fencingToken);
	this.retryingRegistrationConfiguration = checkNotNull(retryingRegistrationConfiguration);

	this.completionFuture = new CompletableFuture<>();
}
 
Example #13
Source File: TaskManagerRunner.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Create a RPC service for the task manager.
 *
 * @param configuration The configuration for the TaskManager.
 * @param haServices to use for the task manager hostname retrieval
 */
@VisibleForTesting
static RpcService createRpcService(
	final Configuration configuration,
	final HighAvailabilityServices haServices) throws Exception {

	checkNotNull(configuration);
	checkNotNull(haServices);

	return AkkaRpcServiceUtils.createRemoteRpcService(
		configuration,
		determineTaskManagerBindAddress(configuration, haServices),
		configuration.getString(TaskManagerOptions.RPC_PORT),
		configuration.getString(TaskManagerOptions.BIND_HOST),
		configuration.getOptional(TaskManagerOptions.RPC_BIND_PORT));
}
 
Example #14
Source File: TaskExecutorToResourceManagerConnection.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public TaskExecutorToResourceManagerConnection(
		Logger log,
		RpcService rpcService,
		String taskManagerAddress,
		ResourceID taskManagerResourceId,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		int dataPort,
		HardwareDescription hardwareDescription,
		String resourceManagerAddress,
		ResourceManagerId resourceManagerId,
		Executor executor,
		RegistrationConnectionListener<TaskExecutorToResourceManagerConnection, TaskExecutorRegistrationSuccess> registrationListener) {

	super(log, resourceManagerAddress, resourceManagerId, executor);

	this.rpcService = checkNotNull(rpcService);
	this.taskManagerAddress = checkNotNull(taskManagerAddress);
	this.taskManagerResourceId = checkNotNull(taskManagerResourceId);
	this.retryingRegistrationConfiguration = checkNotNull(retryingRegistrationConfiguration);
	this.dataPort = dataPort;
	this.hardwareDescription = checkNotNull(hardwareDescription);
	this.registrationListener = checkNotNull(registrationListener);
}
 
Example #15
Source File: StandaloneResourceManager.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public StandaloneResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		jobManagerMetricGroup);
}
 
Example #16
Source File: ActiveResourceManagerFactoryTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected ResourceManager<ResourceID> createActiveResourceManager(
		Configuration configuration,
		ResourceID resourceId,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		MetricRegistry metricRegistry,
		FatalErrorHandler fatalErrorHandler,
		ClusterInformation clusterInformation,
		@Nullable String webInterfaceUrl,
		JobManagerMetricGroup jobManagerMetricGroup) {
	assertThat(configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE), is(true));

	return null;
}
 
Example #17
Source File: AkkaRpcActorTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that actors are properly terminated when the AkkaRpcService is shut down.
 */
@Test
public void testActorTerminationWhenServiceShutdown() throws Exception {
	final ActorSystem rpcActorSystem = AkkaUtils.createDefaultActorSystem();
	final RpcService rpcService = new AkkaRpcService(
		rpcActorSystem, AkkaRpcServiceConfiguration.defaultConfiguration());

	try {
		SimpleRpcEndpoint rpcEndpoint = new SimpleRpcEndpoint(rpcService, SimpleRpcEndpoint.class.getSimpleName());

		rpcEndpoint.start();

		CompletableFuture<Void> terminationFuture = rpcEndpoint.getTerminationFuture();

		rpcService.stopService();

		terminationFuture.get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);
	} finally {
		rpcActorSystem.terminate();
		Await.ready(rpcActorSystem.whenTerminated(), FutureUtils.toFiniteDuration(timeout));
	}
}
 
Example #18
Source File: DispatcherTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public JobManagerRunner createJobManagerRunner(
		JobGraph jobGraph,
		Configuration configuration,
		RpcService rpcService,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		JobManagerSharedServices jobManagerSharedServices,
		JobManagerJobMetricGroupFactory jobManagerJobMetricGroupFactory,
		FatalErrorHandler fatalErrorHandler) throws Exception {
	assertEquals(expectedJobId, jobGraph.getJobID());

	createdJobManagerRunnerLatch.countDown();

	return DefaultJobManagerRunnerFactory.INSTANCE.createJobManagerRunner(
		jobGraph,
		configuration,
		rpcService,
		highAvailabilityServices,
		heartbeatServices,
		jobManagerSharedServices,
		jobManagerJobMetricGroupFactory,
		fatalErrorHandler);
}
 
Example #19
Source File: StandaloneResourceManagerFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected ResourceManager<ResourceID> createResourceManager(
	Configuration configuration,
	ResourceID resourceId,
	RpcService rpcService,
	HighAvailabilityServices highAvailabilityServices,
	HeartbeatServices heartbeatServices,
	FatalErrorHandler fatalErrorHandler,
	ClusterInformation clusterInformation,
	@Nullable String webInterfaceUrl,
	ResourceManagerMetricGroup resourceManagerMetricGroup,
	ResourceManagerRuntimeServices resourceManagerRuntimeServices) {

	final Time standaloneClusterStartupPeriodTime = ConfigurationUtils.getStandaloneClusterStartupPeriodTime(configuration);

	return new StandaloneResourceManager(
		rpcService,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		resourceManagerRuntimeServices.getSlotManager(),
		ResourceManagerPartitionTrackerImpl::new,
		resourceManagerRuntimeServices.getJobLeaderIdService(),
		clusterInformation,
		fatalErrorHandler,
		resourceManagerMetricGroup,
		standaloneClusterStartupPeriodTime,
		AkkaUtils.getTimeoutAsTime(configuration));
}
 
Example #20
Source File: TaskManagerRunnerConfigurationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskManagerRpcServiceShouldBindToHostnameAddress() throws Exception {
	final Configuration config = createFlinkConfigWithHostBindPolicy(HostBindPolicy.NAME);
	final HighAvailabilityServices highAvailabilityServices = createHighAvailabilityServices(config);

	RpcService taskManagerRpcService = null;
	try {
		taskManagerRpcService = TaskManagerRunner.createRpcService(config, highAvailabilityServices);
		assertThat(taskManagerRpcService.getAddress(), not(isEmptyOrNullString()));
	} finally {
		maybeCloseRpcService(taskManagerRpcService);
		highAvailabilityServices.closeAndCleanupAllData();
	}
}
 
Example #21
Source File: StandaloneResourceManager.java    From flink with Apache License 2.0 5 votes vote down vote up
public StandaloneResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		JobManagerMetricGroup jobManagerMetricGroup,
		Time startupPeriodTime) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		jobManagerMetricGroup);
	this.startupPeriodTime = Preconditions.checkNotNull(startupPeriodTime);
}
 
Example #22
Source File: DispatcherHATest.java    From flink with Apache License 2.0 5 votes vote down vote up
HATestingDispatcher(
		RpcService rpcService,
		String endpointId,
		Configuration configuration,
		HighAvailabilityServices highAvailabilityServices,
		GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		JobManagerMetricGroup jobManagerMetricGroup,
		@Nullable String metricQueryServiceAddress,
		ArchivedExecutionGraphStore archivedExecutionGraphStore,
		JobManagerRunnerFactory jobManagerRunnerFactory,
		FatalErrorHandler fatalErrorHandler,
		@Nonnull Queue<DispatcherId> fencingTokens) throws Exception {
	super(
		rpcService,
		endpointId,
		configuration,
		highAvailabilityServices,
		resourceManagerGatewayRetriever,
		blobServer,
		heartbeatServices,
		jobManagerMetricGroup,
		metricQueryServiceAddress,
		archivedExecutionGraphStore,
		jobManagerRunnerFactory,
		fatalErrorHandler);
	this.fencingTokens = fencingTokens;
}
 
Example #23
Source File: StandaloneDispatcher.java    From flink with Apache License 2.0 5 votes vote down vote up
public StandaloneDispatcher(
		RpcService rpcService,
		String endpointId,
		Configuration configuration,
		HighAvailabilityServices highAvailabilityServices,
		GatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
		BlobServer blobServer,
		HeartbeatServices heartbeatServices,
		JobManagerMetricGroup jobManagerMetricGroup,
		@Nullable String metricQueryServiceAddress,
		ArchivedExecutionGraphStore archivedExecutionGraphStore,
		JobManagerRunnerFactory jobManagerRunnerFactory,
		FatalErrorHandler fatalErrorHandler,
		HistoryServerArchivist historyServerArchivist) throws Exception {
	super(
		rpcService,
		endpointId,
		configuration,
		highAvailabilityServices,
		highAvailabilityServices.getSubmittedJobGraphStore(),
		resourceManagerGatewayRetriever,
		blobServer,
		heartbeatServices,
		jobManagerMetricGroup,
		metricQueryServiceAddress,
		archivedExecutionGraphStore,
		jobManagerRunnerFactory,
		fatalErrorHandler,
		historyServerArchivist);
}
 
Example #24
Source File: ResourceManagerFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
protected abstract ResourceManager<T> createResourceManager(
Configuration configuration,
ResourceID resourceId,
RpcService rpcService,
HighAvailabilityServices highAvailabilityServices,
HeartbeatServices heartbeatServices,
FatalErrorHandler fatalErrorHandler,
ClusterInformation clusterInformation,
@Nullable String webInterfaceUrl,
ResourceManagerMetricGroup resourceManagerMetricGroup,
ResourceManagerRuntimeServices resourceManagerRuntimeServices) throws Exception;
 
Example #25
Source File: ResourceManager.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public ResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		JobManagerMetricGroup jobManagerMetricGroup) {

	super(rpcService, resourceManagerEndpointId);

	this.resourceId = checkNotNull(resourceId);
	this.highAvailabilityServices = checkNotNull(highAvailabilityServices);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.slotManager = checkNotNull(slotManager);
	this.metricRegistry = checkNotNull(metricRegistry);
	this.jobLeaderIdService = checkNotNull(jobLeaderIdService);
	this.clusterInformation = checkNotNull(clusterInformation);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.jobManagerMetricGroup = checkNotNull(jobManagerMetricGroup);

	this.jobManagerRegistrations = new HashMap<>(4);
	this.jmResourceIdRegistrations = new HashMap<>(4);
	this.taskExecutors = new HashMap<>(8);
	infoMessageListeners = new ConcurrentHashMap<>(8);
	this.taskExecutorGatewayFutures = new HashMap<>(8);
}
 
Example #26
Source File: ResourceManagerFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
ResourceManager<T> createResourceManager(
Configuration configuration,
ResourceID resourceId,
RpcService rpcService,
HighAvailabilityServices highAvailabilityServices,
HeartbeatServices heartbeatServices,
MetricRegistry metricRegistry,
FatalErrorHandler fatalErrorHandler,
ClusterInformation clusterInformation,
@Nullable String webInterfaceUrl,
JobManagerMetricGroup jobManagerMetricGroup) throws Exception;
 
Example #27
Source File: YarnResourceManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
TestingYarnResourceManager(
		RpcService rpcService,
		ResourceID resourceId,
		Configuration flinkConfig,
		Map<String, String> env,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		@Nullable String webInterfaceUrl,
		ResourceManagerMetricGroup resourceManagerMetricGroup) {
	super(
		rpcService,
		resourceId,
		flinkConfig,
		env,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		NoOpResourceManagerPartitionTracker::get,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		webInterfaceUrl,
		resourceManagerMetricGroup);
	this.testingYarnNMClientAsync = new TestingYarnNMClientAsync(this);
	this.testingYarnAMRMClientAsync = new TestingYarnAMRMClientAsync(this);
}
 
Example #28
Source File: TaskSubmissionTestEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up
private static ShuffleEnvironment<?, ?> createShuffleEnvironment(
		ResourceID taskManagerLocation,
		boolean localCommunication,
		Configuration configuration,
		RpcService testingRpcService,
		boolean mockShuffleEnvironment) throws Exception {
	final ShuffleEnvironment<?, ?> shuffleEnvironment;
	if (mockShuffleEnvironment) {
		shuffleEnvironment = mock(ShuffleEnvironment.class, Mockito.RETURNS_MOCKS);
	} else {
		final InetSocketAddress socketAddress = new InetSocketAddress(
			InetAddress.getByName(testingRpcService.getAddress()), configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT));

		final NettyConfig nettyConfig = new NettyConfig(socketAddress.getAddress(), socketAddress.getPort(),
			ConfigurationParserUtils.getPageSize(configuration), ConfigurationParserUtils.getSlot(configuration), configuration);

		shuffleEnvironment =  new NettyShuffleEnvironmentBuilder()
			.setTaskManagerLocation(taskManagerLocation)
			.setPartitionRequestInitialBackoff(configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_INITIAL))
			.setPartitionRequestMaxBackoff(configuration.getInteger(NettyShuffleEnvironmentOptions.NETWORK_REQUEST_BACKOFF_MAX))
			.setNettyConfig(localCommunication ? null : nettyConfig)
			.build();

		shuffleEnvironment.start();
	}

	return shuffleEnvironment;
}
 
Example #29
Source File: TaskManagerRunnerConfigurationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskManagerRpcServiceShouldBindToHostnameAddress() throws Exception {
	final Configuration config = createFlinkConfigWithHostBindPolicy(HostBindPolicy.NAME);
	final HighAvailabilityServices highAvailabilityServices = createHighAvailabilityServices(config);

	RpcService taskManagerRpcService = null;
	try {
		taskManagerRpcService = TaskManagerRunner.createRpcService(config, highAvailabilityServices);
		assertThat(taskManagerRpcService.getAddress(), not(isEmptyOrNullString()));
	} finally {
		maybeCloseRpcService(taskManagerRpcService);
		highAvailabilityServices.closeAndCleanupAllData();
	}
}
 
Example #30
Source File: MiniCluster.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public RpcService createRpcService() {
	final RpcService rpcService = MiniCluster.this.createRpcService(akkaRpcServiceConfig, true, jobManagerBindAddress);

	synchronized (lock) {
		rpcServices.add(rpcService);
	}

	return rpcService;
}