org.apache.flink.runtime.registration.RetryingRegistrationConfiguration Java Examples

The following examples show how to use org.apache.flink.runtime.registration.RetryingRegistrationConfiguration. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TaskExecutorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private TaskExecutorTestingContext createTaskExecutorTestingContext(final TaskSlotTable<Task> taskSlotTable) throws IOException {
	final OneShotLatch offerSlotsLatch = new OneShotLatch();
	final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder()
		.setOfferSlotsFunction((resourceID, slotOffers) -> {
			offerSlotsLatch.trigger();
			return CompletableFuture.completedFuture(slotOffers);
		}).build();
	rpc.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);

	final JobLeaderService jobLeaderService = new DefaultJobLeaderService(
		unresolvedTaskManagerLocation,
		RetryingRegistrationConfiguration.defaultConfiguration());

	TaskExecutorLocalStateStoresManager stateStoresManager = createTaskExecutorLocalStateStoresManager();
	final TestingTaskExecutor taskExecutor = createTestingTaskExecutor(new TaskManagerServicesBuilder()
		.setTaskSlotTable(taskSlotTable)
		.setJobLeaderService(jobLeaderService)
		.setTaskStateManager(stateStoresManager)
		.build());

	jobManagerLeaderRetriever.notifyListener(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());
	return new TaskExecutorTestingContext(jobMasterGateway, taskSlotTable, taskExecutor);
}
 
Example #2
Source File: TaskExecutorToResourceManagerConnection.java    From flink with Apache License 2.0 6 votes vote down vote up
public TaskExecutorToResourceManagerConnection(
		Logger log,
		RpcService rpcService,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		String resourceManagerAddress,
		ResourceManagerId resourceManagerId,
		Executor executor,
		RegistrationConnectionListener<TaskExecutorToResourceManagerConnection, TaskExecutorRegistrationSuccess> registrationListener,
		TaskExecutorRegistration taskExecutorRegistration) {

	super(log, resourceManagerAddress, resourceManagerId, executor);

	this.rpcService = checkNotNull(rpcService);
	this.retryingRegistrationConfiguration = checkNotNull(retryingRegistrationConfiguration);
	this.registrationListener = checkNotNull(registrationListener);
	this.taskExecutorRegistration = checkNotNull(taskExecutorRegistration);
}
 
Example #3
Source File: JobMasterConfiguration.java    From flink with Apache License 2.0 6 votes vote down vote up
public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}
 
Example #4
Source File: DefaultJobLeaderService.java    From flink with Apache License 2.0 6 votes vote down vote up
public DefaultJobLeaderService(
		UnresolvedTaskManagerLocation location,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration) {
	this.ownLocation = Preconditions.checkNotNull(location);
	this.retryingRegistrationConfiguration = Preconditions.checkNotNull(retryingRegistrationConfiguration);

	// Has to be a concurrent hash map because tests might access this service
	// concurrently via containsJob
	jobLeaderServices = new ConcurrentHashMap<>(4);

	state = DefaultJobLeaderService.State.CREATED;

	ownerAddress = null;
	rpcService = null;
	highAvailabilityServices = null;
	jobLeaderListener = null;
}
 
Example #5
Source File: DefaultJobLeaderService.java    From flink with Apache License 2.0 6 votes vote down vote up
JobManagerRetryingRegistration(
		Logger log,
		RpcService rpcService,
		String targetName,
		Class<JobMasterGateway> targetType,
		String targetAddress,
		JobMasterId jobMasterId,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		String taskManagerRpcAddress,
		UnresolvedTaskManagerLocation unresolvedTaskManagerLocation) {
	super(
		log,
		rpcService,
		targetName,
		targetType,
		targetAddress,
		jobMasterId,
		retryingRegistrationConfiguration);

	this.taskManagerRpcAddress = taskManagerRpcAddress;
	this.unresolvedTaskManagerLocation = Preconditions.checkNotNull(unresolvedTaskManagerLocation);
}
 
Example #6
Source File: TaskManagerServicesBuilder.java    From flink with Apache License 2.0 6 votes vote down vote up
public TaskManagerServicesBuilder() {
	taskManagerLocation = new LocalTaskManagerLocation();
	memoryManager = new MemoryManager(
		MemoryManager.MIN_PAGE_SIZE,
		1,
		MemoryManager.MIN_PAGE_SIZE,
		MemoryType.HEAP,
		false);
	ioManager = mock(IOManager.class);
	shuffleEnvironment = mock(ShuffleEnvironment.class);
	kvStateService = new KvStateService(new KvStateRegistry(), null, null);
	broadcastVariableManager = new BroadcastVariableManager();
	taskEventDispatcher = new TaskEventDispatcher();
	taskSlotTable = mock(TaskSlotTable.class);
	jobManagerTable = new JobManagerTable();
	jobLeaderService = new JobLeaderService(taskManagerLocation, RetryingRegistrationConfiguration.defaultConfiguration());
	taskStateManager = mock(TaskExecutorLocalStateStoresManager.class);
}
 
Example #7
Source File: JobLeaderService.java    From flink with Apache License 2.0 6 votes vote down vote up
JobManagerRetryingRegistration(
		Logger log,
		RpcService rpcService,
		String targetName,
		Class<JobMasterGateway> targetType,
		String targetAddress,
		JobMasterId jobMasterId,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		String taskManagerRpcAddress,
		TaskManagerLocation taskManagerLocation) {
	super(
		log,
		rpcService,
		targetName,
		targetType,
		targetAddress,
		jobMasterId,
		retryingRegistrationConfiguration);

	this.taskManagerRpcAddress = taskManagerRpcAddress;
	this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
}
 
Example #8
Source File: JobLeaderService.java    From flink with Apache License 2.0 6 votes vote down vote up
public JobLeaderService(
		TaskManagerLocation location,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration) {
	this.ownLocation = Preconditions.checkNotNull(location);
	this.retryingRegistrationConfiguration = Preconditions.checkNotNull(retryingRegistrationConfiguration);

	// Has to be a concurrent hash map because tests might access this service
	// concurrently via containsJob
	jobLeaderServices = new ConcurrentHashMap<>(4);

	state = JobLeaderService.State.CREATED;

	ownerAddress = null;
	rpcService = null;
	highAvailabilityServices = null;
	jobLeaderListener = null;
}
 
Example #9
Source File: TaskExecutorToResourceManagerConnection.java    From flink with Apache License 2.0 6 votes vote down vote up
ResourceManagerRegistration(
		Logger log,
		RpcService rpcService,
		String targetAddress,
		ResourceManagerId resourceManagerId,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		String taskExecutorAddress,
		ResourceID resourceID,
		int dataPort,
		HardwareDescription hardwareDescription) {

	super(log, rpcService, "ResourceManager", ResourceManagerGateway.class, targetAddress, resourceManagerId, retryingRegistrationConfiguration);
	this.taskExecutorAddress = checkNotNull(taskExecutorAddress);
	this.resourceID = checkNotNull(resourceID);
	this.dataPort = dataPort;
	this.hardwareDescription = checkNotNull(hardwareDescription);
}
 
Example #10
Source File: TaskExecutorToResourceManagerConnection.java    From flink with Apache License 2.0 6 votes vote down vote up
public TaskExecutorToResourceManagerConnection(
		Logger log,
		RpcService rpcService,
		String taskManagerAddress,
		ResourceID taskManagerResourceId,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		int dataPort,
		HardwareDescription hardwareDescription,
		String resourceManagerAddress,
		ResourceManagerId resourceManagerId,
		Executor executor,
		RegistrationConnectionListener<TaskExecutorToResourceManagerConnection, TaskExecutorRegistrationSuccess> registrationListener) {

	super(log, resourceManagerAddress, resourceManagerId, executor);

	this.rpcService = checkNotNull(rpcService);
	this.taskManagerAddress = checkNotNull(taskManagerAddress);
	this.taskManagerResourceId = checkNotNull(taskManagerResourceId);
	this.retryingRegistrationConfiguration = checkNotNull(retryingRegistrationConfiguration);
	this.dataPort = dataPort;
	this.hardwareDescription = checkNotNull(hardwareDescription);
	this.registrationListener = checkNotNull(registrationListener);
}
 
Example #11
Source File: JobMasterConfiguration.java    From flink with Apache License 2.0 6 votes vote down vote up
public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}
 
Example #12
Source File: TaskExecutorToResourceManagerConnectionTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private TaskExecutorToResourceManagerConnection createTaskExecutorToResourceManagerConnection() {
	final TaskExecutorRegistration taskExecutorRegistration = new TaskExecutorRegistration(
		TASK_MANAGER_ADDRESS,
		TASK_MANAGER_RESOURCE_ID,
		TASK_MANAGER_DATA_PORT,
		TASK_MANAGER_HARDWARE_DESCRIPTION,
		ResourceProfile.ZERO,
		ResourceProfile.ZERO
	);
	return new TaskExecutorToResourceManagerConnection(
		LOGGER,
		rpcService,
		RetryingRegistrationConfiguration.defaultConfiguration(),
		RESOURCE_MANAGER_ADDRESS,
		RESOURCE_MANAGER_ID,
		Executors.directExecutor(),
		new TestRegistrationConnectionListener<>(),
		taskExecutorRegistration);
}
 
Example #13
Source File: TaskManagerServicesBuilder.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public TaskManagerServicesBuilder() {
	taskManagerLocation = new LocalTaskManagerLocation();
	memoryManager = new MemoryManager(
		MemoryManager.MIN_PAGE_SIZE,
		1,
		MemoryManager.MIN_PAGE_SIZE,
		MemoryType.HEAP,
		false);
	ioManager = mock(IOManager.class);
	networkEnvironment = mock(NetworkEnvironment.class);
	broadcastVariableManager = new BroadcastVariableManager();
	taskSlotTable = mock(TaskSlotTable.class);
	jobManagerTable = new JobManagerTable();
	jobLeaderService = new JobLeaderService(taskManagerLocation, RetryingRegistrationConfiguration.defaultConfiguration());
	taskStateManager = mock(TaskExecutorLocalStateStoresManager.class);
}
 
Example #14
Source File: JobMasterConfiguration.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}
 
Example #15
Source File: TaskExecutorToResourceManagerConnection.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public TaskExecutorToResourceManagerConnection(
		Logger log,
		RpcService rpcService,
		String taskManagerAddress,
		ResourceID taskManagerResourceId,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		int dataPort,
		HardwareDescription hardwareDescription,
		String resourceManagerAddress,
		ResourceManagerId resourceManagerId,
		Executor executor,
		RegistrationConnectionListener<TaskExecutorToResourceManagerConnection, TaskExecutorRegistrationSuccess> registrationListener) {

	super(log, resourceManagerAddress, resourceManagerId, executor);

	this.rpcService = checkNotNull(rpcService);
	this.taskManagerAddress = checkNotNull(taskManagerAddress);
	this.taskManagerResourceId = checkNotNull(taskManagerResourceId);
	this.retryingRegistrationConfiguration = checkNotNull(retryingRegistrationConfiguration);
	this.dataPort = dataPort;
	this.hardwareDescription = checkNotNull(hardwareDescription);
	this.registrationListener = checkNotNull(registrationListener);
}
 
Example #16
Source File: TaskExecutorToResourceManagerConnection.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
ResourceManagerRegistration(
		Logger log,
		RpcService rpcService,
		String targetAddress,
		ResourceManagerId resourceManagerId,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		String taskExecutorAddress,
		ResourceID resourceID,
		int dataPort,
		HardwareDescription hardwareDescription) {

	super(log, rpcService, "ResourceManager", ResourceManagerGateway.class, targetAddress, resourceManagerId, retryingRegistrationConfiguration);
	this.taskExecutorAddress = checkNotNull(taskExecutorAddress);
	this.resourceID = checkNotNull(resourceID);
	this.dataPort = dataPort;
	this.hardwareDescription = checkNotNull(hardwareDescription);
}
 
Example #17
Source File: JobLeaderService.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
JobManagerRetryingRegistration(
		Logger log,
		RpcService rpcService,
		String targetName,
		Class<JobMasterGateway> targetType,
		String targetAddress,
		JobMasterId jobMasterId,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		String taskManagerRpcAddress,
		TaskManagerLocation taskManagerLocation) {
	super(
		log,
		rpcService,
		targetName,
		targetType,
		targetAddress,
		jobMasterId,
		retryingRegistrationConfiguration);

	this.taskManagerRpcAddress = taskManagerRpcAddress;
	this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
}
 
Example #18
Source File: JobLeaderService.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public JobLeaderService(
		TaskManagerLocation location,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration) {
	this.ownLocation = Preconditions.checkNotNull(location);
	this.retryingRegistrationConfiguration = Preconditions.checkNotNull(retryingRegistrationConfiguration);

	// Has to be a concurrent hash map because tests might access this service
	// concurrently via containsJob
	jobLeaderServices = new ConcurrentHashMap<>(4);

	state = JobLeaderService.State.CREATED;

	ownerAddress = null;
	rpcService = null;
	highAvailabilityServices = null;
	jobLeaderListener = null;
}
 
Example #19
Source File: JobMasterConfiguration.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public JobMasterConfiguration(
		Time rpcTimeout,
		Time slotRequestTimeout,
		String tmpDirectory,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		Configuration configuration) {
	this.rpcTimeout = Preconditions.checkNotNull(rpcTimeout);
	this.slotRequestTimeout = Preconditions.checkNotNull(slotRequestTimeout);
	this.tmpDirectory = Preconditions.checkNotNull(tmpDirectory);
	this.retryingRegistrationConfiguration = retryingRegistrationConfiguration;
	this.configuration = Preconditions.checkNotNull(configuration);
}
 
Example #20
Source File: TaskExecutorToResourceManagerConnection.java    From flink with Apache License 2.0 5 votes vote down vote up
ResourceManagerRegistration(
		Logger log,
		RpcService rpcService,
		String targetAddress,
		ResourceManagerId resourceManagerId,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		TaskExecutorRegistration taskExecutorRegistration) {

	super(log, rpcService, "ResourceManager", ResourceManagerGateway.class, targetAddress, resourceManagerId, retryingRegistrationConfiguration);
	this.taskExecutorRegistration = taskExecutorRegistration;
}
 
Example #21
Source File: TaskManagerConfiguration.java    From flink with Apache License 2.0 5 votes vote down vote up
public TaskManagerConfiguration(
		int numberSlots,
		ResourceProfile defaultSlotResourceProfile,
		ResourceProfile totalResourceProfile,
		String[] tmpDirectories,
		Time timeout,
		@Nullable Time maxRegistrationDuration,
		Configuration configuration,
		boolean exitJvmOnOutOfMemory,
		@Nullable String taskManagerLogPath,
		@Nullable String taskManagerStdoutPath,
		@Nullable String taskManagerLogDir,
		String taskManagerExternalAddress,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration) {

	this.numberSlots = numberSlots;
	this.defaultSlotResourceProfile = defaultSlotResourceProfile;
	this.totalResourceProfile = totalResourceProfile;
	this.tmpDirectories = Preconditions.checkNotNull(tmpDirectories);
	this.timeout = Preconditions.checkNotNull(timeout);
	this.maxRegistrationDuration = maxRegistrationDuration;
	this.configuration = new UnmodifiableConfiguration(Preconditions.checkNotNull(configuration));
	this.exitJvmOnOutOfMemory = exitJvmOnOutOfMemory;
	this.taskManagerLogPath = taskManagerLogPath;
	this.taskManagerStdoutPath = taskManagerStdoutPath;
	this.taskManagerLogDir = taskManagerLogDir;
	this.taskManagerExternalAddress = taskManagerExternalAddress;
	this.retryingRegistrationConfiguration = retryingRegistrationConfiguration;
}
 
Example #22
Source File: JobMasterConfiguration.java    From flink with Apache License 2.0 5 votes vote down vote up
public JobMasterConfiguration(
		Time rpcTimeout,
		Time slotRequestTimeout,
		String tmpDirectory,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		Configuration configuration) {
	this.rpcTimeout = Preconditions.checkNotNull(rpcTimeout);
	this.slotRequestTimeout = Preconditions.checkNotNull(slotRequestTimeout);
	this.tmpDirectory = Preconditions.checkNotNull(tmpDirectory);
	this.retryingRegistrationConfiguration = retryingRegistrationConfiguration;
	this.configuration = Preconditions.checkNotNull(configuration);
}
 
Example #23
Source File: TaskManagerConfiguration.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public TaskManagerConfiguration(
		int numberSlots,
		String[] tmpDirectories,
		Time timeout,
		@Nullable Time maxRegistrationDuration,
		Time initialRegistrationPause,
		Time maxRegistrationPause,
		Time refusedRegistrationPause,
		Configuration configuration,
		boolean exitJvmOnOutOfMemory,
		FlinkUserCodeClassLoaders.ResolveOrder classLoaderResolveOrder,
		String[] alwaysParentFirstLoaderPatterns,
		@Nullable String taskManagerLogPath,
		@Nullable String taskManagerStdoutPath,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration) {

	this.numberSlots = numberSlots;
	this.tmpDirectories = Preconditions.checkNotNull(tmpDirectories);
	this.timeout = Preconditions.checkNotNull(timeout);
	this.maxRegistrationDuration = maxRegistrationDuration;
	this.initialRegistrationPause = Preconditions.checkNotNull(initialRegistrationPause);
	this.maxRegistrationPause = Preconditions.checkNotNull(maxRegistrationPause);
	this.refusedRegistrationPause = Preconditions.checkNotNull(refusedRegistrationPause);
	this.configuration = new UnmodifiableConfiguration(Preconditions.checkNotNull(configuration));
	this.exitJvmOnOutOfMemory = exitJvmOnOutOfMemory;
	this.classLoaderResolveOrder = classLoaderResolveOrder;
	this.alwaysParentFirstLoaderPatterns = alwaysParentFirstLoaderPatterns;
	this.taskManagerLogPath = taskManagerLogPath;
	this.taskManagerStdoutPath = taskManagerStdoutPath;
	this.retryingRegistrationConfiguration = retryingRegistrationConfiguration;
}
 
Example #24
Source File: TaskExecutorToResourceManagerConnectionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private TaskExecutorToResourceManagerConnection createTaskExecutorToResourceManagerConnection() {
	return new TaskExecutorToResourceManagerConnection(
		LOGGER,
		rpcService,
		TASK_MANAGER_ADDRESS,
		TASK_MANAGER_RESOURCE_ID,
		RetryingRegistrationConfiguration.defaultConfiguration(),
		TASK_MANAGER_DATA_PORT,
		TASK_MANAGER_HARDWARE_DESCRIPTION,
		RESOURCE_MANAGER_ADDRESS,
		RESOURCE_MANAGER_ID,
		Executors.directExecutor(),
		new TestRegistrationConnectionListener<>());
}
 
Example #25
Source File: NetworkBufferCalculationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a task manager services configuration for the tests
 *
 * @param managedMemory         see {@link TaskManagerOptions#MANAGED_MEMORY_SIZE}
 * @param managedMemoryFraction see {@link TaskManagerOptions#MANAGED_MEMORY_FRACTION}
 * @param networkBufFraction	see {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_FRACTION}
 * @param networkBufMin			see {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MIN}
 * @param networkBufMax			see {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MAX}
 * @param memType				on-heap or off-heap
 *
 * @return configuration object
 */
private static TaskManagerServicesConfiguration getTmConfig(
	final long managedMemory, final float managedMemoryFraction, float networkBufFraction,
	long networkBufMin, long networkBufMax,
	final MemoryType memType) {

	final NetworkEnvironmentConfiguration networkConfig = new NetworkEnvironmentConfiguration(
		networkBufFraction,
		networkBufMin,
		networkBufMax,
		checkedDownCast(MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes()),
		null,
		TaskManagerOptions.NETWORK_REQUEST_BACKOFF_INITIAL.defaultValue(),
		TaskManagerOptions.NETWORK_REQUEST_BACKOFF_MAX.defaultValue(),
		TaskManagerOptions.NETWORK_BUFFERS_PER_CHANNEL.defaultValue(),
		TaskManagerOptions.NETWORK_EXTRA_BUFFERS_PER_GATE.defaultValue(),
		null);

	return new TaskManagerServicesConfiguration(
		InetAddress.getLoopbackAddress(),
		new String[] {},
		new String[] {},
		false,
		networkConfig,
		QueryableStateConfiguration.disabled(),
		1,
		managedMemory,
		memType,
		false,
		managedMemoryFraction,
		0,
		RetryingRegistrationConfiguration.defaultConfiguration(),
		Optional.empty());
}
 
Example #26
Source File: TaskExecutorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private TaskManagerServices createTaskManagerServicesWithTaskSlotTable(
		TaskSlotTable<Task> taskSlotTable) throws IOException {
	return new TaskManagerServicesBuilder()
		.setUnresolvedTaskManagerLocation(unresolvedTaskManagerLocation)
		.setShuffleEnvironment(nettyShuffleEnvironment)
		.setTaskSlotTable(taskSlotTable)
		.setJobLeaderService(new DefaultJobLeaderService(
			unresolvedTaskManagerLocation,
			RetryingRegistrationConfiguration.defaultConfiguration()))
		.setTaskStateManager(createTaskExecutorLocalStateStoresManager())
		.build();
}
 
Example #27
Source File: TaskManagerConfiguration.java    From flink with Apache License 2.0 5 votes vote down vote up
public TaskManagerConfiguration(
		int numberSlots,
		String[] tmpDirectories,
		Time timeout,
		@Nullable Time maxRegistrationDuration,
		Time initialRegistrationPause,
		Time maxRegistrationPause,
		Time refusedRegistrationPause,
		Configuration configuration,
		boolean exitJvmOnOutOfMemory,
		FlinkUserCodeClassLoaders.ResolveOrder classLoaderResolveOrder,
		String[] alwaysParentFirstLoaderPatterns,
		@Nullable String taskManagerLogPath,
		@Nullable String taskManagerStdoutPath,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration) {

	this.numberSlots = numberSlots;
	this.tmpDirectories = Preconditions.checkNotNull(tmpDirectories);
	this.timeout = Preconditions.checkNotNull(timeout);
	this.maxRegistrationDuration = maxRegistrationDuration;
	this.initialRegistrationPause = Preconditions.checkNotNull(initialRegistrationPause);
	this.maxRegistrationPause = Preconditions.checkNotNull(maxRegistrationPause);
	this.refusedRegistrationPause = Preconditions.checkNotNull(refusedRegistrationPause);
	this.configuration = new UnmodifiableConfiguration(Preconditions.checkNotNull(configuration));
	this.exitJvmOnOutOfMemory = exitJvmOnOutOfMemory;
	this.classLoaderResolveOrder = classLoaderResolveOrder;
	this.alwaysParentFirstLoaderPatterns = alwaysParentFirstLoaderPatterns;
	this.taskManagerLogPath = taskManagerLogPath;
	this.taskManagerStdoutPath = taskManagerStdoutPath;
	this.retryingRegistrationConfiguration = retryingRegistrationConfiguration;
}
 
Example #28
Source File: TaskManagerServicesConfiguration.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public TaskManagerServicesConfiguration(
		InetAddress taskManagerAddress,
		String[] tmpDirPaths,
		String[] localRecoveryStateRootDirectories,
		boolean localRecoveryEnabled,
		NetworkEnvironmentConfiguration networkConfig,
		@Nullable QueryableStateConfiguration queryableStateConfig,
		int numberOfSlots,
		long configuredMemory,
		MemoryType memoryType,
		boolean preAllocateMemory,
		float memoryFraction,
		long timerServiceShutdownTimeout,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		Optional<Time> systemResourceMetricsProbingInterval) {

	this.taskManagerAddress = checkNotNull(taskManagerAddress);
	this.tmpDirPaths = checkNotNull(tmpDirPaths);
	this.localRecoveryStateRootDirectories = checkNotNull(localRecoveryStateRootDirectories);
	this.localRecoveryEnabled = checkNotNull(localRecoveryEnabled);
	this.networkConfig = checkNotNull(networkConfig);
	this.queryableStateConfig = queryableStateConfig;
	this.numberOfSlots = checkNotNull(numberOfSlots);

	this.configuredMemory = configuredMemory;
	this.memoryType = checkNotNull(memoryType);
	this.preAllocateMemory = preAllocateMemory;
	this.memoryFraction = memoryFraction;

	checkArgument(timerServiceShutdownTimeout >= 0L, "The timer " +
		"service shutdown timeout must be greater or equal to 0.");
	this.timerServiceShutdownTimeout = timerServiceShutdownTimeout;
	this.retryingRegistrationConfiguration = checkNotNull(retryingRegistrationConfiguration);

	this.systemResourceMetricsProbingInterval = checkNotNull(systemResourceMetricsProbingInterval);
}
 
Example #29
Source File: TaskExecutorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testShouldShutDownTaskManagerServicesInPostStop() throws Exception {
	final TaskSlotTableImpl<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(1);

	final JobLeaderService jobLeaderService = new DefaultJobLeaderService(unresolvedTaskManagerLocation, RetryingRegistrationConfiguration.defaultConfiguration());

	final IOManager ioManager = new IOManagerAsync(tmp.newFolder().getAbsolutePath());

	final TaskExecutorLocalStateStoresManager localStateStoresManager = new TaskExecutorLocalStateStoresManager(
		false,
		ioManager.getSpillingDirectories(),
		Executors.directExecutor());

	nettyShuffleEnvironment.start();

	final KvStateService kvStateService = new KvStateService(new KvStateRegistry(), null, null);
	kvStateService.start();

	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder()
		.setUnresolvedTaskManagerLocation(unresolvedTaskManagerLocation)
		.setIoManager(ioManager)
		.setShuffleEnvironment(nettyShuffleEnvironment)
		.setKvStateService(kvStateService)
		.setTaskSlotTable(taskSlotTable)
		.setJobLeaderService(jobLeaderService)
		.setTaskStateManager(localStateStoresManager)
		.build();

	final TaskExecutor taskManager = createTaskExecutor(taskManagerServices);

	try {
		taskManager.start();
	} finally {
		RpcUtils.terminateRpcEndpoint(taskManager, timeout);
	}

	assertThat(taskSlotTable.isClosed(), is(true));
	assertThat(nettyShuffleEnvironment.isClosed(), is(true));
	assertThat(kvStateService.isShutdown(), is(true));
}
 
Example #30
Source File: JobMasterConfiguration.java    From flink with Apache License 2.0 5 votes vote down vote up
public JobMasterConfiguration(
		Time rpcTimeout,
		Time slotRequestTimeout,
		String tmpDirectory,
		RetryingRegistrationConfiguration retryingRegistrationConfiguration,
		Configuration configuration) {
	this.rpcTimeout = Preconditions.checkNotNull(rpcTimeout);
	this.slotRequestTimeout = Preconditions.checkNotNull(slotRequestTimeout);
	this.tmpDirectory = Preconditions.checkNotNull(tmpDirectory);
	this.retryingRegistrationConfiguration = retryingRegistrationConfiguration;
	this.configuration = Preconditions.checkNotNull(configuration);
}