org.apache.flink.configuration.ClusterOptions Java Examples

The following examples show how to use org.apache.flink.configuration.ClusterOptions. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RetryingRegistrationConfigurationTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testConfigurationParsing() {
	final Configuration configuration = new Configuration();

	final long initialRegistrationTimeout = 1L;
	final long maxRegistrationTimeout = 2L;
	final long refusedRegistrationDelay = 3L;
	final long errorRegistrationDelay = 4L;

	configuration.setLong(ClusterOptions.INITIAL_REGISTRATION_TIMEOUT, initialRegistrationTimeout);
	configuration.setLong(ClusterOptions.MAX_REGISTRATION_TIMEOUT, maxRegistrationTimeout);
	configuration.setLong(ClusterOptions.REFUSED_REGISTRATION_DELAY, refusedRegistrationDelay);
	configuration.setLong(ClusterOptions.ERROR_REGISTRATION_DELAY, errorRegistrationDelay);

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	assertThat(retryingRegistrationConfiguration.getInitialRegistrationTimeoutMillis(), is(initialRegistrationTimeout));
	assertThat(retryingRegistrationConfiguration.getMaxRegistrationTimeoutMillis(), is(maxRegistrationTimeout));
	assertThat(retryingRegistrationConfiguration.getRefusedDelayMillis(), is(refusedRegistrationDelay));
	assertThat(retryingRegistrationConfiguration.getErrorDelayMillis(), is(errorRegistrationDelay));
}
Example #2
Source File: RetryingRegistrationConfigurationTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testConfigurationParsing() {
	final Configuration configuration = new Configuration();

	final long initialRegistrationTimeout = 1L;
	final long maxRegistrationTimeout = 2L;
	final long refusedRegistrationDelay = 3L;
	final long errorRegistrationDelay = 4L;

	configuration.setLong(ClusterOptions.INITIAL_REGISTRATION_TIMEOUT, initialRegistrationTimeout);
	configuration.setLong(ClusterOptions.MAX_REGISTRATION_TIMEOUT, maxRegistrationTimeout);
	configuration.setLong(ClusterOptions.REFUSED_REGISTRATION_DELAY, refusedRegistrationDelay);
	configuration.setLong(ClusterOptions.ERROR_REGISTRATION_DELAY, errorRegistrationDelay);

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	assertThat(retryingRegistrationConfiguration.getInitialRegistrationTimeoutMillis(), is(initialRegistrationTimeout));
	assertThat(retryingRegistrationConfiguration.getMaxRegistrationTimeoutMillis(), is(maxRegistrationTimeout));
	assertThat(retryingRegistrationConfiguration.getRefusedDelayMillis(), is(refusedRegistrationDelay));
	assertThat(retryingRegistrationConfiguration.getErrorDelayMillis(), is(errorRegistrationDelay));
}
Example #3
Source File: DefaultSchedulerFactory.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Nonnull
private static SlotSelectionStrategy selectSlotSelectionStrategy(@Nonnull Configuration configuration) {
	final boolean evenlySpreadOutSlots = configuration.getBoolean(ClusterOptions.EVENLY_SPREAD_OUT_SLOTS_STRATEGY);

	final SlotSelectionStrategy locationPreferenceSlotSelectionStrategy;

	if (evenlySpreadOutSlots) {
		locationPreferenceSlotSelectionStrategy = LocationPreferenceSlotSelectionStrategy.createEvenlySpreadOut();
	} else {
		locationPreferenceSlotSelectionStrategy = LocationPreferenceSlotSelectionStrategy.createDefault();
	}

	if (configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY)) {
		return PreviousAllocationSlotSelectionStrategy.create(locationPreferenceSlotSelectionStrategy);
	} else {
		return locationPreferenceSlotSelectionStrategy;
	}
}
Example #4
Source File: RetryingRegistrationConfigurationTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testConfigurationParsing() {
	final Configuration configuration = new Configuration();

	final long initialRegistrationTimeout = 1L;
	final long maxRegistrationTimeout = 2L;
	final long refusedRegistrationDelay = 3L;
	final long errorRegistrationDelay = 4L;

	configuration.setLong(ClusterOptions.INITIAL_REGISTRATION_TIMEOUT, initialRegistrationTimeout);
	configuration.setLong(ClusterOptions.MAX_REGISTRATION_TIMEOUT, maxRegistrationTimeout);
	configuration.setLong(ClusterOptions.REFUSED_REGISTRATION_DELAY, refusedRegistrationDelay);
	configuration.setLong(ClusterOptions.ERROR_REGISTRATION_DELAY, errorRegistrationDelay);

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	assertThat(retryingRegistrationConfiguration.getInitialRegistrationTimeoutMillis(), is(initialRegistrationTimeout));
	assertThat(retryingRegistrationConfiguration.getMaxRegistrationTimeoutMillis(), is(maxRegistrationTimeout));
	assertThat(retryingRegistrationConfiguration.getRefusedDelayMillis(), is(refusedRegistrationDelay));
	assertThat(retryingRegistrationConfiguration.getErrorDelayMillis(), is(errorRegistrationDelay));
}
Example #5
Source File: RetryingRegistrationConfigurationTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testConfigurationWithDeprecatedOptions() {
	final Configuration configuration = new Configuration();

	final Duration refusedRegistrationBackoff = Duration.ofMinutes(42L);
	final Duration registrationMaxBackoff = Duration.ofSeconds(1L);
	final Duration initialRegistrationBackoff = Duration.ofHours(1337L);

	configuration.set(TaskManagerOptions.REFUSED_REGISTRATION_BACKOFF, refusedRegistrationBackoff);
	configuration.set(TaskManagerOptions.REGISTRATION_MAX_BACKOFF, registrationMaxBackoff);
	configuration.set(TaskManagerOptions.INITIAL_REGISTRATION_BACKOFF, initialRegistrationBackoff);

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	assertThat(retryingRegistrationConfiguration.getInitialRegistrationTimeoutMillis(), is(ClusterOptions.INITIAL_REGISTRATION_TIMEOUT.defaultValue()));
	assertThat(retryingRegistrationConfiguration.getRefusedDelayMillis(), is(ClusterOptions.REFUSED_REGISTRATION_DELAY.defaultValue()));
	assertThat(retryingRegistrationConfiguration.getMaxRegistrationTimeoutMillis(), is(ClusterOptions.MAX_REGISTRATION_TIMEOUT.defaultValue()));
}
Example #6
Source File: RetryingRegistrationConfiguration.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public static RetryingRegistrationConfiguration fromConfiguration(final Configuration configuration) {
	long initialRegistrationTimeoutMillis = configuration.getLong(ClusterOptions.INITIAL_REGISTRATION_TIMEOUT);
	long maxRegistrationTimeoutMillis = configuration.getLong(ClusterOptions.MAX_REGISTRATION_TIMEOUT);
	long errorDelayMillis = configuration.getLong(ClusterOptions.ERROR_REGISTRATION_DELAY);
	long refusedDelayMillis = configuration.getLong(ClusterOptions.REFUSED_REGISTRATION_DELAY);

	return new RetryingRegistrationConfiguration(
		initialRegistrationTimeoutMillis,
		maxRegistrationTimeoutMillis,
		errorDelayMillis,
		refusedDelayMillis);
}
Example #7
Source File: RetryingRegistrationConfiguration.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static RetryingRegistrationConfiguration fromConfiguration(final Configuration configuration) {
	long initialRegistrationTimeoutMillis = configuration.getLong(ClusterOptions.INITIAL_REGISTRATION_TIMEOUT);
	long maxRegistrationTimeoutMillis = configuration.getLong(ClusterOptions.MAX_REGISTRATION_TIMEOUT);
	long errorDelayMillis = configuration.getLong(ClusterOptions.ERROR_REGISTRATION_DELAY);
	long refusedDelayMillis = configuration.getLong(ClusterOptions.REFUSED_REGISTRATION_DELAY);

	return new RetryingRegistrationConfiguration(
		initialRegistrationTimeoutMillis,
		maxRegistrationTimeoutMillis,
		errorDelayMillis,
		refusedDelayMillis);
}
Example #8
Source File: SlotManagerConfiguration.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static SlotManagerConfiguration fromConfiguration(
		Configuration configuration,
		WorkerResourceSpec defaultWorkerResourceSpec) throws ConfigurationException {

	final Time rpcTimeout;
	try {
		rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);
	} catch (IllegalArgumentException e) {
		throw new ConfigurationException("Could not parse the resource manager's timeout " +
			"value " + AkkaOptions.ASK_TIMEOUT + '.', e);
	}

	final Time slotRequestTimeout = getSlotRequestTimeout(configuration);
	final Time taskManagerTimeout = Time.milliseconds(
			configuration.getLong(ResourceManagerOptions.TASK_MANAGER_TIMEOUT));

	boolean waitResultConsumedBeforeRelease =
		configuration.getBoolean(ResourceManagerOptions.TASK_MANAGER_RELEASE_WHEN_RESULT_CONSUMED);

	boolean evenlySpreadOutSlots = configuration.getBoolean(ClusterOptions.EVENLY_SPREAD_OUT_SLOTS_STRATEGY);
	final SlotMatchingStrategy slotMatchingStrategy = evenlySpreadOutSlots ?
		LeastUtilizationSlotMatchingStrategy.INSTANCE : AnyMatchingSlotMatchingStrategy.INSTANCE;

	int numSlotsPerWorker = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS);

	int maxSlotNum = configuration.getInteger(ResourceManagerOptions.MAX_SLOT_NUM);

	return new SlotManagerConfiguration(
		rpcTimeout,
		slotRequestTimeout,
		taskManagerTimeout,
		waitResultConsumedBeforeRelease,
		slotMatchingStrategy,
		defaultWorkerResourceSpec,
		numSlotsPerWorker,
		maxSlotNum);
}
Example #9
Source File: RetryingRegistrationConfiguration.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static RetryingRegistrationConfiguration fromConfiguration(final Configuration configuration) {
	long initialRegistrationTimeoutMillis = configuration.getLong(ClusterOptions.INITIAL_REGISTRATION_TIMEOUT);
	long maxRegistrationTimeoutMillis = configuration.getLong(ClusterOptions.MAX_REGISTRATION_TIMEOUT);
	long errorDelayMillis = configuration.getLong(ClusterOptions.ERROR_REGISTRATION_DELAY);
	long refusedDelayMillis = configuration.getLong(ClusterOptions.REFUSED_REGISTRATION_DELAY);

	return new RetryingRegistrationConfiguration(
		initialRegistrationTimeoutMillis,
		maxRegistrationTimeoutMillis,
		errorDelayMillis,
		refusedDelayMillis);
}
Example #10
Source File: ClusterEntrypointUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Gets and verify the io-executor pool size based on configuration.
 *
 * @param config The configuration to read.
 * @return The legal io-executor pool size.
 */
public static int getPoolSize(Configuration config) {
	final int poolSize = config.getInteger(ClusterOptions.CLUSTER_IO_EXECUTOR_POOL_SIZE, 4 * Hardware.getNumberCPUCores());
	Preconditions.checkArgument(poolSize > 0,
		String.format("Illegal pool size (%s) of io-executor, please re-configure '%s'.",
			poolSize, ClusterOptions.CLUSTER_IO_EXECUTOR_POOL_SIZE.key()));
	return poolSize;
}
Example #11
Source File: MiniCluster.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Shuts down the mini cluster, failing all currently executing jobs.
 * The mini cluster can be started again by calling the {@link #start()} method again.
 *
 * <p>This method shuts down all started services and components,
 * even if an exception occurs in the process of shutting down some component.
 *
 * @return Future which is completed once the MiniCluster has been completely shut down
 */
@Override
public CompletableFuture<Void> closeAsync() {
	synchronized (lock) {
		if (running) {
			LOG.info("Shutting down Flink Mini Cluster");
			try {
				final long shutdownTimeoutMillis = miniClusterConfiguration.getConfiguration().getLong(ClusterOptions.CLUSTER_SERVICES_SHUTDOWN_TIMEOUT);
				final int numComponents = 2 + miniClusterConfiguration.getNumTaskManagers();
				final Collection<CompletableFuture<Void>> componentTerminationFutures = new ArrayList<>(numComponents);

				componentTerminationFutures.addAll(terminateTaskExecutors());

				componentTerminationFutures.add(shutDownResourceManagerComponents());

				final FutureUtils.ConjunctFuture<Void> componentsTerminationFuture = FutureUtils.completeAll(componentTerminationFutures);

				final CompletableFuture<Void> metricSystemTerminationFuture = FutureUtils.composeAfterwards(
					componentsTerminationFuture,
					this::closeMetricSystem);

				final CompletableFuture<Void> rpcServicesTerminationFuture = FutureUtils.composeAfterwards(
					metricSystemTerminationFuture,
					this::terminateRpcServices);

				final CompletableFuture<Void> remainingServicesTerminationFuture = FutureUtils.runAfterwards(
					rpcServicesTerminationFuture,
					this::terminateMiniClusterServices);

				final CompletableFuture<Void> executorsTerminationFuture = FutureUtils.composeAfterwards(
					remainingServicesTerminationFuture,
					() -> terminateExecutors(shutdownTimeoutMillis));

				executorsTerminationFuture.whenComplete(
						(Void ignored, Throwable throwable) -> {
							if (throwable != null) {
								terminationFuture.completeExceptionally(ExceptionUtils.stripCompletionException(throwable));
							} else {
								terminationFuture.complete(null);
							}
						});
			} finally {
				running = false;
			}
		}

		return terminationFuture;
	}
}