org.apache.flink.configuration.ConfigOption Java Examples

@Test
public void testConfigNetworkMemoryLegacyNumOfBuffers() {
	final MemorySize pageSize = MemorySize.parse("32k");
	final int numOfBuffers = 1024;
	final MemorySize networkSize = pageSize.multiply(numOfBuffers);

	@SuppressWarnings("deprecation")
	final ConfigOption<Integer> legacyOption = NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS;

	Configuration conf = new Configuration();
	conf.set(TaskManagerOptions.MEMORY_SEGMENT_SIZE, pageSize);
	conf.setInteger(legacyOption, numOfBuffers);

	// validate in configurations without explicit total flink/process memory, otherwise explicit configured
	// network memory size might conflict with total flink/process memory minus other memory sizes
	validateInConfigWithExplicitTaskHeapAndManagedMem(conf, taskExecutorProcessSpec ->
		assertThat(taskExecutorProcessSpec.getNetworkMemSize(), is(networkSize)));
	validateInConfigurationsWithoutExplicitTaskHeapMem(conf, taskExecutorProcessSpec ->
		assertThat(taskExecutorProcessSpec.getNetworkMemSize(), is(networkSize)));
}
 

public Configuration getConfWithLegacyHeapSizeMappedToNewConfigOption(
	Configuration configuration,
	ConfigOption<MemorySize> configOption) {
	if (configuration.contains(configOption)) {
		return configuration;
	}
	return getLegacyHeapMemoryIfExplicitlyConfigured(configuration)
		.map(legacyHeapSize -> {
			Configuration copiedConfig = new Configuration(configuration);
			copiedConfig.set(configOption, legacyHeapSize);
			LOG.info(
				"'{}' is not specified, use the configured deprecated task manager heap value ({}) for it.",
				configOption.key(),
				legacyHeapSize.toHumanReadableString());
			return copiedConfig;
		}).orElse(configuration);
}
 

@Nullable
private static Path parameterOrConfigured(@Nullable Path path, Configuration config, ConfigOption<String> option) {
	if (path != null) {
		return path;
	}
	else {
		String configValue = config.getString(option);
		try {
			return configValue == null ? null : new Path(configValue);
		}
		catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException("Cannot parse value for " + option.key() +
					" : " + configValue + " . Not a valid path.");
		}
	}
}
 

private <F extends Factory> Optional<F> discoverOptionalFormatFactory(
		Class<F> formatFactoryClass,
		ConfigOption<String> formatOption) {
	final String identifier = allOptions.get(formatOption);
	if (identifier == null) {
		return Optional.empty();
	}
	final F factory = discoverFactory(
		context.getClassLoader(),
		formatFactoryClass,
		identifier);
	String formatPrefix = formatPrefix(factory, formatOption);
	// log all used options of other factories
	consumedOptionKeys.addAll(
		factory.requiredOptions().stream()
			.map(ConfigOption::key)
			.map(k -> formatPrefix + k)
			.collect(Collectors.toSet()));
	consumedOptionKeys.addAll(
		factory.optionalOptions().stream()
			.map(ConfigOption::key)
			.map(k -> formatPrefix + k)
			.collect(Collectors.toSet()));
	return Optional.of(factory);
}
 

@Override
public Set<ConfigOption<?>> optionalOptions() {
	return new HashSet<>(Arrays.asList(
		DATA_ID,
		CHANGELOG_MODE,
		BOUNDED,
		RUNTIME_SOURCE,
		TABLE_SOURCE_CLASS,
		LOOKUP_FUNCTION_CLASS,
		ASYNC_ENABLED,
		TABLE_SOURCE_CLASS,
		SINK_INSERT_ONLY,
		RUNTIME_SINK,
		SINK_EXPECTED_MESSAGES_NUM,
		NESTED_PROJECTION_SUPPORTED));
}
 

private void testIsActiveHelper(final String executorOption) throws CliArgsException {
	final String expectedExecutorName = "test-executor";
	final ConfigOption<Integer> configOption = key("test.int").intType().noDefaultValue();
	final int expectedValue = 42;

	final GenericCLI cliUnderTest = new GenericCLI(
			new Configuration(),
			tmp.getRoot().getAbsolutePath());

	final String[] args = {executorOption, expectedExecutorName, "-D" + configOption.key() + "=" + expectedValue};
	final CommandLine commandLine = CliFrontendParser.parse(testOptions, args, true);

	final Configuration configuration = cliUnderTest.applyCommandLineOptionsToConfiguration(commandLine);
	assertEquals(expectedExecutorName, configuration.get(DeploymentOptions.TARGET));
	assertEquals(expectedValue, configuration.getInteger(configOption));
}
 

private static Configuration setAllRequiredOptionsExceptOne(ConfigOption<?> optionToNotSet) {
	Configuration configuration = new Configuration();
	if (!TaskManagerOptions.CPU_CORES.equals(optionToNotSet)) {
		configuration.set(TaskManagerOptions.CPU_CORES, 1.0);
	}

	// skip network to exclude min/max mismatch config failure
	MemorySize network = MemorySize.ofMebiBytes(3);
	configuration.set(TaskManagerOptions.NETWORK_MEMORY_MIN, network);
	configuration.set(TaskManagerOptions.NETWORK_MEMORY_MAX, network);

	//noinspection unchecked
	TaskExecutorResourceUtils.CONFIG_OPTIONS
		.stream()
		.filter(option -> !option.equals(TaskManagerOptions.CPU_CORES))
		.filter(option -> !option.equals(optionToNotSet))
		.forEach(option -> configuration.set((ConfigOption<MemorySize>) option, MemorySize.ofMebiBytes(1)));

	return configuration;
}
 

/**
 * Validates the required and optional {@link ConfigOption}s of a factory.
 *
 * <p>Note: It does not check for left-over options.
 */
public static void validateFactoryOptions(Factory factory, ReadableConfig options) {
	// currently Flink's options have no validation feature which is why we access them eagerly
	// to provoke a parsing error

	final List<String> missingRequiredOptions = factory.requiredOptions().stream()
		.filter(option -> readOption(options, option) == null)
		.map(ConfigOption::key)
		.sorted()
		.collect(Collectors.toList());

	if (!missingRequiredOptions.isEmpty()) {
		throw new ValidationException(
			String.format(
				"One or more required options are missing.\n\n" +
				"Missing required options are:\n\n" +
				"%s",
				String.join("\n", missingRequiredOptions)));
	}

	factory.optionalOptions()
		.forEach(option -> readOption(options, option));
}
 

public static RangeFraction getRangeFraction(
		MemorySize minSize,
		MemorySize maxSize,
		ConfigOption<Float> fractionOption,
		Configuration config) {
	double fraction = config.getFloat(fractionOption);
	try {
		return new RangeFraction(minSize, maxSize, fraction);
	} catch (IllegalArgumentException e) {
		throw new IllegalConfigurationException(
			String.format(
				"Inconsistently configured %s (%s) and its min (%s), max (%s) value",
				fractionOption,
				fraction,
				minSize.toHumanReadableString(),
				maxSize.toHumanReadableString()),
			e);
	}
}
 

private TableFactoryHelper(DynamicTableFactory tableFactory, DynamicTableFactory.Context context) {
	this.tableFactory = tableFactory;
	this.context = context;
	this.allOptions = asConfiguration(context.getCatalogTable().getOptions());
	this.consumedOptionKeys = new HashSet<>();
	this.consumedOptionKeys.add(PROPERTY_VERSION.key());
	this.consumedOptionKeys.add(CONNECTOR.key());
	this.consumedOptionKeys.addAll(
		tableFactory.requiredOptions().stream()
			.map(ConfigOption::key)
			.collect(Collectors.toSet()));
	this.consumedOptionKeys.addAll(
		tableFactory.optionalOptions().stream()
			.map(ConfigOption::key)
			.collect(Collectors.toSet()));
}
 

/**
 * Discovers a {@link EncodingFormat} of the given type using the given option (if present) as factory
 * identifier.
 */
public <I, F extends EncodingFormatFactory<I>> Optional<EncodingFormat<I>> discoverOptionalEncodingFormat(
		Class<F> formatFactoryClass,
		ConfigOption<String> formatOption) {
	return discoverOptionalFormatFactory(formatFactoryClass, formatOption)
		.map(formatFactory -> {
			String formatPrefix = formatPrefix(formatFactory, formatOption);
			try {
				return formatFactory.createEncodingFormat(context, projectOptions(formatPrefix));
			} catch (Throwable t) {
				throw new ValidationException(
					String.format(
						"Error creating sink format '%s' in option space '%s'.",
						formatFactory.factoryIdentifier(),
						formatPrefix),
					t);
			}
		});
}
 

/**
 * Transforms option to table row.
 *
 * @param optionWithMetaInfo option to transform
 * @return row with the option description
 */
private static String toHtmlString(final OptionWithMetaInfo optionWithMetaInfo) {
	ConfigOption<?> option = optionWithMetaInfo.option;
	String defaultValue = stringifyDefault(optionWithMetaInfo);
	Documentation.TableOption tableOption = optionWithMetaInfo.field.getAnnotation(Documentation.TableOption.class);
	StringBuilder execModeStringBuilder = new StringBuilder();
	if (tableOption != null) {
		Documentation.ExecMode execMode = tableOption.execMode();
		if (Documentation.ExecMode.BATCH_STREAMING.equals(execMode)) {
			execModeStringBuilder.append("<br> <span class=\"label label-primary\">")
					.append(Documentation.ExecMode.BATCH.toString())
					.append("</span> <span class=\"label label-primary\">")
					.append(Documentation.ExecMode.STREAMING.toString())
					.append("</span>");
		} else {
			execModeStringBuilder.append("<br> <span class=\"label label-primary\">")
					.append(execMode.toString())
					.append("</span>");
		}
	}

	return "" +
		"        <tr>\n" +
		"            <td><h5>" + escapeCharacters(option.key()) + "</h5>" + execModeStringBuilder.toString() + "</td>\n" +
		"            <td style=\"word-wrap: break-word;\">" + escapeCharacters(addWordBreakOpportunities(defaultValue)) + "</td>\n" +
		"            <td>" + formatter.format(option.description()) + "</td>\n" +
		"        </tr>\n";
}
 

private static void assertEqualsOrDefault(Configuration sslConfig, ConfigOption<Integer> option, long actual) {
	long expected = sslConfig.getInteger(option);
	if (expected != option.defaultValue()) {
		assertEquals(expected, actual);
	} else {
		assertTrue("default value (" + option.defaultValue() + ") should not be propagated",
			actual >= 0);
	}
}
 

/**
 * Transforms option to table row.
 *
 * @param optionWithMetaInfo option to transform
 * @return row with the option description
 */
private static String toHtmlString(final OptionWithMetaInfo optionWithMetaInfo) {
	ConfigOption<?> option = optionWithMetaInfo.option;
	String defaultValue = stringifyDefault(optionWithMetaInfo);
	String type = typeToHtml(optionWithMetaInfo);
	Documentation.TableOption tableOption = optionWithMetaInfo.field.getAnnotation(Documentation.TableOption.class);
	StringBuilder execModeStringBuilder = new StringBuilder();
	if (tableOption != null) {
		Documentation.ExecMode execMode = tableOption.execMode();
		if (Documentation.ExecMode.BATCH_STREAMING.equals(execMode)) {
			execModeStringBuilder.append("<br> <span class=\"label label-primary\">")
					.append(Documentation.ExecMode.BATCH.toString())
					.append("</span> <span class=\"label label-primary\">")
					.append(Documentation.ExecMode.STREAMING.toString())
					.append("</span>");
		} else {
			execModeStringBuilder.append("<br> <span class=\"label label-primary\">")
					.append(execMode.toString())
					.append("</span>");
		}
	}

	return "" +
		"        <tr>\n" +
		"            <td><h5>" + escapeCharacters(option.key()) + "</h5>" + execModeStringBuilder.toString() + "</td>\n" +
		"            <td style=\"word-wrap: break-word;\">" + escapeCharacters(addWordBreakOpportunities(defaultValue)) + "</td>\n" +
		"            <td>" + type + "</td>\n" +
		"            <td>" + formatter.format(option.description()) + "</td>\n" +
		"        </tr>\n";
}
 

public static <T> T getSerializedInstance(
    ClassLoader classLoader, Configuration configuration, ConfigOption<byte[]> option) {
  final byte[] bytes = configuration.getBytes(option.key(), option.defaultValue());
  try {
    return InstantiationUtil.deserializeObject(bytes, classLoader, false);
  } catch (IOException | ClassNotFoundException e) {
    throw new IllegalStateException("Unable to initialize.", e);
  }
}
 

public static void storeSerializedInstance(
    Configuration configuration, ConfigOption<byte[]> option, Object instance) {
  try {
    byte[] bytes = InstantiationUtil.serializeObject(instance);
    configuration.setBytes(option.key(), bytes);
  } catch (IOException e) {
    throw new IllegalStateException(e);
  }
}
 

@Test
public void testConfigManagedMemoryLegacySize() {
	final MemorySize managedMemSize = MemorySize.parse("100m");

	@SuppressWarnings("deprecation")
	final ConfigOption<MemorySize> legacyOption = TaskManagerOptions.MANAGED_MEMORY_SIZE;

	Configuration conf = new Configuration();
	conf.set(legacyOption, managedMemSize);

	// validate in configurations without explicit managed memory size,
	// to avoid checking against overwritten managed memory size
	validateInConfigurationsWithoutExplicitManagedMem(conf, taskExecutorProcessSpec -> assertThat(taskExecutorProcessSpec.getManagedMemorySize(), is(managedMemSize)));
}
 

public ProcessMemoryOptions(
		List<ConfigOption<MemorySize>> requiredFineGrainedOptions,
		ConfigOption<MemorySize> totalFlinkMemoryOption,
		ConfigOption<MemorySize> totalProcessMemoryOption,
		JvmMetaspaceAndOverheadOptions jvmOptions) {
	this.requiredFineGrainedOptions = new ArrayList<>(checkNotNull(requiredFineGrainedOptions));
	this.totalFlinkMemoryOption = checkNotNull(totalFlinkMemoryOption);
	this.totalProcessMemoryOption = checkNotNull(totalProcessMemoryOption);
	this.jvmOptions = checkNotNull(jvmOptions);
}
 

public long getExtraConfLong(ConfigOption<Long> config) {
  Object o = extras().get(config.key());
  if (config.hasDefaultValue()) {
    return o == null ? config.defaultValue() : Long.parseLong(o.toString());
  } else {
    throw new IllegalArgumentException("Unspecified config " + config.key());
  }
}
 

/**
 * Helper method to check whether the (key,value) is valid through given configuration and returns the formatted value.
 *
 * @param option The configuration key which is configurable in {@link RocksDBConfigurableOptions}.
 * @param value The value within given configuration.
 */
private static void checkArgumentValid(ConfigOption<?> option, Object value) {
	final String key = option.key();

	if (POSITIVE_INT_CONFIG_SET.contains(option)) {
		Preconditions.checkArgument((Integer) value  > 0,
			"Configured value for key: " + key + " must be larger than 0.");
	} else if (SIZE_CONFIG_SET.contains(option)) {
		Preconditions.checkArgument(((MemorySize) value).getBytes() > 0,
			"Configured size for key" + key + " must be larger than 0.");
	}
}
 

/**
 * Parse a valid port for the config option. A fixed port is expected, and do not support a range of ports.
 *
 * @param flinkConfig flink config
 * @param port port config option
 * @return valid port
 */
public static Integer parsePort(Configuration flinkConfig, ConfigOption<String> port) {
	checkNotNull(flinkConfig.get(port), port.key() + " should not be null.");

	try {
		return Integer.parseInt(flinkConfig.get(port));
	} catch (NumberFormatException ex) {
		throw new FlinkRuntimeException(
			port.key() + " should be specified to a fixed port. Do not support a range of ports.",
			ex);
	}
}
 

private void checkAllOrNone(ReadableConfig config, ConfigOption<?>[] configOptions) {
	int presentCount = 0;
	for (ConfigOption configOption : configOptions) {
		if (config.getOptional(configOption).isPresent()) {
			presentCount++;
		}
	}
	String[] propertyNames = Arrays.stream(configOptions).map(ConfigOption::key).toArray(String[]::new);
	Preconditions.checkArgument(configOptions.length == presentCount || presentCount == 0,
		"Either all or none of the following options should be provided:\n" + String.join("\n", propertyNames));
}
 

@Override
public Set<ConfigOption<?>> requiredOptions() {
	final Set<ConfigOption<?>> options = new HashSet<>();
	options.add(HOSTNAME);
	options.add(PORT);
	options.add(FactoryUtil.FORMAT); // use pre-defined option for format
	return options;
}
 

@Override
public Set<ConfigOption<?>> optionalOptions() {
	Set<ConfigOption<?>> set = new HashSet<>();
	set.add(ZOOKEEPER_ZNODE_PARENT);
	set.add(NULL_STRING_LITERAL);
	set.add(SINK_BUFFER_FLUSH_MAX_SIZE);
	set.add(SINK_BUFFER_FLUSH_MAX_ROWS);
	set.add(SINK_BUFFER_FLUSH_INTERVAL);
	return set;
}
 

@Override
public Set<ConfigOption<?>> optionalOptions() {
	Set<ConfigOption<?>> options = new HashSet<>();
	options.add(FIELD_DELIMITER);
	options.add(LINE_DELIMITER);
	options.add(DISABLE_QUOTE_CHARACTER);
	options.add(QUOTE_CHARACTER);
	options.add(ALLOW_COMMENTS);
	options.add(IGNORE_PARSE_ERRORS);
	options.add(ARRAY_ELEMENT_DELIMITER);
	options.add(ESCAPE_CHARACTER);
	options.add(NULL_LITERAL);
	return options;
}
 

private String formatPrefix(Factory formatFactory, ConfigOption<String> formatOption) {
	String identifier = formatFactory.factoryIdentifier();
	if (formatOption.key().equals(FORMAT_KEY)) {
		return identifier + ".";
	} else if (formatOption.key().endsWith(FORMAT_SUFFIX)) {
		// extract the key prefix, e.g. extract 'key' from 'key.format'
		String keyPrefix = formatOption.key().substring(0, formatOption.key().length() - FORMAT_SUFFIX.length());
		return keyPrefix + "." + identifier + ".";
	} else {
		throw new ValidationException(
			"Format identifier key should be 'format' or suffix with '.format', " +
				"don't support format identifier key '" + formatOption.key() + "'.");
	}
}
 

public static MemorySize getMemorySizeFromConfig(Configuration config, ConfigOption<MemorySize> option) {
	try {
		return Preconditions.checkNotNull(config.get(option), "The memory option is not set and has no default value.");
	} catch (Throwable t) {
		throw new IllegalConfigurationException("Cannot read memory size from config option '" + option.key() + "'.", t);
	}
}
 

@Override
public Set<ConfigOption<?>> optionalOptions() {
	final Set<ConfigOption<?>> options = new HashSet<>();
	options.add(PROPS_GROUP_ID);
	options.add(SCAN_STARTUP_MODE);
	options.add(SCAN_STARTUP_SPECIFIC_OFFSETS);
	options.add(SCAN_STARTUP_TIMESTAMP_MILLIS);
	options.add(SINK_PARTITIONER);
	return options;
}
 

@Override
public Set<ConfigOption<?>> optionalOptions() {
	Set<ConfigOption<?>> options = new HashSet<>();
	options.add(UTC_TIMEZONE);
	// support "parquet.*"
	return options;
}
 

/**
 * Discovers a {@link EncodingFormat} of the given type using the given option as factory identifier.
 */
public <I, F extends EncodingFormatFactory<I>> EncodingFormat<I> discoverEncodingFormat(
		Class<F> formatFactoryClass,
		ConfigOption<String> formatOption) {
	return discoverOptionalEncodingFormat(formatFactoryClass, formatOption)
		.orElseThrow(() ->
			new ValidationException(
				String.format("Could not find required sink format '%s'.", formatOption.key())));
}