Java Code Examples for org.apache.flink.configuration.Configuration#setBoolean()

StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) {
	String dbPath;
	String checkpointPath;
	try {
		dbPath = tempFolder.newFolder().getAbsolutePath();
		checkpointPath = tempFolder.newFolder().toURI().toString();
	} catch (IOException e) {
		throw new FlinkRuntimeException("Failed to init rocksdb test state backend");
	}
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
	Configuration config = new Configuration();
	config.setBoolean(TTL_COMPACT_FILTER_ENABLED, true);
	backend = backend.configure(config, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 

@Test
public void testNativeMetricsConfigurable() {
	for (RocksDBProperty property : RocksDBProperty.values()) {
		Configuration config = new Configuration();
		config.setBoolean(property.getConfigKey(), true);

		RocksDBNativeMetricOptions options = RocksDBNativeMetricOptions.fromConfig(config);

		Assert.assertTrue(
			String.format("Failed to enable native metrics with property %s", property.getConfigKey()),
			options.isEnabled());

		Assert.assertTrue(
			String.format("Failed to enable native metric %s using config", property.getConfigKey()),
			options.getProperties().contains(property.getRocksDBProperty())
		);
	}
}
 

@Test
public final void executeTest() throws Exception {
	EventTimeWindowCheckpointingITCase.tempFolder.create();
	EventTimeWindowCheckpointingITCase windowChkITCase =
		new EventTimeWindowCheckpointingITCase() {

			@Override
			protected StateBackendEnum getStateBackend() {
				return backendEnum;
			}

			@Override
			protected Configuration createClusterConfig() throws IOException {
				Configuration config = super.createClusterConfig();

				config.setBoolean(
					CheckpointingOptions.LOCAL_RECOVERY,
					localRecoveryEnabled);

				return config;
			}
		};

	executeTest(windowChkITCase);
}
 

@Override
public void writeParametersToConfig(Configuration config) {
	for (int i = 0; i < this.positions.length; i++) {
		if (this.positions[i] < 0) {
			throw new IllegalArgumentException("The key position " + i + " is invalid: " + this.positions[i]);
		}
		if (this.types[i] == null || !Value.class.isAssignableFrom(this.types[i])) {
			throw new IllegalArgumentException("The key type " + i + " is null or not implenting the interface " + 
				Value.class.getName() + ".");
		}
	}
	
	// write the config
	config.setInteger(NUM_KEYS, this.positions.length);
	for (int i = 0; i < this.positions.length; i++) {
		config.setInteger(KEY_POS_PREFIX + i, this.positions[i]);
		config.setString(KEY_CLASS_PREFIX + i, this.types[i].getName());
		config.setBoolean(KEY_SORT_DIRECTION_PREFIX + i, this.sortDirections[i]);
	}
}
 

private static Configuration getClusterConfiguration() {
	Configuration config = new Configuration();
	try {
		File logDir = File.createTempFile("TestBaseUtils-logdir", null);
		assertTrue("Unable to delete temp file", logDir.delete());
		assertTrue("Unable to create temp directory", logDir.mkdir());
		File logFile = new File(logDir, "jobmanager.log");
		File outFile = new File(logDir, "jobmanager.out");

		Files.createFile(logFile.toPath());
		Files.createFile(outFile.toPath());

		config.setString(WebOptions.LOG_PATH, logFile.getAbsolutePath());
		config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, logFile.getAbsolutePath());
	} catch (Exception e) {
		throw new AssertionError("Could not setup test.", e);
	}
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "12m");
	config.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);

	return config;
}
 

private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}
 

@Test
public void testFailedToInitWithTwoProtocolsSet() {
	final Configuration config = new Configuration();

	config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, true);
	config.setString(SecurityOptions.SSL_KEYSTORE,
		getClass().getResource("/local127.keystore").getPath());
	config.setString(SecurityOptions.SSL_KEYSTORE_PASSWORD, "password");
	config.setString(SecurityOptions.SSL_KEY_PASSWORD, "password");
	config.setString(SecurityOptions.SSL_TRUSTSTORE,
		getClass().getResource("/local127.truststore").getPath());

	config.setString(SecurityOptions.SSL_TRUSTSTORE_PASSWORD, "password");
	config.setString(SecurityOptions.SSL_ALGORITHMS, "TLSv1,TLSv1.1");

	try (final BlobServer ignored = new BlobServer(config, new VoidBlobStore())) {
		fail();
	} catch (Exception e) {
		findThrowable(e, IOException.class);
		findThrowableWithMessage(e, "Unable to open BLOB Server in specified port range: 0");
	}
}
 

private Configuration createInternalSslConfigWithKeyStore() {
	final Configuration config = new Configuration();
	config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, true);
	addSslProviderConfig(config, sslProvider);
	addInternalKeyStoreConfig(config);
	return config;
}
 

public static void toConfiguration(final SavepointRestoreSettings savepointRestoreSettings, final Configuration configuration) {
	configuration.setBoolean(SavepointConfigOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE, savepointRestoreSettings.allowNonRestoredState());
	final String savepointPath = savepointRestoreSettings.getRestorePath();
	if (savepointPath != null) {
		configuration.setString(SavepointConfigOptions.SAVEPOINT_PATH, savepointPath);
	}
}
 

/**
 * Creates a {@link LocalEnvironment} for local program execution that also starts the
 * web monitoring UI.
 *
 * <p>The local execution environment will run the program in a multi-threaded fashion in
 * the same JVM as the environment was created in. It will use the parallelism specified in the
 * parameter.
 *
 * <p>If the configuration key 'rest.port' was set in the configuration, that particular
 * port will be used for the web UI. Otherwise, the default port (8081) will be used.
 */
@PublicEvolving
public static ExecutionEnvironment createLocalEnvironmentWithWebUI(Configuration conf) {
	checkNotNull(conf, "conf");

	conf.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);

	if (!conf.contains(RestOptions.PORT)) {
		// explicitly set this option so that it's not set to 0 later
		conf.setInteger(RestOptions.PORT, RestOptions.PORT.defaultValue());
	}

	return createLocalEnvironment(conf, -1);
}
 

/**
 * Set temporary configuration directories if necessary.
 *
 * @param configuration flink config to patch
 * @param defaultDirs in case no tmp directories is set, next directories will be applied
 */
public static void updateTmpDirectoriesInConfiguration(
		Configuration configuration,
		@Nullable String defaultDirs) {
	if (configuration.contains(CoreOptions.TMP_DIRS)) {
		LOG.info("Overriding Fink's temporary file directories with those " +
			"specified in the Flink config: {}", configuration.getValue(CoreOptions.TMP_DIRS));
	} else if (defaultDirs != null) {
		LOG.info("Setting directories for temporary files to: {}", defaultDirs);
		configuration.setString(CoreOptions.TMP_DIRS, defaultDirs);
		configuration.setBoolean(USE_LOCAL_DEFAULT_TMP_DIRS, true);
	}
}
 

private static Configuration getConfiguration() {
	verifyJvmOptions();
	Configuration config = new Configuration();
	config.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true);
	config.setString(AkkaOptions.ASK_TIMEOUT, TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT());
	config.set(TaskManagerOptions.MEMORY_SEGMENT_SIZE, MemorySize.parse("4096"));
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 2048);

	return config;
}
 

/**
 * Validates loading a memory state backend with additional parameters from the cluster configuration.
 */
@Test
public void testLoadMemoryStateWithParameters() throws Exception {
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();
	final Path expectedCheckpointPath = new Path(checkpointDir);
	final Path expectedSavepointPath = new Path(savepointDir);

	final boolean async = !CheckpointingOptions.ASYNC_SNAPSHOTS.defaultValue();

	// we configure with the explicit string (rather than AbstractStateBackend#X_STATE_BACKEND_NAME)
	// to guard against config-breaking changes of the name

	final Configuration config1 = new Configuration();
	config1.setString(backendKey, "jobmanager");
	config1.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
	config1.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config1.setBoolean(CheckpointingOptions.ASYNC_SNAPSHOTS, async);

	final Configuration config2 = new Configuration();
	config2.setString(backendKey, MemoryStateBackendFactory.class.getName());
	config2.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
	config2.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config2.setBoolean(CheckpointingOptions.ASYNC_SNAPSHOTS, async);

	MemoryStateBackend backend1 = (MemoryStateBackend)
			StateBackendLoader.loadStateBackendFromConfig(config1, cl, null);
	MemoryStateBackend backend2 = (MemoryStateBackend)
			StateBackendLoader.loadStateBackendFromConfig(config2, cl, null);

	assertNotNull(backend1);
	assertNotNull(backend2);

	assertEquals(expectedCheckpointPath, backend1.getCheckpointPath());
	assertEquals(expectedCheckpointPath, backend2.getCheckpointPath());
	assertEquals(expectedSavepointPath, backend1.getSavepointPath());
	assertEquals(expectedSavepointPath, backend2.getSavepointPath());
	assertEquals(async, backend1.isUsingAsynchronousSnapshots());
	assertEquals(async, backend2.isUsingAsynchronousSnapshots());
}
 

@Test
public void testFailedToInitWithMissingMandatorySslConfiguration() {
	final Configuration config = new Configuration();

	config.setBoolean(SecurityOptions.SSL_INTERNAL_ENABLED, true);

	try (final BlobServer ignored = new BlobServer(config, new VoidBlobStore())) {
		fail();
	} catch (Exception e) {
		findThrowable(e, IOException.class);
		findThrowableWithMessage(e, "Failed to initialize SSL for the blob server");
	}
}
 

private static Configuration getConfig() {
	Configuration config = new Configuration();
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
	config.setBoolean(WebOptions.SUBMIT_ENABLE, false);
	return config;
}
 

/**
 * This tests that the creation of {@link TaskManagerServices} correctly creates the local state root directory
 * for the {@link TaskExecutorLocalStateStoresManager} with the configured root directory.
 */
@Test
public void testCreationFromConfig() throws Exception {

	final Configuration config = new Configuration();

	File newFolder = temporaryFolder.newFolder();
	String tmpDir = newFolder.getAbsolutePath() + File.separator;
	final String rootDirString = "__localStateRoot1,__localStateRoot2,__localStateRoot3".replaceAll("__", tmpDir);

	// test configuration of the local state directories
	config.setString(CheckpointingOptions.LOCAL_RECOVERY_TASK_MANAGER_STATE_ROOT_DIRS, rootDirString);

	// test configuration of the local state mode
	config.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, true);

	TaskManagerServices taskManagerServices = createTaskManagerServices(createTaskManagerServiceConfiguration(config));

	try {
		TaskExecutorLocalStateStoresManager taskStateManager = taskManagerServices.getTaskManagerStateStore();

		// verify configured directories for local state
		String[] split = rootDirString.split(",");
		File[] rootDirectories = taskStateManager.getLocalStateRootDirectories();
		for (int i = 0; i < split.length; ++i) {
			Assert.assertEquals(
				new File(split[i], TaskManagerServices.LOCAL_STATE_SUB_DIRECTORY_ROOT),
				rootDirectories[i]);
		}

		// verify local recovery mode
		Assert.assertTrue(taskStateManager.isLocalRecoveryEnabled());

		Assert.assertEquals("localState", TaskManagerServices.LOCAL_STATE_SUB_DIRECTORY_ROOT);
		for (File rootDirectory : rootDirectories) {
			FileUtils.deleteFileOrDirectory(rootDirectory);
		}
	} finally {
		taskManagerServices.shutDown();
	}
}
 

@Test
public void testConnectFailure() throws Exception {
	RestClient restClient = null;
	RestServerEndpoint serverEndpoint = null;

	try {
		final Configuration baseConfig = new Configuration();
		baseConfig.setString(RestOptions.BIND_PORT, "0");
		baseConfig.setString(RestOptions.ADDRESS, "localhost");
		baseConfig.setBoolean(SecurityOptions.SSL_REST_ENABLED, true);
		baseConfig.setBoolean(SecurityOptions.SSL_REST_AUTHENTICATION_ENABLED, true);
		baseConfig.setString(SecurityOptions.SSL_ALGORITHMS, "TLS_RSA_WITH_AES_128_CBC_SHA");

		Configuration serverConfig = new Configuration(baseConfig);
		serverConfig.setString(SecurityOptions.SSL_REST_TRUSTSTORE, TRUST_STORE_FILE);
		serverConfig.setString(SecurityOptions.SSL_REST_TRUSTSTORE_PASSWORD, "password");
		serverConfig.setString(SecurityOptions.SSL_REST_KEYSTORE, KEY_STORE_FILE);
		serverConfig.setString(SecurityOptions.SSL_REST_KEYSTORE_PASSWORD, "password");
		serverConfig.setString(SecurityOptions.SSL_REST_KEY_PASSWORD, "password");

		Configuration clientConfig = new Configuration(baseConfig);
		clientConfig.setString(SecurityOptions.SSL_REST_TRUSTSTORE, UNTRUSTED_KEY_STORE_FILE);
		clientConfig.setString(SecurityOptions.SSL_REST_TRUSTSTORE_PASSWORD, "password");
		clientConfig.setString(SecurityOptions.SSL_REST_KEYSTORE, KEY_STORE_FILE);
		clientConfig.setString(SecurityOptions.SSL_REST_KEYSTORE_PASSWORD, "password");
		clientConfig.setString(SecurityOptions.SSL_REST_KEY_PASSWORD, "password");

		RestServerEndpointConfiguration restServerConfig = RestServerEndpointConfiguration.fromConfiguration(serverConfig);
		RestClientConfiguration restClientConfig = RestClientConfiguration.fromConfiguration(clientConfig);

		RestfulGateway restfulGateway = TestingRestfulGateway.newBuilder().build();
		RestServerEndpointITCase.TestVersionHandler testVersionHandler = new RestServerEndpointITCase.TestVersionHandler(
			() -> CompletableFuture.completedFuture(restfulGateway),
			RpcUtils.INF_TIMEOUT);

		serverEndpoint = new RestServerEndpointITCase.TestRestServerEndpoint(
			restServerConfig,
			Arrays.asList(Tuple2.of(testVersionHandler.getMessageHeaders(), testVersionHandler)));
		restClient = new RestServerEndpointITCase.TestRestClient(restClientConfig);
		serverEndpoint.start();

		CompletableFuture<EmptyResponseBody> response = restClient.sendRequest(
			serverEndpoint.getServerAddress().getHostName(),
			serverEndpoint.getServerAddress().getPort(),
			RestServerEndpointITCase.TestVersionHeaders.INSTANCE,
			EmptyMessageParameters.getInstance(),
			EmptyRequestBody.getInstance(),
			Collections.emptyList()
		);
		response.get(60, TimeUnit.SECONDS);

		fail("should never complete normally");
	} catch (ExecutionException exception) {
		// that is what we want
		assertTrue(ExceptionUtils.findThrowable(exception, SSLHandshakeException.class).isPresent());
	} finally {
		if (restClient != null) {
			restClient.shutdown(timeout);
		}

		if (serverEndpoint != null) {
			serverEndpoint.close();
		}
	}
}
 

/**
 * Validates loading a file system state backend with additional parameters from the cluster configuration.
 */
@Test
public void testLoadFileSystemStateBackend() throws Exception {
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();
	final Path expectedCheckpointsPath = new Path(checkpointDir);
	final Path expectedSavepointsPath = new Path(savepointDir);
	final int threshold = 1000000;
	final boolean async = !CheckpointingOptions.ASYNC_SNAPSHOTS.defaultValue();

	// we configure with the explicit string (rather than AbstractStateBackend#X_STATE_BACKEND_NAME)
	// to guard against config-breaking changes of the name 
	final Configuration config1 = new Configuration();
	config1.setString(backendKey, "filesystem");
	config1.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
	config1.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config1.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, threshold);
	config1.setBoolean(CheckpointingOptions.ASYNC_SNAPSHOTS, async);

	final Configuration config2 = new Configuration();
	config2.setString(backendKey, FsStateBackendFactory.class.getName());
	config2.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
	config2.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config2.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, threshold);
	config2.setBoolean(CheckpointingOptions.ASYNC_SNAPSHOTS, async);

	StateBackend backend1 = StateBackendLoader.loadStateBackendFromConfig(config1, cl, null);
	StateBackend backend2 = StateBackendLoader.loadStateBackendFromConfig(config2, cl, null);

	assertTrue(backend1 instanceof FsStateBackend);
	assertTrue(backend2 instanceof FsStateBackend);

	FsStateBackend fs1 = (FsStateBackend) backend1;
	FsStateBackend fs2 = (FsStateBackend) backend2;

	assertEquals(expectedCheckpointsPath, fs1.getCheckpointPath());
	assertEquals(expectedCheckpointsPath, fs2.getCheckpointPath());
	assertEquals(expectedSavepointsPath, fs1.getSavepointPath());
	assertEquals(expectedSavepointsPath, fs2.getSavepointPath());
	assertEquals(threshold, fs1.getMinFileSizeThreshold());
	assertEquals(threshold, fs2.getMinFileSizeThreshold());
	assertEquals(async, fs1.isUsingAsynchronousSnapshots());
	assertEquals(async, fs2.isUsingAsynchronousSnapshots());
}
 

private static Configuration createConfiguration() {
	Configuration configuration = new Configuration();
	configuration.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, true);
	configuration.setString(EXECUTION_FAILOVER_STRATEGY.key(), "region");
	return configuration;
}
 

@Test
public void testNestedFileRead() throws IOException {
	String[] dirs = new String[] {"first", "second"};
	List<String> expectedFiles = new ArrayList<>();

	File parentDir = temporaryFolder.getRoot();
	for (String dir: dirs) {
		// create input file
		File tmpDir = temporaryFolder.newFolder(dir);

		File tempFile = File.createTempFile("TextInputFormatTest", ".tmp", tmpDir);

		expectedFiles.add(new Path(tempFile.getAbsolutePath()).makeQualified(FileSystem.getLocalFileSystem()).toString());
	}

	TextInputFormat inputFormat = new TextInputFormat(new Path(parentDir.toURI()));
	inputFormat.setNestedFileEnumeration(true);
	inputFormat.setNumLineSamples(10);

	// this is to check if the setter overrides the configuration (as expected)
	Configuration config = new Configuration();
	config.setBoolean("recursive.file.enumeration", false);
	config.setString("delimited-format.numSamples", "20");
	inputFormat.configure(config);

	assertTrue(inputFormat.getNestedFileEnumeration());
	assertEquals(10, inputFormat.getNumLineSamples());

	FileInputSplit[] splits = inputFormat.createInputSplits(expectedFiles.size());

	List<String> paths = new ArrayList<>();
	for (FileInputSplit split: splits) {
		paths.add(split.getPath().toString());
	}

	Collections.sort(expectedFiles);
	Collections.sort(paths);
	for (int i = 0; i < expectedFiles.size(); i++) {
		assertEquals(expectedFiles.get(i), paths.get(i));
	}
}