Java Code Examples for org.apache.flink.configuration.Configuration#getString()

public static ClusterConfigurationInfo from(Configuration config) {
	ClusterConfigurationInfo clusterConfig = new ClusterConfigurationInfo(config.keySet().size());

	for (String key : config.keySet()) {
		String value = config.getString(key, null);

		// Mask key values which contain sensitive information
		if (value != null && GlobalConfiguration.isSensitive(key)) {
			value = GlobalConfiguration.HIDDEN_CONTENT;
		}

		clusterConfig.add(new ClusterConfigurationInfoEntry(key, value));
	}

	return clusterConfig;
}
 

private static Optional<MetricReporter> loadReporter(
		final String reporterName,
		final Configuration reporterConfig,
		final Map<String, MetricReporterFactory> reporterFactories)
		throws ClassNotFoundException, IllegalAccessException, InstantiationException {

	final String reporterClassName = reporterConfig.getString(ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, null);
	final String factoryClassName = reporterConfig.getString(ConfigConstants.METRICS_REPORTER_FACTORY_CLASS_SUFFIX, null);

	if (factoryClassName != null) {
		return loadViaFactory(factoryClassName, reporterName, reporterConfig, reporterFactories);
	}

	if (reporterClassName != null) {
		return loadViaReflection(reporterClassName, reporterName, reporterConfig, reporterFactories);
	}

	LOG.warn("No reporter class nor factory set for reporter {}. Metrics might not be exposed/reported.", reporterName);
	return Optional.empty();
}
 

public static RestHandlerConfiguration fromConfiguration(Configuration configuration) {
	final long refreshInterval = configuration.getLong(WebOptions.REFRESH_INTERVAL);

	final int maxCheckpointStatisticCacheEntries = configuration.getInteger(WebOptions.CHECKPOINTS_HISTORY_SIZE);

	final Time timeout = Time.milliseconds(configuration.getLong(WebOptions.TIMEOUT));

	final String rootDir = "flink-web-ui";
	final File webUiDir = new File(configuration.getString(WebOptions.TMP_DIR), rootDir);

	final boolean webSubmitEnabled = configuration.getBoolean(WebOptions.SUBMIT_ENABLE);

	return new RestHandlerConfiguration(
		refreshInterval,
		maxCheckpointStatisticCacheEntries,
		timeout,
		webUiDir,
		webSubmitEnabled);
}
 

public static String stsValidate(String accessId, String accessKey, String localErrorMsg, Configuration properties) {
	String stsRoleArn = properties.getString(BlinkOptions.STS.STS_ROLE_ARN);
	String stsAccessId = properties.getString(BlinkOptions.STS.STS_ACCESS_ID);
	String stsAccessKey = properties.getString(BlinkOptions.STS.STS_ACCESS_KEY);
	String stsUid = properties.getString(BlinkOptions.STS.STS_UID);
	if (BlinkStringUtil.isNotEmpty(accessId, accessKey) || BlinkStringUtil.isNotEmpty(stsRoleArn, stsAccessId, stsAccessKey, stsUid)){
		return null;
	} else if (properties.containsKey(BlinkOptions.STS.STS_ROLE_ARN.key())){
		throw new NotEnoughParamsException(String.format("Lack necessary arguments: {0}", BlinkOptions.STS.STS_PARAMS_HELP_MSG));
	} else {
		throw new NotEnoughParamsException(localErrorMsg);
	}
}
 

public static void checkJobManagerAddress(Configuration config, String expectedAddress, int expectedPort) {
	String jobManagerAddress = config.getString(JobManagerOptions.ADDRESS);
	int jobManagerPort = config.getInteger(JobManagerOptions.PORT, -1);

	assertEquals(expectedAddress, jobManagerAddress);
	assertEquals(expectedPort, jobManagerPort);
}
 

/**
 * Calculates the heap size via
 * {@link TaskManagerServices#calculateHeapSizeMB(long, Configuration)} and the shell script
 * and verifies that these are equal.
 *
 * @param config     flink configuration
 * @param tolerance  tolerate values that are off by this factor (0.01 = 1%)
 */
private void compareHeapSizeJavaVsScript(final Configuration config, float tolerance)
		throws IOException {

	final long totalJavaMemorySizeMB = config.getLong(KEY_TASKM_MEM_SIZE, 0L);

	long javaHeapSizeMB = TaskManagerServices.calculateHeapSizeMB(totalJavaMemorySizeMB, config);

	String[] command = {"src/test/bin/calcTMHeapSizeMB.sh",
		totalJavaMemorySizeMB + "m",
		String.valueOf(config.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP)),
		String.valueOf(config.getFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION)),
		config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN),
		config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX),
		config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE),
		String.valueOf(config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION))};
	String scriptOutput = executeScript(command);

	// we need a tolerance of at least one, to compensate for MB/byte conversion rounding errors
	long absoluteTolerance = Math.max(1L, (long) (javaHeapSizeMB * tolerance));

	Long scriptHeapSizeMB = Long.valueOf(scriptOutput);
	assertThat(
		"Different heap sizes (Java: " + javaHeapSizeMB + ", Script: " + scriptHeapSizeMB +
			") with configuration: " + config.toString(), scriptHeapSizeMB,
		allOf(greaterThanOrEqualTo(javaHeapSizeMB - absoluteTolerance),
			lessThanOrEqualTo(javaHeapSizeMB + absoluteTolerance)));
}
 

@Test
public void testPythonBufferMemorySize() {
	final Configuration configuration = new Configuration();
	final String defaultPythonBufferMemorySize = configuration.getString(PythonOptions.PYTHON_DATA_BUFFER_MEMORY_SIZE);
	assertThat(defaultPythonBufferMemorySize, is(equalTo(PythonOptions.PYTHON_DATA_BUFFER_MEMORY_SIZE.defaultValue())));

	final String expectedPythonBufferMemorySize = "100mb";
	configuration.setString(PythonOptions.PYTHON_DATA_BUFFER_MEMORY_SIZE, expectedPythonBufferMemorySize);

	final String actualPythonBufferMemorySize = configuration.getString(PythonOptions.PYTHON_DATA_BUFFER_MEMORY_SIZE);
	assertThat(actualPythonBufferMemorySize, is(equalTo(expectedPythonBufferMemorySize)));
}
 

/**
 * Configures this input format by reading the path to the file from the configuration and the string that
 * defines the record delimiter.
 * 
 * @param parameters The configuration object to read the parameters from.
 */
@Override
public void configure(Configuration parameters) {
	super.configure(parameters);

	// the if() clauses are to prevent the configure() method from
	// overwriting the values set by the setters

	if (Arrays.equals(delimiter, new byte[] {'\n'})) {
		String delimString = parameters.getString(RECORD_DELIMITER, null);
		if (delimString != null) {
			setDelimiter(delimString);
		}
	}
	
	// set the number of samples
	if (numLineSamples == NUM_SAMPLES_UNDEFINED) {
		String samplesString = parameters.getString(NUM_STATISTICS_SAMPLES, null);
		if (samplesString != null) {
			try {
				setNumLineSamples(Integer.parseInt(samplesString));
			} catch (NumberFormatException e) {
				if (LOG.isWarnEnabled()) {
					LOG.warn("Invalid value for number of samples to take: " + samplesString + ". Skipping sampling.");
				}
				setNumLineSamples(0);
			}
		}
	}
}
 

/**
 * Create a security configuration from the global configuration.
 * @param flinkConf the Flink global configuration.
 * @param securityModuleFactories the security modules to apply.
 */
public SecurityConfiguration(Configuration flinkConf,
		List<SecurityModuleFactory> securityModuleFactories) {
	this.isZkSaslDisable = flinkConf.getBoolean(SecurityOptions.ZOOKEEPER_SASL_DISABLE);
	this.keytab = flinkConf.getString(SecurityOptions.KERBEROS_LOGIN_KEYTAB);
	this.principal = flinkConf.getString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL);
	this.useTicketCache = flinkConf.getBoolean(SecurityOptions.KERBEROS_LOGIN_USETICKETCACHE);
	this.loginContextNames = parseList(flinkConf.getString(SecurityOptions.KERBEROS_LOGIN_CONTEXTS));
	this.zkServiceName = flinkConf.getString(SecurityOptions.ZOOKEEPER_SASL_SERVICE_NAME);
	this.zkLoginContextName = flinkConf.getString(SecurityOptions.ZOOKEEPER_SASL_LOGIN_CONTEXT_NAME);
	this.securityModuleFactories = Collections.unmodifiableList(securityModuleFactories);
	this.flinkConfig = checkNotNull(flinkConf);
	validate();
}
 

/**
 * Creates a {@link MesosServices} instance depending on the high availability settings.
 *
 * @param configuration containing the high availability settings
 * @param hostname the hostname to advertise to remote clients
 * @return a mesos services instance
 * @throws Exception if the mesos services instance could not be created
 */
public static MesosServices createMesosServices(Configuration configuration, String hostname) throws Exception {

	ActorSystem localActorSystem = AkkaUtils.createLocalActorSystem(configuration);

	MesosArtifactServer artifactServer = createArtifactServer(configuration, hostname);

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			return new StandaloneMesosServices(localActorSystem, artifactServer);

		case ZOOKEEPER:
			final String zkMesosRootPath = configuration.getString(
				HighAvailabilityOptions.HA_ZOOKEEPER_MESOS_WORKERS_PATH);

			ZooKeeperUtilityFactory zooKeeperUtilityFactory = new ZooKeeperUtilityFactory(
				configuration,
				zkMesosRootPath);

			return new ZooKeeperMesosServices(localActorSystem, artifactServer, zooKeeperUtilityFactory);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}
 

public static String getLoggingYarnCommand(final Configuration configuration) {
	checkNotNull(configuration);

	final String logConfigFilePath = configuration.getString(YarnConfigOptionsInternal.APPLICATION_LOG_CONFIG_FILE);
	if (logConfigFilePath == null) {
		return "";
	}

	String logCommand = getLog4jCommand(logConfigFilePath);
	if (logCommand.isEmpty()) {
		logCommand = getLogBackCommand(logConfigFilePath);
	}
	return logCommand;
}
 

private static void addStatefulFunctionsConfiguration(Configuration configuration) {
  String parentFirst =
      configuration.getString(CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL, "");
  if (parentFirst.isEmpty()) {
    parentFirst = Constants.STATEFUL_FUNCTIONS_PACKAGE;
  } else if (parentFirst.endsWith(";")) {
    parentFirst = parentFirst + Constants.STATEFUL_FUNCTIONS_PACKAGE;
  } else {
    parentFirst = parentFirst + ";" + Constants.STATEFUL_FUNCTIONS_PACKAGE;
  }
  configuration.setString(
      CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL, parentFirst);
}
 

/**
 * Creates a {@link ZooKeeperCheckpointIDCounter} instance.
 *
 * @param client        The {@link CuratorFramework} ZooKeeper client to use
 * @param configuration {@link Configuration} object
 * @param jobId         ID of job to create the instance for
 * @return {@link ZooKeeperCheckpointIDCounter} instance
 */
public static ZooKeeperCheckpointIDCounter createCheckpointIDCounter(
		CuratorFramework client,
		Configuration configuration,
		JobID jobId) {

	String checkpointIdCounterPath = configuration.getString(
			HighAvailabilityOptions.HA_ZOOKEEPER_CHECKPOINT_COUNTER_PATH);

	checkpointIdCounterPath += ZooKeeperJobGraphStore.getPathForJob(jobId);

	return new ZooKeeperCheckpointIDCounter(client, checkpointIdCounterPath, new DefaultLastStateConnectionStateListener());
}
 

static HistoryServerArchivist createHistoryServerArchivist(Configuration configuration, JsonArchivist jsonArchivist, Executor ioExecutor) {
	final String configuredArchivePath = configuration.getString(JobManagerOptions.ARCHIVE_DIR);

	if (configuredArchivePath != null) {
		final Path archivePath = WebMonitorUtils.validateAndNormalizeUri(new Path(configuredArchivePath).toUri());

		return new JsonResponseHistoryServerArchivist(jsonArchivist, archivePath, ioExecutor);
	} else {
		return VoidHistoryServerArchivist.INSTANCE;
	}
}
 

/**
 * Helper to test that the {@link BlobServer} recovery from its HA store works.
 *
 * <p>Uploads two BLOBs to one {@link BlobServer} and expects a second one to be able to retrieve
 * them via a shared HA store upon request of a {@link BlobCacheService}.
 *
 * @param config
 * 		blob server configuration (including HA settings like {@link HighAvailabilityOptions#HA_STORAGE_PATH}
 * 		and {@link HighAvailabilityOptions#HA_CLUSTER_ID}) used to set up <tt>blobStore</tt>
 * @param blobStore
 * 		shared HA blob store to use
 *
 * @throws IOException
 * 		in case of failures
 */
public static void testBlobServerRecovery(final Configuration config, final BlobStore blobStore) throws IOException {
	final String clusterId = config.getString(HighAvailabilityOptions.HA_CLUSTER_ID);
	String storagePath = config.getString(HighAvailabilityOptions.HA_STORAGE_PATH) + "/" + clusterId;
	Random rand = new Random();

	try (
		BlobServer server0 = new BlobServer(config, blobStore);
		BlobServer server1 = new BlobServer(config, blobStore);
		// use VoidBlobStore as the HA store to force download from server[1]'s HA store
		BlobCacheService cache1 = new BlobCacheService(
			config, new VoidBlobStore(), new InetSocketAddress("localhost", server1.getPort())
		)) {

		server0.start();
		server1.start();

		// Random data
		byte[] expected = new byte[1024];
		rand.nextBytes(expected);
		byte[] expected2 = Arrays.copyOfRange(expected, 32, 288);

		BlobKey[] keys = new BlobKey[2];
		BlobKey nonHAKey;

		// Put job-related HA data
		JobID[] jobId = new JobID[] { new JobID(), new JobID() };
		keys[0] = put(server0, jobId[0], expected, PERMANENT_BLOB); // Request 1
		keys[1] = put(server0, jobId[1], expected2, PERMANENT_BLOB); // Request 2

		// put non-HA data
		nonHAKey = put(server0, jobId[0], expected2, TRANSIENT_BLOB);
		verifyKeyDifferentHashEquals(keys[1], nonHAKey);

		// check that the storage directory exists
		final Path blobServerPath = new Path(storagePath, "blob");
		FileSystem fs = blobServerPath.getFileSystem();
		assertTrue("Unknown storage dir: " + blobServerPath, fs.exists(blobServerPath));

		// Verify HA requests from cache1 (connected to server1) with no immediate access to the file
		verifyContents(cache1, jobId[0], keys[0], expected);
		verifyContents(cache1, jobId[1], keys[1], expected2);

		// Verify non-HA file is not accessible from server1
		verifyDeleted(cache1, jobId[0], nonHAKey);

		// Remove again
		server1.cleanupJob(jobId[0], true);
		server1.cleanupJob(jobId[1], true);

		// Verify everything is clean
		assertTrue("HA storage directory does not exist", fs.exists(new Path(storagePath)));
		if (fs.exists(blobServerPath)) {
			final org.apache.flink.core.fs.FileStatus[] recoveryFiles =
				fs.listStatus(blobServerPath);
			ArrayList<String> filenames = new ArrayList<>(recoveryFiles.length);
			for (org.apache.flink.core.fs.FileStatus file: recoveryFiles) {
				filenames.add(file.toString());
			}
			fail("Unclean state backend: " + filenames);
		}
	}
}
 

/**
 * Creates an SSL context for the external REST SSL.
 * If mutual authentication is configured the client and the server side configuration are identical.
 */
@Nullable
private static SSLContext createRestSSLContext(Configuration config, RestSSLContextConfigMode configMode) throws Exception {
	checkNotNull(config, "config");

	if (!isRestSSLEnabled(config)) {
		return null;
	}

	KeyManager[] keyManagers = null;
	if (configMode == RestSSLContextConfigMode.SERVER || configMode == RestSSLContextConfigMode.MUTUAL) {
		String keystoreFilePath = getAndCheckOption(
			config, SecurityOptions.SSL_REST_KEYSTORE, SecurityOptions.SSL_KEYSTORE);

		String keystorePassword = getAndCheckOption(
			config, SecurityOptions.SSL_REST_KEYSTORE_PASSWORD, SecurityOptions.SSL_KEYSTORE_PASSWORD);

		String certPassword = getAndCheckOption(
			config, SecurityOptions.SSL_REST_KEY_PASSWORD, SecurityOptions.SSL_KEY_PASSWORD);

		KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
		try (InputStream keyStoreFile = Files.newInputStream(new File(keystoreFilePath).toPath())) {
			keyStore.load(keyStoreFile, keystorePassword.toCharArray());
		}

		KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
		kmf.init(keyStore, certPassword.toCharArray());

		keyManagers = kmf.getKeyManagers();
	}

	TrustManager[] trustManagers = null;
	if (configMode == RestSSLContextConfigMode.CLIENT || configMode == RestSSLContextConfigMode.MUTUAL) {
		String trustStoreFilePath = getAndCheckOption(
			config, SecurityOptions.SSL_REST_TRUSTSTORE, SecurityOptions.SSL_TRUSTSTORE);

		String trustStorePassword = getAndCheckOption(
			config, SecurityOptions.SSL_REST_TRUSTSTORE_PASSWORD, SecurityOptions.SSL_TRUSTSTORE_PASSWORD);

		KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
		try (InputStream trustStoreFile = Files.newInputStream(new File(trustStoreFilePath).toPath())) {
			trustStore.load(trustStoreFile, trustStorePassword.toCharArray());
		}

		TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
		tmf.init(trustStore);

		trustManagers = tmf.getTrustManagers();
	}

	String sslProtocolVersion = config.getString(SecurityOptions.SSL_PROTOCOL);
	SSLContext sslContext = SSLContext.getInstance(sslProtocolVersion);
	sslContext.init(keyManagers, trustManagers, null);

	return sslContext;
}
 

public ZooKeeperRunningJobsRegistry(final CuratorFramework client, final Configuration configuration) {
	this.client = checkNotNull(client, "client");
	this.runningJobPath = configuration.getString(HighAvailabilityOptions.ZOOKEEPER_RUNNING_JOB_REGISTRY_PATH);
}
 

public ZooKeeperRunningJobsRegistry(final CuratorFramework client, final Configuration configuration) {
	this.client = checkNotNull(client, "client");
	this.runningJobPath = configuration.getString(HighAvailabilityOptions.ZOOKEEPER_RUNNING_JOB_REGISTRY_PATH);
}
 

@Override
public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode)
		throws CompilerException
{
	// see if an internal hint dictates the strategy to use
	final Configuration conf = getOperator().getParameters();
	final String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null);
	final ShipStrategyType preSet;
	
	if (shipStrategy != null) {
		if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH)) {
			preSet = ShipStrategyType.PARTITION_HASH;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE)) {
			preSet = ShipStrategyType.PARTITION_RANGE;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_FORWARD)) {
			preSet = ShipStrategyType.FORWARD;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
			preSet = ShipStrategyType.PARTITION_RANDOM;
		} else {
			throw new CompilerException("Unrecognized ship strategy hint: " + shipStrategy);
		}
	} else {
		preSet = null;
	}
	
	// get the predecessor node
	Operator<?> children = ((SingleInputOperator<?, ?, ?>) getOperator()).getInput();
	
	OptimizerNode pred;
	DagConnection conn;
	if (children == null) {
		throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input.");
	} else {
		pred = contractToNode.get(children);
		conn = new DagConnection(pred, this, defaultExchangeMode);
		if (preSet != null) {
			conn.setShipStrategy(preSet);
		}
	}
	
	// create the connection and add it
	setIncomingConnection(conn);
	pred.addOutgoingConnection(conn);
}
 

private Configuration generateClusterConfiguration(Configuration configuration) {
	final Configuration resultConfiguration = new Configuration(Preconditions.checkNotNull(configuration));

	final String webTmpDir = configuration.getString(WebOptions.TMP_DIR);
	final File uniqueWebTmpDir = new File(webTmpDir, "flink-web-" + UUID.randomUUID());

	resultConfiguration.setString(WebOptions.TMP_DIR, uniqueWebTmpDir.getAbsolutePath());

	return resultConfiguration;
}