Java Code Examples for org.apache.flink.configuration.GlobalConfiguration#loadConfiguration()

/**
 * Merge HadoopConfiguration into Configuration. This is necessary for the HDFS configuration.
 */
public static void mergeHadoopConf(Configuration hadoopConfig) {

	// we have to load the global configuration here, because the HadoopInputFormatBase does not
	// have access to a Flink configuration object
	org.apache.flink.configuration.Configuration flinkConfiguration = GlobalConfiguration.loadConfiguration();

	Configuration hadoopConf =
		org.apache.flink.api.java.hadoop.mapred.utils.HadoopUtils.getHadoopConfiguration(flinkConfiguration);

	for (Map.Entry<String, String> e : hadoopConf) {
		if (hadoopConfig.get(e.getKey()) == null) {
			hadoopConfig.set(e.getKey(), e.getValue());
		}
	}
}
 

/**
 * Generate configuration from only the config file and dynamic properties.
 * @param args the commandline arguments
 * @param cmdLineSyntax the syntax for this application
 * @return generated configuration
 * @throws FlinkParseException if the configuration cannot be generated
 */
public static Configuration loadCommonConfiguration(String[] args, String cmdLineSyntax) throws FlinkParseException {
	final CommandLineParser<ClusterConfiguration> commandLineParser = new CommandLineParser<>(new ClusterConfigurationParserFactory());

	final ClusterConfiguration clusterConfiguration;

	try {
		clusterConfiguration = commandLineParser.parse(args);
	} catch (FlinkParseException e) {
		LOG.error("Could not parse the command line options.", e);
		commandLineParser.printHelp(cmdLineSyntax);
		throw e;
	}

	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(clusterConfiguration.getDynamicProperties());
	return GlobalConfiguration.loadConfiguration(clusterConfiguration.getConfigDir(), dynamicProperties);
}
 

public static void main(final String[] args) {
	final String configurationDirectory = CliFrontend.getConfigurationDirectoryFromEnv();

	final Configuration flinkConfiguration = GlobalConfiguration.loadConfiguration();

	int retCode;

	try {
		final FlinkYarnSessionCli cli = new FlinkYarnSessionCli(
			flinkConfiguration,
			configurationDirectory,
			"",
			""); // no prefix for the YARN session

		SecurityUtils.install(new SecurityConfiguration(flinkConfiguration));

		retCode = SecurityUtils.getInstalledContext().runSecured(() -> cli.run(args));
	} catch (CliArgsException e) {
		retCode = handleCliArgsException(e, LOG);
	} catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		retCode = handleError(strippedThrowable, LOG);
	}

	System.exit(retCode);
}
 

public FlinkPlanner(List<String> classPath) {
	super();
	this.classPath = classPath;

	env.getConfig().disableSysoutLogging();
	if (env.getParallelism() <= 0) {
		// load the default parallelism from config
		GlobalConfiguration.loadConfiguration(new File(CliFrontend.getConfigurationDirectoryFromEnv()).getAbsolutePath());
		org.apache.flink.configuration.Configuration configuration = GlobalConfiguration.getConfiguration();
		int parallelism = configuration.getInteger(ConfigConstants.DEFAULT_PARALLELISM_KEY, -1);
		if (parallelism <= 0) {
			throw new RuntimeException("Please set the default parallelism via the -p command-line flag");
		} else {
			env.setParallelism(parallelism);
		}
	}

}
 

/**
 * Entry point for the execution of a python plan.
 *
 * @param args planPath[ package1[ packageX[ - parameter1[ parameterX]]]]
 * @throws Exception
 */
public static void main(String[] args) throws Exception {
	Configuration globalConfig = GlobalConfiguration.loadConfiguration();
	PythonPlanBinder binder = new PythonPlanBinder(globalConfig);
	try {
		binder.runPlan(args);
	} catch (Exception e) {
		System.out.println("Failed to run plan: " + e.getMessage());
		LOG.error("Failed to run plan.", e);
	}
}
 

FlinkDistribution(Path distributionDir) {
	bin = distributionDir.resolve("bin");
	opt = distributionDir.resolve("opt");
	lib = distributionDir.resolve("lib");
	conf = distributionDir.resolve("conf");
	log = distributionDir.resolve("log");
	plugins = distributionDir.resolve("plugins");

	defaultConfig = new UnmodifiableConfiguration(GlobalConfiguration.loadConfiguration(conf.toAbsolutePath().toString()));
}
 

/**
 * The instance entry point for the YARN task executor. Obtains user group information and calls
 * the main work method {@link TaskManagerRunner#runTaskManager(Configuration, ResourceID)}  as a
 * privileged action.
 *
 * @param args The command line arguments.
 */
private static void run(String[] args) {
	try {
		LOG.debug("All environment variables: {}", ENV);

		final String currDir = ENV.get(Environment.PWD.key());
		LOG.info("Current working Directory: {}", currDir);

		final Configuration configuration = GlobalConfiguration.loadConfiguration(currDir);

		FileSystem.initialize(configuration, PluginUtils.createPluginManagerFromRootFolder(configuration));

		setupConfigurationAndInstallSecurityContext(configuration, currDir, ENV);

		final String containerId = ENV.get(YarnResourceManager.ENV_FLINK_CONTAINER_ID);
		Preconditions.checkArgument(containerId != null,
			"ContainerId variable %s not set", YarnResourceManager.ENV_FLINK_CONTAINER_ID);

		SecurityUtils.getInstalledContext().runSecured((Callable<Void>) () -> {
			TaskManagerRunner.runTaskManager(configuration, new ResourceID(containerId));
			return null;
		});
	}
	catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		// make sure that everything whatever ends up in the log
		LOG.error("YARN TaskManager initialization failed.", strippedThrowable);
		System.exit(INIT_ERROR_EXIT_CODE);
	}
}
 

/**
 * Loads the global configuration, adds the given dynamic properties configuration, and sets
 * the temp directory paths.
 *
 * @param dynamicProperties dynamic properties to integrate
 * @param log logger instance
 * @return the loaded and adapted global configuration
 */
public static Configuration loadConfiguration(Configuration dynamicProperties, Logger log) {
	Configuration configuration =
		GlobalConfiguration.loadConfiguration(dynamicProperties);

	// read the environment variables
	final Map<String, String> envs = System.getenv();
	final String tmpDirs = envs.get(MesosConfigKeys.ENV_FLINK_TMP_DIR);

	BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, tmpDirs);

	return configuration;
}
 

/**
 * Submits the job based on the arguments.
 */
public static void main(final String[] args) {
	EnvironmentInformation.logEnvironmentInfo(LOG, "Command Line Client", args);

	// 1. find the configuration directory
	final String configurationDirectory = getConfigurationDirectoryFromEnv();

	// 2. load the global configuration
	final Configuration configuration = GlobalConfiguration.loadConfiguration(configurationDirectory);

	// 3. load the custom command lines
	final List<CustomCommandLine<?>> customCommandLines = loadCustomCommandLines(
		configuration,
		configurationDirectory);

	try {
		final CliFrontend cli = new CliFrontend(
			configuration,
			customCommandLines);

		SecurityUtils.install(new SecurityConfiguration(cli.configuration));
		int retCode = SecurityUtils.getInstalledContext()
				.runSecured(() -> cli.parseParameters(args));
		System.exit(retCode);
	}
	catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		LOG.error("Fatal error while running command line interface.", strippedThrowable);
		strippedThrowable.printStackTrace();
		System.exit(31);
	}
}
 

/**
 * Merge HadoopConfiguration into JobConf. This is necessary for the HDFS configuration.
 */
public static void mergeHadoopConf(JobConf jobConf) {
	// we have to load the global configuration here, because the HadoopInputFormatBase does not
	// have access to a Flink configuration object
	org.apache.flink.configuration.Configuration flinkConfiguration = GlobalConfiguration.loadConfiguration();

	Configuration hadoopConf = getHadoopConfiguration(flinkConfiguration);
	for (Map.Entry<String, String> e : hadoopConf) {
		if (jobConf.get(e.getKey()) == null) {
			jobConf.set(e.getKey(), e.getValue());
		}
	}
}
 

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);
	String configDir = pt.getRequired("configDir");

	LOG.info("Loading configuration from {}", configDir);
	final Configuration flinkConfig = GlobalConfiguration.loadConfiguration(configDir);

	FileSystem.initialize(flinkConfig, PluginUtils.createPluginManagerFromRootFolder(flinkConfig));

	// run the history server
	SecurityUtils.install(new SecurityConfiguration(flinkConfig));

	try {
		SecurityUtils.getInstalledContext().runSecured(new Callable<Integer>() {
			@Override
			public Integer call() throws Exception {
				HistoryServer hs = new HistoryServer(flinkConfig);
				hs.run();
				return 0;
			}
		});
		System.exit(0);
	} catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		LOG.error("Failed to run HistoryServer.", strippedThrowable);
		strippedThrowable.printStackTrace();
		System.exit(1);
	}
}
 

/**
 * Submits the job based on the arguments.
 */
public static void main(final String[] args) {
	EnvironmentInformation.logEnvironmentInfo(LOG, "Command Line Client", args);

	// 1. find the configuration directory
	final String configurationDirectory = getConfigurationDirectoryFromEnv();

	// 2. load the global configuration
	final Configuration configuration = GlobalConfiguration.loadConfiguration(configurationDirectory);

	// 3. load the custom command lines
	final List<CustomCommandLine<?>> customCommandLines = loadCustomCommandLines(
		configuration,
		configurationDirectory);

	try {
		final CliFrontend cli = new CliFrontend(
			configuration,
			customCommandLines);

		SecurityUtils.install(new SecurityConfiguration(cli.configuration));
		int retCode = SecurityUtils.getInstalledContext()
				.runSecured(() -> cli.parseParameters(args));
		System.exit(retCode);
	}
	catch (Throwable t) {
		final Throwable strippedThrowable = ExceptionUtils.stripException(t, UndeclaredThrowableException.class);
		LOG.error("Fatal error while running command line interface.", strippedThrowable);
		strippedThrowable.printStackTrace();
		System.exit(31);
	}
}
 

@Override
protected void before() throws IOException {
	defaultConfig = new UnmodifiableConfiguration(GlobalConfiguration.loadConfiguration(conf.toAbsolutePath().toString()));
	final Path originalConfig = conf.resolve(FLINK_CONF_YAML);
	final Path backupConfig = conf.resolve(FLINK_CONF_YAML_BACKUP);
	Files.copy(originalConfig, backupConfig);
	filesToDelete.add(new AutoClosablePath(backupConfig));
}
 

/**
 * Merge HadoopConfiguration into JobConf. This is necessary for the HDFS configuration.
 */
public static void mergeHadoopConf(JobConf jobConf) {
	// we have to load the global configuration here, because the HadoopInputFormatBase does not
	// have access to a Flink configuration object
	org.apache.flink.configuration.Configuration flinkConfiguration = GlobalConfiguration.loadConfiguration();

	Configuration hadoopConf = getHadoopConfiguration(flinkConfiguration);
	for (Map.Entry<String, String> e : hadoopConf) {
		if (jobConf.get(e.getKey()) == null) {
			jobConf.set(e.getKey(), e.getValue());
		}
	}
}
 

/**
 * Merge HadoopConfiguration into JobConf. This is necessary for the HDFS configuration.
 */
public static void mergeHadoopConf(JobConf jobConf) {
	// we have to load the global configuration here, because the HadoopInputFormatBase does not
	// have access to a Flink configuration object
	org.apache.flink.configuration.Configuration flinkConfiguration = GlobalConfiguration.loadConfiguration();

	Configuration hadoopConf = getHadoopConfiguration(flinkConfiguration);
	for (Map.Entry<String, String> e : hadoopConf) {
		if (jobConf.get(e.getKey()) == null) {
			jobConf.set(e.getKey(), e.getValue());
		}
	}
}
 

/**
 * Loads the global configuration, adds the given dynamic properties configuration, and sets
 * the temp directory paths.
 *
 * @param dynamicProperties dynamic properties to integrate
 * @param log logger instance
 * @return the loaded and adapted global configuration
 */
public static Configuration loadConfiguration(Configuration dynamicProperties, Logger log) {
	Configuration configuration =
		GlobalConfiguration.loadConfiguration(dynamicProperties);

	// read the environment variables
	final Map<String, String> envs = System.getenv();
	final String tmpDirs = envs.get(MesosConfigKeys.ENV_FLINK_TMP_DIR);

	BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, tmpDirs);

	return configuration;
}
 

protected Configuration getConfiguration() {
	final Configuration configuration = GlobalConfiguration
		.loadConfiguration(CliFrontendTestUtils.getConfigDir());
	return configuration;
}
 

protected Configuration getConfiguration() {
	final Configuration configuration = GlobalConfiguration
		.loadConfiguration(CliFrontendTestUtils.getConfigDir());
	return configuration;
}
 

public static void prepare(TemporaryFolder tempFolder) {

		try {
			File baseDirForSecureRun = tempFolder.newFolder();
			LOG.info("Base Directory for Secure Environment: {}", baseDirForSecureRun);

			String hostName = "localhost";
			Properties kdcConf = MiniKdc.createConf();
			if (LOG.isDebugEnabled()) {
				kdcConf.setProperty(MiniKdc.DEBUG, "true");
			}
			kdcConf.setProperty(MiniKdc.KDC_BIND_ADDRESS, hostName);
			kdc = new MiniKdc(kdcConf, baseDirForSecureRun);
			kdc.start();
			LOG.info("Started Mini KDC");

			File keytabFile = new File(baseDirForSecureRun, "test-users.keytab");
			testKeytab = keytabFile.getAbsolutePath();
			testZkServerPrincipal = "zookeeper/127.0.0.1";
			testZkClientPrincipal = "zk-client/127.0.0.1";
			testKafkaServerPrincipal = "kafka/" + hostName;
			hadoopServicePrincipal = "hadoop/" + hostName;
			testPrincipal = "client/" + hostName;

			kdc.createPrincipal(keytabFile, testPrincipal, testZkServerPrincipal,
					hadoopServicePrincipal,
					testZkClientPrincipal,
					testKafkaServerPrincipal);

			testPrincipal = testPrincipal + "@" + kdc.getRealm();
			testZkServerPrincipal = testZkServerPrincipal + "@" + kdc.getRealm();
			testZkClientPrincipal = testZkClientPrincipal + "@" + kdc.getRealm();
			testKafkaServerPrincipal = testKafkaServerPrincipal + "@" + kdc.getRealm();
			hadoopServicePrincipal = hadoopServicePrincipal + "@" + kdc.getRealm();

			LOG.info("-------------------------------------------------------------------");
			LOG.info("Test Principal: {}", testPrincipal);
			LOG.info("Test ZK Server Principal: {}", testZkServerPrincipal);
			LOG.info("Test ZK Client Principal: {}", testZkClientPrincipal);
			LOG.info("Test Kafka Server Principal: {}", testKafkaServerPrincipal);
			LOG.info("Test Hadoop Service Principal: {}", hadoopServicePrincipal);
			LOG.info("Test Keytab: {}", testKeytab);
			LOG.info("-------------------------------------------------------------------");

			//Security Context is established to allow non hadoop applications that requires JAAS
			//based SASL/Kerberos authentication to work. However, for Hadoop specific applications
			//the context can be reinitialized with Hadoop configuration by calling
			//ctx.setHadoopConfiguration() for the UGI implementation to work properly.
			//See Yarn test case module for reference
			Configuration flinkConfig = GlobalConfiguration.loadConfiguration();
			flinkConfig.setBoolean(SecurityOptions.ZOOKEEPER_SASL_DISABLE, false);
			flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, testKeytab);
			flinkConfig.setBoolean(SecurityOptions.KERBEROS_LOGIN_USETICKETCACHE, false);
			flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, testPrincipal);
			flinkConfig.setString(SecurityOptions.KERBEROS_LOGIN_CONTEXTS, "Client,KafkaClient");
			SecurityConfiguration ctx = new SecurityConfiguration(flinkConfig);
			TestingSecurityContext.install(ctx, getClientSecurityConfigurationMap());

			populateJavaPropertyVariables();

		} catch (Exception e) {
			throw new RuntimeException("Exception occured while preparing secure environment.", e);
		}

	}
 

@Override
protected Configuration loadClientConfiguration() {
	return GlobalConfiguration.loadConfiguration(env.get(ApplicationConstants.Environment.PWD.key()));
}