org.apache.flink.yarn.Utils Java Examples

The following examples show how to use org.apache.flink.yarn.Utils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AthenaXYarnClusterDescriptor.java    From AthenaX with Apache License 2.0 5 votes vote down vote up 
private Map<String, String> setUpAmEnvironment(
    ApplicationId appId,
    String amClassPath,
    String shipFiles,
    String dynamicProperties) throws IOException {
  final Map<String, String> env = new HashMap<>();

  // set Flink app class path
  env.put(YarnConfigKeys.ENV_FLINK_CLASSPATH, amClassPath);

  // set Flink on YARN internal configuration values
  env.put(YarnConfigKeys.ENV_TM_COUNT, String.valueOf(job.taskManagerCount()));
  env.put(YarnConfigKeys.ENV_TM_MEMORY, String.valueOf(job.taskManagerMemoryMb()));
  env.put(YarnConfigKeys.FLINK_JAR_PATH, clusterConf.flinkUberJar().toString());
  env.put(YarnConfigKeys.ENV_APP_ID, appId.toString());
  env.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, clusterConf.homeDir());
  env.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, shipFiles);
  env.put(YarnConfigKeys.ENV_SLOTS, "-1");
  env.put(YarnConfigKeys.ENV_DETACHED, "true");

  // https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md#identity-on-an-insecure-cluster-hadoop_user_name
  env.put(YarnConfigKeys.ENV_HADOOP_USER_NAME,
      UserGroupInformation.getCurrentUser().getUserName());

  if (dynamicProperties != null) {
    env.put(YarnConfigKeys.ENV_DYNAMIC_PROPERTIES, dynamicProperties);
  }

  // set classpath from YARN configuration
  Utils.setupYarnClassPath(clusterConf.conf(), env);

  return env;
}
Example #2
Source File: YarnEntrypointUtils.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public static Configuration loadConfiguration(String workingDirectory, Map<String, String> env, Logger log) {
		Configuration configuration = GlobalConfiguration.loadConfiguration(workingDirectory);

		final String remoteKeytabPrincipal = env.get(YarnConfigKeys.KEYTAB_PRINCIPAL);

		final String zooKeeperNamespace = env.get(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE);

		final Map<String, String> dynamicProperties = FlinkYarnSessionCli.getDynamicProperties(
			env.get(YarnConfigKeys.ENV_DYNAMIC_PROPERTIES));

		final String hostname = env.get(ApplicationConstants.Environment.NM_HOST.key());
		Preconditions.checkState(
			hostname != null,
			"ApplicationMaster hostname variable %s not set",
			ApplicationConstants.Environment.NM_HOST.key());

		configuration.setString(JobManagerOptions.ADDRESS, hostname);
		configuration.setString(RestOptions.ADDRESS, hostname);

		// TODO: Support port ranges for the AM
//		final String portRange = configuration.getString(
//			ConfigConstants.YARN_APPLICATION_MASTER_PORT,
//			ConfigConstants.DEFAULT_YARN_JOB_MANAGER_PORT);

		for (Map.Entry<String, String> property : dynamicProperties.entrySet()) {
			configuration.setString(property.getKey(), property.getValue());
		}

		if (zooKeeperNamespace != null) {
			configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zooKeeperNamespace);
		}

		// if a web monitor shall be started, set the port to random binding
		if (configuration.getInteger(WebOptions.PORT, 0) >= 0) {
			configuration.setInteger(WebOptions.PORT, 0);
		}

		if (!configuration.contains(RestOptions.BIND_PORT)) {
			// set the REST port to 0 to select it randomly
			configuration.setString(RestOptions.BIND_PORT, "0");
		}

		// if the user has set the deprecated YARN-specific config keys, we add the
		// corresponding generic config keys instead. that way, later code needs not
		// deal with deprecated config keys

		BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
			ConfigConstants.YARN_APPLICATION_MASTER_ENV_PREFIX,
			ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX);

		BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
			ConfigConstants.YARN_TASK_MANAGER_ENV_PREFIX,
			ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX);

		final String keytabPath;

		if (env.get(YarnConfigKeys.KEYTAB_PATH) == null) {
			keytabPath = null;
		} else {
			File f = new File(workingDirectory, Utils.KEYTAB_FILE_NAME);
			keytabPath = f.getAbsolutePath();
		}

		if (keytabPath != null && remoteKeytabPrincipal != null) {
			configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
			configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
		}

		final String localDirs = env.get(ApplicationConstants.Environment.LOCAL_DIRS.key());
		BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, localDirs);

		return configuration;
	}
Example #3
Source File: YarnEntrypointUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
public static Configuration loadConfiguration(String workingDirectory, Map<String, String> env, Logger log) {
		Configuration configuration = GlobalConfiguration.loadConfiguration(workingDirectory);

		final String remoteKeytabPrincipal = env.get(YarnConfigKeys.KEYTAB_PRINCIPAL);

		final String zooKeeperNamespace = env.get(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE);

		final Map<String, String> dynamicProperties = FlinkYarnSessionCli.getDynamicProperties(
			env.get(YarnConfigKeys.ENV_DYNAMIC_PROPERTIES));

		final String hostname = env.get(ApplicationConstants.Environment.NM_HOST.key());
		Preconditions.checkState(
			hostname != null,
			"ApplicationMaster hostname variable %s not set",
			ApplicationConstants.Environment.NM_HOST.key());

		configuration.setString(JobManagerOptions.ADDRESS, hostname);
		configuration.setString(RestOptions.ADDRESS, hostname);

		// TODO: Support port ranges for the AM
//		final String portRange = configuration.getString(
//			ConfigConstants.YARN_APPLICATION_MASTER_PORT,
//			ConfigConstants.DEFAULT_YARN_JOB_MANAGER_PORT);

		for (Map.Entry<String, String> property : dynamicProperties.entrySet()) {
			configuration.setString(property.getKey(), property.getValue());
		}

		if (zooKeeperNamespace != null) {
			configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zooKeeperNamespace);
		}

		// if a web monitor shall be started, set the port to random binding
		if (configuration.getInteger(WebOptions.PORT, 0) >= 0) {
			configuration.setInteger(WebOptions.PORT, 0);
		}

		if (!configuration.contains(RestOptions.BIND_PORT)) {
			// set the REST port to 0 to select it randomly
			configuration.setString(RestOptions.BIND_PORT, "0");
		}

		// if the user has set the deprecated YARN-specific config keys, we add the
		// corresponding generic config keys instead. that way, later code needs not
		// deal with deprecated config keys

		BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
			ConfigConstants.YARN_APPLICATION_MASTER_ENV_PREFIX,
			ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX);

		BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
			ConfigConstants.YARN_TASK_MANAGER_ENV_PREFIX,
			ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX);

		final String keytabPath;

		if (env.get(YarnConfigKeys.KEYTAB_PATH) == null) {
			keytabPath = null;
		} else {
			File f = new File(workingDirectory, Utils.KEYTAB_FILE_NAME);
			keytabPath = f.getAbsolutePath();
		}

		if (keytabPath != null && remoteKeytabPrincipal != null) {
			configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
			configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
		}

		final String localDirs = env.get(ApplicationConstants.Environment.LOCAL_DIRS.key());
		BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, localDirs);

		return configuration;
	}
Example #4
Source File: YarnEntrypointUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
public static Configuration loadConfiguration(String workingDirectory, Map<String, String> env) {
	Configuration configuration = GlobalConfiguration.loadConfiguration(workingDirectory);

	final String keytabPrincipal = env.get(YarnConfigKeys.KEYTAB_PRINCIPAL);

	final String zooKeeperNamespace = env.get(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE);

	final String hostname = env.get(ApplicationConstants.Environment.NM_HOST.key());
	Preconditions.checkState(
		hostname != null,
		"ApplicationMaster hostname variable %s not set",
		ApplicationConstants.Environment.NM_HOST.key());

	configuration.setString(JobManagerOptions.ADDRESS, hostname);
	configuration.setString(RestOptions.ADDRESS, hostname);

	if (zooKeeperNamespace != null) {
		configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zooKeeperNamespace);
	}

	// if a web monitor shall be started, set the port to random binding
	if (configuration.getInteger(WebOptions.PORT, 0) >= 0) {
		configuration.setInteger(WebOptions.PORT, 0);
	}

	if (!configuration.contains(RestOptions.BIND_PORT)) {
		// set the REST port to 0 to select it randomly
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	// if the user has set the deprecated YARN-specific config keys, we add the
	// corresponding generic config keys instead. that way, later code needs not
	// deal with deprecated config keys

	BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
		ConfigConstants.YARN_APPLICATION_MASTER_ENV_PREFIX,
		ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX);

	BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
		ConfigConstants.YARN_TASK_MANAGER_ENV_PREFIX,
		ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX);

	final String keytabPath = Utils.resolveKeytabPath(workingDirectory, env.get(YarnConfigKeys.LOCAL_KEYTAB_PATH));

	if (keytabPath != null && keytabPrincipal != null) {
		configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
		configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, keytabPrincipal);
	}

	final String localDirs = env.get(ApplicationConstants.Environment.LOCAL_DIRS.key());
	BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, localDirs);

	return configuration;
}