org.apache.flink.yarn.YarnConfigKeys Java Examples

The following examples show how to use org.apache.flink.yarn.YarnConfigKeys. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: YarnEntrypointUtils.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public static void logYarnEnvironmentInformation(Map<String, String> env, Logger log) throws IOException {
	final String yarnClientUsername = env.get(YarnConfigKeys.ENV_HADOOP_USER_NAME);
	Preconditions.checkArgument(
		yarnClientUsername != null,
		"YARN client user name environment variable %s not set",
		YarnConfigKeys.ENV_HADOOP_USER_NAME);

	UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();

	log.info("YARN daemon is running as: {} Yarn client user obtainer: {}",
		currentUser.getShortUserName(), yarnClientUsername);
}
Example #2
Source File: YarnEntrypointUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static void logYarnEnvironmentInformation(Map<String, String> env, Logger log) throws IOException {
	final String yarnClientUsername = env.get(YarnConfigKeys.ENV_HADOOP_USER_NAME);
	Preconditions.checkArgument(
		yarnClientUsername != null,
		"YARN client user name environment variable %s not set",
		YarnConfigKeys.ENV_HADOOP_USER_NAME);

	UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();

	log.info("YARN daemon is running as: {} Yarn client user obtainer: {}",
		currentUser.getShortUserName(), yarnClientUsername);
}
Example #3
Source File: YarnJobDescriptor.java    From sylph with Apache License 2.0 5 votes vote down vote up 
private Map<String, String> setUpAmEnvironment(
        Path uploadingDir,
        ApplicationId appId,
        String amClassPath,
        String shipFiles,
        String dynamicProperties)
        throws IOException
{
    final Map<String, String> appMasterEnv = new HashMap<>();

    appMasterEnv.put(YarnConfigKeys.ENV_FLINK_CLASSPATH, amClassPath);

    // set Flink on YARN internal configuration values
    appMasterEnv.put(YarnConfigKeys.ENV_TM_COUNT, String.valueOf(appConf.getTaskManagerCount()));
    appMasterEnv.put(YarnConfigKeys.ENV_TM_MEMORY, String.valueOf(appConf.getTaskManagerMemoryMb()));
    appMasterEnv.put(YarnConfigKeys.FLINK_JAR_PATH, flinkJar.toString());
    appMasterEnv.put(YarnConfigKeys.ENV_APP_ID, appId.toString());
    appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, uploadingDir.getParent().toString()); //$home/.flink/appid 这个目录里面存放临时数据
    appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, shipFiles);
    appMasterEnv.put(YarnConfigKeys.ENV_SLOTS, String.valueOf(appConf.getTaskManagerSlots()));
    appMasterEnv.put(YarnConfigKeys.ENV_DETACHED, String.valueOf(true));  //是否分离 分离就cluser模式 否则是client模式
    appMasterEnv.put(YarnConfigKeys.FLINK_YARN_FILES, uploadingDir.toUri().toString());
    appMasterEnv.put(YarnConfigKeys.ENV_HADOOP_USER_NAME, UserGroupInformation.getCurrentUser().getUserName());

    if (dynamicProperties != null) {
        appMasterEnv.put(YarnConfigKeys.ENV_DYNAMIC_PROPERTIES, dynamicProperties);
    }
    return appMasterEnv;
}
Example #4
Source File: AthenaXYarnClusterDescriptor.java    From AthenaX with Apache License 2.0 5 votes vote down vote up 
private Map<String, String> setUpAmEnvironment(
    ApplicationId appId,
    String amClassPath,
    String shipFiles,
    String dynamicProperties) throws IOException {
  final Map<String, String> env = new HashMap<>();

  // set Flink app class path
  env.put(YarnConfigKeys.ENV_FLINK_CLASSPATH, amClassPath);

  // set Flink on YARN internal configuration values
  env.put(YarnConfigKeys.ENV_TM_COUNT, String.valueOf(job.taskManagerCount()));
  env.put(YarnConfigKeys.ENV_TM_MEMORY, String.valueOf(job.taskManagerMemoryMb()));
  env.put(YarnConfigKeys.FLINK_JAR_PATH, clusterConf.flinkUberJar().toString());
  env.put(YarnConfigKeys.ENV_APP_ID, appId.toString());
  env.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, clusterConf.homeDir());
  env.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, shipFiles);
  env.put(YarnConfigKeys.ENV_SLOTS, "-1");
  env.put(YarnConfigKeys.ENV_DETACHED, "true");

  // https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md#identity-on-an-insecure-cluster-hadoop_user_name
  env.put(YarnConfigKeys.ENV_HADOOP_USER_NAME,
      UserGroupInformation.getCurrentUser().getUserName());

  if (dynamicProperties != null) {
    env.put(YarnConfigKeys.ENV_DYNAMIC_PROPERTIES, dynamicProperties);
  }

  // set classpath from YARN configuration
  Utils.setupYarnClassPath(clusterConf.conf(), env);

  return env;
}
Example #5
Source File: YarnEntrypointUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static void logYarnEnvironmentInformation(Map<String, String> env, Logger log) throws IOException {
	final String yarnClientUsername = env.get(YarnConfigKeys.ENV_HADOOP_USER_NAME);
	Preconditions.checkArgument(
		yarnClientUsername != null,
		"YARN client user name environment variable %s not set",
		YarnConfigKeys.ENV_HADOOP_USER_NAME);

	UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();

	log.info("YARN daemon is running as: {} Yarn client user obtainer: {}",
		currentUser.getShortUserName(), yarnClientUsername);
}
Example #6
Source File: YarnEntrypointUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testParsingValidKerberosEnv() throws IOException {
	final Configuration initialConfiguration = new Configuration();
	Map<String, String> env = new HashMap<>();
	File keytabFile = TEMPORARY_FOLDER.newFile();
	env.put(YarnConfigKeys.LOCAL_KEYTAB_PATH, keytabFile.getAbsolutePath());
	env.put(YarnConfigKeys.KEYTAB_PRINCIPAL, "starlord");

	Configuration configuration = loadConfiguration(initialConfiguration, env);

	assertThat(configuration.get(SecurityOptions.KERBEROS_LOGIN_KEYTAB), is(keytabFile.getAbsolutePath()));
	assertThat(configuration.get(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL), is("starlord"));
}
Example #7
Source File: YarnEntrypointUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testParsingKerberosEnvWithMissingKeytab() throws IOException {
	final Configuration initialConfiguration = new Configuration();
	Map<String, String> env = new HashMap<>();
	env.put(YarnConfigKeys.LOCAL_KEYTAB_PATH, "/hopefully/doesnt/exist");
	env.put(YarnConfigKeys.KEYTAB_PRINCIPAL, "starlord");

	Configuration configuration = loadConfiguration(initialConfiguration, env);

	// both keytab and principal should be null
	assertThat(configuration.get(SecurityOptions.KERBEROS_LOGIN_KEYTAB), nullValue());
	assertThat(configuration.get(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL), nullValue());
}
Example #8
Source File: YarnEntrypointUtils.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public static Configuration loadConfiguration(String workingDirectory, Map<String, String> env, Logger log) {
		Configuration configuration = GlobalConfiguration.loadConfiguration(workingDirectory);

		final String remoteKeytabPrincipal = env.get(YarnConfigKeys.KEYTAB_PRINCIPAL);

		final String zooKeeperNamespace = env.get(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE);

		final Map<String, String> dynamicProperties = FlinkYarnSessionCli.getDynamicProperties(
			env.get(YarnConfigKeys.ENV_DYNAMIC_PROPERTIES));

		final String hostname = env.get(ApplicationConstants.Environment.NM_HOST.key());
		Preconditions.checkState(
			hostname != null,
			"ApplicationMaster hostname variable %s not set",
			ApplicationConstants.Environment.NM_HOST.key());

		configuration.setString(JobManagerOptions.ADDRESS, hostname);
		configuration.setString(RestOptions.ADDRESS, hostname);

		// TODO: Support port ranges for the AM
//		final String portRange = configuration.getString(
//			ConfigConstants.YARN_APPLICATION_MASTER_PORT,
//			ConfigConstants.DEFAULT_YARN_JOB_MANAGER_PORT);

		for (Map.Entry<String, String> property : dynamicProperties.entrySet()) {
			configuration.setString(property.getKey(), property.getValue());
		}

		if (zooKeeperNamespace != null) {
			configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zooKeeperNamespace);
		}

		// if a web monitor shall be started, set the port to random binding
		if (configuration.getInteger(WebOptions.PORT, 0) >= 0) {
			configuration.setInteger(WebOptions.PORT, 0);
		}

		if (!configuration.contains(RestOptions.BIND_PORT)) {
			// set the REST port to 0 to select it randomly
			configuration.setString(RestOptions.BIND_PORT, "0");
		}

		// if the user has set the deprecated YARN-specific config keys, we add the
		// corresponding generic config keys instead. that way, later code needs not
		// deal with deprecated config keys

		BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
			ConfigConstants.YARN_APPLICATION_MASTER_ENV_PREFIX,
			ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX);

		BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
			ConfigConstants.YARN_TASK_MANAGER_ENV_PREFIX,
			ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX);

		final String keytabPath;

		if (env.get(YarnConfigKeys.KEYTAB_PATH) == null) {
			keytabPath = null;
		} else {
			File f = new File(workingDirectory, Utils.KEYTAB_FILE_NAME);
			keytabPath = f.getAbsolutePath();
		}

		if (keytabPath != null && remoteKeytabPrincipal != null) {
			configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
			configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
		}

		final String localDirs = env.get(ApplicationConstants.Environment.LOCAL_DIRS.key());
		BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, localDirs);

		return configuration;
	}
Example #9
Source File: YarnEntrypointUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
public static Configuration loadConfiguration(String workingDirectory, Map<String, String> env, Logger log) {
		Configuration configuration = GlobalConfiguration.loadConfiguration(workingDirectory);

		final String remoteKeytabPrincipal = env.get(YarnConfigKeys.KEYTAB_PRINCIPAL);

		final String zooKeeperNamespace = env.get(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE);

		final Map<String, String> dynamicProperties = FlinkYarnSessionCli.getDynamicProperties(
			env.get(YarnConfigKeys.ENV_DYNAMIC_PROPERTIES));

		final String hostname = env.get(ApplicationConstants.Environment.NM_HOST.key());
		Preconditions.checkState(
			hostname != null,
			"ApplicationMaster hostname variable %s not set",
			ApplicationConstants.Environment.NM_HOST.key());

		configuration.setString(JobManagerOptions.ADDRESS, hostname);
		configuration.setString(RestOptions.ADDRESS, hostname);

		// TODO: Support port ranges for the AM
//		final String portRange = configuration.getString(
//			ConfigConstants.YARN_APPLICATION_MASTER_PORT,
//			ConfigConstants.DEFAULT_YARN_JOB_MANAGER_PORT);

		for (Map.Entry<String, String> property : dynamicProperties.entrySet()) {
			configuration.setString(property.getKey(), property.getValue());
		}

		if (zooKeeperNamespace != null) {
			configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zooKeeperNamespace);
		}

		// if a web monitor shall be started, set the port to random binding
		if (configuration.getInteger(WebOptions.PORT, 0) >= 0) {
			configuration.setInteger(WebOptions.PORT, 0);
		}

		if (!configuration.contains(RestOptions.BIND_PORT)) {
			// set the REST port to 0 to select it randomly
			configuration.setString(RestOptions.BIND_PORT, "0");
		}

		// if the user has set the deprecated YARN-specific config keys, we add the
		// corresponding generic config keys instead. that way, later code needs not
		// deal with deprecated config keys

		BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
			ConfigConstants.YARN_APPLICATION_MASTER_ENV_PREFIX,
			ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX);

		BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
			ConfigConstants.YARN_TASK_MANAGER_ENV_PREFIX,
			ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX);

		final String keytabPath;

		if (env.get(YarnConfigKeys.KEYTAB_PATH) == null) {
			keytabPath = null;
		} else {
			File f = new File(workingDirectory, Utils.KEYTAB_FILE_NAME);
			keytabPath = f.getAbsolutePath();
		}

		if (keytabPath != null && remoteKeytabPrincipal != null) {
			configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
			configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
		}

		final String localDirs = env.get(ApplicationConstants.Environment.LOCAL_DIRS.key());
		BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, localDirs);

		return configuration;
	}
Example #10
Source File: YarnEntrypointUtils.java    From flink with Apache License 2.0 4 votes vote down vote up 
public static Configuration loadConfiguration(String workingDirectory, Map<String, String> env) {
	Configuration configuration = GlobalConfiguration.loadConfiguration(workingDirectory);

	final String keytabPrincipal = env.get(YarnConfigKeys.KEYTAB_PRINCIPAL);

	final String zooKeeperNamespace = env.get(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE);

	final String hostname = env.get(ApplicationConstants.Environment.NM_HOST.key());
	Preconditions.checkState(
		hostname != null,
		"ApplicationMaster hostname variable %s not set",
		ApplicationConstants.Environment.NM_HOST.key());

	configuration.setString(JobManagerOptions.ADDRESS, hostname);
	configuration.setString(RestOptions.ADDRESS, hostname);

	if (zooKeeperNamespace != null) {
		configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zooKeeperNamespace);
	}

	// if a web monitor shall be started, set the port to random binding
	if (configuration.getInteger(WebOptions.PORT, 0) >= 0) {
		configuration.setInteger(WebOptions.PORT, 0);
	}

	if (!configuration.contains(RestOptions.BIND_PORT)) {
		// set the REST port to 0 to select it randomly
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	// if the user has set the deprecated YARN-specific config keys, we add the
	// corresponding generic config keys instead. that way, later code needs not
	// deal with deprecated config keys

	BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
		ConfigConstants.YARN_APPLICATION_MASTER_ENV_PREFIX,
		ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX);

	BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
		ConfigConstants.YARN_TASK_MANAGER_ENV_PREFIX,
		ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX);

	final String keytabPath = Utils.resolveKeytabPath(workingDirectory, env.get(YarnConfigKeys.LOCAL_KEYTAB_PATH));

	if (keytabPath != null && keytabPrincipal != null) {
		configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
		configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, keytabPrincipal);
	}

	final String localDirs = env.get(ApplicationConstants.Environment.LOCAL_DIRS.key());
	BootstrapTools.updateTmpDirectoriesInConfiguration(configuration, localDirs);

	return configuration;
}