Java Code Examples for org.apache.hadoop.yarn.conf.HAUtil#getRMHAIds()

The following examples show how to use org.apache.hadoop.yarn.conf.HAUtil#getRMHAIds() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ConfiguredRMFailoverProxyProvider.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
public void init(Configuration configuration, RMProxy<T> rmProxy,
                  Class<T> protocol) {
  this.rmProxy = rmProxy;
  this.protocol = protocol;
  this.rmProxy.checkAllowedProtocols(this.protocol);
  this.conf = new YarnConfiguration(configuration);
  Collection<String> rmIds = HAUtil.getRMHAIds(conf);
  this.rmServiceIds = rmIds.toArray(new String[rmIds.size()]);
  conf.set(YarnConfiguration.RM_HA_ID, rmServiceIds[currentProxyIndex]);

  conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
      conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES,
          YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES));

  conf.setInt(CommonConfigurationKeysPublic.
      IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
      conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS,
          YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS));
}
Example 2
Source File: ClientRMProxy.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Unstable
public static Text getTokenService(Configuration conf, String address,
    String defaultAddr, int defaultPort) {
  if (HAUtil.isHAEnabled(conf)) {
    // Build a list of service addresses to form the service name
    ArrayList<String> services = new ArrayList<String>();
    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    for (String rmId : HAUtil.getRMHAIds(conf)) {
      // Set RM_ID to get the corresponding RM_ADDRESS
      yarnConf.set(YarnConfiguration.RM_HA_ID, rmId);
      services.add(SecurityUtil.buildTokenService(
          yarnConf.getSocketAddr(address, defaultAddr, defaultPort))
          .toString());
    }
    return new Text(Joiner.on(',').join(services));
  }

  // Non-HA case - no need to set RM_ID
  return SecurityUtil.buildTokenService(conf.getSocketAddr(address,
    defaultAddr, defaultPort));
}
Example 3
Source File: TestZKRMStateStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private Configuration createHARMConf(
    String rmIds, String rmId, int adminPort) {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.set(YarnConfiguration.RM_HA_IDS, rmIds);
  conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
  conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName());
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
  conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, ZK_TIMEOUT_MS);
  conf.set(YarnConfiguration.RM_HA_ID, rmId);
  conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "localhost:0");

  for (String rpcAddress : YarnConfiguration.getServiceAddressConfKeys(conf)) {
    for (String id : HAUtil.getRMHAIds(conf)) {
      conf.set(HAUtil.addSuffix(rpcAddress, id), "localhost:0");
    }
  }
  conf.set(HAUtil.addSuffix(YarnConfiguration.RM_ADMIN_ADDRESS, rmId),
      "localhost:" + adminPort);
  return conf;
}
Example 4
Source File: RMAdminCLI.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
protected HAServiceTarget resolveTarget(String rmId) {
  Collection<String> rmIds = HAUtil.getRMHAIds(getConf());
  if (!rmIds.contains(rmId)) {
    StringBuilder msg = new StringBuilder();
    msg.append(rmId + " is not a valid serviceId. It should be one of ");
    for (String id : rmIds) {
      msg.append(id + " ");
    }
    throw new IllegalArgumentException(msg.toString());
  }
  try {
    YarnConfiguration conf = new YarnConfiguration(getConf());
    conf.set(YarnConfiguration.RM_HA_ID, rmId);
    return new RMHAServiceTarget(conf);
  } catch (IllegalArgumentException iae) {
    throw new YarnRuntimeException("Could not connect to " + rmId +
        "; the configuration for it might be missing");
  } catch (IOException ioe) {
    throw new YarnRuntimeException(
        "Could not connect to RM HA Admin for node " + rmId);
  }
}
Example 5
Source File: ConfiguredRMFailoverProxyProvider.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
public void init(Configuration configuration, RMProxy<T> rmProxy,
                  Class<T> protocol) {
  this.rmProxy = rmProxy;
  this.protocol = protocol;
  this.rmProxy.checkAllowedProtocols(this.protocol);
  this.conf = new YarnConfiguration(configuration);
  Collection<String> rmIds = HAUtil.getRMHAIds(conf);
  this.rmServiceIds = rmIds.toArray(new String[rmIds.size()]);
  conf.set(YarnConfiguration.RM_HA_ID, rmServiceIds[currentProxyIndex]);

  conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
      conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES,
          YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES));

  conf.setInt(CommonConfigurationKeysPublic.
      IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
      conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS,
          YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS));
}
Example 6
Source File: ClientRMProxy.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Unstable
public static Text getTokenService(Configuration conf, String address,
    String defaultAddr, int defaultPort) {
  if (HAUtil.isHAEnabled(conf)) {
    // Build a list of service addresses to form the service name
    ArrayList<String> services = new ArrayList<String>();
    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    for (String rmId : HAUtil.getRMHAIds(conf)) {
      // Set RM_ID to get the corresponding RM_ADDRESS
      yarnConf.set(YarnConfiguration.RM_HA_ID, rmId);
      services.add(SecurityUtil.buildTokenService(
          yarnConf.getSocketAddr(address, defaultAddr, defaultPort))
          .toString());
    }
    return new Text(Joiner.on(',').join(services));
  }

  // Non-HA case - no need to set RM_ID
  return SecurityUtil.buildTokenService(conf.getSocketAddr(address,
    defaultAddr, defaultPort));
}
Example 7
Source File: TestZKRMStateStore.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private Configuration createHARMConf(
    String rmIds, String rmId, int adminPort) {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.set(YarnConfiguration.RM_HA_IDS, rmIds);
  conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
  conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName());
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
  conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, ZK_TIMEOUT_MS);
  conf.set(YarnConfiguration.RM_HA_ID, rmId);
  conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "localhost:0");

  for (String rpcAddress : YarnConfiguration.getServiceAddressConfKeys(conf)) {
    for (String id : HAUtil.getRMHAIds(conf)) {
      conf.set(HAUtil.addSuffix(rpcAddress, id), "localhost:0");
    }
  }
  conf.set(HAUtil.addSuffix(YarnConfiguration.RM_ADMIN_ADDRESS, rmId),
      "localhost:" + adminPort);
  return conf;
}
Example 8
Source File: RMAdminCLI.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
protected HAServiceTarget resolveTarget(String rmId) {
  Collection<String> rmIds = HAUtil.getRMHAIds(getConf());
  if (!rmIds.contains(rmId)) {
    StringBuilder msg = new StringBuilder();
    msg.append(rmId + " is not a valid serviceId. It should be one of ");
    for (String id : rmIds) {
      msg.append(id + " ");
    }
    throw new IllegalArgumentException(msg.toString());
  }
  try {
    YarnConfiguration conf = new YarnConfiguration(getConf());
    conf.set(YarnConfiguration.RM_HA_ID, rmId);
    return new RMHAServiceTarget(conf);
  } catch (IllegalArgumentException iae) {
    throw new YarnRuntimeException("Could not connect to " + rmId +
        "; the configuration for it might be missing");
  } catch (IOException ioe) {
    throw new YarnRuntimeException(
        "Could not connect to RM HA Admin for node " + rmId);
  }
}
Example 9
Source File: MiniYARNCluster.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
public void serviceInit(Configuration conf) throws Exception {
  useFixedPorts = conf.getBoolean(
      YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS,
      YarnConfiguration.DEFAULT_YARN_MINICLUSTER_FIXED_PORTS);
  useRpc = conf.getBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC,
      YarnConfiguration.DEFAULT_YARN_MINICLUSTER_USE_RPC);
  failoverTimeout = conf.getInt(YarnConfiguration.RM_ZK_TIMEOUT_MS,
      YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS);

  if (useRpc && !useFixedPorts) {
    throw new YarnRuntimeException("Invalid configuration!" +
        " Minicluster can use rpc only when configured to use fixed ports");
  }

  conf.setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true);
  if (resourceManagers.length > 1) {
    conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
    if (conf.get(YarnConfiguration.RM_HA_IDS) == null) {
      StringBuilder rmIds = new StringBuilder();
      for (int i = 0; i < resourceManagers.length; i++) {
        if (i != 0) {
          rmIds.append(",");
        }
        rmIds.append("rm" + i);
      }
      conf.set(YarnConfiguration.RM_HA_IDS, rmIds.toString());
    }
    Collection<String> rmIdsCollection = HAUtil.getRMHAIds(conf);
    rmIds = rmIdsCollection.toArray(new String[rmIdsCollection.size()]);
  }

  for (int i = 0; i < resourceManagers.length; i++) {
    resourceManagers[i] = createResourceManager();
    if (!useFixedPorts) {
      if (HAUtil.isHAEnabled(conf)) {
        setHARMConfiguration(i, conf);
      } else {
        setNonHARMConfiguration(conf);
      }
    }
    addService(new ResourceManagerWrapper(i));
  }
  for(int index = 0; index < nodeManagers.length; index++) {
    nodeManagers[index] =
        useRpc ? new CustomNodeManager() : new ShortCircuitedNodeManager();
    addService(new NodeManagerWrapper(index));
  }

  if (enableAHS) {
    addService(new ApplicationHistoryServerWrapper());
  }
  
  super.serviceInit(
      conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf));
}
Example 10
Source File: Hadoop23YarnAppClient.java    From twill with Apache License 2.0 4 votes vote down vote up 
/**
 * Overrides parent method to adds RM delegation token to the given context. If YARN is running with HA RM,
 * delegation tokens for each RM service will be added.
 */
protected void addRMToken(ContainerLaunchContext context, YarnClient yarnClient, ApplicationId appId) {
  if (!UserGroupInformation.isSecurityEnabled()) {
    return;
  }

  try {
    Text renewer = new Text(UserGroupInformation.getCurrentUser().getShortUserName());
    org.apache.hadoop.yarn.api.records.Token rmDelegationToken = yarnClient.getRMDelegationToken(renewer);

    // The following logic is copied from ClientRMProxy.getRMDelegationTokenService, which is not available in
    // YARN older than 2.4
    List<String> services = new ArrayList<>();
    if (HAUtil.isHAEnabled(configuration)) {
      // If HA is enabled, we need to enumerate all RM hosts
      // and add the corresponding service name to the token service
      // Copy the yarn conf since we need to modify it to get the RM addresses
      YarnConfiguration yarnConf = new YarnConfiguration(configuration);
      for (String rmId : HAUtil.getRMHAIds(configuration)) {
        yarnConf.set(YarnConfiguration.RM_HA_ID, rmId);
        InetSocketAddress address = yarnConf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
                                                           YarnConfiguration.DEFAULT_RM_ADDRESS,
                                                           YarnConfiguration.DEFAULT_RM_PORT);
        services.add(SecurityUtil.buildTokenService(address).toString());
      }
    } else {
      services.add(SecurityUtil.buildTokenService(YarnUtils.getRMAddress(configuration)).toString());
    }

    Credentials credentials = YarnUtils.decodeCredentials(context.getTokens());

    // casting needed for later Hadoop version
    @SuppressWarnings("RedundantCast")
    Token<TokenIdentifier> token = ConverterUtils.convertFromYarn(rmDelegationToken, (InetSocketAddress) null);

    token.setService(new Text(Joiner.on(',').join(services)));
    credentials.addToken(new Text(token.getService()), token);

    LOG.debug("Added RM delegation token {} for application {}", token, appId);
    credentials.addToken(token.getService(), token);

    context.setTokens(YarnUtils.encodeCredentials(credentials));

  } catch (Exception e) {
    throw Throwables.propagate(e);
  }
}
Example 11
Source File: MiniYARNCluster.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
public void serviceInit(Configuration conf) throws Exception {
  useFixedPorts = conf.getBoolean(
      YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS,
      YarnConfiguration.DEFAULT_YARN_MINICLUSTER_FIXED_PORTS);
  useRpc = conf.getBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC,
      YarnConfiguration.DEFAULT_YARN_MINICLUSTER_USE_RPC);
  failoverTimeout = conf.getInt(YarnConfiguration.RM_ZK_TIMEOUT_MS,
      YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS);

  if (useRpc && !useFixedPorts) {
    throw new YarnRuntimeException("Invalid configuration!" +
        " Minicluster can use rpc only when configured to use fixed ports");
  }

  conf.setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true);
  if (resourceManagers.length > 1) {
    conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
    if (conf.get(YarnConfiguration.RM_HA_IDS) == null) {
      StringBuilder rmIds = new StringBuilder();
      for (int i = 0; i < resourceManagers.length; i++) {
        if (i != 0) {
          rmIds.append(",");
        }
        rmIds.append("rm" + i);
      }
      conf.set(YarnConfiguration.RM_HA_IDS, rmIds.toString());
    }
    Collection<String> rmIdsCollection = HAUtil.getRMHAIds(conf);
    rmIds = rmIdsCollection.toArray(new String[rmIdsCollection.size()]);
  }

  for (int i = 0; i < resourceManagers.length; i++) {
    resourceManagers[i] = createResourceManager();
    if (!useFixedPorts) {
      if (HAUtil.isHAEnabled(conf)) {
        setHARMConfiguration(i, conf);
      } else {
        setNonHARMConfiguration(conf);
      }
    }
    addService(new ResourceManagerWrapper(i));
  }
  for(int index = 0; index < nodeManagers.length; index++) {
    nodeManagers[index] =
        useRpc ? new CustomNodeManager() : new ShortCircuitedNodeManager();
    addService(new NodeManagerWrapper(index));
  }

  if (enableAHS) {
    addService(new ApplicationHistoryServerWrapper());
  }
  
  super.serviceInit(
      conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf));
}
Example 12
Source File: MiniYARNClusterSplice.java    From spliceengine with GNU Affero General Public License v3.0 4 votes vote down vote up 
@Override
public void serviceInit(Configuration conf) throws Exception {
    useFixedPorts = conf.getBoolean(
        YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS,
        YarnConfiguration.DEFAULT_YARN_MINICLUSTER_FIXED_PORTS);
    useRpc = conf.getBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC,
                             YarnConfiguration.DEFAULT_YARN_MINICLUSTER_USE_RPC);
    failoverTimeout = conf.getInt(YarnConfiguration.RM_ZK_TIMEOUT_MS,
                                  YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS);

    if (useRpc && !useFixedPorts) {
        throw new YarnRuntimeException("Invalid configuration!" +
                                           " Minicluster can use rpc only when configured to use fixed ports");
    }

    conf.setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true);
    if (resourceManagers.length > 1) {
        conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
        if (conf.get(YarnConfiguration.RM_HA_IDS) == null) {
            StringBuilder rmIds = new StringBuilder();
            for (int i = 0; i < resourceManagers.length; i++) {
                if (i != 0) {
                    rmIds.append(",");
                }
                rmIds.append("rm" + i);
            }
            conf.set(YarnConfiguration.RM_HA_IDS, rmIds.toString());
        }
        Collection<String> rmIdsCollection = HAUtil.getRMHAIds(conf);
        rmIds = rmIdsCollection.toArray(new String[rmIdsCollection.size()]);
    }

    for (int i = 0; i < resourceManagers.length; i++) {
        resourceManagers[i] = createResourceManager();
        if (!useFixedPorts) {
            if (HAUtil.isHAEnabled(conf)) {
                setHARMConfiguration(i, conf);
            } else {
                setNonHARMConfiguration(conf);
            }
        }
        addService(new ResourceManagerWrapper(i));
    }
    for(int index = 0; index < nodeManagers.length; index++) {
        nodeManagers[index] =
            useRpc ? new CustomNodeManager() : new ShortCircuitedNodeManager();
        addService(new NodeManagerWrapper(index));
    }

    if (enableAHS) {
        addService(new ApplicationHistoryServerWrapper());
    }

    super.serviceInit(
        conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf));
}