Java Code Examples for org.apache.hadoop.conf.Configuration#getSocketAddr()

public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf,
    Policy httpPolicy) {
  InetSocketAddress address = null;
  if (httpPolicy == Policy.HTTPS_ONLY) {
    address =
        conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT);
  } else {
    address =
        conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);      
  }
  return getResolvedAddress(address);
}
 

@Before
public void startUp() {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.SCM_STORE_CLASS,
      InMemorySCMStore.class.getName());
  conf.set(YarnConfiguration.SHARED_CACHE_ROOT, testDir.getPath());
  AppChecker appChecker = spy(new DummyAppChecker());
  store = new InMemorySCMStore(appChecker);
  store.init(conf);
  store.start();

  service = new SharedCacheUploaderService(store);
  service.init(conf);
  service.start();

  YarnRPC rpc = YarnRPC.create(new Configuration());

  InetSocketAddress scmAddress =
      conf.getSocketAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_PORT);

  proxy =
      (SCMUploaderProtocol) rpc.getProxy(
          SCMUploaderProtocol.class, scmAddress, conf);
}
 

private int refreshLoadedJobCache() throws IOException {
  // Refresh the loaded job cache
  Configuration conf = getConf();
  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  HSAdminRefreshProtocol refreshProtocol = HSProxies.createProxy(conf,
      address, HSAdminRefreshProtocol.class,
      UserGroupInformation.getCurrentUser());

  refreshProtocol.refreshLoadedJobCache();
  return 0;
}
 

private int refreshJobRetentionSettings() throws IOException {
  // Refresh job retention settings
  Configuration conf = getConf();
  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  HSAdminRefreshProtocol refreshProtocol = HSProxies.createProxy(conf,
      address, HSAdminRefreshProtocol.class,
      UserGroupInformation.getCurrentUser());

  refreshProtocol.refreshJobRetentionSettings();
  return 0;
}
 

public static String getResolvedRemoteRMWebAppURLWithoutScheme(Configuration conf,
    Policy httpPolicy) {
  InetSocketAddress address = null;
  String rmId = null;
  if (HAUtil.isHAEnabled(conf)) {
    // If HA enabled, pick one of the RM-IDs and rely on redirect to go to
    // the Active RM
    rmId = (String) HAUtil.getRMHAIds(conf).toArray()[0];
  }

  if (httpPolicy == Policy.HTTPS_ONLY) {
    address =
        conf.getSocketAddr(
            rmId == null
                ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS
                : HAUtil.addSuffix(
                YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, rmId),
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT);
  } else {
    address =
        conf.getSocketAddr(
            rmId == null
                ? YarnConfiguration.RM_WEBAPP_ADDRESS
                : HAUtil.addSuffix(
                YarnConfiguration.RM_WEBAPP_ADDRESS, rmId),
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
  }
  return getResolvedAddress(address);
}
 

public static String getResolvedMRHistoryWebAppURLWithoutScheme(
    Configuration conf, boolean isSSLEnabled) {
  InetSocketAddress address = null;
  if (isSSLEnabled) {
    address =
        conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
            JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
            JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT);
  } else {
    address =
        conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
            JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS,
            JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT);    }
  address = NetUtils.getConnectAddress(address);
  StringBuffer sb = new StringBuffer();
  InetAddress resolved = address.getAddress();
  if (resolved == null || resolved.isAnyLocalAddress() || 
      resolved.isLoopbackAddress()) {
    String lh = address.getHostName();
    try {
      lh = InetAddress.getLocalHost().getCanonicalHostName();
    } catch (UnknownHostException e) {
      //Ignore and fallback.
    }
    sb.append(lh);
  } else {
    sb.append(address.getHostName());
  }
  sb.append(":").append(address.getPort());
  return sb.toString();
}
 

private int refreshLogRetentionSettings() throws IOException {
  // Refresh log retention settings
  Configuration conf = getConf();
  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  HSAdminRefreshProtocol refreshProtocol = HSProxies.createProxy(conf,
      address, HSAdminRefreshProtocol.class,
      UserGroupInformation.getCurrentUser());

  refreshProtocol.refreshLogRetentionSettings();
  return 0;
}
 

@Override
protected GetUserMappingsProtocol getUgmProtocol() throws IOException {
  Configuration conf = getConf();
  
  final InetSocketAddress addr = conf.getSocketAddr(
      YarnConfiguration.RM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADMIN_PORT);
  final YarnRPC rpc = YarnRPC.create(conf);
  
  ResourceManagerAdministrationProtocol adminProtocol = (ResourceManagerAdministrationProtocol) rpc.getProxy(
      ResourceManagerAdministrationProtocol.class, addr, getConf());

  return adminProtocol;
}
 

private int getGroups(String[] usernames) throws IOException {
  // Get groups users belongs to
  if (usernames.length == 0) {
    usernames = new String[] { UserGroupInformation.getCurrentUser()
        .getUserName() };
  }

  // Get the current configuration
  Configuration conf = getConf();

  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  GetUserMappingsProtocol getUserMappingProtocol = HSProxies.createProxy(
      conf, address, GetUserMappingsProtocol.class,
      UserGroupInformation.getCurrentUser());
  for (String username : usernames) {
    StringBuilder sb = new StringBuilder();
    sb.append(username + " :");
    for (String group : getUserMappingProtocol.getGroupsForUser(username)) {
      sb.append(" ");
      sb.append(group);
    }
    System.out.println(sb);
  }

  return 0;
}
 

InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(
          YarnConfiguration.RM_BIND_HOST,
          YarnConfiguration.RM_ADDRESS,
          YarnConfiguration.DEFAULT_RM_ADDRESS,
          YarnConfiguration.DEFAULT_RM_PORT);
}
 

@Override
protected void serviceInit(Configuration conf) throws Exception {
  resourceTrackerAddress = conf.getSocketAddr(
      YarnConfiguration.RM_BIND_HOST,
      YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);

  RackResolver.init(conf);
  nextHeartBeatInterval =
      conf.getLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS,
          YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS);
  if (nextHeartBeatInterval <= 0) {
    throw new YarnRuntimeException("Invalid Configuration. "
        + YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS
        + " should be larger than 0.");
  }

  minAllocMb = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
  minAllocVcores = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);

  minimumNodeManagerVersion = conf.get(
      YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,
      YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION);

  super.serviceInit(conf);
}
 

InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(YarnConfiguration.SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_PORT);
}
 

@Test
public void testContainerLocalizer() throws Exception {

  Assume.assumeTrue(shouldRun());

  String locId = "container_01_01";
  Path nmPrivateContainerTokensPath =
      dirsHandler
        .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
            + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId));
  files.create(nmPrivateContainerTokensPath, EnumSet.of(CREATE, OVERWRITE));
  Configuration config = new YarnConfiguration(conf);
  InetSocketAddress nmAddr =
      config.getSocketAddr(YarnConfiguration.NM_BIND_HOST,
        YarnConfiguration.NM_LOCALIZER_ADDRESS,
        YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
        YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
  String appId = "application_01_01";
  exec = new LinuxContainerExecutor() {
    @Override
    public void buildMainArgs(List<String> command, String user,
        String appId, String locId, InetSocketAddress nmAddr,
        List<String> localDirs) {
      MockContainerLocalizer.buildMainArgs(command, user, appId, locId,
        nmAddr, localDirs);
    }
  };
  exec.setConf(conf);

  exec.startLocalizer(new LocalizerStartContext.Builder()
      .setNmPrivateContainerTokens(nmPrivateContainerTokensPath)
      .setNmAddr(nmAddr)
      .setUser(appSubmitter)
      .setAppId(appId)
      .setLocId(locId)
      .setDirsHandler(dirsHandler)
      .build());

  String locId2 = "container_01_02";
  Path nmPrivateContainerTokensPath2 =
      dirsHandler
        .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
            + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId2));
  files.create(nmPrivateContainerTokensPath2, EnumSet.of(CREATE, OVERWRITE));
  exec.startLocalizer(new LocalizerStartContext.Builder()
          .setNmPrivateContainerTokens(nmPrivateContainerTokensPath2)
          .setNmAddr(nmAddr)
          .setUser(appSubmitter)
          .setAppId(appId)
          .setLocId(locId2)
          .setDirsHandler(dirsHandler)
          .build());


  cleanupUserAppCache(appSubmitter);
}
 

InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(YarnConfiguration.SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_PORT);
}
 

@Test
public void testUnauthorizedAccess() throws Exception {
  MyContainerManager containerManager = new MyContainerManager();
  rm = new MockRMWithAMS(conf, containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  RMApp app = rm.submitApp(1024);

  nm1.nodeHeartbeat(true);

  int waitCount = 0;
  while (containerManager.containerTokens == null && waitCount++ < 40) {
    LOG.info("Waiting for AM Launch to happen..");
    Thread.sleep(1000);
  }
  Assert.assertNotNull(containerManager.containerTokens);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  final Configuration conf = rm.getConfig();
  final YarnRPC rpc = YarnRPC.create(conf);
  final InetSocketAddress serviceAddr = conf.getSocketAddr(
      YarnConfiguration.RM_SCHEDULER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);

  UserGroupInformation currentUser = UserGroupInformation
      .createRemoteUser(applicationAttemptId.toString());

  // First try contacting NM without tokens
  ApplicationMasterProtocol client = currentUser
      .doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class,
              serviceAddr, conf);
        }
      });
  
  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  try {
    client.registerApplicationMaster(request);
    Assert.fail("Should fail with authorization error");
  } catch (Exception e) {
    if (isCause(AccessControlException.class, e)) {
      // Because there are no tokens, the request should be rejected as the
      // server side will assume we are trying simple auth.
      String expectedMessage = "";
      if (UserGroupInformation.isSecurityEnabled()) {
        expectedMessage = "Client cannot authenticate via:[TOKEN]";
      } else {
        expectedMessage =
            "SIMPLE authentication is not enabled.  Available:[TOKEN]";
      }
      Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage)); 
    } else {
      throw e;
    }
  }

  // TODO: Add validation of invalid authorization when there's more data in
  // the AMRMToken
}
 

@Test
public void testContainerLocalizer() throws Exception {
  if (!shouldRun()) {
    return;
  }
  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();
  for (String localDir : localDirs) {
    Path userDir =
        new Path(localDir, ContainerLocalizer.USERCACHE);
    files.mkdir(userDir, new FsPermission("777"), false);
    // $local/filecache
    Path fileDir =
        new Path(localDir, ContainerLocalizer.FILECACHE);
    files.mkdir(fileDir, new FsPermission("777"), false);
  }
  String locId = "container_01_01";
  Path nmPrivateContainerTokensPath =
      dirsHandler.getLocalPathForWrite(
          ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
                locId));
  files.create(nmPrivateContainerTokensPath, EnumSet.of(CREATE, OVERWRITE));
  Configuration config = new YarnConfiguration(conf);
  InetSocketAddress nmAddr = config.getSocketAddr(
    YarnConfiguration.NM_BIND_HOST,
    YarnConfiguration.NM_LOCALIZER_ADDRESS,
    YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
    YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
  String appId = "application_01_01";
  exec = new LinuxContainerExecutor() {
    @Override
    public void buildMainArgs(List<String> command, String user, String appId,
        String locId, InetSocketAddress nmAddr, List<String> localDirs) {
      MockContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
        localDirs);
    }
  };
  exec.setConf(conf);

  exec.startLocalizer(nmPrivateContainerTokensPath, nmAddr, appSubmitter,
    appId, locId, dirsHandler);

  String locId2 = "container_01_02";
  Path nmPrivateContainerTokensPath2 =
      dirsHandler
        .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
            + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId2));
  files.create(nmPrivateContainerTokensPath2, EnumSet.of(CREATE, OVERWRITE));
  exec.startLocalizer(nmPrivateContainerTokensPath2, nmAddr, appSubmitter,
    appId, locId2, dirsHandler);
}
 

public static InetSocketAddress getMasterAddress(Configuration conf) {
  return conf
      .getSocketAddr(YarnConfiguration.RM_ADDRESS,
          YarnConfiguration.DEFAULT_RM_ADDRESS,
          YarnConfiguration.DEFAULT_RM_PORT);
}
 

/**
 * Retrieve the timeline server bind address from configuration
 *
 * @param conf
 * @return InetSocketAddress
 */
private static InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);
}
 

/**
 * Retrieve PROXY bind address from configuration
 *
 * @param conf
 * @return InetSocketAddress
 */
public static InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(YarnConfiguration.PROXY_ADDRESS,
    YarnConfiguration.DEFAULT_PROXY_ADDRESS,
    YarnConfiguration.DEFAULT_PROXY_PORT);
}
 

/**
 * Retrieve RM bind address from configuration
 * 
 * @param conf
 * @return InetSocketAddress
 */
public static InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
    YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT);
}