org.apache.hadoop.yarn.client.ClientRMProxy Java Examples

The following examples show how to use org.apache.hadoop.yarn.client.ClientRMProxy. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RMDelegationTokenIdentifier.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private static ApplicationClientProtocol getRmClient(Token<?> token,
    Configuration conf) throws IOException {
  String[] services = token.getService().toString().split(",");
  for (String service : services) {
    InetSocketAddress addr = NetUtils.createSocketAddr(service);
    if (localSecretManager != null) {
      // return null if it's our token
      if (localServiceAddress.getAddress().isAnyLocalAddress()) {
        if (NetUtils.isLocalAddress(addr.getAddress()) &&
            addr.getPort() == localServiceAddress.getPort()) {
          return null;
        }
      } else if (addr.equals(localServiceAddress)) {
        return null;
      }
    }
  }
  return ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class);
}
Example #2
Source File: TestUnmanagedAMLauncher.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
  if (args[0].equals("success")) {
    ApplicationMasterProtocol client = ClientRMProxy.createRMProxy(conf,
        ApplicationMasterProtocol.class);
    client.registerApplicationMaster(RegisterApplicationMasterRequest
        .newInstance(NetUtils.getHostname(), -1, ""));
    Thread.sleep(1000);
    FinishApplicationMasterResponse resp =
        client.finishApplicationMaster(FinishApplicationMasterRequest
          .newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
    assertTrue(resp.getIsUnregistered());
    System.exit(0);
  } else {
    System.exit(1);
  }
}
Example #3
Source File: YarnClientImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
protected void serviceStart() throws Exception {
  try {
    rmClient = ClientRMProxy.createRMProxy(getConfig(),
        ApplicationClientProtocol.class);
    if (historyServiceEnabled) {
      historyClient.start();
    }
    if (timelineServiceEnabled) {
      timelineClient.start();
    }
  } catch (IOException e) {
    throw new YarnRuntimeException(e);
  }
  super.serviceStart();
}
Example #4
Source File: RMDelegationTokenIdentifier.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private static ApplicationClientProtocol getRmClient(Token<?> token,
    Configuration conf) throws IOException {
  String[] services = token.getService().toString().split(",");
  for (String service : services) {
    InetSocketAddress addr = NetUtils.createSocketAddr(service);
    if (localSecretManager != null) {
      // return null if it's our token
      if (localServiceAddress.getAddress().isAnyLocalAddress()) {
        if (NetUtils.isLocalAddress(addr.getAddress()) &&
            addr.getPort() == localServiceAddress.getPort()) {
          return null;
        }
      } else if (addr.equals(localServiceAddress)) {
        return null;
      }
    }
  }
  return ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class);
}
Example #5
Source File: TestUnmanagedAMLauncher.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
  if (args[0].equals("success")) {
    ApplicationMasterProtocol client = ClientRMProxy.createRMProxy(conf,
        ApplicationMasterProtocol.class);
    client.registerApplicationMaster(RegisterApplicationMasterRequest
        .newInstance(NetUtils.getHostname(), -1, ""));
    Thread.sleep(1000);
    FinishApplicationMasterResponse resp =
        client.finishApplicationMaster(FinishApplicationMasterRequest
          .newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
    assertTrue(resp.getIsUnregistered());
    System.exit(0);
  } else {
    System.exit(1);
  }
}
Example #6
Source File: YarnClientImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
protected void serviceStart() throws Exception {
  try {
    rmClient = ClientRMProxy.createRMProxy(getConfig(),
        ApplicationClientProtocol.class);
    if (historyServiceEnabled) {
      historyClient.start();
    }
    if (timelineServiceEnabled) {
      timelineClient.start();
    }
  } catch (IOException e) {
    throw new YarnRuntimeException(e);
  }
  super.serviceStart();
}
Example #7
Source File: RMAdminCLI.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
protected ResourceManagerAdministrationProtocol createAdminProtocol()
    throws IOException {
  // Get the current configuration
  final YarnConfiguration conf = new YarnConfiguration(getConf());
  return ClientRMProxy.createRMProxy(conf,
      ResourceManagerAdministrationProtocol.class);
}
Example #8
Source File: RMAdminCLI.java    From big-c with Apache License 2.0 5 votes vote down vote up 
protected ResourceManagerAdministrationProtocol createAdminProtocol()
    throws IOException {
  // Get the current configuration
  final YarnConfiguration conf = new YarnConfiguration(getConf());
  return ClientRMProxy.createRMProxy(conf,
      ResourceManagerAdministrationProtocol.class);
}
Example #9
Source File: AppReportFetcher.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Create a new Connection to the RM to fetch Application reports.
 * @param conf the conf to use to know where the RM is.
 */
public AppReportFetcher(Configuration conf) {
  this.conf = conf;
  try {
    applicationsManager = ClientRMProxy.createRMProxy(conf,
        ApplicationClientProtocol.class);
  } catch (IOException e) {
    throw new YarnRuntimeException(e);
  }
}
Example #10
Source File: AMRMClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void updateAMRMToken(Token token) throws IOException {
  org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken =
      new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token
        .getIdentifier().array(), token.getPassword().array(), new Text(
        token.getKind()), new Text(token.getService()));
  // Preserve the token service sent by the RM when adding the token
  // to ensure we replace the previous token setup by the RM.
  // Afterwards we can update the service address for the RPC layer.
  UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
  currentUGI.addToken(amrmToken);
  amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
}
Example #11
Source File: AMRMClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
protected void serviceStart() throws Exception {
  final YarnConfiguration conf = new YarnConfiguration(getConfig());
  try {
    rmClient =
        ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
  } catch (IOException e) {
    throw new YarnRuntimeException(e);
  }
  super.serviceStart();
}
Example #12
Source File: AppReportFetcher.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Create a new Connection to the RM to fetch Application reports.
 * @param conf the conf to use to know where the RM is.
 */
public AppReportFetcher(Configuration conf) {
  this.conf = conf;
  try {
    applicationsManager = ClientRMProxy.createRMProxy(conf,
        ApplicationClientProtocol.class);
  } catch (IOException e) {
    throw new YarnRuntimeException(e);
  }
}
Example #13
Source File: LocalContainerAllocator.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void updateAMRMToken(Token token) throws IOException {
  org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken =
      new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token
        .getIdentifier().array(), token.getPassword().array(), new Text(
        token.getKind()), new Text(token.getService()));
  UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
  currentUGI.addToken(amrmToken);
  amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
}
Example #14
Source File: AMRMClientImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
protected void serviceStart() throws Exception {
  final YarnConfiguration conf = new YarnConfiguration(getConfig());
  try {
    rmClient =
        ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
  } catch (IOException e) {
    throw new YarnRuntimeException(e);
  }
  super.serviceStart();
}
Example #15
Source File: RMContainerAllocator.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void updateAMRMToken(Token token) throws IOException {
  org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken =
      new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token
        .getIdentifier().array(), token.getPassword().array(), new Text(
        token.getKind()), new Text(token.getService()));
  UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
  currentUGI.addToken(amrmToken);
  amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
}
Example #16
Source File: RMCommunicator.java    From big-c with Apache License 2.0 5 votes vote down vote up 
protected ApplicationMasterProtocol createSchedulerProxy() {
  final Configuration conf = getConfig();

  try {
    return ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
  } catch (IOException e) {
    throw new YarnRuntimeException(e);
  }
}
Example #17
Source File: RMCommunicator.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
protected ApplicationMasterProtocol createSchedulerProxy() {
  final Configuration conf = getConfig();

  try {
    return ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
  } catch (IOException e) {
    throw new YarnRuntimeException(e);
  }
}
Example #18
Source File: RMContainerAllocator.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void updateAMRMToken(Token token) throws IOException {
  org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken =
      new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token
        .getIdentifier().array(), token.getPassword().array(), new Text(
        token.getKind()), new Text(token.getService()));
  UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
  currentUGI.addToken(amrmToken);
  amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
}
Example #19
Source File: LocalContainerAllocator.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void updateAMRMToken(Token token) throws IOException {
  org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken =
      new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token
        .getIdentifier().array(), token.getPassword().array(), new Text(
        token.getKind()), new Text(token.getService()));
  UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
  currentUGI.addToken(amrmToken);
  amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
}
Example #20
Source File: AMRMClientImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void updateAMRMToken(Token token) throws IOException {
  org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken =
      new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(token
        .getIdentifier().array(), token.getPassword().array(), new Text(
        token.getKind()), new Text(token.getService()));
  // Preserve the token service sent by the RM when adding the token
  // to ensure we replace the previous token setup by the RM.
  // Afterwards we can update the service address for the RPC layer.
  UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
  currentUGI.addToken(amrmToken);
  amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
}
Example #21
Source File: TestLocalContainerAllocator.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testAMRMTokenUpdate() throws Exception {
  Configuration conf = new Configuration();
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(1, 1), 1);
  AMRMTokenIdentifier oldTokenId = new AMRMTokenIdentifier(attemptId, 1);
  AMRMTokenIdentifier newTokenId = new AMRMTokenIdentifier(attemptId, 2);
  Token<AMRMTokenIdentifier> oldToken = new Token<AMRMTokenIdentifier>(
      oldTokenId.getBytes(), "oldpassword".getBytes(), oldTokenId.getKind(),
      new Text());
  Token<AMRMTokenIdentifier> newToken = new Token<AMRMTokenIdentifier>(
      newTokenId.getBytes(), "newpassword".getBytes(), newTokenId.getKind(),
      new Text());

  MockScheduler scheduler = new MockScheduler();
  scheduler.amToken = newToken;

  final LocalContainerAllocator lca =
      new StubbedLocalContainerAllocator(scheduler);
  lca.init(conf);
  lca.start();

  UserGroupInformation testUgi = UserGroupInformation.createUserForTesting(
      "someuser", new String[0]);
  testUgi.addToken(oldToken);
  testUgi.doAs(new PrivilegedExceptionAction<Void>() {
        @Override
        public Void run() throws Exception {
          lca.heartbeat();
          return null;
        }
  });
  lca.close();

  // verify there is only one AMRM token in the UGI and it matches the
  // updated token from the RM
  int tokenCount = 0;
  Token<? extends TokenIdentifier> ugiToken = null;
  for (Token<? extends TokenIdentifier> token : testUgi.getTokens()) {
    if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind())) {
      ugiToken = token;
      ++tokenCount;
    }
  }

  Assert.assertEquals("too many AMRM tokens", 1, tokenCount);
  Assert.assertArrayEquals("token identifier not updated",
      newToken.getIdentifier(), ugiToken.getIdentifier());
  Assert.assertArrayEquals("token password not updated",
      newToken.getPassword(), ugiToken.getPassword());
  Assert.assertEquals("AMRM token service not updated",
      new Text(ClientRMProxy.getAMRMTokenService(conf)),
      ugiToken.getService());
}
Example #22
Source File: ResourceMgrDelegate.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public Text getRMDelegationTokenService() {
  if (rmDTService == null) {
    rmDTService = ClientRMProxy.getRMDelegationTokenService(conf);
  }
  return rmDTService;
}
Example #23
Source File: TestAMRMClient.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Before
public void startApp() throws Exception {
  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer =
      BuilderUtils.newContainerLaunchContext(
        Collections.<String, LocalResource> emptyMap(),
        new HashMap<String, String>(), Arrays.asList("sleep", "100"),
        new HashMap<String, ByteBuffer>(), null,
        new HashMap<ApplicationAccessType, String>());
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1));
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  RMAppAttempt appAttempt = null;
  while (true) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
  }
  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));

  // emulate RM setup of AMRM token in credentials by adding the token
  // *before* setting the token service
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
  appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf));
}
Example #24
Source File: AggregatedLogDeletionService.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@VisibleForTesting
protected ApplicationClientProtocol creatRMClient() throws IOException {
  return ClientRMProxy.createRMProxy(getConfig(),
    ApplicationClientProtocol.class);
}
Example #25
Source File: ResourceMgrDelegate.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public Text getRMDelegationTokenService() {
  if (rmDTService == null) {
    rmDTService = ClientRMProxy.getRMDelegationTokenService(conf);
  }
  return rmDTService;
}
Example #26
Source File: TestLocalContainerAllocator.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testAMRMTokenUpdate() throws Exception {
  Configuration conf = new Configuration();
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(1, 1), 1);
  AMRMTokenIdentifier oldTokenId = new AMRMTokenIdentifier(attemptId, 1);
  AMRMTokenIdentifier newTokenId = new AMRMTokenIdentifier(attemptId, 2);
  Token<AMRMTokenIdentifier> oldToken = new Token<AMRMTokenIdentifier>(
      oldTokenId.getBytes(), "oldpassword".getBytes(), oldTokenId.getKind(),
      new Text());
  Token<AMRMTokenIdentifier> newToken = new Token<AMRMTokenIdentifier>(
      newTokenId.getBytes(), "newpassword".getBytes(), newTokenId.getKind(),
      new Text());

  MockScheduler scheduler = new MockScheduler();
  scheduler.amToken = newToken;

  final LocalContainerAllocator lca =
      new StubbedLocalContainerAllocator(scheduler);
  lca.init(conf);
  lca.start();

  UserGroupInformation testUgi = UserGroupInformation.createUserForTesting(
      "someuser", new String[0]);
  testUgi.addToken(oldToken);
  testUgi.doAs(new PrivilegedExceptionAction<Void>() {
        @Override
        public Void run() throws Exception {
          lca.heartbeat();
          return null;
        }
  });
  lca.close();

  // verify there is only one AMRM token in the UGI and it matches the
  // updated token from the RM
  int tokenCount = 0;
  Token<? extends TokenIdentifier> ugiToken = null;
  for (Token<? extends TokenIdentifier> token : testUgi.getTokens()) {
    if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind())) {
      ugiToken = token;
      ++tokenCount;
    }
  }

  Assert.assertEquals("too many AMRM tokens", 1, tokenCount);
  Assert.assertArrayEquals("token identifier not updated",
      newToken.getIdentifier(), ugiToken.getIdentifier());
  Assert.assertArrayEquals("token password not updated",
      newToken.getPassword(), ugiToken.getPassword());
  Assert.assertEquals("AMRM token service not updated",
      new Text(ClientRMProxy.getAMRMTokenService(conf)),
      ugiToken.getService());
}
Example #27
Source File: TestAMRMClient.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Before
public void startApp() throws Exception {
  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer =
      BuilderUtils.newContainerLaunchContext(
        Collections.<String, LocalResource> emptyMap(),
        new HashMap<String, String>(), Arrays.asList("sleep", "100"),
        new HashMap<String, ByteBuffer>(), null,
        new HashMap<ApplicationAccessType, String>());
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1, 1));
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  RMAppAttempt appAttempt = null;
  while (true) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
  }
  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));

  // emulate RM setup of AMRM token in credentials by adding the token
  // *before* setting the token service
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
  appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf));
}
Example #28
Source File: AggregatedLogDeletionService.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@VisibleForTesting
protected ApplicationClientProtocol creatRMClient() throws IOException {
  return ClientRMProxy.createRMProxy(getConfig(),
    ApplicationClientProtocol.class);
}