org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse Java Exaples

Source File: TestRPC.java From hadoop with Apache License 2.0

6 votes

@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException {
  StartContainersResponse response =
      recordFactory.newRecordInstance(StartContainersResponse.class);
  for (StartContainerRequest request : requests.getStartContainerRequests()) {
    Token containerToken = request.getContainerToken();
    ContainerTokenIdentifier tokenId = null;

    try {
      tokenId = newContainerTokenIdentifier(containerToken);
    } catch (IOException e) {
      throw RPCUtil.getRemoteException(e);
    }
    ContainerStatus status =
        recordFactory.newRecordInstance(ContainerStatus.class);
    status.setState(ContainerState.RUNNING);
    status.setContainerId(tokenId.getContainerID());
    status.setExitStatus(0);
    statuses.add(status);

  }
  return response;
}

Source File: TestContainerManagerRecovery.java From big-c with Apache License 2.0

6 votes

private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}

Source File: TestContainerManagerSecurity.java From big-c with Apache License 2.0

6 votes

private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}

Source File: TestRPC.java From big-c with Apache License 2.0

6 votes

@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException {
  StartContainersResponse response =
      recordFactory.newRecordInstance(StartContainersResponse.class);
  for (StartContainerRequest request : requests.getStartContainerRequests()) {
    Token containerToken = request.getContainerToken();
    ContainerTokenIdentifier tokenId = null;

    try {
      tokenId = newContainerTokenIdentifier(containerToken);
    } catch (IOException e) {
      throw RPCUtil.getRemoteException(e);
    }
    ContainerStatus status =
        recordFactory.newRecordInstance(ContainerStatus.class);
    status.setState(ContainerState.RUNNING);
    status.setContainerId(tokenId.getContainerID());
    status.setExitStatus(0);
    statuses.add(status);

  }
  return response;
}

Source File: TestContainerManagerRecovery.java From hadoop with Apache License 2.0

6 votes

private StartContainersResponse startContainer(Context context,
    final ContainerManagerImpl cm, ContainerId cid,
    ContainerLaunchContext clc, LogAggregationContext logAggregationContext)
        throws Exception {
  UserGroupInformation user = UserGroupInformation.createRemoteUser(
      cid.getApplicationAttemptId().toString());
  StartContainerRequest scReq = StartContainerRequest.newInstance(
      clc, TestContainerManager.createContainerToken(cid, 0,
          context.getNodeId(), user.getShortUserName(),
          context.getContainerTokenSecretManager(), logAggregationContext));
  final List<StartContainerRequest> scReqList =
      new ArrayList<StartContainerRequest>();
  scReqList.add(scReq);
  NMTokenIdentifier nmToken = new NMTokenIdentifier(
      cid.getApplicationAttemptId(), context.getNodeId(),
      user.getShortUserName(),
      context.getNMTokenSecretManager().getCurrentKey().getKeyId());
  user.addTokenIdentifier(nmToken);
  return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() {
    @Override
    public StartContainersResponse run() throws Exception {
      return cm.startContainers(
          StartContainersRequest.newInstance(scReqList));
    }
  });
}

Source File: TestContainerManagerSecurity.java From hadoop with Apache License 2.0

6 votes

private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}

Source File: TestAMAuthorization.java From big-c with Apache License 2.0

5 votes

@Override
public StartContainersResponse
    startContainers(StartContainersRequest request)
        throws YarnException {
  containerTokens = request.getStartContainerRequests().get(0).getContainerLaunchContext().getTokens();
  return StartContainersResponse.newInstance(null, null, null);
}

Source File: AMLauncher.java From big-c with Apache License 2.0

5 votes

private void launch() throws IOException, YarnException {
  connect();
  ContainerId masterContainerID = masterContainer.getId();
  ApplicationSubmissionContext applicationContext =
    application.getSubmissionContext();
  LOG.info("Setting up container " + masterContainer
      + " for AM " + application.getAppAttemptId());  
  ContainerLaunchContext launchContext =
      createAMContainerLaunchContext(applicationContext, masterContainerID);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(launchContext,
        masterContainer.getContainerToken());
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);

  StartContainersResponse response =
      containerMgrProxy.startContainers(allRequests);
  if (response.getFailedRequests() != null
      && response.getFailedRequests().containsKey(masterContainerID)) {
    Throwable t =
        response.getFailedRequests().get(masterContainerID).deSerialize();
    parseAndThrowException(t);
  } else {
    LOG.info("Done launching container " + masterContainer + " for AM "
        + application.getAppAttemptId());
  }
}

Source File: TestApplicationMasterLauncher.java From big-c with Apache License 2.0

5 votes

@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}

Source File: ContainerManagementProtocolPBClientImpl.java From hadoop with Apache License 2.0

5 votes

@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests) throws YarnException,
        IOException {
  StartContainersRequestProto requestProto =
      ((StartContainersRequestPBImpl) requests).getProto();
  try {
    return new StartContainersResponsePBImpl(proxy.startContainers(null,
      requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}

Source File: TestContainerLaunchRPC.java From hadoop with Apache License 2.0

5 votes

@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException, IOException {
  try {
    // make the thread sleep to look like its not going to respond
    Thread.sleep(10000);
  } catch (Exception e) {
    LOG.error(e);
    throw new YarnException(e);
  }
  throw new YarnException("Shouldn't happen!!");
}

Source File: TestContainerLaunchRPC.java From big-c with Apache License 2.0

5 votes

@Override
public StartContainersResponse startContainers(
    StartContainersRequest requests) throws YarnException, IOException {
  try {
    // make the thread sleep to look like its not going to respond
    Thread.sleep(10000);
  } catch (Exception e) {
    LOG.error(e);
    throw new YarnException(e);
  }
  throw new YarnException("Shouldn't happen!!");
}

Source File: ContainerManagementProtocolPBClientImpl.java From big-c with Apache License 2.0

5 votes

@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests) throws YarnException,
        IOException {
  StartContainersRequestProto requestProto =
      ((StartContainersRequestPBImpl) requests).getProto();
  try {
    return new StartContainersResponsePBImpl(proxy.startContainers(null,
      requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}

Source File: TestContainerLauncher.java From big-c with Apache License 2.0

5 votes

@Override
public StartContainersResponse startContainers(StartContainersRequest requests)
    throws IOException {

  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  ContainerTokenIdentifier containerTokenIdentifier =
      MRApp.newContainerTokenIdentifier(request.getContainerToken());

  // Validate that the container is what RM is giving.
  Assert.assertEquals(MRApp.NM_HOST + ":" + MRApp.NM_PORT,
    containerTokenIdentifier.getNmHostAddress());

  StartContainersResponse response = recordFactory
      .newRecordInstance(StartContainersResponse.class);
  status = recordFactory.newRecordInstance(ContainerStatus.class);
  try {
    // make the thread sleep to look like its not going to respond
    Thread.sleep(15000);
  } catch (Exception e) {
    LOG.error(e);
    throw new UndeclaredThrowableException(e);
  }
  status.setState(ContainerState.RUNNING);
  status.setContainerId(containerTokenIdentifier.getContainerID());
  status.setExitStatus(0);
  return response;
}

Source File: TestContainerLauncher.java From hadoop with Apache License 2.0

5 votes

@Override
public StartContainersResponse startContainers(StartContainersRequest requests)
    throws IOException {

  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  ContainerTokenIdentifier containerTokenIdentifier =
      MRApp.newContainerTokenIdentifier(request.getContainerToken());

  // Validate that the container is what RM is giving.
  Assert.assertEquals(MRApp.NM_HOST + ":" + MRApp.NM_PORT,
    containerTokenIdentifier.getNmHostAddress());

  StartContainersResponse response = recordFactory
      .newRecordInstance(StartContainersResponse.class);
  status = recordFactory.newRecordInstance(ContainerStatus.class);
  try {
    // make the thread sleep to look like its not going to respond
    Thread.sleep(15000);
  } catch (Exception e) {
    LOG.error(e);
    throw new UndeclaredThrowableException(e);
  }
  status.setState(ContainerState.RUNNING);
  status.setContainerId(containerTokenIdentifier.getContainerID());
  status.setExitStatus(0);
  return response;
}

Source File: TestApplicationMasterLauncher.java From hadoop with Apache License 2.0

5 votes

@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}

Source File: TestAMAuthorization.java From hadoop with Apache License 2.0

5 votes

@Override
public StartContainersResponse
    startContainers(StartContainersRequest request)
        throws YarnException {
  containerTokens = request.getStartContainerRequests().get(0).getContainerLaunchContext().getTokens();
  return StartContainersResponse.newInstance(null, null, null);
}

Source File: AMLauncher.java From hadoop with Apache License 2.0

5 votes

private void launch() throws IOException, YarnException {
  connect();
  ContainerId masterContainerID = masterContainer.getId();
  ApplicationSubmissionContext applicationContext =
    application.getSubmissionContext();
  LOG.info("Setting up container " + masterContainer
      + " for AM " + application.getAppAttemptId());  
  ContainerLaunchContext launchContext =
      createAMContainerLaunchContext(applicationContext, masterContainerID);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(launchContext,
        masterContainer.getContainerToken());
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);

  StartContainersResponse response =
      containerMgrProxy.startContainers(allRequests);
  if (response.getFailedRequests() != null
      && response.getFailedRequests().containsKey(masterContainerID)) {
    Throwable t =
        response.getFailedRequests().get(masterContainerID).deSerialize();
    parseAndThrowException(t);
  } else {
    LOG.info("Done launching container " + masterContainer + " for AM "
        + application.getAppAttemptId());
  }
}

Source File: TestContainerLauncherImpl.java From big-c with Apache License 2.0

4 votes

@Test(timeout = 5000)
public void testOutOfOrder() throws Exception {
  LOG.info("STARTING testOutOfOrder");
  AppContext mockContext = mock(AppContext.class);
  @SuppressWarnings("rawtypes")
  EventHandler mockEventHandler = mock(EventHandler.class);
  when(mockContext.getEventHandler()).thenReturn(mockEventHandler);

  ContainerManagementProtocolClient mockCM =
      mock(ContainerManagementProtocolClient.class);
  ContainerLauncherImplUnderTest ut =
      new ContainerLauncherImplUnderTest(mockContext, mockCM);
  
  Configuration conf = new Configuration();
  ut.init(conf);
  ut.start();
  try {
    ContainerId contId = makeContainerId(0l, 0, 0, 1);
    TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
    String cmAddress = "127.0.0.1:8000";
    StartContainersResponse startResp =
      recordFactory.newRecordInstance(StartContainersResponse.class);
    startResp.setAllServicesMetaData(serviceResponse);

    LOG.info("inserting cleanup event");
    ContainerLauncherEvent mockCleanupEvent = 
      mock(ContainerLauncherEvent.class);
    when(mockCleanupEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
    when(mockCleanupEvent.getContainerID())
      .thenReturn(contId);
    when(mockCleanupEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockCleanupEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    ut.handle(mockCleanupEvent);
    
    ut.waitForPoolToIdle();
    
    verify(mockCM, never()).stopContainers(any(StopContainersRequest.class));

    LOG.info("inserting launch event");
    ContainerRemoteLaunchEvent mockLaunchEvent = 
      mock(ContainerRemoteLaunchEvent.class);
    when(mockLaunchEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
    when(mockLaunchEvent.getContainerID())
      .thenReturn(contId);
    when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp);
    when(mockLaunchEvent.getContainerToken()).thenReturn(
        createNewContainerToken(contId, cmAddress));
    ut.handle(mockLaunchEvent);
    
    ut.waitForPoolToIdle();
    
    verify(mockCM, never()).startContainers(any(StartContainersRequest.class));
  } finally {
    ut.stop();
  }
}

Source File: TestContainerLauncherImpl.java From big-c with Apache License 2.0

4 votes

@Test(timeout = 5000)
public void testHandle() throws Exception {
  LOG.info("STARTING testHandle");
  AppContext mockContext = mock(AppContext.class);
  @SuppressWarnings("rawtypes")
  EventHandler mockEventHandler = mock(EventHandler.class);
  when(mockContext.getEventHandler()).thenReturn(mockEventHandler);
  String cmAddress = "127.0.0.1:8000";
  ContainerManagementProtocolClient mockCM =
      mock(ContainerManagementProtocolClient.class);
  ContainerLauncherImplUnderTest ut =
      new ContainerLauncherImplUnderTest(mockContext, mockCM);
  
  Configuration conf = new Configuration();
  ut.init(conf);
  ut.start();
  try {
    ContainerId contId = makeContainerId(0l, 0, 0, 1);
    TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
    StartContainersResponse startResp =
      recordFactory.newRecordInstance(StartContainersResponse.class);
    startResp.setAllServicesMetaData(serviceResponse);
    

    LOG.info("inserting launch event");
    ContainerRemoteLaunchEvent mockLaunchEvent = 
      mock(ContainerRemoteLaunchEvent.class);
    when(mockLaunchEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
    when(mockLaunchEvent.getContainerID())
      .thenReturn(contId);
    when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp);
    when(mockLaunchEvent.getContainerToken()).thenReturn(
        createNewContainerToken(contId, cmAddress));
    ut.handle(mockLaunchEvent);
    
    ut.waitForPoolToIdle();
    
    verify(mockCM).startContainers(any(StartContainersRequest.class));
    
    LOG.info("inserting cleanup event");
    ContainerLauncherEvent mockCleanupEvent = 
      mock(ContainerLauncherEvent.class);
    when(mockCleanupEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
    when(mockCleanupEvent.getContainerID())
      .thenReturn(contId);
    when(mockCleanupEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockCleanupEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    ut.handle(mockCleanupEvent);
    
    ut.waitForPoolToIdle();
    
    verify(mockCM).stopContainers(any(StopContainersRequest.class));
  } finally {
    ut.stop();
  }
}

Source File: TestContainerLauncherImpl.java From big-c with Apache License 2.0

4 votes

@Test(timeout = 5000)
public void testMyShutdown() throws Exception {
  LOG.info("in test Shutdown");

  AppContext mockContext = mock(AppContext.class);
  @SuppressWarnings("rawtypes")
  EventHandler mockEventHandler = mock(EventHandler.class);
  when(mockContext.getEventHandler()).thenReturn(mockEventHandler);

  ContainerManagementProtocolClient mockCM =
      mock(ContainerManagementProtocolClient.class);
  ContainerLauncherImplUnderTest ut =
      new ContainerLauncherImplUnderTest(mockContext, mockCM);

  Configuration conf = new Configuration();
  ut.init(conf);
  ut.start();
  try {
    ContainerId contId = makeContainerId(0l, 0, 0, 1);
    TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
    String cmAddress = "127.0.0.1:8000";
    StartContainersResponse startResp =
      recordFactory.newRecordInstance(StartContainersResponse.class);
    startResp.setAllServicesMetaData(serviceResponse);

    LOG.info("inserting launch event");
    ContainerRemoteLaunchEvent mockLaunchEvent =
      mock(ContainerRemoteLaunchEvent.class);
    when(mockLaunchEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
    when(mockLaunchEvent.getContainerID())
      .thenReturn(contId);
    when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp);
    when(mockLaunchEvent.getContainerToken()).thenReturn(
        createNewContainerToken(contId, cmAddress));
    ut.handle(mockLaunchEvent);

    ut.waitForPoolToIdle();

    verify(mockCM).startContainers(any(StartContainersRequest.class));

    // skip cleanup and make sure stop kills the container

  } finally {
    ut.stop();
    verify(mockCM).stopContainers(any(StopContainersRequest.class));
  }
}

Source File: TestContainerLauncherImpl.java From big-c with Apache License 2.0

4 votes

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test(timeout = 5000)
public void testContainerCleaned() throws Exception {
  LOG.info("STARTING testContainerCleaned");
  
  CyclicBarrier startLaunchBarrier = new CyclicBarrier(2);
  CyclicBarrier completeLaunchBarrier = new CyclicBarrier(2);

  AppContext mockContext = mock(AppContext.class);
  
  EventHandler mockEventHandler = mock(EventHandler.class);
  when(mockContext.getEventHandler()).thenReturn(mockEventHandler);

  ContainerManagementProtocolClient mockCM =
      new ContainerManagerForTest(startLaunchBarrier, completeLaunchBarrier);
  ContainerLauncherImplUnderTest ut =
      new ContainerLauncherImplUnderTest(mockContext, mockCM);
  
  Configuration conf = new Configuration();
  ut.init(conf);
  ut.start();
  try {
    ContainerId contId = makeContainerId(0l, 0, 0, 1);
    TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
    String cmAddress = "127.0.0.1:8000";
    StartContainersResponse startResp =
      recordFactory.newRecordInstance(StartContainersResponse.class);
    startResp.setAllServicesMetaData(serviceResponse);
    
   
    LOG.info("inserting launch event");
    ContainerRemoteLaunchEvent mockLaunchEvent = 
      mock(ContainerRemoteLaunchEvent.class);
    when(mockLaunchEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
    when(mockLaunchEvent.getContainerID())
      .thenReturn(contId);
    when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    when(mockLaunchEvent.getContainerToken()).thenReturn(
        createNewContainerToken(contId, cmAddress));
    ut.handle(mockLaunchEvent);
    
    startLaunchBarrier.await();
    
         
    LOG.info("inserting cleanup event");
    ContainerLauncherEvent mockCleanupEvent = 
      mock(ContainerLauncherEvent.class);
    when(mockCleanupEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
    when(mockCleanupEvent.getContainerID())
      .thenReturn(contId);
    when(mockCleanupEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockCleanupEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    ut.handle(mockCleanupEvent);

    completeLaunchBarrier.await();
   
    ut.waitForPoolToIdle();
    
    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
    verify(mockEventHandler, atLeast(2)).handle(arg.capture());
    boolean containerCleaned = false;
    
    for (int i =0; i < arg.getAllValues().size(); i++) {
      LOG.info(arg.getAllValues().get(i).toString());
      Event currentEvent = arg.getAllValues().get(i);
      if (currentEvent.getType() == TaskAttemptEventType.TA_CONTAINER_CLEANED) {
        containerCleaned = true;
      }
    }
    assert(containerCleaned);
    
  } finally {
    ut.stop();
  }
}

Source File: YarnContainerProxy.java From incubator-tajo with Apache License 2.0

4 votes

@Override
@SuppressWarnings("unchecked")
public synchronized void launch(ContainerLaunchContext commonContainerLaunchContext) {
  LOG.info("Launching Container with Id: " + containerID);
  if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
    state = ContainerState.DONE;
    LOG.error("Container (" + containerID + " was killed before it was launched");
    return;
  }

  ContainerManagementProtocol proxy = null;
  try {

    proxy = getCMProxy(containerID, containerMgrAddress,
        containerToken);

    // Construct the actual Container
    ContainerLaunchContext containerLaunchContext = createContainerLaunchContext(commonContainerLaunchContext);

    // Now launch the actual container
    List<StartContainerRequest> startRequestList = new ArrayList<StartContainerRequest>();
    StartContainerRequest startRequest = Records
        .newRecord(StartContainerRequest.class);
    startRequest.setContainerLaunchContext(containerLaunchContext);
    startRequestList.add(startRequest);
    StartContainersRequest startRequests = Records.newRecord(StartContainersRequest.class);
    startRequests.setStartContainerRequests(startRequestList);
    StartContainersResponse response = proxy.startContainers(startRequests);

    ByteBuffer portInfo = response.getAllServicesMetaData().get(PullServerAuxService.PULLSERVER_SERVICEID);

    if(portInfo != null) {
      port = PullServerAuxService.deserializeMetaData(portInfo);
    }

    LOG.info("PullServer port returned by ContainerManager for "
        + containerID + " : " + port);

    if(port < 0) {
      this.state = ContainerState.FAILED;
      throw new IllegalStateException("Invalid shuffle port number "
          + port + " returned for " + containerID);
    }

    this.state = ContainerState.RUNNING;
    this.hostName = containerMgrAddress.split(":")[0];
    context.getResourceAllocator().addContainer(containerID, this);
  } catch (Throwable t) {
    String message = "Container launch failed for " + containerID + " : "
        + StringUtils.stringifyException(t);
    this.state = ContainerState.FAILED;
    LOG.error(message);
  } finally {
    if (proxy != null) {
      yarnRPC.stopProxy(proxy, conf);
    }
  }
}

Source File: ContainerLauncherImpl.java From big-c with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
public synchronized void launch(ContainerRemoteLaunchEvent event) {
  LOG.info("Launching " + taskAttemptID);
  if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
    state = ContainerState.DONE;
    sendContainerLaunchFailedMsg(taskAttemptID, 
        "Container was killed before it was launched");
    return;
  }
  
  ContainerManagementProtocolProxyData proxy = null;
  try {

    proxy = getCMProxy(containerMgrAddress, containerID);

    // Construct the actual Container
    ContainerLaunchContext containerLaunchContext =
      event.getContainerLaunchContext();

    // Now launch the actual container
    StartContainerRequest startRequest =
        StartContainerRequest.newInstance(containerLaunchContext,
          event.getContainerToken());
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(startRequest);
    StartContainersRequest requestList = StartContainersRequest.newInstance(list);
    StartContainersResponse response =
        proxy.getContainerManagementProtocol().startContainers(requestList);
    if (response.getFailedRequests() != null
        && response.getFailedRequests().containsKey(containerID)) {
      throw response.getFailedRequests().get(containerID).deSerialize();
    }
    ByteBuffer portInfo =
        response.getAllServicesMetaData().get(
            ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID);
    int port = -1;
    if(portInfo != null) {
      port = ShuffleHandler.deserializeMetaData(portInfo);
    }
    LOG.info("Shuffle port returned by ContainerManager for "
        + taskAttemptID + " : " + port);

    if(port < 0) {
      this.state = ContainerState.FAILED;
      throw new IllegalStateException("Invalid shuffle port number "
          + port + " returned for " + taskAttemptID);
    }

    // after launching, send launched event to task attempt to move
    // it from ASSIGNED to RUNNING state
    context.getEventHandler().handle(
        new TaskAttemptContainerLaunchedEvent(taskAttemptID, port));
    this.state = ContainerState.RUNNING;
  } catch (Throwable t) {
    String message = "Container launch failed for " + containerID + " : "
        + StringUtils.stringifyException(t);
    this.state = ContainerState.FAILED;
    sendContainerLaunchFailedMsg(taskAttemptID, message);
  } finally {
    if (proxy != null) {
      cmProxy.mayBeCloseProxy(proxy);
    }
  }
}

Source File: ContainerLauncherImpl.java From incubator-tez with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
public synchronized void launch(NMCommunicatorLaunchRequestEvent event) {
  LOG.info("Launching Container with Id: " + event.getContainerId());
  if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
    state = ContainerState.DONE;
    sendContainerLaunchFailedMsg(event.getContainerId(),
        "Container was killed before it was launched");
    return;
  }

  ContainerManagementProtocolProxyData proxy = null;
  try {

    proxy = getCMProxy(containerID, containerMgrAddress,
        containerToken);

    // Construct the actual Container
    ContainerLaunchContext containerLaunchContext =
      event.getContainerLaunchContext();

    // Now launch the actual container
    StartContainerRequest startRequest = Records
      .newRecord(StartContainerRequest.class);
    startRequest.setContainerToken(event.getContainerToken());
    startRequest.setContainerLaunchContext(containerLaunchContext);

    StartContainersResponse response =
        proxy.getContainerManagementProtocol().startContainers(
            StartContainersRequest.newInstance(
                Collections.singletonList(startRequest)));
    if (response.getFailedRequests() != null
        && !response.getFailedRequests().isEmpty()) {
      throw response.getFailedRequests().get(containerID).deSerialize();
    }

    // after launching, send launched event to task attempt to move
    // it from ASSIGNED to RUNNING state
    context.getEventHandler().handle(
        new AMContainerEventLaunched(containerID));
    ContainerLaunchedEvent lEvt = new ContainerLaunchedEvent(
        containerID, clock.getTime(), context.getApplicationAttemptId());
    context.getHistoryHandler().handle(new DAGHistoryEvent(
        null, lEvt));

    this.state = ContainerState.RUNNING;
  } catch (Throwable t) {
    String message = "Container launch failed for " + containerID + " : "
        + StringUtils.stringifyException(t);
    this.state = ContainerState.FAILED;
    sendContainerLaunchFailedMsg(containerID, message);
  } finally {
    if (proxy != null) {
      cmProxy.mayBeCloseProxy(proxy);
    }
  }
}

Source File: TezContainerLauncherImpl.java From tez with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
public synchronized void launch(ContainerLaunchRequest event) {
  LOG.info("Launching " + event.getContainerId());
  if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
    state = ContainerState.DONE;
    sendContainerLaunchFailedMsg(event.getContainerId(),
        "Container was killed before it was launched");
    return;
  }

  ContainerManagementProtocolProxyData proxy = null;
  try {

    proxy = getCMProxy(containerID, containerMgrAddress,
        containerToken);

    // Construct the actual Container
    ContainerLaunchContext containerLaunchContext =
      event.getContainerLaunchContext();

    // Now launch the actual container
    StartContainerRequest startRequest = Records
      .newRecord(StartContainerRequest.class);
    startRequest.setContainerToken(event.getContainerToken());
    startRequest.setContainerLaunchContext(containerLaunchContext);

    StartContainersResponse response =
        proxy.getContainerManagementProtocol().startContainers(
            StartContainersRequest.newInstance(
                Collections.singletonList(startRequest)));
    if (response.getFailedRequests() != null
        && !response.getFailedRequests().isEmpty()) {
      throw response.getFailedRequests().get(containerID).deSerialize();
    }

    // after launching, send launched event to task attempt to move
    // it from ASSIGNED to RUNNING state
    getContext().containerLaunched(containerID);
    this.state = ContainerState.RUNNING;

    int shufflePort = TezRuntimeUtils.INVALID_PORT;
    Map<String, java.nio.ByteBuffer> servicesMetaData = response.getAllServicesMetaData();
    if (servicesMetaData != null) {
      String auxiliaryService = conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID,
          TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT);
      ByteBuffer portInfo = servicesMetaData.get(auxiliaryService);
      if (portInfo != null) {
        DataInputByteBuffer in = new DataInputByteBuffer();
        in.reset(portInfo);
        shufflePort = in.readInt();
      } else {
        LOG.warn("Shuffle port for {} is not present is the services metadata response", auxiliaryService);
      }
    } else {
      LOG.warn("Shuffle port cannot be found since services metadata response is missing");
    }
    if (deletionTracker != null) {
      deletionTracker.addNodeShufflePort(event.getNodeId(), shufflePort);
    }
  } catch (Throwable t) {
    String message = "Container launch failed for " + containerID + " : "
        + ExceptionUtils.getStackTrace(t);
    this.state = ContainerState.FAILED;
    sendContainerLaunchFailedMsg(containerID, message);
  } finally {
    if (proxy != null) {
      cmProxy.mayBeCloseProxy(proxy);
    }
  }
}

Source File: TestContainerManager.java From big-c with Apache License 2.0

4 votes

@Test
public void testStartContainerFailureWithUnknownAuxService() throws Exception {
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
      new String[] { "existService" });
  conf.setClass(
      String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "existService"),
      ServiceA.class, Service.class);
  containerManager.start();

  List<StartContainerRequest> startRequest =
      new ArrayList<StartContainerRequest>();

  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
  String serviceName = "non_exist_auxService";
  serviceData.put(serviceName, ByteBuffer.wrap(serviceName.getBytes()));
  containerLaunchContext.setServiceData(serviceData);

  ContainerId cId = createContainerId(0);
  String user = "start_container_fail";
  Token containerToken =
      createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager());
  StartContainerRequest request =
      StartContainerRequest.newInstance(containerLaunchContext,
          containerToken);

  // start containers
  startRequest.add(request);
  StartContainersRequest requestList =
      StartContainersRequest.newInstance(startRequest);

  StartContainersResponse response =
      containerManager.startContainers(requestList);
  Assert.assertTrue(response.getFailedRequests().size() == 1);
  Assert.assertTrue(response.getSuccessfullyStartedContainers().size() == 0);
  Assert.assertTrue(response.getFailedRequests().containsKey(cId));
  Assert.assertTrue(response.getFailedRequests().get(cId).getMessage()
      .contains("The auxService:" + serviceName + " does not exist"));
}

Source File: TestContainerManager.java From big-c with Apache License 2.0

4 votes

@Test
public void testMultipleContainersLaunch() throws Exception {
  containerManager.start();

  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  for (int i = 0; i < 10; i++) {
    ContainerId cId = createContainerId(i);
    long identifier = 0;
    if ((i & 1) == 0)
      // container with even id fail
      identifier = ResourceManagerConstants.RM_INVALID_IDENTIFIER;
    else
      identifier = DUMMY_RM_IDENTIFIER;
    Token containerToken =
        createContainerToken(cId, identifier, context.getNodeId(), user,
          context.getContainerTokenSecretManager());
    StartContainerRequest request =
        StartContainerRequest.newInstance(containerLaunchContext,
          containerToken);
    list.add(request);
  }
  StartContainersRequest requestList =
      StartContainersRequest.newInstance(list);

  StartContainersResponse response =
      containerManager.startContainers(requestList);

  Assert.assertEquals(5, response.getSuccessfullyStartedContainers().size());
  for (ContainerId id : response.getSuccessfullyStartedContainers()) {
    // Containers with odd id should succeed.
    Assert.assertEquals(1, id.getContainerId() & 1);
  }
  Assert.assertEquals(5, response.getFailedRequests().size());
  for (Map.Entry<ContainerId, SerializedException> entry : response
    .getFailedRequests().entrySet()) {
    // Containers with even id should fail.
    Assert.assertEquals(0, entry.getKey().getContainerId() & 1);
    Assert.assertTrue(entry.getValue().getMessage()
      .contains(
        "Container " + entry.getKey() + " rejected as it is allocated by a previous RM"));
  }
}

Source File: TestContainerLauncherImpl.java From hadoop with Apache License 2.0

4 votes

@SuppressWarnings({ "rawtypes", "unchecked" })
@Test(timeout = 5000)
public void testContainerCleaned() throws Exception {
  LOG.info("STARTING testContainerCleaned");
  
  CyclicBarrier startLaunchBarrier = new CyclicBarrier(2);
  CyclicBarrier completeLaunchBarrier = new CyclicBarrier(2);

  AppContext mockContext = mock(AppContext.class);
  
  EventHandler mockEventHandler = mock(EventHandler.class);
  when(mockContext.getEventHandler()).thenReturn(mockEventHandler);

  ContainerManagementProtocolClient mockCM =
      new ContainerManagerForTest(startLaunchBarrier, completeLaunchBarrier);
  ContainerLauncherImplUnderTest ut =
      new ContainerLauncherImplUnderTest(mockContext, mockCM);
  
  Configuration conf = new Configuration();
  ut.init(conf);
  ut.start();
  try {
    ContainerId contId = makeContainerId(0l, 0, 0, 1);
    TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
    String cmAddress = "127.0.0.1:8000";
    StartContainersResponse startResp =
      recordFactory.newRecordInstance(StartContainersResponse.class);
    startResp.setAllServicesMetaData(serviceResponse);
    
   
    LOG.info("inserting launch event");
    ContainerRemoteLaunchEvent mockLaunchEvent = 
      mock(ContainerRemoteLaunchEvent.class);
    when(mockLaunchEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
    when(mockLaunchEvent.getContainerID())
      .thenReturn(contId);
    when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    when(mockLaunchEvent.getContainerToken()).thenReturn(
        createNewContainerToken(contId, cmAddress));
    ut.handle(mockLaunchEvent);
    
    startLaunchBarrier.await();
    
         
    LOG.info("inserting cleanup event");
    ContainerLauncherEvent mockCleanupEvent = 
      mock(ContainerLauncherEvent.class);
    when(mockCleanupEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
    when(mockCleanupEvent.getContainerID())
      .thenReturn(contId);
    when(mockCleanupEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockCleanupEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    ut.handle(mockCleanupEvent);

    completeLaunchBarrier.await();
   
    ut.waitForPoolToIdle();
    
    ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
    verify(mockEventHandler, atLeast(2)).handle(arg.capture());
    boolean containerCleaned = false;
    
    for (int i =0; i < arg.getAllValues().size(); i++) {
      LOG.info(arg.getAllValues().get(i).toString());
      Event currentEvent = arg.getAllValues().get(i);
      if (currentEvent.getType() == TaskAttemptEventType.TA_CONTAINER_CLEANED) {
        containerCleaned = true;
      }
    }
    assert(containerCleaned);
    
  } finally {
    ut.stop();
  }
}

Source File: TestContainerLauncherImpl.java From hadoop with Apache License 2.0

4 votes

@Test(timeout = 5000)
public void testMyShutdown() throws Exception {
  LOG.info("in test Shutdown");

  AppContext mockContext = mock(AppContext.class);
  @SuppressWarnings("rawtypes")
  EventHandler mockEventHandler = mock(EventHandler.class);
  when(mockContext.getEventHandler()).thenReturn(mockEventHandler);

  ContainerManagementProtocolClient mockCM =
      mock(ContainerManagementProtocolClient.class);
  ContainerLauncherImplUnderTest ut =
      new ContainerLauncherImplUnderTest(mockContext, mockCM);

  Configuration conf = new Configuration();
  ut.init(conf);
  ut.start();
  try {
    ContainerId contId = makeContainerId(0l, 0, 0, 1);
    TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
    String cmAddress = "127.0.0.1:8000";
    StartContainersResponse startResp =
      recordFactory.newRecordInstance(StartContainersResponse.class);
    startResp.setAllServicesMetaData(serviceResponse);

    LOG.info("inserting launch event");
    ContainerRemoteLaunchEvent mockLaunchEvent =
      mock(ContainerRemoteLaunchEvent.class);
    when(mockLaunchEvent.getType())
      .thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
    when(mockLaunchEvent.getContainerID())
      .thenReturn(contId);
    when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
    when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
    when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp);
    when(mockLaunchEvent.getContainerToken()).thenReturn(
        createNewContainerToken(contId, cmAddress));
    ut.handle(mockLaunchEvent);

    ut.waitForPoolToIdle();

    verify(mockCM).startContainers(any(StartContainersRequest.class));

    // skip cleanup and make sure stop kills the container

  } finally {
    ut.stop();
    verify(mockCM).stopContainers(any(StopContainersRequest.class));
  }
}

org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse Java Examples