org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes Java Examples

@Override
public void start() throws Exception {
  super.start();
  client.start();
  if (shouldReuseContainers) {
    reuseExecutor = createExecutor();
  }
  TaskSchedulerContext ctx = getContext();
  RegisterApplicationMasterResponse response = client.registerApplicationMaster(
      ctx.getAppHostName(), ctx.getAppClientPort(), ctx.getAppTrackingUrl());
  ctx.setApplicationRegistrationData(response.getMaximumResourceCapability(),
      response.getApplicationACLs(), response.getClientToAMTokenMasterKey(),
      response.getQueue());
  if (response.getSchedulerResourceTypes().contains(SchedulerResourceTypes.CPU)) {
    resourceCalculator = new MemCpuResourceCalculator();
  } else {
    resourceCalculator = new MemResourceCalculator();
  }
}
 

private void addSchedulerResourceTypes() {
  maybeInitBuilder();
  builder.clearSchedulerResourceTypes();
  if (schedulerResourceTypes == null) {
    return;
  }
  Iterable<? extends SchedulerResourceTypes> values =
      new Iterable<SchedulerResourceTypes>() {

        @Override
        public Iterator<SchedulerResourceTypes> iterator() {
          return new Iterator<SchedulerResourceTypes>() {
            Iterator<SchedulerResourceTypes> settingsIterator =
                schedulerResourceTypes.iterator();

            @Override
            public boolean hasNext() {
              return settingsIterator.hasNext();
            }

            @Override
            public SchedulerResourceTypes next() {
              return settingsIterator.next();
            }

            @Override
            public void remove() {
              throw new UnsupportedOperationException();
            }
          };
        }
      };
  this.builder.addAllSchedulerResourceTypes(values);
}
 

@Override
public RegisterApplicationMasterResponse registerApplicationMaster(String appHostName, int appHostPort,
    String appTrackingUrl) {
  mockRegResponse = mock(RegisterApplicationMasterResponse.class);
  Resource mockMaxResource = Resources.createResource(1024*1024, 1024);
  Map<ApplicationAccessType, String> mockAcls = Collections.emptyMap();
  when(mockRegResponse.getMaximumResourceCapability()).thenReturn(
      mockMaxResource);
  when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls);
  when(mockRegResponse.getSchedulerResourceTypes()).thenReturn(
      EnumSet.of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU));
  return mockRegResponse;
}
 

@Override
public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) {
  EnumSet<SchedulerResourceTypes> supportedResourceTypes = response.getSchedulerResourceTypes();
  Set<String> supportedTypes = new HashSet<String>();
  for (SchedulerResourceTypes resourceType : supportedResourceTypes) {
    supportedTypes.add(resourceType.name());
  }
  return supportedTypes;
}
 

@Override
public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) {
  EnumSet<SchedulerResourceTypes> supportedResourceTypes = response.getSchedulerResourceTypes();
  Set<String> supportedTypes = new HashSet<String>();
  for (SchedulerResourceTypes resourceType : supportedResourceTypes) {
    supportedTypes.add(resourceType.name());
  }
  return supportedTypes;
}
 

public RMCommunicator(ClientService clientService, AppContext context) {
  super("RMCommunicator");
  this.clientService = clientService;
  this.context = context;
  this.eventHandler = context.getEventHandler();
  this.applicationId = context.getApplicationID();
  this.stopped = new AtomicBoolean(false);
  this.heartbeatCallbacks = new ConcurrentLinkedQueue<Runnable>();
  this.schedulerResourceTypes = EnumSet.of(SchedulerResourceTypes.MEMORY);
}
 

public static int divideAndCeilContainers(Resource required, Resource factor,
    EnumSet<SchedulerResourceTypes> resourceTypes) {
  if (resourceTypes.contains(SchedulerResourceTypes.CPU)) {
    return Math.max(divideAndCeil(required.getMemory(), factor.getMemory()),
      divideAndCeil(required.getVirtualCores(), factor.getVirtualCores()));
  }
  return divideAndCeil(required.getMemory(), factor.getMemory());
}
 

public static int computeAvailableContainers(Resource available,
    Resource required, EnumSet<SchedulerResourceTypes> resourceTypes) {
  if (resourceTypes.contains(SchedulerResourceTypes.CPU)) {
    return Math.min(available.getMemory() / required.getMemory(),
      available.getVirtualCores() / required.getVirtualCores());
  }
  return available.getMemory() / required.getMemory();
}
 

/** {@inheritDoc} */
@Override
public EnumSet<SchedulerResourceTypes> getSchedulingResourceTypes() {
  if (calculator.getClass().getName()
    .equals(DefaultResourceCalculator.class.getName())) {
    return EnumSet.of(SchedulerResourceTypes.MEMORY);
  }
  return EnumSet
    .of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU);
}
 

@Override
public void setSchedulerResourceTypes(EnumSet<SchedulerResourceTypes> types) {
  if (types == null) {
    return;
  }
  initSchedulerResourceTypes();
  this.schedulerResourceTypes.clear();
  this.schedulerResourceTypes.addAll(types);
}
 

private void initSchedulerResourceTypes() {
  if (this.schedulerResourceTypes != null) {
    return;
  }
  RegisterApplicationMasterResponseProtoOrBuilder p =
      viaProto ? proto : builder;

  List<SchedulerResourceTypes> list = p.getSchedulerResourceTypesList();
  if (list.isEmpty()) {
    this.schedulerResourceTypes =
        EnumSet.noneOf(SchedulerResourceTypes.class);
  } else {
    this.schedulerResourceTypes = EnumSet.copyOf(list);
  }
}
 

public RMCommunicator(ClientService clientService, AppContext context) {
  super("RMCommunicator");
  this.clientService = clientService;
  this.context = context;
  this.eventHandler = context.getEventHandler();
  this.applicationId = context.getApplicationID();
  this.stopped = new AtomicBoolean(false);
  this.heartbeatCallbacks = new ConcurrentLinkedQueue<Runnable>();
  this.schedulerResourceTypes = EnumSet.of(SchedulerResourceTypes.MEMORY);
}
 

public static int computeAvailableContainers(Resource available,
    Resource required, EnumSet<SchedulerResourceTypes> resourceTypes) {
  if (resourceTypes.contains(SchedulerResourceTypes.CPU)) {
    if (required.getGpuCores() == 0) {
      return Math.min(available.getMemory() / required.getMemory(),
              available.getVirtualCores() / required.getVirtualCores());
    }
    else {
      return Math.min(available.getMemory() / required.getMemory(),
              Math.min(available.getVirtualCores() / required.getVirtualCores(),
                      available.getGpuCores() / required.getGpuCores()));
    }
  }
  return available.getMemory() / required.getMemory();
}
 

private void initSchedulerResourceTypes() {
  if (this.schedulerResourceTypes != null) {
    return;
  }
  RegisterApplicationMasterResponseProtoOrBuilder p =
      viaProto ? proto : builder;

  List<SchedulerResourceTypes> list = p.getSchedulerResourceTypesList();
  if (list.isEmpty()) {
    this.schedulerResourceTypes =
        EnumSet.noneOf(SchedulerResourceTypes.class);
  } else {
    this.schedulerResourceTypes = EnumSet.copyOf(list);
  }
}
 

private void addSchedulerResourceTypes() {
  maybeInitBuilder();
  builder.clearSchedulerResourceTypes();
  if (schedulerResourceTypes == null) {
    return;
  }
  Iterable<? extends SchedulerResourceTypes> values =
      new Iterable<SchedulerResourceTypes>() {

        @Override
        public Iterator<SchedulerResourceTypes> iterator() {
          return new Iterator<SchedulerResourceTypes>() {
            Iterator<SchedulerResourceTypes> settingsIterator =
                schedulerResourceTypes.iterator();

            @Override
            public boolean hasNext() {
              return settingsIterator.hasNext();
            }

            @Override
            public SchedulerResourceTypes next() {
              return settingsIterator.next();
            }

            @Override
            public void remove() {
              throw new UnsupportedOperationException();
            }
          };
        }
      };
  this.builder.addAllSchedulerResourceTypes(values);
}
 

@Override
public void setSchedulerResourceTypes(EnumSet<SchedulerResourceTypes> types) {
  if (types == null) {
    return;
  }
  initSchedulerResourceTypes();
  this.schedulerResourceTypes.clear();
  this.schedulerResourceTypes.addAll(types);
}
 

/** {@inheritDoc} */
@Override
public EnumSet<SchedulerResourceTypes> getSchedulingResourceTypes() {
  if (calculator.getClass().getName()
    .equals(DefaultResourceCalculator.class.getName())) {
    return EnumSet.of(SchedulerResourceTypes.MEMORY);
  }
  return EnumSet
    .of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU, SchedulerResourceTypes.GPU);
}
 

public static int divideAndCeilContainers(Resource required, Resource factor,
    EnumSet<SchedulerResourceTypes> resourceTypes) {
  if (resourceTypes.contains(SchedulerResourceTypes.CPU)) {
    if (factor.getGpuCores() == 0) {
      return Math.max(divideAndCeil(required.getMemory(), factor.getMemory()),
              divideAndCeil(required.getVirtualCores(), factor.getVirtualCores()));
    }
    else {
      return Math.max(divideAndCeil(required.getMemory(), factor.getMemory()),
              Math.max(divideAndCeil(required.getVirtualCores(), factor.getVirtualCores()),
                      divideAndCeil(required.getGpuCores(), factor.getGpuCores())));
    }
  }
  return divideAndCeil(required.getMemory(), factor.getMemory());
}
 

public EnumSet<SchedulerResourceTypes> getSchedulerResourceTypes() {
  return schedulerResourceTypes;
}
 

/** {@inheritDoc} */
@Override
public EnumSet<SchedulerResourceTypes> getSchedulingResourceTypes() {
  return EnumSet
    .of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU);
}
 

/** {@inheritDoc} */
@Override
public EnumSet<SchedulerResourceTypes> getSchedulingResourceTypes() {
  return EnumSet.of(SchedulerResourceTypes.MEMORY);
}
 

@Test(timeout = 3000000)
public void testResourceTypes() throws Exception {
  HashMap<YarnConfiguration, EnumSet<SchedulerResourceTypes>> driver =
      new HashMap<YarnConfiguration, EnumSet<SchedulerResourceTypes>>();

  CapacitySchedulerConfiguration csconf =
      new CapacitySchedulerConfiguration();
  csconf.setResourceComparator(DominantResourceCalculator.class);
  YarnConfiguration testCapacityDRConf = new YarnConfiguration(csconf);
  testCapacityDRConf.setClass(YarnConfiguration.RM_SCHEDULER,
    CapacityScheduler.class, ResourceScheduler.class);
  YarnConfiguration testCapacityDefConf = new YarnConfiguration();
  testCapacityDefConf.setClass(YarnConfiguration.RM_SCHEDULER,
    CapacityScheduler.class, ResourceScheduler.class);
  YarnConfiguration testFairDefConf = new YarnConfiguration();
  testFairDefConf.setClass(YarnConfiguration.RM_SCHEDULER,
    FairScheduler.class, ResourceScheduler.class);

  driver.put(conf, EnumSet.of(SchedulerResourceTypes.MEMORY));
  driver.put(testCapacityDRConf,
    EnumSet.of(SchedulerResourceTypes.CPU, SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.GPU));
  driver.put(testCapacityDefConf, EnumSet.of(SchedulerResourceTypes.MEMORY));
  driver.put(testFairDefConf,
    EnumSet.of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU, SchedulerResourceTypes.GPU));

  for (Map.Entry<YarnConfiguration, EnumSet<SchedulerResourceTypes>> entry : driver
    .entrySet()) {
    EnumSet<SchedulerResourceTypes> expectedValue = entry.getValue();
    MockRM rm = new MockRM(entry.getKey());
    rm.start();
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
    RMApp app1 = rm.submitApp(2048);
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
    MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
    RegisterApplicationMasterResponse resp = am1.registerAppAttempt();
    EnumSet<SchedulerResourceTypes> types = resp.getSchedulerResourceTypes();
    LOG.info("types = " + types.toString());
    Assert.assertEquals(expectedValue, types);
    rm.stop();
  }
}
 

@Test(timeout = 3000000)
public void testResourceTypes() throws Exception {
  HashMap<YarnConfiguration, EnumSet<SchedulerResourceTypes>> driver =
      new HashMap<YarnConfiguration, EnumSet<SchedulerResourceTypes>>();

  CapacitySchedulerConfiguration csconf =
      new CapacitySchedulerConfiguration();
  csconf.setResourceComparator(DominantResourceCalculator.class);
  YarnConfiguration testCapacityDRConf = new YarnConfiguration(csconf);
  testCapacityDRConf.setClass(YarnConfiguration.RM_SCHEDULER,
    CapacityScheduler.class, ResourceScheduler.class);
  YarnConfiguration testCapacityDefConf = new YarnConfiguration();
  testCapacityDefConf.setClass(YarnConfiguration.RM_SCHEDULER,
    CapacityScheduler.class, ResourceScheduler.class);
  YarnConfiguration testFairDefConf = new YarnConfiguration();
  testFairDefConf.setClass(YarnConfiguration.RM_SCHEDULER,
    FairScheduler.class, ResourceScheduler.class);

  driver.put(conf, EnumSet.of(SchedulerResourceTypes.MEMORY));
  driver.put(testCapacityDRConf,
    EnumSet.of(SchedulerResourceTypes.CPU, SchedulerResourceTypes.MEMORY));
  driver.put(testCapacityDefConf, EnumSet.of(SchedulerResourceTypes.MEMORY));
  driver.put(testFairDefConf,
    EnumSet.of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU));

  for (Map.Entry<YarnConfiguration, EnumSet<SchedulerResourceTypes>> entry : driver
    .entrySet()) {
    EnumSet<SchedulerResourceTypes> expectedValue = entry.getValue();
    MockRM rm = new MockRM(entry.getKey());
    rm.start();
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
    RMApp app1 = rm.submitApp(2048);
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
    MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
    RegisterApplicationMasterResponse resp = am1.registerAppAttempt();
    EnumSet<SchedulerResourceTypes> types = resp.getSchedulerResourceTypes();
    LOG.info("types = " + types.toString());
    Assert.assertEquals(expectedValue, types);
    rm.stop();
  }
}
 

@Override
public EnumSet<SchedulerResourceTypes> getSchedulerResourceTypes() {
  initSchedulerResourceTypes();
  return this.schedulerResourceTypes;
}
 

@Override
public EnumSet<SchedulerResourceTypes> getSchedulerResourceTypes() {
  initSchedulerResourceTypes();
  return this.schedulerResourceTypes;
}
 

/** {@inheritDoc} */
@Override
public EnumSet<SchedulerResourceTypes> getSchedulingResourceTypes() {
  return EnumSet.of(SchedulerResourceTypes.MEMORY);
}
 

/** {@inheritDoc} */
@Override
public EnumSet<SchedulerResourceTypes> getSchedulingResourceTypes() {
  return EnumSet
    .of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU, SchedulerResourceTypes.GPU);
}
 

public EnumSet<SchedulerResourceTypes> getSchedulerResourceTypes() {
  return schedulerResourceTypes;
}
 

/**
 * Set the resource types used by the scheduler.
 *
 * @param types
 *          a set of the resource types that the scheduler considers during
 *          scheduling
 */
@Private
@Unstable
public abstract void setSchedulerResourceTypes(
    EnumSet<SchedulerResourceTypes> types);
 

/**
 * Return a collection of the resource types that are considered when
 * scheduling
 *
 * @return an EnumSet containing the resource types
 */
public EnumSet<SchedulerResourceTypes> getSchedulingResourceTypes();