org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler Java Examples

@GET
@Path("/scheduler")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public SchedulerTypeInfo getSchedulerInfo() {
  init();
  ResourceScheduler rs = rm.getResourceScheduler();
  SchedulerInfo sinfo;
  if (rs instanceof CapacityScheduler) {
    CapacityScheduler cs = (CapacityScheduler) rs;
    CSQueue root = cs.getRootQueue();
    sinfo =
        new CapacitySchedulerInfo(root, cs, new NodeLabel(
            RMNodeLabelsManager.NO_LABEL));
  } else if (rs instanceof FairScheduler) {
    FairScheduler fs = (FairScheduler) rs;
    sinfo = new FairSchedulerInfo(fs);
  } else if (rs instanceof FifoScheduler) {
    sinfo = new FifoSchedulerInfo(this.rm);
  } else {
    throw new NotFoundException("Unknown scheduler configured");
  }
  return new SchedulerTypeInfo(sinfo);
}
 

public static FairScheduler setupFairScheduler(
    ReservationSystemTestUtil testUtil,
    RMContext rmContext, Configuration conf, int numContainers) throws
    IOException {
  FairScheduler scheduler = new FairScheduler();
  scheduler.setRMContext(rmContext);

  when(rmContext.getScheduler()).thenReturn(scheduler);

  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);


  Resource resource = testUtil.calculateClusterResource(numContainers);
  RMNode node1 = MockNodes.newNodeInfo(1, resource, 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  return scheduler;
}
 

@GET
@Path("/scheduler")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public SchedulerTypeInfo getSchedulerInfo() {
  init();
  ResourceScheduler rs = rm.getResourceScheduler();
  SchedulerInfo sinfo;
  if (rs instanceof CapacityScheduler) {
    CapacityScheduler cs = (CapacityScheduler) rs;
    CSQueue root = cs.getRootQueue();
    sinfo = new CapacitySchedulerInfo(root);
  } else if (rs instanceof FairScheduler) {
    FairScheduler fs = (FairScheduler) rs;
    sinfo = new FairSchedulerInfo(fs);
  } else if (rs instanceof FifoScheduler) {
    sinfo = new FifoSchedulerInfo(this.rm);
  } else {
    throw new NotFoundException("Unknown scheduler configured");
  }
  return new SchedulerTypeInfo(sinfo);
}
 

public static FairScheduler setupFairScheduler(
    ReservationSystemTestUtil testUtil,
    RMContext rmContext, Configuration conf, int numContainers) throws
    IOException {
  FairScheduler scheduler = new FairScheduler();
  scheduler.setRMContext(rmContext);

  when(rmContext.getScheduler()).thenReturn(scheduler);

  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);


  Resource resource = testUtil.calculateClusterResource(numContainers);
  RMNode node1 = MockNodes.newNodeInfo(1, resource, 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  return scheduler;
}
 

@Override
public void configureScheduler() {
  try {
    PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("<queue name=\"root\">");
    out.println("  <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  <queue name=\"default\">");
    out.println("    <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  </queue>");
    out.println("  <queue name=\"test\">");
    out.println("    <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  </queue>");
    out.println("</queue>");
    out.println("</allocations>");
    out.close();
  } catch(IOException e) {
  }
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
  conf.set("yarn.resourcemanager.scheduler.class",
      FairScheduler.class.getName());
}
 

@Override
public void configureScheduler() {
  try {
    PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("<queue name=\"root\">");
    out.println("  <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  <queue name=\"default\">");
    out.println("    <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  </queue>");
    out.println("  <queue name=\"test\">");
    out.println("    <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  </queue>");
    out.println("</queue>");
    out.println("</allocations>");
    out.close();
  } catch(IOException e) {
  }
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
  conf.set("yarn.resourcemanager.scheduler.class",
      FairScheduler.class.getName());
}
 

@Inject
public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx,
    Configuration conf) {
  super(ctx);
  FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
  fsinfo = new FairSchedulerInfo(scheduler);
  apps = new ConcurrentHashMap<ApplicationId, RMApp>();
  for (Map.Entry<ApplicationId, RMApp> entry : rm.getRMContext().getRMApps()
      .entrySet()) {
    if (!(RMAppState.NEW.equals(entry.getValue().getState())
        || RMAppState.NEW_SAVING.equals(entry.getValue().getState())
        || RMAppState.SUBMITTED.equals(entry.getValue().getState()))) {
      apps.put(entry.getKey(), entry.getValue());
    }
  }
  this.conf = conf;
  this.rm = rm;
}
 

private void configureFairScheduler(YarnConfiguration conf) throws IOException {
  // Disable queueMaxAMShare limitation for fair scheduler
  PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queueMaxAMShareDefault>-1.0</queueMaxAMShareDefault>");
  out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
  out.println("<queue name=\"root\">");
  out.println("  <schedulingPolicy>drf</schedulingPolicy>");
  out.println("  <weight>1.0</weight>");
  out.println("  <fairSharePreemptionTimeout>100</fairSharePreemptionTimeout>");
  out.println("  <minSharePreemptionTimeout>120</minSharePreemptionTimeout>");
  out.println("  <fairSharePreemptionThreshold>.5</fairSharePreemptionThreshold>");
  out.println("</queue>");
  out.println("</allocations>");
  out.close();

  conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
}
 

private void configureFairScheduler(YarnConfiguration conf) throws IOException {
  // Disable queueMaxAMShare limitation for fair scheduler
  PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queueMaxAMShareDefault>-1.0</queueMaxAMShareDefault>");
  out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
  out.println("<queue name=\"root\">");
  out.println("  <schedulingPolicy>drf</schedulingPolicy>");
  out.println("  <weight>1.0</weight>");
  out.println("  <fairSharePreemptionTimeout>100</fairSharePreemptionTimeout>");
  out.println("  <minSharePreemptionTimeout>120</minSharePreemptionTimeout>");
  out.println("  <fairSharePreemptionThreshold>.5</fairSharePreemptionThreshold>");
  out.println("</queue>");
  out.println("</allocations>");
  out.close();

  conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
}
 

public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 

public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 

@Inject
public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx,
    Configuration conf) {
  super(ctx);
  FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
  fsinfo = new FairSchedulerInfo(scheduler);
  apps = new ConcurrentHashMap<ApplicationId, RMApp>();
  for (Map.Entry<ApplicationId, RMApp> entry : rm.getRMContext().getRMApps()
      .entrySet()) {
    if (!(RMAppState.NEW.equals(entry.getValue().getState())
        || RMAppState.NEW_SAVING.equals(entry.getValue().getState())
        || RMAppState.SUBMITTED.equals(entry.getValue().getState()))) {
      apps.put(entry.getKey(), entry.getValue());
    }
  }
  this.conf = conf;
  this.rm = rm;
}
 

public SchedulerInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();

  if (rs instanceof CapacityScheduler) {
    this.schedulerName = "Capacity Scheduler";
  } else if (rs instanceof FairScheduler) {
    this.schedulerName = "Fair Scheduler";
  } else if (rs instanceof FifoScheduler) {
    this.schedulerName = "Fifo Scheduler";
  }
  this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
  this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());
  this.schedulingResourceTypes = rs.getSchedulingResourceTypes();
}
 

@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
    ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 

private static FairScheduler mockFairScheduler() throws IOException {
  FairScheduler fs = new FairScheduler();
  FairSchedulerConfiguration conf = new FairSchedulerConfiguration();
  fs.setRMContext(new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null));
  fs.init(conf);
  return fs;
}
 

@Override
public void reinitialize(Configuration conf, RMContext rmContext)
    throws YarnException {
  // Validate if the scheduler is fair scheduler
  ResourceScheduler scheduler = rmContext.getScheduler();
  if (!(scheduler instanceof FairScheduler)) {
    throw new YarnRuntimeException("Class "
        + scheduler.getClass().getCanonicalName() + " not instance of "
        + FairScheduler.class.getCanonicalName());
  }
  fairScheduler = (FairScheduler) scheduler;
  this.conf = conf;
  super.reinitialize(conf, rmContext);
}
 

private String getDefaultPlanFollower() {
  // currently only capacity scheduler is supported
  if (scheduler instanceof CapacityScheduler) {
    return CapacitySchedulerPlanFollower.class.getName();
  } else if (scheduler instanceof FairScheduler) {
    return FairSchedulerPlanFollower.class.getName();
  }
  return null;
}
 

/**
 * Get the default reservation system corresponding to the scheduler
 * 
 * @param scheduler the scheduler for which the reservation system is required
 */
public static String getDefaultReservationSystem(ResourceScheduler scheduler) {
  if (scheduler instanceof CapacityScheduler) {
    return CapacityReservationSystem.class.getName();
  } else if (scheduler instanceof FairScheduler) {
    return FairReservationSystem.class.getName();
  }
  return null;
}
 

@Override
public void init(Clock clock, ResourceScheduler sched,
    Collection<Plan> plans) {
  super.init(clock, sched, plans);
  fs = (FairScheduler)sched;
  LOG.info("Initializing Plan Follower Policy:"
      + this.getClass().getCanonicalName());
}
 

@Before
public void setup() {
  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);
  ExitUtil.disableSystemExit();
  conf = new YarnConfiguration();
  UserGroupInformation.setConfiguration(conf);
  conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
  conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
}
 

private void checkFSQueue(ResourceManager rm,
    SchedulerApplication  schedulerApp, Resource usedResources,
    Resource availableResources) throws Exception {
  // waiting for RM's scheduling apps
  int retry = 0;
  Resource assumedFairShare = Resource.newInstance(8192, 8);
  while (true) {
    Thread.sleep(100);
    if (assumedFairShare.equals(((FairScheduler)rm.getResourceScheduler())
        .getQueueManager().getRootQueue().getFairShare())) {
      break;
    }
    retry++;
    if (retry > 30) {
      Assert.fail("Apps are not scheduled within assumed timeout");
    }
  }

  FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
  FSParentQueue root = scheduler.getQueueManager().getRootQueue();
  // ************ check cluster used Resources ********
  assertTrue(root.getPolicy() instanceof DominantResourceFairnessPolicy);
  assertEquals(usedResources,root.getResourceUsage());

  // ************ check app headroom ****************
  FSAppAttempt schedulerAttempt =
      (FSAppAttempt) schedulerApp.getCurrentAppAttempt();
  assertEquals(availableResources, schedulerAttempt.getHeadroom());

  // ************ check queue metrics ****************
  QueueMetrics queueMetrics = scheduler.getRootQueueMetrics();
  assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
      availableResources.getVirtualCores(), usedResources.getMemory(),
      usedResources.getVirtualCores());
}
 

protected Configuration createConfiguration() {
  Configuration conf = testHelper.createConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
  return conf;
}
 

protected Configuration createConfiguration() {
  Configuration conf = testHelper.createConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
  return conf;
}
 

private static FairScheduler mockFairScheduler() throws IOException {
  FairScheduler fs = new FairScheduler();
  FairSchedulerConfiguration conf = new FairSchedulerConfiguration();
  fs.setRMContext(new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null));
  fs.init(conf);
  return fs;
}
 

@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
    ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 

@Before
public void setup() {
  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);
  ExitUtil.disableSystemExit();
  conf = new YarnConfiguration();
  UserGroupInformation.setConfiguration(conf);
  conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
  conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
}
 

@Override
public void init(Clock clock, ResourceScheduler sched,
    Collection<Plan> plans) {
  super.init(clock, sched, plans);
  fs = (FairScheduler)sched;
  LOG.info("Initializing Plan Follower Policy:"
      + this.getClass().getCanonicalName());
}
 

/**
 * Get the default reservation system corresponding to the scheduler
 * 
 * @param scheduler the scheduler for which the reservation system is required
 */
public static String getDefaultReservationSystem(ResourceScheduler scheduler) {
  if (scheduler instanceof CapacityScheduler) {
    return CapacityReservationSystem.class.getName();
  } else if (scheduler instanceof FairScheduler) {
    return FairReservationSystem.class.getName();
  }
  return null;
}
 

private String getDefaultPlanFollower() {
  // currently only capacity scheduler is supported
  if (scheduler instanceof CapacityScheduler) {
    return CapacitySchedulerPlanFollower.class.getName();
  } else if (scheduler instanceof FairScheduler) {
    return FairSchedulerPlanFollower.class.getName();
  }
  return null;
}
 

@Override
public void reinitialize(Configuration conf, RMContext rmContext)
    throws YarnException {
  // Validate if the scheduler is fair scheduler
  ResourceScheduler scheduler = rmContext.getScheduler();
  if (!(scheduler instanceof FairScheduler)) {
    throw new YarnRuntimeException("Class "
        + scheduler.getClass().getCanonicalName() + " not instance of "
        + FairScheduler.class.getCanonicalName());
  }
  fairScheduler = (FairScheduler) scheduler;
  this.conf = conf;
  super.reinitialize(conf, rmContext);
}