org.apache.hadoop.yarn.api.records.QueueInfo Java Examples

/**
 * Utility method to validate a list resource requests, by insuring that the
 * requested memory/vcore is non-negative and not greater than max
 */
public static void normalizeAndValidateRequests(List<ResourceRequest> ask,
    Resource maximumResource, String queueName, YarnScheduler scheduler,
    RMContext rmContext)
    throws InvalidResourceRequestException {

  QueueInfo queueInfo = null;
  try {
    queueInfo = scheduler.getQueueInfo(queueName, false, false);
  } catch (IOException e) {
  }

  for (ResourceRequest resReq : ask) {
    SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource,
        queueName, scheduler, rmContext, queueInfo);
  }
}
 

private static void normalizeNodeLabelExpressionInRequest(
    ResourceRequest resReq, QueueInfo queueInfo) {

  String labelExp = resReq.getNodeLabelExpression();

  // if queue has default label expression, and RR doesn't have, use the
  // default label expression of queue
  if (labelExp == null && queueInfo != null && ResourceRequest.ANY
      .equals(resReq.getResourceName())) {
    labelExp = queueInfo.getDefaultNodeLabelExpression();
  }

  // If labelExp still equals to null, set it to be NO_LABEL
  if (labelExp == null) {
    labelExp = RMNodeLabelsManager.NO_LABEL;
  }
  resReq.setNodeLabelExpression(labelExp);
}
 

@Test
public void testDefaultNodeLabelExpressionQueueConfig() throws Exception {
  CapacityScheduler cs = new CapacityScheduler();
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);
  conf.setDefaultNodeLabelExpression("root.a", " x");
  conf.setDefaultNodeLabelExpression("root.b", " y ");
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(conf);
  cs.start();

  QueueInfo queueInfoA = cs.getQueueInfo("a", true, false);
  Assert.assertEquals(queueInfoA.getQueueName(), "a");
  Assert.assertEquals(queueInfoA.getDefaultNodeLabelExpression(), "x");

  QueueInfo queueInfoB = cs.getQueueInfo("b", true, false);
  Assert.assertEquals(queueInfoB.getQueueName(), "b");
  Assert.assertEquals(queueInfoB.getDefaultNodeLabelExpression(), "y");
}
 

@Override
public QueueInfo getQueueInfo( 
    boolean includeChildQueues, boolean recursive) {
  QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
  queueInfo.setQueueName(DEFAULT_QUEUE.getQueueName());
  queueInfo.setCapacity(1.0f);
  if (clusterResource.getMemory() == 0) {
    queueInfo.setCurrentCapacity(0.0f);
  } else {
    queueInfo.setCurrentCapacity((float) usedResource.getMemory()
        / clusterResource.getMemory());
  }
  queueInfo.setMaximumCapacity(1.0f);
  queueInfo.setChildQueues(new ArrayList<QueueInfo>());
  queueInfo.setQueueState(QueueState.RUNNING);
  return queueInfo;
}
 

/**
 * Utility method to validate a list resource requests, by insuring that the
 * requested memory/vcore is non-negative and not greater than max
 */
public static void normalizeAndValidateRequests(List<ResourceRequest> ask,
    Resource maximumResource, String queueName, YarnScheduler scheduler,
    RMContext rmContext)
    throws InvalidResourceRequestException {

  QueueInfo queueInfo = null;
  try {
    queueInfo = scheduler.getQueueInfo(queueName, false, false);
  } catch (IOException e) {
  }

  for (ResourceRequest resReq : ask) {
    SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource,
        queueName, scheduler, rmContext, queueInfo);
  }
}
 

@Override
public synchronized QueueInfo getQueueInfo( 
    boolean includeChildQueues, boolean recursive) {
  QueueInfo queueInfo = getQueueInfo();

  List<QueueInfo> childQueuesInfo = new ArrayList<QueueInfo>();
  if (includeChildQueues) {
    for (CSQueue child : childQueues) {
      // Get queue information recursively?
      childQueuesInfo.add(
          child.getQueueInfo(recursive, recursive));
    }
  }
  queueInfo.setChildQueues(childQueuesInfo);
  
  return queueInfo;
}
 

@Override
public void trackQueue(String queueName) {
  trackedQueues.add(queueName);
  FifoScheduler fifo = (FifoScheduler) scheduler;
  // for FifoScheduler, only DEFAULT_QUEUE
  // here the three parameters doesn't affect results
  final QueueInfo queue = fifo.getQueueInfo(queueName, false, false);
  // track currentCapacity, maximumCapacity (always 1.0f)
  metrics.register("variable.queue." + queueName + ".currentcapacity",
    new Gauge<Float>() {
      @Override
      public Float getValue() {
        return queue.getCurrentCapacity();
      }
    }
  );
  metrics.register("variable.queue." + queueName + ".",
    new Gauge<Float>() {
      @Override
      public Float getValue() {
        return queue.getCurrentCapacity();
      }
    }
  );
}
 

@Override
public QueueInfo getQueueInfo( 
    boolean includeChildQueues, boolean recursive) {
  QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
  queueInfo.setQueueName(DEFAULT_QUEUE.getQueueName());
  queueInfo.setCapacity(1.0f);
  if (clusterResource.getMemory() == 0) {
    queueInfo.setCurrentCapacity(0.0f);
  } else {
    queueInfo.setCurrentCapacity((float) usedResource.getMemory()
        / clusterResource.getMemory());
  }
  queueInfo.setMaximumCapacity(1.0f);
  queueInfo.setChildQueues(new ArrayList<QueueInfo>());
  queueInfo.setQueueState(QueueState.RUNNING);
  return queueInfo;
}
 

/**
 	 * Traverse queues bf.
 	 *
 	 * @param queueName the queue name
 	 * @param queueAlert the queue alert
* @param rmCommunicator 
 	 * @throws YarnException the yarn exception
 	 * @throws IOException Signals that an I/O exception has occurred.
 	 */
 	private void traverseQueuesBF(String queueName, List<AlertInfo> queueAlert, RMCommunicator rmCommunicator) throws YarnException, IOException {
   final String parentQueue = queueName;
   QueueInfo qi = rmCommunicator.getQueueInfo(queueName);
      List<String> queueNames = new ArrayList<String>(5); 
   float childrenCapacity = 0.0f;
   for (QueueInfo info : qi.getChildQueues()) {
	   queueNames.add(info.getQueueName());
	   childrenCapacity += info.getCapacity();
   }
	if(childrenCapacity > 1.0){
		AlertInfo alertInfo = new AlertInfo (ExtendedConstants.WARNING_LEVEL,ExtendedConstants.HYPHEN,"Queue "+parentQueue+":"+"child capacity exceeded 100 percent", getDate());
		queueAlert.add(alertInfo);
	}
   for(String name : queueNames) {
	   traverseQueuesBF(name, queueAlert, rmCommunicator);
   }
   
  }
 

/**
 * Lists the Queue Information matching the given queue name
 * 
 * @param queueName
 * @throws YarnException
 * @throws IOException
 */
private int listQueue(String queueName) throws YarnException, IOException {
  int rc;
  PrintWriter writer = new PrintWriter(
      new OutputStreamWriter(sysout, Charset.forName("UTF-8")));

  QueueInfo queueInfo = client.getQueueInfo(queueName);
  if (queueInfo != null) {
    writer.println("Queue Information : ");
    printQueueInfo(writer, queueInfo);
    rc = 0;
  } else {
    writer.println("Cannot get queue from RM by queueName = " + queueName
        + ", please check.");
    rc = -1;
  }
  writer.flush();
  return rc;
}
 

private void mockRMContext(YarnScheduler yarnScheduler, RMContext rmContext)
    throws IOException {
  Dispatcher dispatcher = mock(Dispatcher.class);
  when(rmContext.getDispatcher()).thenReturn(dispatcher);
  EventHandler eventHandler = mock(EventHandler.class);
  when(dispatcher.getEventHandler()).thenReturn(eventHandler);
  QueueInfo queInfo = recordFactory.newRecordInstance(QueueInfo.class);
  queInfo.setQueueName("testqueue");
  when(yarnScheduler.getQueueInfo(eq("testqueue"), anyBoolean(), anyBoolean()))
      .thenReturn(queInfo);
  when(yarnScheduler.getQueueInfo(eq("nonexistentqueue"), anyBoolean(), anyBoolean()))
      .thenThrow(new IOException("queue does not exist"));
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
      yarnScheduler);
  when(rmContext.getRMApps()).thenReturn(apps);
  when(yarnScheduler.getAppsInQueue(eq("testqueue"))).thenReturn(
      getSchedulerApps(apps));
   ResourceScheduler rs = mock(ResourceScheduler.class);
   when(rmContext.getScheduler()).thenReturn(rs);
}
 

@Test
public void testGetQueueInfo() throws Exception {
  QueueCLI cli = createAndGetQueueCLI();
  Set<String> nodeLabels = new HashSet<String>();
  nodeLabels.add("GPU");
  nodeLabels.add("JDK_7");
  QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f,
      null, null, QueueState.RUNNING, nodeLabels, "GPU");
  when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
  int result = cli.run(new String[] { "-status", "queueA" });
  assertEquals(0, result);
  verify(client).getQueueInfo("queueA");
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  PrintWriter pw = new PrintWriter(baos);
  pw.println("Queue Information : ");
  pw.println("Queue Name : " + "queueA");
  pw.println("\tState : " + "RUNNING");
  pw.println("\tCapacity : " + "40.0%");
  pw.println("\tCurrent Capacity : " + "50.0%");
  pw.println("\tMaximum Capacity : " + "80.0%");
  pw.println("\tDefault Node Label expression : " + "GPU");
  pw.println("\tAccessible Node Labels : " + "JDK_7,GPU");
  pw.close();
  String queueInfoStr = baos.toString("UTF-8");
  Assert.assertEquals(queueInfoStr, sysOutStream.toString());
}
 

@Test
public void testGetQueueInfoWithEmptyNodeLabel() throws Exception {
  QueueCLI cli = createAndGetQueueCLI();
  QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f,
      null, null, QueueState.RUNNING, null, null);
  when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
  int result = cli.run(new String[] { "-status", "queueA" });
  assertEquals(0, result);
  verify(client).getQueueInfo("queueA");
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  PrintWriter pw = new PrintWriter(baos);
  pw.println("Queue Information : ");
  pw.println("Queue Name : " + "queueA");
  pw.println("\tState : " + "RUNNING");
  pw.println("\tCapacity : " + "40.0%");
  pw.println("\tCurrent Capacity : " + "50.0%");
  pw.println("\tMaximum Capacity : " + "80.0%");
  pw.println("\tDefault Node Label expression : ");
  pw.println("\tAccessible Node Labels : ");
  pw.close();
  String queueInfoStr = baos.toString("UTF-8");
  Assert.assertEquals(queueInfoStr, sysOutStream.toString());
}
 

@Test
public void testGetQueueInfo() throws Exception {
  QueueCLI cli = createAndGetQueueCLI();
  Set<String> nodeLabels = new HashSet<String>();
  nodeLabels.add("GPU");
  nodeLabels.add("JDK_7");
  QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f,
      null, null, QueueState.RUNNING, nodeLabels, "GPU");
  when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
  int result = cli.run(new String[] { "-status", "queueA" });
  assertEquals(0, result);
  verify(client).getQueueInfo("queueA");
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  PrintWriter pw = new PrintWriter(baos);
  pw.println("Queue Information : ");
  pw.println("Queue Name : " + "queueA");
  pw.println("\tState : " + "RUNNING");
  pw.println("\tCapacity : " + "40.0%");
  pw.println("\tCurrent Capacity : " + "50.0%");
  pw.println("\tMaximum Capacity : " + "80.0%");
  pw.println("\tDefault Node Label expression : " + "GPU");
  pw.println("\tAccessible Node Labels : " + "JDK_7,GPU");
  pw.close();
  String queueInfoStr = baos.toString("UTF-8");
  Assert.assertEquals(queueInfoStr, sysOutStream.toString());
}
 

/**
 * Lists the Queue Information matching the given queue name
 * 
 * @param queueName
 * @throws YarnException
 * @throws IOException
 */
private int listQueue(String queueName) throws YarnException, IOException {
  int rc;
  PrintWriter writer = new PrintWriter(
      new OutputStreamWriter(sysout, Charset.forName("UTF-8")));

  QueueInfo queueInfo = client.getQueueInfo(queueName);
  if (queueInfo != null) {
    writer.println("Queue Information : ");
    printQueueInfo(writer, queueInfo);
    rc = 0;
  } else {
    writer.println("Cannot get queue from RM by queueName = " + queueName
        + ", please check.");
    rc = -1;
  }
  writer.flush();
  return rc;
}
 

@Override
public void trackQueue(String queueName) {
  trackedQueues.add(queueName);
  FifoScheduler fifo = (FifoScheduler) scheduler;
  // for FifoScheduler, only DEFAULT_QUEUE
  // here the three parameters doesn't affect results
  final QueueInfo queue = fifo.getQueueInfo(queueName, false, false);
  // track currentCapacity, maximumCapacity (always 1.0f)
  metrics.register("variable.queue." + queueName + ".currentcapacity",
    new Gauge<Float>() {
      @Override
      public Float getValue() {
        return queue.getCurrentCapacity();
      }
    }
  );
  metrics.register("variable.queue." + queueName + ".",
    new Gauge<Float>() {
      @Override
      public Float getValue() {
        return queue.getCurrentCapacity();
      }
    }
  );
}
 

@Test
public void testDefaultNodeLabelExpressionQueueConfig() throws Exception {
  CapacityScheduler cs = new CapacityScheduler();
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);
  conf.setDefaultNodeLabelExpression("root.a", " x");
  conf.setDefaultNodeLabelExpression("root.b", " y ");
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(conf);
  cs.start();

  QueueInfo queueInfoA = cs.getQueueInfo("a", true, false);
  Assert.assertEquals(queueInfoA.getQueueName(), "a");
  Assert.assertEquals(queueInfoA.getDefaultNodeLabelExpression(), "x");

  QueueInfo queueInfoB = cs.getQueueInfo("b", true, false);
  Assert.assertEquals(queueInfoB.getQueueName(), "b");
  Assert.assertEquals(queueInfoB.getDefaultNodeLabelExpression(), "y");
}
 

@Override
public void setChildQueues(List<QueueInfo> childQueues) {
  if (childQueues == null) {
    builder.clearChildQueues();
  }
  this.childQueuesList = childQueues;
}
 

@Override
public List<QueueInfo> getAllQueues() throws YarnException,
    IOException {
  List<QueueInfo> queues = new ArrayList<QueueInfo>();

  QueueInfo rootQueue =
      rmClient.getQueueInfo(getQueueInfoRequest(ROOT, false, true, true))
        .getQueueInfo();
  getChildQueues(rootQueue, queues, true);
  return queues;
}
 

private void addChildQueuesInfoToProto() {
  maybeInitBuilder();
  builder.clearChildQueues();
  if (childQueuesList == null)
    return;
  Iterable<QueueInfoProto> iterable = new Iterable<QueueInfoProto>() {
    @Override
    public Iterator<QueueInfoProto> iterator() {
      return new Iterator<QueueInfoProto>() {

        Iterator<QueueInfo> iter = childQueuesList.iterator();

        @Override
        public boolean hasNext() {
          return iter.hasNext();
        }

        @Override
        public QueueInfoProto next() {
          return convertToProtoFormat(iter.next());
        }

        @Override
        public void remove() {
          throw new UnsupportedOperationException();

        }
      };

    }
  };
  builder.addAllChildQueues(iterable);
}
 

@Override
public QueueInfo getQueueInfo() {
  if (this.queueInfo != null) {
    return this.queueInfo;
  }

  GetQueueInfoResponseProtoOrBuilder p = viaProto ? proto : builder;
  if (!p.hasQueueInfo()) {
    return null;
  }
  this.queueInfo = convertFromProtoFormat(p.getQueueInfo());
  return this.queueInfo;
}
 

private void initLocalChildQueuesList() {
  if (this.childQueuesList != null) {
    return;
  }
  QueueInfoProtoOrBuilder p = viaProto ? proto : builder;
  List<QueueInfoProto> list = p.getChildQueuesList();
  childQueuesList = new ArrayList<QueueInfo>();

  for (QueueInfoProto a : list) {
    childQueuesList.add(convertFromProtoFormat(a));
  }
}
 

public static void normalizeAndvalidateRequest(ResourceRequest resReq,
    Resource maximumResource, String queueName, YarnScheduler scheduler,
    RMContext rmContext, QueueInfo queueInfo)
    throws InvalidResourceRequestException {
  normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler,
      false, rmContext, queueInfo);
}
 

private void getChildQueues(QueueInfo parent, List<QueueInfo> queues,
    boolean recursive) {
  List<QueueInfo> childQueues = parent.getChildQueues();

  for (QueueInfo child : childQueues) {
    queues.add(child);
    if (recursive) {
      getChildQueues(child, queues, recursive);
    }
  }
}
 

@Override
public List<QueueInfo> getRootQueueInfos() throws YarnException,
    IOException {
  List<QueueInfo> queues = new ArrayList<QueueInfo>();

  QueueInfo rootQueue =
      rmClient.getQueueInfo(getQueueInfoRequest(ROOT, false, true, true))
        .getQueueInfo();
  getChildQueues(rootQueue, queues, false);
  return queues;
}
 

@Override
public String getClusterDescription() {

	try {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		PrintStream ps = new PrintStream(baos);

		YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();

		ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers());
		List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);
		final String format = "|%-16s |%-16s %n";
		ps.printf("|Property         |Value          %n");
		ps.println("+---------------------------------------+");
		int totalMemory = 0;
		int totalCores = 0;
		for (NodeReport rep : nodes) {
			final Resource res = rep.getCapability();
			totalMemory += res.getMemory();
			totalCores += res.getVirtualCores();
			ps.format(format, "NodeID", rep.getNodeId());
			ps.format(format, "Memory", res.getMemory() + " MB");
			ps.format(format, "vCores", res.getVirtualCores());
			ps.format(format, "HealthReport", rep.getHealthReport());
			ps.format(format, "Containers", rep.getNumContainers());
			ps.println("+---------------------------------------+");
		}
		ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores);
		List<QueueInfo> qInfo = yarnClient.getAllQueues();
		for (QueueInfo q : qInfo) {
			ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " +
				q.getMaximumCapacity() + " Applications: " + q.getApplications().size());
		}
		return baos.toString();
	} catch (Exception e) {
		throw new RuntimeException("Couldn't get cluster description", e);
	}
}
 

@Override
public QueueInfo getQueueInfo(String queueName) throws YarnException,
    IOException {
  GetQueueInfoRequest request =
      getQueueInfoRequest(queueName, true, false, false);
  Records.newRecord(GetQueueInfoRequest.class);
  return rmClient.getQueueInfo(request).getQueueInfo();
}
 

private void addLeafQueue(QueueInfo queueInfo, List<String> list) {
	
	if (queueInfo.getChildQueues() == null || queueInfo.getChildQueues().size() == 0) {
		// If leaf queue then
		list.add(queueInfo.getQueueName());
	} else {
		// If not leaf queue
		for (QueueInfo childQueue : queueInfo.getChildQueues()) {
			addLeafQueue(childQueue, list);
		}
	}
}
 

protected QueueInfo getQueueInfo() {
  QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
  queueInfo.setQueueName(queueName);
  queueInfo.setAccessibleNodeLabels(accessibleLabels);
  queueInfo.setCapacity(queueCapacities.getCapacity());
  queueInfo.setMaximumCapacity(queueCapacities.getMaximumCapacity());
  queueInfo.setQueueState(state);
  queueInfo.setDefaultNodeLabelExpression(defaultLabelExpression);
  queueInfo.setCurrentCapacity(getUsedCapacity());
  return queueInfo;
}
 

private void getChildQueues(QueueInfo parent, List<QueueInfo> queues,
    boolean recursive) {
  List<QueueInfo> childQueues = parent.getChildQueues();

  for (QueueInfo child : childQueues) {
    queues.add(child);
    if (recursive) {
      getChildQueues(child, queues, recursive);
    }
  }
}