Java Code Examples for org.codehaus.jackson.node.ArrayNode#get()

public static void visitOperators(ObjectNode programNode,
                                  ObjectNode jobNode,
                                  OperatorVisitor tracerObj,
                                  boolean reverse)
{
    ArrayNode jobs = (ArrayNode) programNode.get("jobs");
    int si = (reverse ? jobs.size() - 1 : 0);
    int ei = (reverse ? -1 : jobs.size());

    for (int i = si; i != ei; i = i + increment(reverse))
    {
        if (jobNode != null && jobNode != jobs.get(i))
            continue;

        if (!visitOperatorsInJob(programNode,
                                 (ObjectNode) jobs.get(i),
                                 tracerObj,
                                 reverse))
            return;
    }
}
 

private static List<Integer>[] getOperatorDependency(ArrayNode operatorsJson)
{
    Map<String, Integer> relationName2OperatorID = new HashMap<String, Integer>();
    int numOperators = operatorsJson.size();
    List<Integer>[] inputOperatorIDs = new ArrayList[numOperators];

    for (int i = 0; i < numOperators; i++)
    {
        JsonNode operatorJson = operatorsJson.get(i);
        if (!operatorJson.has("operator"))
            continue;

        OperatorType type =
                OperatorType.valueOf(operatorJson.get("operator").getTextValue());
        String outputName = operatorJson.get("output").getTextValue();

        if (type.isTupleOperator())
        {
            inputOperatorIDs[i] =
                    getInputOperatorIDs(relationName2OperatorID, operatorJson);
            relationName2OperatorID.put(outputName, i);
        }
    }

    return inputOperatorIDs;
}
 

public static int getIndexFromArray(ArrayNode anode, JsonNode element)
{
    for (int i = 0; i < anode.size(); i++)
    {
        if (anode.get(i) == element)
            return i;
    }
    return -1;
}
 

private ObjectNode getOperatorSourceInPhase(ObjectNode jobNode,
                                            JsonNode phaseNode,
                                            ObjectNode opNode,
                                            String inputRelation)
{
    ArrayNode opsNode =
            isReducePhase(phaseNode) ? (ArrayNode) phaseNode
                    : (ArrayNode) (((ObjectNode) phaseNode).get("operators"));

    boolean reachedDest = false;

    // don't expect to find store in the list of oeprators.
    if (isReducePhase(phaseNode) && isStoreCommand(jobNode, phaseNode, opNode))
        reachedDest = true;

    for (int i = opsNode.size() - 1; i >= 0; i--)
    {
        ObjectNode candidateOp = (ObjectNode) (opsNode.get(i));
        if (candidateOp == opNode)
        {
            reachedDest = true;
            continue;
        }

        if (!reachedDest)
            continue;
        if (isOutputOf(candidateOp, inputRelation))
            return candidateOp;
    }
    return null;
}
 

public List<ObjectNode> findOperatorInputSources(ObjectNode opNode,
                                                 String inputRelation)
{
    int opSequence = this.getOpSequence(opNode);
    ObjectNode jobNode = (ObjectNode) (this.getJobPhase(opNode).getFirst());
    JsonNode phaseNode = this.getJobPhase(opNode).getSecond();
    List<ObjectNode> result = new ArrayList<ObjectNode>();

    if (isReducePhase(phaseNode)
            && (opSequence == 0 || getOperatorSourceInPhase(jobNode,
                                                            (ArrayNode) phaseNode,
                                                            opNode,
                                                            inputRelation) == null))
    {
        // if either first operator in reduce phase or a matching source within
        // the same phase cannot be found,
        // look inside all the map jobs.
        ArrayNode mapsArray = (ArrayNode) getJobPhase(opNode).getFirst().get("map");
        for (JsonNode mapNode : mapsArray)
        {
            ArrayNode mapOps = (ArrayNode) ((ObjectNode) mapNode).get("operators");
            if (mapOps == null || mapOps.size() == 0)
                continue;
            ObjectNode lastOp = (ObjectNode) mapOps.get(mapOps.size() - 1);
            result.add(findOperatorSourcePrior(getOpSequence(lastOp), inputRelation));
        }
    }
    else
        result.add(findOperatorSource(opNode, inputRelation));

    return result;

}
 

private void updateCounter()
{
    long[] operatorTime = getOperatorTime();

    String profileCounterGroupName =
            PhaseContext.isMapper() ? mapperProfileCounterGroupName
                    : reducerProfileCounterGroupName;

    ArrayNode operatorsJson = multipassOperatorsJson.get(currentPassIndex);
    for (int i = 0; i < operatorTime.length; i++)
    {
        if (operatorTime[i] > 0)
        {
            JsonNode operatorJson = operatorsJson.get(i);

            OperatorType type =
                    OperatorType.valueOf(operatorJson.get("operator").getTextValue());
            String outputName = operatorJson.get("output").getTextValue();

            String counterName =
                    String.format("P%d-O%d-%s-%s",
                                  currentPassIndex,
                                  i,
                                  type,
                                  outputName);
            Counter profileCounter =
                    PhaseContext.getCounter(profileCounterGroupName, counterName);
            profileCounter.increment(operatorTime[i]);
        }
    }
}
 

public void printQueueInfo(HttpClient client, ObjectMapper mapper, String schedulerResourceURL) {
	
	//http://sandbox.hortonworks.com:8088/ws/v1/cluster/scheduler in case of HDP
	GetMethod get = new GetMethod(schedulerResourceURL);
    get.setRequestHeader("Accept", "application/json");
    try {
        int statusCode = client.executeMethod(get);

        if (statusCode != HttpStatus.SC_OK) {
        	LOGGER.error("Method failed: " + get.getStatusLine());
        }
        
		InputStream in = get.getResponseBodyAsStream();
		
		JsonNode jsonNode = mapper.readValue(in, JsonNode.class);
		ArrayNode queues = (ArrayNode) jsonNode.path("scheduler").path("schedulerInfo").path("queues").get("queue");
		for (int i = 0; i < queues.size(); i++) {
			JsonNode queueNode = queues.get(i);						
			LOGGER.info("queueName / usedCapacity / absoluteUsedCap / absoluteCapacity / absMaxCapacity: " + 
					queueNode.findValue("queueName") + " / " +
					queueNode.findValue("usedCapacity") + " / " + 
					queueNode.findValue("absoluteUsedCapacity") + " / " + 
					queueNode.findValue("absoluteCapacity") + " / " +
					queueNode.findValue("absoluteMaxCapacity"));
		}
	} catch (IOException e) {
		LOGGER.error("Exception occured", e);
	} finally {	        
		get.releaseConnection();
	}	      
        
}
 

/**
 * 查找ObjectNode下propertyName属性为name的json节点
 * @param name
 * @param propertyName
 * @param objectNode
 * @return
 */
public static JsonNode getChildElementByProperty(String name,String propertyName,ArrayNode objectNode){
 JsonNode jsonNode = null;
 for( int i = 0; i< objectNode.size();i++){
  JsonNode childNode = objectNode.get(i);
  if(childNode.get(propertyName)!=null && name.equals(childNode.get(propertyName).asText())){
	  return childNode;
  }
 }
 return jsonNode;
}
 

/**
 * Method to get all Pods and Containers associated to the deployment.
 * 
 * @param authToken
 * @param eVo
 * @param depLabelMap
 * @param externalPodsMap
 *
 */
private static void getPodsAndContainersOfDeployments(String authToken, EnvironmentVO eVo,
		Map<String, List<ServiceLabelVO>> depLabelMap, Map<String, ArrayList<Integer>> externalPodsMap) throws IOException {
	String podApiUrl = eVo.getK8sUrl() + PropertyUtil.getInstance().getValue(SurveillerConstants.K8S_API_PODS_PATH);
	/* Call - the Kube Pod api to get all the Pods in the Cluster */
	String podJson = RestCommunicationHandler.getResponse(podApiUrl, true, SurveillerConstants.BEARER_TOKEN, authToken);
	ObjectMapper mapper = new ObjectMapper();
	JsonNode podRootNode = mapper.readTree(podJson);
	ArrayNode appsNode = (ArrayNode) podRootNode.get("items");
	Iterator<JsonNode> podsIterator = appsNode.getElements();
	while (podsIterator.hasNext()) {
		JsonNode appsInNode = podsIterator.next();
		JsonNode metadataNode = appsInNode.get("metadata");
		JsonNode nameNode = metadataNode.get("name");
		String podName = nameNode.getValueAsText();
		JsonNode namespaceNode = metadataNode.get("namespace");
		String namespace = namespaceNode.getValueAsText();
		if (namespace.equals("default") || !namespace.contains("-")) {
			logger.info("Excluding Pods - " + podName + " in the namespace - " + namespace);
			continue;
		}
		JsonNode specNode = appsInNode.get("spec");
		ArrayNode contArrayNode = (ArrayNode) specNode.get("containers");
		/* Number of containers in Pod */
		int numCount = contArrayNode.size();
		//JsonNode ownerReferencesNode = 
		ArrayNode ownerReferencesArray = (ArrayNode) metadataNode.get("ownerReferences");
		if(ownerReferencesArray == null){
			loadPodsAndContainer("Kube-Systems",namespace, externalPodsMap, numCount);
			continue;
		}
		JsonNode ownerReferencesNode = ownerReferencesArray.get(0);
		JsonNode kindNode = ownerReferencesNode.get("kind");
		String kind = kindNode.getTextValue();
		if(!kind.equalsIgnoreCase("ReplicaSet")){
			loadPodsAndContainer(kind,namespace , externalPodsMap, numCount);
			continue;
		}
		
		JsonNode labelsNode = metadataNode.get("labels");
		if (labelsNode == null) {
			logger.info("The labelsNode is null for the pod - " + podName + " in the namespace - " + namespace);
			continue;
		}
		Map<String, String> podLabelMap = mapper.convertValue(labelsNode, Map.class);
		List<ServiceLabelVO> servLblList = depLabelMap.get(namespace);
		if(servLblList == null){
			continue;
		}
		serviceLabelLoop: for (ServiceLabelVO servLabelVo : servLblList) {
			int labelCount = 0;
			for (Entry<String, String> labelInfo : servLabelVo.getLabel().entrySet()) {
				if (podLabelMap.containsKey(labelInfo.getKey())) {
					if (podLabelMap.get(labelInfo.getKey()).equals(labelInfo.getValue())) {
						labelCount = labelCount + 1;
					} else {
						continue serviceLabelLoop;
					}
				} else {
					continue serviceLabelLoop;
				}
			}
			if (servLabelVo.getLabel().size() == labelCount) {
				
				servLabelVo.setNumOfContainers(servLabelVo.getNumOfContainers() + numCount);
				servLabelVo.setNumOfPods(servLabelVo.getNumOfPods() + 1);
				break;
			}
		}
	}
	
}
 

private void incrementalizeInputLoad(ObjectNode programNode,
                                     ObjectNode inputNode,
                                     ObjectNode cubeOperatorNode,
                                     String mvName,
                                     String mvPath) throws IOException,
        AggregateRewriteException
{

    // extract input paths from inputNode and adjust start-date to MV refresh date+1.
    ArrayNode paths = (ArrayNode) inputNode.get("path");
    System.out.println("Incrementalize InputNode = " + inputNode.toString());
    int newMvRefreshTime = 0;
    for (int i = 0; i < paths.size(); i++)
    {
        JsonNode pathNode = paths.get(i);
        if (pathNode instanceof ObjectNode)
        {
            String startDate =
                    ((ObjectNode) pathNode).get("startDate").getTextValue();
            // System.out.println("startDate = " + startDate);
            DateTime loadStart = DateTimeUtilities.getDateTime((startDate));
            String endDate = ((ObjectNode) pathNode).get("endDate").getTextValue();
            DateTime loadEnd = DateTimeUtilities.getDateTime(endDate);

            if (mvRefreshDate != 0 && incLoadDate != null)
            {
                if (loadStart.isBefore(incLoadDate) && loadEnd.isAfter(incLoadDate))
                {
                    ((ObjectNode) pathNode).put("origStartDate", startDate);
                    ((ObjectNode) pathNode).put("startDate",
                                                Integer.toString(DateTimeUtilities.asInt(incLoadDate)));
                }
                else
                    throw new AggregateRewriteException(String.format("MV date range mis-matches load range[%s, %s] ",
                                                                      startDate,
                                                                      endDate));
            }

            newMvRefreshTime =
                    Math.max(Integer.parseInt(((ObjectNode) pathNode).get("endDate")
                                                                     .getTextValue()),
                             newMvRefreshTime);
        }
    }

    System.out.println("Setting MV refresh time for " + mvName + " to "
            + newMvRefreshTime);
    mvRefreshMap.put(mvName, newMvRefreshTime);

}
 

public void rewriteFactBlockgenPath(ObjectNode cubeNode,
                                    LineagePath opSequencePath,
                                    ObjectNode factNode,
                                    Pair<ObjectNode, ObjectNode> bgInfo) throws AggregateRewriteException
{
    // if CREATE_BLOCK is not one of the ultimate or penultimate opNodes, report
    // rewrite exception
    ObjectNode bgNode = bgInfo.getSecond();
    ArrayNode phaseOps = lineage.getPhaseOperators(lineage.getPhase(bgNode));
    int len = phaseOps.size();
    if (phaseOps.get(len - 1) != bgNode)
        throw new AggregateRewriteException("Xformed Fact CREATE_BLOCK should be last operator before store");

    /*
     * boolean formattedDateColumn = false; boolean regenerateAfterJoin = false;
     * String dateColumnAfterJoin = null;
     */
    getFactTimeSpecNode(factNode, cubeNode);
    formattedDateColumn = false;
    dateColumnAlias = tNode.get("dateColumn").getTextValue();

    for (LineageGraphVertex opNodeVertex : opSequencePath.nodes)
    {
        ObjectNode opNode = ((OperatorLineage) opNodeVertex).node;

        // if incompatible operator, trigger exception
        if (incompatibleWithBlockgenPath(opNode))
          throw new AggregateRewriteException("Found incompatible operator along fact blockgen path " + opNode);

        // If generate inject date column
        if (opNode.get("operator") != null
                && opNode.get("operator").getTextValue().equalsIgnoreCase("GENERATE"))
        {
            ArrayNode generateArgs = (ArrayNode) (opNode.get("outputTuple"));
            if (!formattedDateColumn)
            {
                JsonNode dateColumnNode =
                        generateDateColumn(cubeNode, factNode, bgInfo);

                generateArgs.add(dateColumnNode);
                formattedDateColumn = true;
            }
            else
            {

                generateArgs.add(regenerateDateColumn(dateColumnAlias));

            }
            dateColumnAlias = DATE_COLUMN_NAME;
        }

        // If join operator, stash the renamed date column
        if (lineage.isJoinOperator(opNode))
        {
            BlockSchema joinSchema = new BlockSchema(opNode.get("schema"));
            String[] colNames = joinSchema.getColumnNames();
            String newAlias = dateColumnAlias;

            String[] joinInputs = JsonUtils.asArray(opNode.get("input"));
            int factIndex =
                    lineage.getDescendantInputIndex(joinInputs, opNode, factNode);
            newAlias = joinInputs[factIndex] + "___" + dateColumnAlias;

            // TODO add an explict generate if the very next statement after JOIN is
            // not a GENERATE.
            if (addGenerateAfterJoin(opNode))
                continue;
            dateColumnAlias = newAlias;

        }

        // if group by on all columns OR distinct that is either immediately before
        // or after the CREATE_BLOCK, remove
        if (opNode.get("operator") != null
                && (opNode.get("operator")
                          .getTextValue()
                          .equalsIgnoreCase("DISTINCT") || lineage.isDistinctGroupBy(opNode))
                && lineage.isParentOperator(opNode, bgInfo.getSecond()))
        {
            JsonNode phaseNode = lineage.getPhase(opNode);
            JsonUtils.deleteFromArrayNode(phaseOps, opNode);

            // remember new input to bgNode
            this.factBgInput = opNode.get("input").getTextValue();
            bgInfo.getSecond().put("input", factBgInput);
        }
    }
}
 

private void insertIncrementalMultipleDayGroupBy(ObjectNode programNode,
                                                 Pair<ObjectNode, ObjectNode> bgInfo) throws AggregateRewriteException
{
    String[] factColumns = lineage.getSchemaOutputColumns(bgInfo.getSecond());
    String[] sortKeys;
    String[] groupByColumns;

    if (lineage.isBlockgenByIndex(bgInfo.getSecond()))
    {
        ObjectNode jobNode = lineage.getOperatorJobNode(bgInfo.getSecond());
        sortKeys =      JsonUtils.asArray(((ObjectNode) (jobNode.get("shuffle"))).get("pivotKeys"));
        groupByColumns =
                (String[]) ArrayUtils.addAll(new String[] { "BLOCK_ID" }, factColumns);
    }
    else {
        sortKeys = JsonUtils.asArray(bgInfo.getSecond().get("pivotKeys"));
        groupByColumns = factColumns;
    }

    // check sort key condition
    if (!CommonUtils.isPrefix(sortKeys, groupByColumns))
      throw new AggregateRewriteException("Blockgen of union fact not sorted by fact collumns");

    ArrayNode aggsNode = JsonUtils.createArrayNode();
    ArrayNode udafArgsNode = JsonUtils.createArrayNode();

    String startDateHyphenated = DateTimeUtilities.getHyphenated(this.factStartDate);
    udafArgsNode.add(startDateHyphenated);
    aggsNode.add(RewriteUtils.createObjectNode("type",
                                               "USER_DEFINED_AGGREGATION",
                                               "udaf",
                                               "com.linkedin.cubert.operator.aggregate.PresenceBitmapUDAF",
                                               "constructorArgs",
                                               udafArgsNode,
                                               "input",
                                               DATE_COLUMN_NAME,
                                               "output",
                                               BITMAP_COLUMN_NAME));
    String blockgenInputRelation =
            lineage.getOperatorSources(bgInfo.getSecond())
                   .get(0)
                   .get("output")
                   .getTextValue();
    ObjectNode groupByNode =
            RewriteUtils.createObjectNode("operator",
                                          "GROUP_BY",
                                          "input",
                                          blockgenInputRelation,
                                          "output",
                                          blockgenInputRelation,
                                          "groupBy",
                                          JsonUtils.createArrayNode(groupByColumns),
                                          "aggregates",
                                          aggsNode);

    ArrayNode phaseOperators =
            lineage.getPhaseOperators(lineage.getPhase(bgInfo.getSecond()));
    int blockGenIndex = 0;
    for (; blockGenIndex < phaseOperators.size(); blockGenIndex++)
    {
        if (phaseOperators.get(blockGenIndex) == bgInfo.getSecond())
            break;
    }
    if (blockGenIndex == phaseOperators.size())
        throw new RuntimeException("Cannot find CREATE_BLOCK operator in phase operator list");
    phaseOperators.insert(blockGenIndex, groupByNode);
    // phaseOperators.insert(blockGenIndex + 1, generateNode);

}