Java Code Examples for org.codehaus.jackson.node.ObjectNode#path()

private String[] getCountDistinctMeasuresForGroupBy(ObjectNode operatorNode)
{
    ArrayList<String> measureCols = new ArrayList<String>();
    if (!operatorNode.get("operator").getTextValue().equalsIgnoreCase("GROUP_BY"))
        return null;

    if (!operatorNode.has("aggregates"))
        return null;

    for (JsonNode aggregateJson : operatorNode.path("aggregates"))
    {
        // Create the aggregator object
        AggregationType aggType =
                AggregationType.valueOf(JsonUtils.getText(aggregateJson, "type"));
        if (!aggType.toString().equalsIgnoreCase("COUNT_DISTINCT"))
            continue;
        String[] measureColumns = JsonUtils.asArray(aggregateJson.get("input"));
        measureCols.addAll(Arrays.asList(measureColumns));
    }
    return measureCols.toArray(new String[measureCols.size()]);
}
 

public static boolean isCountDistinctAggregate(ObjectNode operatorNode)
{
    if (operatorNode.get("operator") == null)
      return false;

    String type = operatorNode.get("operator").getTextValue();
    if (!type.equals("GROUP_BY") && !type.equals("CUBE"))
        return false;

    if (!operatorNode.has("aggregates"))
        return false;

    for (JsonNode aggregateJson : operatorNode.path("aggregates"))
    {
        // Create the aggregator object
        JsonNode typeNode = aggregateJson.get("type");

        // Group by case
        if (typeNode.isTextual()){
          AggregationType aggType =
            AggregationType.valueOf(JsonUtils.getText(aggregateJson, "type"));
          String measureColumn = JsonUtils.getText(aggregateJson, "input");
          if (aggType != AggregationType.COUNT_DISTINCT)
            return false;
        }
        else if (typeNode instanceof ArrayNode){
          String[] typeArray = JsonUtils.asArray(aggregateJson, "type");
          if (!typeArray[0].equals("SUM") || !typeArray[1].equals("COUNT_TO_ONE"))
            return false;
        }
    }

    return true;
}
 

private JsonNode rewriteBlockgenByIndex(JsonNode job)
{
    ObjectNode newJob = (ObjectNode) cloneNode(job);
    ObjectNode shuffle = (ObjectNode) newJob.get("shuffle");

    String path = getText(shuffle, "relation");

    // add a cache index
    String indexName = generateVariableName(namesUsed);
    if (!newJob.has("cacheIndex") || newJob.get("cacheIndex").isNull())
        newJob.put("cacheIndex", mapper.createArrayNode());
    ArrayNode cacheIndex = (ArrayNode) newJob.get("cacheIndex");
    cacheIndex.add(createObjectNode("name", indexName, "path", path));

    // create BLOCK-INDEX-JOIN operator
    ObjectNode blockIndexJoin =
            createObjectNode("operator",
                             "BLOCK_INDEX_JOIN",
                             "input",
                             shuffle.get("name"),
                             "output",
                             shuffle.get("name"),
                             "partitionKeys",
                             shuffle.get("partitionKeys"),
                             "index",
                             indexName);
    copyLine(shuffle, blockIndexJoin, "[MAP] ");
    // add it as the last operator for all mapper
    for (JsonNode map : newJob.path("map"))
    {
        if (!map.has("operators") || map.get("operators").isNull())
            ((ObjectNode) map).put("operators", mapper.createArrayNode());
        ArrayNode operators = (ArrayNode) map.get("operators");
        // we need unique references for all blockIndexJoin
        operators.add(JsonUtils.cloneNode(blockIndexJoin));
    }

    // create CREATE-BLOCK operator
    ObjectNode createBlock =
            createObjectNode("operator",
                             "CREATE_BLOCK",
                             "input",
                             shuffle.get("name"),
                             "output",
                             shuffle.get("name"),
                             "blockgenType",
                             "BY_INDEX",
                             "index",
                             indexName,
                             "partitionKeys",
                             createArrayNode("BLOCK_ID"),
                             "indexPath",
                             path);

    copyLine(shuffle, createBlock, "[REDUCE] ");
    // add it as first operator in reduce
    if (!newJob.has("reduce") || newJob.get("reduce").isNull())
        newJob.put("reduce", mapper.createArrayNode());
    ArrayNode reduce = (ArrayNode) newJob.get("reduce");
    reduce.insert(0, createBlock);

    // add DISTINCT operator, if requested
    boolean isDistinct =
            shuffle.has("distinct") && shuffle.get("distinct").getBooleanValue();

    if (isDistinct)
    {
        ObjectNode distinct =
                createObjectNode("operator",
                                 "DISTINCT",
                                 "input",
                                 shuffle.get("name"),
                                 "output",
                                 shuffle.get("name"));
        copyLine(shuffle, distinct, "[REDUCE DISTINCT] ");
        reduce.insert(0, distinct);
    }

    // blockgen by index uses a different partitioner
    shuffle.put("partitionerClass",
                "com.linkedin.cubert.plan.physical.ByIndexPartitioner");

    // clean up shuffle
    shuffle.put("type", "SHUFFLE");
    shuffle.put("partitionKeys", createArrayNode("BLOCK_ID"));
    shuffle.put("distinct", isDistinct);
    shuffle.put("index", indexName);
    shuffle.remove("blockgenType");
    shuffle.remove("relation");

    ArrayNode pivotKeys = mapper.createArrayNode();
    pivotKeys.add("BLOCK_ID");
    if (shuffle.has("pivotKeys"))
    {
        for (JsonNode key : shuffle.path("pivotKeys"))
            pivotKeys.add(key);
    }
    shuffle.put("pivotKeys", pivotKeys);

    return newJob;
}
 

private JsonNode rewriteCube(JsonNode job)
{
    ObjectNode newJob = (ObjectNode) cloneNode(job);
    ObjectNode shuffle = (ObjectNode) newJob.get("shuffle");
    String name = getText(shuffle, "name");
    JsonNode aggregates = shuffle.get("aggregates");

    // create the OLAP_CUBE_COUNT_DISTINCT operator
    ObjectNode cube =
            createObjectNode("operator",
                             "CUBE",
                             "input",
                             name,
                             "output",
                             name,
                             "dimensions",
                             shuffle.get("dimensions"),
                             "aggregates",
                             cloneNode(aggregates));

    if (shuffle.has("groupingSets"))
        cube.put("groupingSets", shuffle.get("groupingSets"));
    if (shuffle.has("innerDimensions"))
        cube.put("innerDimensions", shuffle.get("innerDimensions"));
    if (shuffle.has("hashTableSize"))
        cube.put("hashTableSize", shuffle.get("hashTableSize"));
    copyLine(shuffle, cube, "[MAP] ");

    // add it as the last operator for all mapper
    for (JsonNode map : newJob.path("map"))
    {
        if (!map.has("operators") || map.get("operators").isNull())
            ((ObjectNode) map).put("operators", mapper.createArrayNode());
        ArrayNode operators = (ArrayNode) map.get("operators");
        operators.add(cube);
    }

    rewriteGroupByAggregateForCube(aggregates);

    // create the GROUP BY operator at the reducer
    ObjectNode groupBy =
            createObjectNode("operator",
                             "GROUP_BY",
                             "input",
                             name,
                             "output",
                             name,
                             "groupBy",
                             shuffle.get("dimensions"),
                             "aggregates",
                             aggregates);
    copyLine(shuffle, groupBy, "[REDUCE] ");
    // add it as first operator in reduce
    if (!newJob.has("reduce") || newJob.get("reduce").isNull())
        newJob.put("reduce", mapper.createArrayNode());
    ArrayNode reduce = (ArrayNode) newJob.get("reduce");
    reduce.insert(0, groupBy);

    // clean up shuffle
    shuffle.put("type", "SHUFFLE");
    shuffle.put("aggregates", aggregates);
    shuffle.put("partitionKeys", shuffle.get("dimensions"));
    shuffle.put("pivotKeys", shuffle.get("dimensions"));
    shuffle.remove("dimensions");
    shuffle.remove("groupingSets");
    shuffle.remove("innerDimensions");

    return newJob;
}