Java Code Examples for org.apache.kylin.metadata.model.MeasureDesc#getFunction()

@Override
public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
    for (MeasureDesc measureDesc : measureDescs) {
        if (!sqlDigest.involvedMeasure.contains(measureDesc)) {
            continue;
        }
        FunctionDesc extendColumnFunc = measureDesc.getFunction();
        List<TblColRef> hosts = getExtendedColumnHosts(extendColumnFunc);
        TblColRef extended = getExtendedColumn(extendColumnFunc);

        if (!sqlDigest.groupbyColumns.contains(extended)) {
            continue;
        }

        sqlDigest.aggregations.add(extendColumnFunc);
        sqlDigest.groupbyColumns.remove(extended);
        sqlDigest.groupbyColumns.addAll(hosts);
        sqlDigest.metricColumns.add(extended);
    }
}
 

FunctionDesc findInMeasures(FunctionDesc aggFunc, List<MeasureDesc> measures) {
    for (MeasureDesc m : measures) {
        if (aggFunc.equals(m.getFunction())) {
            return m.getFunction();
        }
    }

    // no count(col) measure found, use count(1) to replace it.
    if (aggFunc.isCount()) {
        FunctionDesc func = findCountConstantFunc(measures);
        if (func != null)
            return func;
    }

    return aggFunc;
}
 

public static String createMeasureStatement(List<MeasureDesc> measureList) {
    StringBuilder sql = new StringBuilder();

    for (MeasureDesc measureDesc : measureList) {
        FunctionDesc functionDesc = measureDesc.getFunction();
        if (functionDesc.isSum() || functionDesc.isMax() || functionDesc.isMin()) {
            sql.append("," + functionDesc.getExpression() + "(" + functionDesc.getParameter().getValue() + ")\n");
            break;
        } else if (functionDesc.isCountDistinct()) {
            sql.append(",COUNT" + "(DISTINCT " + functionDesc.getParameter().getValue() + ")\n");
            break;
        }
    }

    return sql.toString();
}
 

public List<MeasureDesc> listEffectiveRewriteMeasures(String project, String table, boolean onlyRewriteMeasure) {
    Set<IRealization> realizations = getRealizationsByTable(project, table);
    List<MeasureDesc> result = Lists.newArrayList();
    for (IRealization r : realizations) {
        if (!r.isReady())
            continue;

        for (MeasureDesc m : r.getMeasures()) {
            FunctionDesc func = m.getFunction();
            if (belongToTable(func, table, r.getModel())) {
                if (!onlyRewriteMeasure || func.needRewrite()) {
                    result.add(m);
                }
            }
        }
    }
    return result;
}
 

private void checkHiveGlobalDictionaryColumn(){
    Set<String> mrDictColumnSet = new HashSet<>();
    if (kylinConfig.getMrHiveDictColumns() != null) {
        Collections.addAll(mrDictColumnSet, kylinConfig.getMrHiveDictColumns());
    }

    for (MeasureDesc measure : measureDescList) {
        if (measure.getFunction().getExpression().equalsIgnoreCase(FunctionDesc.FUNC_COUNT_DISTINCT)) {
            FunctionDesc functionDesc = measure.getFunction();
            TblColRef colRef = functionDesc.getParameter().getColRefs().get(0);
            if (mrDictColumnSet.contains(JoinedFlatTable.colName(colRef, true))) {
                functionDesc.setMrDict(true);
                logger.info("Enable hive global dictionary for {}", colRef);
                measure.setFunction(functionDesc);
            }
        }
    }
}
 

private Object buildValueOf(int idxOfMeasure, List<String> row) {
    MeasureDesc measure = parsedStreamingCubeInfo.measureDescs[idxOfMeasure];
    FunctionDesc function = measure.getFunction();
    int[] colIdxOnFlatTable = parsedStreamingCubeInfo.intermediateTableDesc.getMeasureColumnIndexes()[idxOfMeasure];

    int paramCount = function.getParameterCount();
    String[] inputToMeasure = new String[paramCount];

    // pick up parameter values
    ParameterDesc param = function.getParameter();
    int paramColIdx = 0; // index among parameters of column type
    for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
        String value;
        if (function.isCount()) {
            value = "1";
        } else if (param.isColumnType()) {
            value = row.get(colIdxOnFlatTable[paramColIdx++]);
        } else {
            value = param.getValue();
        }
        inputToMeasure[i] = value;
    }
    return parsedStreamingCubeInfo.measureIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, dictionaryMap);
}
 

private void checkMrDictClolumn(){
    Set<String> mrDictColumnSet = new HashSet<>();
    if (kylinConfig.getMrHiveDictColumns() != null) {
        Collections.addAll(mrDictColumnSet, kylinConfig.getMrHiveDictColumns());
    }

    for (MeasureDesc measure : measureDescList) {
        if (measure.getFunction().getExpression().equalsIgnoreCase(FunctionDesc.FUNC_COUNT_DISTINCT)) {
            FunctionDesc functionDesc = measure.getFunction();
            TblColRef colRef = functionDesc.getParameter().getColRefs().get(0);
            if (mrDictColumnSet.contains(JoinedFlatTable.colName(colRef, true))) {
                functionDesc.setMrDict(true);
                logger.info("setMrDict for {}", colRef);
                measure.setFunction(functionDesc);
            }
        }
    }
}
 

@SuppressWarnings("deprecation")
private void initMeasureColumns() {
    if (measures == null || measures.isEmpty()) {
        return;
    }

    for (MeasureDesc m : measures) {
        m.setName(m.getName().toUpperCase(Locale.ROOT));

        if (m.getDependentMeasureRef() != null) {
            m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase(Locale.ROOT));
        }

        FunctionDesc func = m.getFunction();
        func.init(model);
        allColumns.addAll(func.getParameter().getColRefs());

        if (ExtendedColumnMeasureType.FUNC_EXTENDED_COLUMN.equalsIgnoreCase(m.getFunction().getExpression())) {
            FunctionDesc functionDesc = m.getFunction();

            List<TblColRef> hosts = ExtendedColumnMeasureType.getExtendedColumnHosts(functionDesc);
            TblColRef extendedColumn = ExtendedColumnMeasureType.getExtendedColumn(functionDesc);
            initExtendedColumnMap(hosts.toArray(new TblColRef[hosts.size()]), extendedColumn);
        }
    }
}
 

private List<ColumnDesc> listSourceColumns() {
    ProjectManager mgr = ProjectManager.getInstance(olapSchema.getConfig());

    List<ColumnDesc> tableColumns = mgr.listExposedColumns(olapSchema.getProjectName(), sourceTable, exposeMore);

    List<ColumnDesc> metricColumns = Lists.newArrayList();
    List<MeasureDesc> countMeasures = mgr.listEffectiveRewriteMeasures(olapSchema.getProjectName(),
            sourceTable.getIdentity());
    HashSet<String> metFields = new HashSet<String>();
    for (MeasureDesc m : countMeasures) {

        FunctionDesc func = m.getFunction();
        String fieldName = func.getRewriteFieldName();
        if (!metFields.contains(fieldName)) {
            metFields.add(fieldName);
            ColumnDesc fakeCountCol = func.newFakeRewriteColumn(sourceTable);
            metricColumns.add(fakeCountCol);
        }
    }

    Collections.sort(tableColumns, new Comparator<ColumnDesc>() {
        @Override
        public int compare(ColumnDesc o1, ColumnDesc o2) {
            return o1.getZeroBasedIndex() - o2.getZeroBasedIndex();
        }
    });
    return Lists.newArrayList(Iterables.concat(tableColumns, metricColumns));
}
 

@Test
public void testDecode() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    Long count = new Long(2);
    Long item_count = new Long(100);
    ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count });

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc aggrFunc = measure.getFunction();
        int index = hbaseCol.findMeasure(aggrFunc);
        rowValueDecoder.setProjectIndex(index);
    }

    rowValueDecoder.decodeAndConvertJavaObj(valueBytes);
    Object[] measureValues = rowValueDecoder.getValues();
    //BigDecimal.ROUND_HALF_EVEN in BigDecimalSerializer
    assertEquals("[333.1235, 333.1111, 333.2000, 2, 100]", Arrays.toString(measureValues));
}
 

private void init(Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics) {
    this.dimensions = new TblColRef[selectedDimensions.size()];
    this.metrics = new FunctionDesc[selectedMetrics.size()];
    this.measures = new MeasureDesc[selectedMetrics.size()];
    this.dimDataTypes = new DataType[dimensions.length];
    this.metricsDataTypes = new DataType[metrics.length];
    // sort dimensions according to the rowKey definition
    dimColIdxMap = Maps.newHashMap();
    RowKeyDesc rowKeyDesc = cubeDesc.getRowkey();
    int colIdx = 0;
    for (RowKeyColDesc rowKeyColDesc : rowKeyDesc.getRowKeyColumns()) {
        TblColRef dimension = rowKeyColDesc.getColRef();
        if (selectedDimensions.contains(dimension)) {
            dimensions[colIdx] = dimension;
            dimDataTypes[colIdx] = dimension.getType();
            dimColIdxMap.put(dimension, colIdx);
            colIdx++;
        }
    }

    nDimensions = colIdx;

    colIdx = 0;
    // metrics
    metricsColIdxMap = Maps.newHashMap();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc func = measure.getFunction();
        if (selectedMetrics.contains(func)) {
            metrics[colIdx] = func;
            measures[colIdx] = measure;
            metricsColIdxMap.put(func.getParameter().getColRef(), colIdx);
            metricsDataTypes[colIdx] = func.getReturnDataType();
            colIdx++;
        }
    }

    nMetrics = colIdx;
}
 

public String[] buildValueOf(int idxOfMeasure, String[] row) {
    MeasureDesc measure = cubeDesc.getMeasures().get(idxOfMeasure);
    FunctionDesc function = measure.getFunction();
    int[] colIdxOnFlatTable = flatDesc.getMeasureColumnIndexes()[idxOfMeasure];

    int paramCount = function.getParameterCount();
    List<String> inputToMeasure = Lists.newArrayListWithExpectedSize(paramCount);

    // pick up parameter values
    ParameterDesc param = function.getParameter();
    int colParamIdx = 0; // index among parameters of column type
    for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
        String value;
        if (param.isColumnType()) {
            value = getCell(colIdxOnFlatTable[colParamIdx++], row);
            if (function.isCount() && value == null) {
                value = ZERO;
            } else if (function.isCount()) {
                value = ONE;
            }
        } else {
            value = param.getValue();
            if (function.isCount()) {
                value = ONE;
            }
        }
        inputToMeasure.add(value);
    }

    return inputToMeasure.toArray(new String[inputToMeasure.size()]);
}
 

private FunctionDesc findAggrFuncFromCubeDesc(FunctionDesc aggrFunc) {
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        if (measure.getFunction().equals(aggrFunc))
            return measure.getFunction();
    }
    return aggrFunc;
}
 

private FunctionDesc findAggrFuncFromCubeDesc(FunctionDesc aggrFunc) {
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        if (measure.getFunction().equals(aggrFunc))
            return measure.getFunction();
    }
    return aggrFunc;
}
 

private FunctionDesc findAggrFuncFromCubeDesc(CubeDesc cubeDesc, FunctionDesc aggrFunc) {
    aggrFunc.init(cubeDesc.getModel());
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        if (measure.getFunction().equals(aggrFunc))
            return measure.getFunction();
    }
    return aggrFunc;
}
 

/**
 * Get columns which need dictionaries during cube building, while don't need to store them
 */
public Set<TblColRef> getAllColumnsNeedDictionaryForBuildingOnly() {
    Set<TblColRef> result = Sets.newHashSet();
    Set<TblColRef> colsNeedDictStored = Sets.newHashSet();

    // dictionaries in measures
    for (MeasureDesc measure : measures) {
        FunctionDesc func = measure.getFunction();
        MeasureType<?> aggrType = func.getMeasureType();

        // cols need dict stored in a measure
        Set<TblColRef> colSet = Sets.newHashSet();
        colSet.addAll(aggrType.getColumnsNeedDictionary(func));
        colSet.removeAll(aggrType.getColumnsNeedDictionaryForBuildingOnly(func));
        colsNeedDictStored.addAll(colSet);

        result.addAll(aggrType.getColumnsNeedDictionaryForBuildingOnly(func));
    }

    // dictionaries in dimensions
    colsNeedDictStored.addAll(getAllDimsHaveDictionary());

    // any additional dictionaries
    if (dictionaries != null) {
        for (DictionaryDesc dictDesc : dictionaries) {
            TblColRef col = dictDesc.getColumnRef();
            colsNeedDictStored.add(col);
        }
    }

    result.removeAll(colsNeedDictStored);

    return result;
}
 

@Override
public void validate(CubeDesc cube, ValidateContext context) {
    List<MeasureDesc> measures = cube.getMeasures();

    if (validateMeasureNamesDuplicated(measures, context)) {
        return;
    }

    List<FunctionDesc> countStarFuncs = new ArrayList<FunctionDesc>();

    Iterator<MeasureDesc> it = measures.iterator();
    while (it.hasNext()) {
        MeasureDesc measure = it.next();
        FunctionDesc func = measure.getFunction();
        ParameterDesc parameter = func.getParameter();
        if (parameter == null) {
            context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        String type = func.getParameter().getType();
        String value = func.getParameter().getValue();
        if (StringUtils.isEmpty(type)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(value)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(func.getReturnType())) {
            context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        if (StringUtils.equalsIgnoreCase(FunctionDesc.PARAMETER_TYPE_COLUMN, type)) {
            validateColumnParameter(context, cube, value);
        } else if (StringUtils.equals(FunctionDesc.PARAMETER_TYPE_CONSTANT, type)) {
            validateCostantParameter(context, cube, value);
        }

        try {
            func.getMeasureType().validate(func);
        } catch (IllegalArgumentException ex) {
            context.addResult(ResultLevel.ERROR, ex.getMessage());
        }

        if (func.isCount() && func.getParameter().isConstant())
            countStarFuncs.add(func);

        if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(func.getExpression())) {
            if (parameter.getNextParameter() == null) {
                context.addResult(ResultLevel.ERROR, "Must define at least 2 parameters for function " + func.getExpression() + " in " + measure.getName());
                return;
            }

            ParameterDesc groupByCol = parameter.getNextParameter();
            List<String> duplicatedCol = Lists.newArrayList();
            while (groupByCol != null) {
                String embeded_groupby = groupByCol.getValue();
                for (DimensionDesc dimensionDesc : cube.getDimensions()) {
                    if (dimensionDesc.getColumn() != null && dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) {
                        duplicatedCol.add(embeded_groupby);
                    }
                }
                groupByCol = groupByCol.getNextParameter();
            }

        }
    }


    if (countStarFuncs.size() != 1) {
        context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are "
                + countStarFuncs.size() + " -- " + countStarFuncs);
    }
}
 

@Test
public void testReducerOnlyAggrInBaseCuboid() throws Exception {
    reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready");
    reduceDriver.getConfiguration().setInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, 1);

    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    MeasureDesc measureDesc = cubeDesc.getMeasures().get(0);
    FunctionDesc functionDesc = measureDesc.getFunction();
    Field field = FunctionDesc.class.getDeclaredField("measureType");
    field.setAccessible(true);
    MeasureType origMeasureType = functionDesc.getMeasureType();
    field.set(functionDesc, new MockUpMeasureType(origMeasureType));

    BufferedMeasureCodec codec = new BufferedMeasureCodec(cubeDesc.getMeasures());

    Text key1 = new Text("72010ustech");
    List<Text> values1 = new ArrayList<Text>();
    values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 100));
    values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 200));
    values1.add(newValueText(codec, "10", "10", "10", 1, 300));

    Text key2 = new Text("1tech");
    List<Text> values2 = new ArrayList<Text>();
    values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 500));
    values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 1000));

    Text key3 = new Text("0");
    List<Text> values3 = new ArrayList<Text>();
    values3.add(newValueText(codec, "146.52", "146.52", "146.52", 0, 0));

    reduceDriver.withInput(key1, values1);
    reduceDriver.withInput(key2, values2);
    reduceDriver.withInput(key3, values3);

    List<Pair<Text, Text>> result = reduceDriver.run();

    Pair<Text, Text> p1 = new Pair<>(new Text("72010ustech"), newValueText(codec, null, "10", "20.34", 3, 600));
    Pair<Text, Text> p2 = new Pair<>(new Text("1tech"), newValueText(codec, null, "15.09", "20.34", 2, 1500));
    Pair<Text, Text> p3 = new Pair<>(new Text("0"), newValueText(codec, null, "146.52", "146.52", 0, 0));

    assertEquals(3, result.size());

    assertTrue(result.contains(p1));
    assertTrue(result.contains(p2));
    assertTrue(result.contains(p3));
}
 

@Override
public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
    // If sqlDiegest is already adjusted, then not to adjust it again.
    if (sqlDigest.isBorrowedContext) {
        return;
    }

    if (sqlDigest.aggregations.size() > 1) {
        return;
    }

    for (MeasureDesc measureDesc : measureDescs) {
        if (!sqlDigest.involvedMeasure.contains(measureDesc)) {
            continue;
        }
        FunctionDesc topnFunc = measureDesc.getFunction();
        List<TblColRef> topnLiteralCol = getTopNLiteralColumn(topnFunc);

        if (sqlDigest.groupbyColumns.containsAll(topnLiteralCol) == false) {
            continue;
        }

        if (sqlDigest.aggregations.size() > 0) {
            FunctionDesc origFunc = sqlDigest.aggregations.iterator().next();
            if (origFunc.isSum() == false && origFunc.isCount() == false) {
                logger.warn("When query with topN, only SUM/Count function is allowed.");
                return;
            }

            if (isTopNCompatibleSum(measureDesc.getFunction(), origFunc) == false) {
                continue;
            }

            // topN not totally match, but have cuboid can answer, not use topN to adjust
            // topN totally match or (topN fuzzy match, but no cuboid can answer), use topN to adjust
            if (!totallyMatchTopN(sqlDigest) && cuboidCanAnswer) {
                continue;
            }

            logger.info("Rewrite function " + origFunc + " to " + topnFunc);
        }


        sqlDigest.aggregations = Lists.newArrayList(topnFunc);
        sqlDigest.groupbyColumns.removeAll(topnLiteralCol);
        sqlDigest.metricColumns.addAll(topnLiteralCol);
        break;
    }
}
 

@Test
public void testReducerOnlyAggrInBaseCuboid() throws Exception {
    reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready");
    reduceDriver.getConfiguration().setInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, 1);

    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    MeasureDesc measureDesc = cubeDesc.getMeasures().get(0);
    FunctionDesc functionDesc = measureDesc.getFunction();
    Field field = FunctionDesc.class.getDeclaredField("measureType");
    field.setAccessible(true);
    MeasureType origMeasureType = functionDesc.getMeasureType();
    field.set(functionDesc, new MockUpMeasureType(origMeasureType));

    BufferedMeasureCodec codec = new BufferedMeasureCodec(cubeDesc.getMeasures());

    Text key1 = new Text("72010ustech");
    List<Text> values1 = new ArrayList<Text>();
    values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 100));
    values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 200));
    values1.add(newValueText(codec, "10", "10", "10", 1, 300));

    Text key2 = new Text("1tech");
    List<Text> values2 = new ArrayList<Text>();
    values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 500));
    values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 1000));

    Text key3 = new Text("0");
    List<Text> values3 = new ArrayList<Text>();
    values3.add(newValueText(codec, "146.52", "146.52", "146.52", 0, 0));

    reduceDriver.withInput(key1, values1);
    reduceDriver.withInput(key2, values2);
    reduceDriver.withInput(key3, values3);

    List<Pair<Text, Text>> result = reduceDriver.run();

    Pair<Text, Text> p1 = new Pair<>(new Text("72010ustech"), newValueText(codec, null, "10", "20.34", 3, 600));
    Pair<Text, Text> p2 = new Pair<>(new Text("1tech"), newValueText(codec, null, "15.09", "20.34", 2, 1500));
    Pair<Text, Text> p3 = new Pair<>(new Text("0"), newValueText(codec, null, "146.52", "146.52", 0, 0));

    assertEquals(3, result.size());

    assertTrue(result.contains(p1));
    assertTrue(result.contains(p2));
    assertTrue(result.contains(p3));
}