Java Code Examples for org.apache.kylin.metadata.model.FunctionDesc#getMeasureType()

/**
 * Get columns which need dictionaries during cube building, while don't need to store them
 */
public Set<TblColRef> getAllColumnsNeedDictionaryForBuildingOnly() {
    Set<TblColRef> result = Sets.newHashSet();
    Set<TblColRef> colsNeedDictStored = Sets.newHashSet();

    // dictionaries in measures
    for (MeasureDesc measure : measures) {
        FunctionDesc func = measure.getFunction();
        MeasureType<?> aggrType = func.getMeasureType();

        // cols need dict stored in a measure
        Set<TblColRef> colSet = Sets.newHashSet();
        colSet.addAll(aggrType.getColumnsNeedDictionary(func));
        colSet.removeAll(aggrType.getColumnsNeedDictionaryForBuildingOnly(func));
        colsNeedDictStored.addAll(colSet);

        result.addAll(aggrType.getColumnsNeedDictionaryForBuildingOnly(func));
    }

    // dictionaries in dimensions
    colsNeedDictStored.addAll(getAllDimsHaveDictionary());

    // any additional dictionaries
    if (dictionaries != null) {
        for (DictionaryDesc dictDesc : dictionaries) {
            TblColRef col = dictDesc.getColumnRef();
            colsNeedDictStored.add(col);
        }
    }

    result.removeAll(colsNeedDictStored);

    return result;
}
 

@Override
public void implementRewrite(RewriteImplementor implementor) {
    // only rewrite the innermost aggregation
    if (needRewrite()) {
        translateAggregation();
        buildRewriteFieldsAndMetricsColumns();
    }

    implementor.visitChild(this, getInput());

    this.rewriting = true;

    // only rewrite the innermost aggregation
    if (needRewrite()) {
        // rewrite the aggCalls
        this.rewriteAggCalls = Lists.newArrayListWithExpectedSize(aggCalls.size());
        for (int i = 0; i < this.aggCalls.size(); i++) {
            AggregateCall aggCall = this.hackAggCalls.get(i) != null ? this.hackAggCalls.get(i)
                    : this.aggCalls.get(i);
            if (SqlStdOperatorTable.GROUPING == aggCall.getAggregation()) {
                this.rewriteAggCalls.add(aggCall);
                continue;
            }

            FunctionDesc cubeFunc = this.context.aggregations.get(i);
            // filter needn,t rewrite aggfunc
            // if it's not a cube, then the "needRewriteField func" should not resort to any rewrite fields,
            // which do not exist at all
            if (!(noPrecaculatedFieldsAvailable() && cubeFunc.needRewriteField())) {
                if (cubeFunc.needRewrite()) {
                    aggCall = rewriteAggregateCall(aggCall, cubeFunc);
                }

                //if not dim as measure (using some measure), differentiate it with a new class
                if (cubeFunc.getMeasureType() != null &&
                // DimCountDistinct case
                        cubeFunc.getMeasureType().needRewriteField()) {
                    aggCall = new KylinAggregateCall(aggCall, cubeFunc);
                }
            } else {
                logger.info(aggCall + "skip rewriteAggregateCall because no pre-aggregated field available");
            }

            this.rewriteAggCalls.add(aggCall);
            this.context.aggrSqlCalls.add(toSqlCall(aggCall));
        }
    }

    // rebuild rowType & columnRowType
    this.rowType = this.deriveRowType();
    this.columnRowType = this.buildColumnRowType();

    this.rewriting = false;
}
 

public CubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, //
        Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, TupleInfo returnTupleInfo) {
    this.cubeSeg = cubeSeg;
    this.cuboid = cuboid;
    this.gtColIdx = gtColIdx;
    this.tupleInfo = returnTupleInfo;
    this.derivedColFillers = Lists.newArrayList();

    nSelectedDims = selectedDimensions.size();
    tupleIdx = new int[selectedDimensions.size() + selectedMetrics.size()];

    // measure types don't have this many, but aligned length make programming easier
    measureTypes = new MeasureType[selectedDimensions.size() + selectedMetrics.size()];

    advMeasureFillers = Lists.newArrayListWithCapacity(1);
    advMeasureIndexInGTValues = Lists.newArrayListWithCapacity(1);
    usedLookupTables = Lists.newArrayList();
    eventTimezone = cubeSeg.getConfig().getStreamingDerivedTimeTimezone();
    autoJustByTimezone = eventTimezone.length() > 0
            && cubeSeg.getCubeDesc().getModel().getRootFactTable().getTableDesc().isStreamingTable();
    if (autoJustByTimezone) {
        logger.debug("Will ajust dimsension for Time Derived Column.");
        timeZoneOffset = TimeZone.getTimeZone(eventTimezone).getRawOffset();
    } else {
        timeZoneOffset = 0;
    }
    ////////////

    int i = 0;

    // pre-calculate dimension index mapping to tuple
    for (TblColRef dim : selectedDimensions) {
        tupleIdx[i] = tupleInfo.hasColumn(dim) ? tupleInfo.getColumnIndex(dim) : -1;
        if (TimeDerivedColumnType.isTimeDerivedColumn(dim.getName())
                && !TimeDerivedColumnType.isTimeDerivedColumnAboveDayLevel(dim.getName())) {
            timestampColumn.add(tupleIdx[i]);
        }
        i++;
    }

    for (FunctionDesc metric : selectedMetrics) {
        if (metric.needRewrite()) {
            String rewriteFieldName = metric.getRewriteFieldName();
            tupleIdx[i] = tupleInfo.hasField(rewriteFieldName) ? tupleInfo.getFieldIndex(rewriteFieldName) : -1;
        } else {
            // a non-rewrite metrics (like sum, or dimension playing as metrics) is like a dimension column
            TblColRef col = metric.getParameter().getColRefs().get(0);
            tupleIdx[i] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
        }

        MeasureType<?> measureType = metric.getMeasureType();
        if (measureType.needAdvancedTupleFilling()) {
            Map<TblColRef, Dictionary<String>> dictionaryMap = buildDictionaryMap(measureType.getColumnsNeedDictionary(metric));
            advMeasureFillers.add(measureType.getAdvancedTupleFiller(metric, returnTupleInfo, dictionaryMap));
            advMeasureIndexInGTValues.add(i);
        } else {
            measureTypes[i] = measureType;
        }

        i++;
    }

    // prepare derived columns and filler
    Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc().getHostToDerivedInfo(cuboid.getColumns(), null);
    for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
        TblColRef[] hostCols = entry.getKey().data;
        for (DeriveInfo deriveInfo : entry.getValue()) {
            IDerivedColumnFiller filler = newDerivedColumnFiller(hostCols, deriveInfo);
            if (filler != null) {
                derivedColFillers.add(filler);
            }
        }
    }
}
 

@Test
public void testReducerOnlyAggrInBaseCuboid() throws Exception {
    reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready");
    reduceDriver.getConfiguration().setInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, 1);

    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    MeasureDesc measureDesc = cubeDesc.getMeasures().get(0);
    FunctionDesc functionDesc = measureDesc.getFunction();
    Field field = FunctionDesc.class.getDeclaredField("measureType");
    field.setAccessible(true);
    MeasureType origMeasureType = functionDesc.getMeasureType();
    field.set(functionDesc, new MockUpMeasureType(origMeasureType));

    BufferedMeasureCodec codec = new BufferedMeasureCodec(cubeDesc.getMeasures());

    Text key1 = new Text("72010ustech");
    List<Text> values1 = new ArrayList<Text>();
    values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 100));
    values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 200));
    values1.add(newValueText(codec, "10", "10", "10", 1, 300));

    Text key2 = new Text("1tech");
    List<Text> values2 = new ArrayList<Text>();
    values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 500));
    values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 1000));

    Text key3 = new Text("0");
    List<Text> values3 = new ArrayList<Text>();
    values3.add(newValueText(codec, "146.52", "146.52", "146.52", 0, 0));

    reduceDriver.withInput(key1, values1);
    reduceDriver.withInput(key2, values2);
    reduceDriver.withInput(key3, values3);

    List<Pair<Text, Text>> result = reduceDriver.run();

    Pair<Text, Text> p1 = new Pair<>(new Text("72010ustech"), newValueText(codec, null, "10", "20.34", 3, 600));
    Pair<Text, Text> p2 = new Pair<>(new Text("1tech"), newValueText(codec, null, "15.09", "20.34", 2, 1500));
    Pair<Text, Text> p3 = new Pair<>(new Text("0"), newValueText(codec, null, "146.52", "146.52", 0, 0));

    assertEquals(3, result.size());

    assertTrue(result.contains(p1));
    assertTrue(result.contains(p2));
    assertTrue(result.contains(p3));
}
 

public StreamingTupleConverter(ResponseResultSchema schema, TupleInfo returnTupleInfo) {
    this.tupleInfo = returnTupleInfo;
    dimCnt = schema.getDimensionCount();
    metricsCnt = schema.getMetricsCount();
    dimTupleIdx = new int[dimCnt];
    metricsTupleIdx = new int[metricsCnt];

    // measure types don't have this many, but aligned length make programming easier
    measureTypes = new MeasureType[metricsCnt];

    advMeasureFillers = Lists.newArrayListWithCapacity(1);
    advMeasureIndexInGTValues = Lists.newArrayListWithCapacity(1);

    int idx = 0;
    // pre-calculate dimension index mapping to tuple
    for (TblColRef dim : schema.getDimensions()) {
        dimTupleIdx[idx] = tupleInfo.hasColumn(dim) ? tupleInfo.getColumnIndex(dim) : -1;
        if (dim.getType().isDateTimeFamily() && TimeDerivedColumnType.isTimeDerivedColumn(dim.getName()))
            timestampColumn.add(dimTupleIdx[idx]);
        idx++;
    }

    idx = 0;
    for (FunctionDesc metric : schema.getMetrics()) {
        if (metric.needRewrite()) {
            String rewriteFieldName = metric.getRewriteFieldName();
            metricsTupleIdx[idx] = tupleInfo.hasField(rewriteFieldName) ? tupleInfo.getFieldIndex(rewriteFieldName) : -1;
        } else { // a non-rewrite metrics (like sum, or dimension playing as metrics) is like a dimension column
            TblColRef col = metric.getParameter().getColRefs().get(0);
            metricsTupleIdx[idx] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
        }

        MeasureType<?> measureType = metric.getMeasureType();
        if (measureType.needAdvancedTupleFilling()) {
            advMeasureFillers.add(measureType.getAdvancedTupleFiller(metric, returnTupleInfo, null));
            advMeasureIndexInGTValues.add(idx);
        } else {
            measureTypes[idx] = measureType;
        }
        idx++;
    }
}
 

@Override
public void implementRewrite(RewriteImplementor implementor) {
    // only rewrite the innermost aggregation
    if (needRewrite()) {
        translateAggregation();
        buildRewriteFieldsAndMetricsColumns();
    }

    implementor.visitChild(this, getInput());

    this.rewriting = true;

    // only rewrite the innermost aggregation
    if (needRewrite()) {
        // rewrite the aggCalls
        this.rewriteAggCalls = Lists.newArrayListWithExpectedSize(aggCalls.size());
        for (int i = 0; i < this.aggCalls.size(); i++) {
            AggregateCall aggCall = this.hackAggCalls.get(i) != null ? this.hackAggCalls.get(i)
                    : this.aggCalls.get(i);
            if (SqlStdOperatorTable.GROUPING == aggCall.getAggregation()) {
                this.rewriteAggCalls.add(aggCall);
                continue;
            }

            FunctionDesc cubeFunc = this.context.aggregations.get(i);
            // filter needn,t rewrite aggfunc
            // if it's not a cube, then the "needRewriteField func" should not resort to any rewrite fields,
            // which do not exist at all
            if (!(noPrecaculatedFieldsAvailable() && cubeFunc.needRewriteField())) {
                if (cubeFunc.needRewrite()) {
                    aggCall = rewriteAggregateCall(aggCall, cubeFunc);
                }

                //if not dim as measure (using some measure), differentiate it with a new class
                if (cubeFunc.getMeasureType() != null &&
                // DimCountDistinct case
                        cubeFunc.getMeasureType().needRewriteField()) {
                    aggCall = new KylinAggregateCall(aggCall, cubeFunc);
                }
            } else {
                logger.info(aggCall + "skip rewriteAggregateCall because no pre-aggregated field available");
            }

            this.rewriteAggCalls.add(aggCall);
            this.context.aggrSqlCalls.add(toSqlCall(aggCall));
        }
    }

    // rebuild rowType & columnRowType
    this.rowType = this.deriveRowType();
    this.columnRowType = this.buildColumnRowType();

    this.rewriting = false;
}
 

public CubeTupleConverter(CubeSegment cubeSeg, Cuboid cuboid, //
        Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics, int[] gtColIdx, TupleInfo returnTupleInfo) {
    this.cubeSeg = cubeSeg;
    this.cuboid = cuboid;
    this.gtColIdx = gtColIdx;
    this.tupleInfo = returnTupleInfo;
    this.derivedColFillers = Lists.newArrayList();

    nSelectedDims = selectedDimensions.size();
    tupleIdx = new int[selectedDimensions.size() + selectedMetrics.size()];

    // measure types don't have this many, but aligned length make programming easier
    measureTypes = new MeasureType[selectedDimensions.size() + selectedMetrics.size()];

    advMeasureFillers = Lists.newArrayListWithCapacity(1);
    advMeasureIndexInGTValues = Lists.newArrayListWithCapacity(1);
    usedLookupTables = Lists.newArrayList();
    eventTimezone = cubeSeg.getConfig().getStreamingDerivedTimeTimezone();
    autoJustByTimezone = eventTimezone.length() > 0
            && cubeSeg.getCubeDesc().getModel().getRootFactTable().getTableDesc().isStreamingTable();
    if (autoJustByTimezone) {
        logger.debug("Will ajust dimsension for Time Derived Column.");
        timeZoneOffset = TimeZone.getTimeZone(eventTimezone).getRawOffset();
    } else {
        timeZoneOffset = 0;
    }
    ////////////

    int i = 0;

    // pre-calculate dimension index mapping to tuple
    for (TblColRef dim : selectedDimensions) {
        tupleIdx[i] = tupleInfo.hasColumn(dim) ? tupleInfo.getColumnIndex(dim) : -1;
        if (TimeDerivedColumnType.isTimeDerivedColumn(dim.getName())
                && !TimeDerivedColumnType.isTimeDerivedColumnAboveDayLevel(dim.getName())) {
            timestampColumn.add(tupleIdx[i]);
        }
        i++;
    }

    for (FunctionDesc metric : selectedMetrics) {
        if (metric.needRewriteField()) {
            String rewriteFieldName = metric.getRewriteFieldName();
            tupleIdx[i] = tupleInfo.hasField(rewriteFieldName) ? tupleInfo.getFieldIndex(rewriteFieldName) : -1;
        } else {
            // a non-rewrite metrics (like sum, or dimension playing as metrics) is like a dimension column
            TblColRef col = metric.getParameter().getColRefs().get(0);
            tupleIdx[i] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
        }

        MeasureType<?> measureType = metric.getMeasureType();
        if (measureType.needAdvancedTupleFilling()) {
            Map<TblColRef, Dictionary<String>> dictionaryMap = buildDictionaryMap(measureType.getColumnsNeedDictionary(metric));
            advMeasureFillers.add(measureType.getAdvancedTupleFiller(metric, returnTupleInfo, dictionaryMap));
            advMeasureIndexInGTValues.add(i);
        } else {
            measureTypes[i] = measureType;
        }

        i++;
    }

    // prepare derived columns and filler
    Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo = cuboid.getCubeDesc().getHostToDerivedInfo(cuboid.getColumns(), null);
    for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
        TblColRef[] hostCols = entry.getKey().data;
        for (DeriveInfo deriveInfo : entry.getValue()) {
            IDerivedColumnFiller filler = newDerivedColumnFiller(hostCols, deriveInfo);
            if (filler != null) {
                derivedColFillers.add(filler);
            }
        }
    }

    rowKeyDesc = cubeSeg.getCubeDesc().getRowkey();
}
 

@Test
public void testReducerOnlyAggrInBaseCuboid() throws Exception {
    reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready");
    reduceDriver.getConfiguration().setInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, 1);

    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    MeasureDesc measureDesc = cubeDesc.getMeasures().get(0);
    FunctionDesc functionDesc = measureDesc.getFunction();
    Field field = FunctionDesc.class.getDeclaredField("measureType");
    field.setAccessible(true);
    MeasureType origMeasureType = functionDesc.getMeasureType();
    field.set(functionDesc, new MockUpMeasureType(origMeasureType));

    BufferedMeasureCodec codec = new BufferedMeasureCodec(cubeDesc.getMeasures());

    Text key1 = new Text("72010ustech");
    List<Text> values1 = new ArrayList<Text>();
    values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 100));
    values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 200));
    values1.add(newValueText(codec, "10", "10", "10", 1, 300));

    Text key2 = new Text("1tech");
    List<Text> values2 = new ArrayList<Text>();
    values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1, 500));
    values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1, 1000));

    Text key3 = new Text("0");
    List<Text> values3 = new ArrayList<Text>();
    values3.add(newValueText(codec, "146.52", "146.52", "146.52", 0, 0));

    reduceDriver.withInput(key1, values1);
    reduceDriver.withInput(key2, values2);
    reduceDriver.withInput(key3, values3);

    List<Pair<Text, Text>> result = reduceDriver.run();

    Pair<Text, Text> p1 = new Pair<>(new Text("72010ustech"), newValueText(codec, null, "10", "20.34", 3, 600));
    Pair<Text, Text> p2 = new Pair<>(new Text("1tech"), newValueText(codec, null, "15.09", "20.34", 2, 1500));
    Pair<Text, Text> p3 = new Pair<>(new Text("0"), newValueText(codec, null, "146.52", "146.52", 0, 0));

    assertEquals(3, result.size());

    assertTrue(result.contains(p1));
    assertTrue(result.contains(p2));
    assertTrue(result.contains(p3));
}
 

public StreamingTupleConverter(ResponseResultSchema schema, TupleInfo returnTupleInfo) {
    this.tupleInfo = returnTupleInfo;
    dimCnt = schema.getDimensionCount();
    metricsCnt = schema.getMetricsCount();
    dimTupleIdx = new int[dimCnt];
    metricsTupleIdx = new int[metricsCnt];

    // measure types don't have this many, but aligned length make programming easier
    measureTypes = new MeasureType[metricsCnt];

    advMeasureFillers = Lists.newArrayListWithCapacity(1);
    advMeasureIndexInGTValues = Lists.newArrayListWithCapacity(1);

    int idx = 0;
    // pre-calculate dimension index mapping to tuple
    for (TblColRef dim : schema.getDimensions()) {
        dimTupleIdx[idx] = tupleInfo.hasColumn(dim) ? tupleInfo.getColumnIndex(dim) : -1;
        if (dim.getType().isDateTimeFamily() && TimeDerivedColumnType.isTimeDerivedColumn(dim.getName()))
            timestampColumn.add(dimTupleIdx[idx]);
        idx++;
    }

    idx = 0;
    for (FunctionDesc metric : schema.getMetrics()) {
        if (metric.needRewrite()) {
            String rewriteFieldName = metric.getRewriteFieldName();
            metricsTupleIdx[idx] = tupleInfo.hasField(rewriteFieldName) ? tupleInfo.getFieldIndex(rewriteFieldName) : -1;
        } else { // a non-rewrite metrics (like sum, or dimension playing as metrics) is like a dimension column
            TblColRef col = metric.getParameter().getColRefs().get(0);
            metricsTupleIdx[idx] = tupleInfo.hasColumn(col) ? tupleInfo.getColumnIndex(col) : -1;
        }

        MeasureType<?> measureType = metric.getMeasureType();
        if (measureType.needAdvancedTupleFilling()) {
            advMeasureFillers.add(measureType.getAdvancedTupleFiller(metric, returnTupleInfo, null));
            advMeasureIndexInGTValues.add(idx);
        } else {
            measureTypes[idx] = measureType;
        }
        idx++;
    }
}