Java Code Examples for org.apache.kylin.metadata.model.FunctionDesc#getParameter()

The following examples show how to use org.apache.kylin.metadata.model.FunctionDesc#getParameter() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TopNMeasureType.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private boolean isTopNCompatibleSum(FunctionDesc topN, FunctionDesc sum) {
    if (sum == null)
        return false;

    if (!isTopN(topN))
        return false;

    TblColRef topnNumCol = getTopNNumericColumn(topN);

    if (topnNumCol == null) {
        if (sum.isCount())
            return true;

        return false;
    }

    if (sum.isSum() == false)
        return false;

    if (sum.getParameter() == null || sum.getParameter().getColRefs() == null
            || sum.getParameter().getColRefs().size() == 0)
        return false;

    TblColRef sumCol = sum.getParameter().getColRefs().get(0);
    return sumCol.equals(topnNumCol);
}
Example 2
Source File: SegmentMemoryStore.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private Object buildValueOf(int idxOfMeasure, List<String> row) {
    MeasureDesc measure = parsedStreamingCubeInfo.measureDescs[idxOfMeasure];
    FunctionDesc function = measure.getFunction();
    int[] colIdxOnFlatTable = parsedStreamingCubeInfo.intermediateTableDesc.getMeasureColumnIndexes()[idxOfMeasure];

    int paramCount = function.getParameterCount();
    String[] inputToMeasure = new String[paramCount];

    // pick up parameter values
    ParameterDesc param = function.getParameter();
    int paramColIdx = 0; // index among parameters of column type
    for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
        String value;
        if (function.isCount()) {
            value = "1";
        } else if (param.isColumnType()) {
            value = row.get(colIdxOnFlatTable[paramColIdx++]);
        } else {
            value = param.getValue();
        }
        inputToMeasure[i] = value;
    }
    return parsedStreamingCubeInfo.measureIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, dictionaryMap);
}
Example 3
Source File: TopNMeasureType.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private boolean isTopNCompatibleSum(FunctionDesc topN, FunctionDesc sum) {
    if (sum == null)
        return false;

    if (!isTopN(topN))
        return false;

    TblColRef topnNumCol = getTopNNumericColumn(topN);

    if (topnNumCol == null) {
        if (sum.isCount())
            return true;

        return false;
    }

    if (sum.isSum() == false)
        return false;

    if (sum.getParameter() == null || sum.getParameter().getColRefs() == null
            || sum.getParameter().getColRefs().size() == 0)
        return false;

    TblColRef sumCol = sum.getParameter().getColRefs().get(0);
    return sumCol.equals(topnNumCol);
}
Example 4
Source File: SegmentMemoryStore.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private Object buildValueOf(int idxOfMeasure, List<String> row) {
    MeasureDesc measure = parsedStreamingCubeInfo.measureDescs[idxOfMeasure];
    FunctionDesc function = measure.getFunction();
    int[] colIdxOnFlatTable = parsedStreamingCubeInfo.intermediateTableDesc.getMeasureColumnIndexes()[idxOfMeasure];

    int paramCount = function.getParameterCount();
    String[] inputToMeasure = new String[paramCount];

    // pick up parameter values
    ParameterDesc param = function.getParameter();
    int paramColIdx = 0; // index among parameters of column type
    for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
        String value;
        if (function.isCount()) {
            value = "1";
        } else if (param.isColumnType()) {
            value = row.get(colIdxOnFlatTable[paramColIdx++]);
        } else {
            value = param.getValue();
        }
        inputToMeasure[i] = value;
    }
    return parsedStreamingCubeInfo.measureIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, dictionaryMap);
}
Example 5
Source File: CubeCapabilityChecker.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, CapabilityResult result,
        Set<TblColRef> dimCols) {

    Iterator<FunctionDesc> it = unmatchedAggregations.iterator();
    while (it.hasNext()) {
        FunctionDesc functionDesc = it.next();

        // let calcite handle count
        if (functionDesc.isCount()) {
            logger.warn("No count measure found for column {}, will use count(1) to replace it, please note that it will count all value(include null value)", functionDesc.getParameter() == null ? "" : functionDesc.getParameter().getColRef().getName());
            it.remove();
            continue;
        }

        // calcite can do aggregation from columns on-the-fly
        ParameterDesc parameterDesc = functionDesc.getParameter();
        if (parameterDesc == null) {
            continue;
        }
        List<TblColRef> neededCols = parameterDesc.getColRefs();
        if (neededCols.size() > 0 && dimCols.containsAll(neededCols)
                && FunctionDesc.BUILT_IN_AGGREGATIONS.contains(functionDesc.getExpression())) {
            result.influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc));
            it.remove();
            continue;
        }
    }
}
Example 6
Source File: KeyValueBuilder.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public String[] buildValueOf(int idxOfMeasure, String[] row) {
    MeasureDesc measure = cubeDesc.getMeasures().get(idxOfMeasure);
    FunctionDesc function = measure.getFunction();
    int[] colIdxOnFlatTable = flatDesc.getMeasureColumnIndexes()[idxOfMeasure];

    int paramCount = function.getParameterCount();
    List<String> inputToMeasure = Lists.newArrayListWithExpectedSize(paramCount);

    // pick up parameter values
    ParameterDesc param = function.getParameter();
    int colParamIdx = 0; // index among parameters of column type
    for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
        String value;
        if (param.isColumnType()) {
            value = getCell(colIdxOnFlatTable[colParamIdx++], row);
            if (function.isCount() && value == null) {
                value = ZERO;
            } else if (function.isCount()) {
                value = ONE;
            }
        } else {
            value = param.getValue();
            if (function.isCount()) {
                value = ONE;
            }
        }
        inputToMeasure.add(value);
    }

    return inputToMeasure.toArray(new String[inputToMeasure.size()]);
}
Example 7
Source File: CubeCapabilityChecker.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private static void tryDimensionAsMeasures(Collection<FunctionDesc> unmatchedAggregations, CapabilityResult result,
        Set<TblColRef> dimCols) {

    Iterator<FunctionDesc> it = unmatchedAggregations.iterator();
    while (it.hasNext()) {
        FunctionDesc functionDesc = it.next();

        // let calcite handle count
        if (functionDesc.isCount()) {
            logger.warn("No count measure found for column {}, will use count(1) to replace it, please note that it will count all value(include null value)", functionDesc.getParameter() == null ? "" : functionDesc.getParameter().getColRef().getName());
            it.remove();
            continue;
        }

        // calcite can do aggregation from columns on-the-fly
        ParameterDesc parameterDesc = functionDesc.getParameter();
        if (parameterDesc == null) {
            continue;
        }

        List<TblColRef> neededCols = functionDesc instanceof ExpressionDynamicFunctionDesc
                ? Lists.newArrayList(ExpressionColCollector
                        .collectColumns(((ExpressionDynamicFunctionDesc) functionDesc).getTupleExpression()))
                : parameterDesc.getColRefs();
        if (neededCols.size() > 0 && dimCols.containsAll(neededCols)
                && FunctionDesc.BUILT_IN_AGGREGATIONS.contains(functionDesc.getExpression())) {
            result.influences.add(new CapabilityResult.DimensionAsMeasure(functionDesc));
            it.remove();
            continue;
        }
    }
}
Example 8
Source File: KeyValueBuilder.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public String[] buildValueOf(int idxOfMeasure, String[] row) {
    MeasureDesc measure = cubeDesc.getMeasures().get(idxOfMeasure);
    FunctionDesc function = measure.getFunction();
    int[] colIdxOnFlatTable = flatDesc.getMeasureColumnIndexes()[idxOfMeasure];

    int paramCount = function.getParameterCount();
    List<String> inputToMeasure = Lists.newArrayListWithExpectedSize(paramCount);

    // pick up parameter values
    ParameterDesc param = function.getParameter();
    int colParamIdx = 0; // index among parameters of column type
    for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
        String value;
        if (param.isColumnType()) {
            value = getCell(colIdxOnFlatTable[colParamIdx++], row);
            if (function.isCount() && value == null) {
                value = ZERO;
            } else if (function.isCount()) {
                value = ONE;
            }
        } else {
            value = param.getValue();
            if (function.isCount()) {
                value = ONE;
            }
        }
        inputToMeasure.add(value);
    }
    if (BitmapMapMeasureType.DATATYPE_BITMAP_MAP.equalsIgnoreCase(function.getReturnType())) {
        inputToMeasure.add(segmentStartTime);
    }

    return inputToMeasure.toArray(new String[inputToMeasure.size()]);
}
Example 9
Source File: BaseCuboidMapper.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private byte[] getValueBytes(SplittedBytes[] splitBuffers, int measureIdx) {
    MeasureDesc desc = cubeDesc.getMeasures().get(measureIdx);
    FunctionDesc func = desc.getFunction();
    ParameterDesc paramDesc = func.getParameter();
    int[] flatTableIdx = intermediateTableDesc.getMeasureColumnIndexes()[measureIdx];

    byte[] result = null;

    // constant
    if (flatTableIdx == null) {
        result = Bytes.toBytes(paramDesc.getValue());
    }
    // column values
    else {
        // for multiple columns, their values are joined
        for (int i = 0; i < flatTableIdx.length; i++) {
            SplittedBytes split = splitBuffers[flatTableIdx[i]];
            if (result == null) {
                result = Arrays.copyOf(split.value, split.length);
            } else {
                byte[] newResult = new byte[result.length + split.length];
                System.arraycopy(result, 0, newResult, 0, result.length);
                System.arraycopy(split.value, 0, newResult, result.length, split.length);
                result = newResult;
            }
        }
    }

    if (func.isCount() || func.isHolisticCountDistinct()) {
        // note for holistic count distinct, this value will be ignored
        result = ONE;
    }

    if (isNull(result)) {
        result = null;
    }

    return result;
}
Example 10
Source File: CubeDesc.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void initMeasureColumns(Map<String, TableDesc> tables) {
    if (measures == null || measures.isEmpty()) {
        return;
    }

    TableDesc factTable = tables.get(getFactTable());
    for (MeasureDesc m : measures) {
        m.setName(m.getName().toUpperCase());

        if (m.getDependentMeasureRef() != null) {
            m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase());
        }

        FunctionDesc f = m.getFunction();
        f.setExpression(f.getExpression().toUpperCase());
        f.setReturnDataType(DataType.getInstance(f.getReturnType()));

        ParameterDesc p = f.getParameter();
        p.normalizeColumnValue();

        if (p.isColumnType()) {
            ArrayList<TblColRef> colRefs = Lists.newArrayList();
            for (String cName : p.getValue().split("\\s*,\\s*")) {
                ColumnDesc sourceColumn = factTable.findColumnByName(cName);
                TblColRef colRef = new TblColRef(sourceColumn);
                colRefs.add(colRef);
                allColumns.add(colRef);
            }
            if (colRefs.isEmpty() == false)
                p.setColRefs(colRefs);
        }
    }
}
Example 11
Source File: CubeDesc.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void initMeasureColumns(Map<String, TableDesc> tables) {
    if (measures == null || measures.isEmpty()) {
        return;
    }

    TableDesc factTable = tables.get(getFactTable());
    for (MeasureDesc m : measures) {
        m.setName(m.getName().toUpperCase());

        if (m.getDependentMeasureRef() != null) {
            m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase());
        }
        
        FunctionDesc f = m.getFunction();
        f.setExpression(f.getExpression().toUpperCase());
        f.setReturnDataType(DataType.getInstance(f.getReturnType()));

        ParameterDesc p = f.getParameter();
        p.normalizeColumnValue();

        if (p.isColumnType()) {
            ArrayList<TblColRef> colRefs = Lists.newArrayList();
            for (String cName : p.getValue().split("\\s*,\\s*")) {
                ColumnDesc sourceColumn = factTable.findColumnByName(cName);
                TblColRef colRef = new TblColRef(sourceColumn);
                colRefs.add(colRef);
                allColumns.add(colRef);
            }
            if (colRefs.isEmpty() == false)
                p.setColRefs(colRefs);
        }
        
        // verify holistic count distinct as a dependent measure
        if (m.isHolisticCountDistinct() && StringUtils.isBlank(m.getDependentMeasureRef())) {
            throw new IllegalStateException(m + " is a holistic count distinct but it has no DependentMeasureRef defined!");
        }
    }
}
Example 12
Source File: FunctionRule.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    List<MeasureDesc> measures = cube.getMeasures();

    if (validateMeasureNamesDuplicated(measures, context)) {
        return;
    }

    List<FunctionDesc> countStarFuncs = new ArrayList<FunctionDesc>();

    Iterator<MeasureDesc> it = measures.iterator();
    while (it.hasNext()) {
        MeasureDesc measure = it.next();
        FunctionDesc func = measure.getFunction();
        ParameterDesc parameter = func.getParameter();
        if (parameter == null) {
            context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        String type = func.getParameter().getType();
        String value = func.getParameter().getValue();
        if (StringUtils.isEmpty(type)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(value)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(func.getReturnType())) {
            context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        if (StringUtils.equalsIgnoreCase(FunctionDesc.PARAMETER_TYPE_COLUMN, type)) {
            validateColumnParameter(context, cube, value);
        } else if (StringUtils.equals(FunctionDesc.PARAMETER_TYPE_CONSTANT, type)) {
            validateCostantParameter(context, cube, value);
        }

        try {
            func.getMeasureType().validate(func);
        } catch (IllegalArgumentException ex) {
            context.addResult(ResultLevel.ERROR, ex.getMessage());
        }

        if (func.isCount() && func.getParameter().isConstant())
            countStarFuncs.add(func);

        if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(func.getExpression())) {
            if (parameter.getNextParameter() == null) {
                context.addResult(ResultLevel.ERROR, "Must define at least 2 parameters for function " + func.getExpression() + " in " + measure.getName());
                return;
            }

            ParameterDesc groupByCol = parameter.getNextParameter();
            List<String> duplicatedCol = Lists.newArrayList();
            while (groupByCol != null) {
                String embeded_groupby = groupByCol.getValue();
                for (DimensionDesc dimensionDesc : cube.getDimensions()) {
                    if (dimensionDesc.getColumn() != null && dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) {
                        duplicatedCol.add(embeded_groupby);
                    }
                }
                groupByCol = groupByCol.getNextParameter();
            }

        }
    }


    if (countStarFuncs.size() != 1) {
        context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are "
                + countStarFuncs.size() + " -- " + countStarFuncs);
    }
}
Example 13
Source File: FunctionRule.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    List<MeasureDesc> measures = cube.getMeasures();

    if (validateMeasureNamesDuplicated(measures, context)) {
        return;
    }

    List<FunctionDesc> countStarFuncs = new ArrayList<FunctionDesc>();

    Iterator<MeasureDesc> it = measures.iterator();
    while (it.hasNext()) {
        MeasureDesc measure = it.next();
        FunctionDesc func = measure.getFunction();
        ParameterDesc parameter = func.getParameter();
        if (parameter == null) {
            context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        String type = func.getParameter().getType();
        String value = func.getParameter().getValue();
        if (StringUtils.isEmpty(type)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(value)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(func.getReturnType())) {
            context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        if (StringUtils.equalsIgnoreCase(FunctionDesc.PARAMETER_TYPE_COLUMN, type)) {
            validateColumnParameter(context, cube, value);
        } else if (StringUtils.equals(FunctionDesc.PARAMETER_TYPE_CONSTANT, type)) {
            validateCostantParameter(context, cube, value);
        }

        try {
            func.getMeasureType().validate(func);
        } catch (IllegalArgumentException ex) {
            context.addResult(ResultLevel.ERROR, ex.getMessage());
        }

        if (func.isCount() && func.getParameter().isConstant())
            countStarFuncs.add(func);

        if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(func.getExpression())) {
            if (parameter.getNextParameter() == null) {
                context.addResult(ResultLevel.ERROR, "Must define at least 2 parameters for function " + func.getExpression() + " in " + measure.getName());
                return;
            }

            ParameterDesc groupByCol = parameter.getNextParameter();
            List<String> duplicatedCol = Lists.newArrayList();
            while (groupByCol != null) {
                String embeded_groupby = groupByCol.getValue();
                for (DimensionDesc dimensionDesc : cube.getDimensions()) {
                    if (dimensionDesc.getColumn() != null && dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) {
                        duplicatedCol.add(embeded_groupby);
                    }
                }
                groupByCol = groupByCol.getNextParameter();
            }

        }
    }


    if (countStarFuncs.size() != 1) {
        context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are "
                + countStarFuncs.size() + " -- " + countStarFuncs);
    }
}
Example 14
Source File: FunctionRule.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    List<MeasureDesc> measures = cube.getMeasures();

    List<FunctionDesc> countFuncs = new ArrayList<FunctionDesc>();

    Iterator<MeasureDesc> it = measures.iterator();
    while (it.hasNext()) {
        MeasureDesc measure = it.next();
        FunctionDesc func = measure.getFunction();
        ParameterDesc parameter = func.getParameter();
        if (parameter == null) {
            context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        String type = func.getParameter().getType();
        String value = func.getParameter().getValue();
        if (StringUtils.isEmpty(type)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(value)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(func.getReturnType())) {
            context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        if (StringUtils.equalsIgnoreCase(FunctionDesc.PARAMETER_TYPE_COLUMN, type)) {
            validateColumnParameter(context, cube, value);
        } else if (StringUtils.equals(FunctionDesc.PARAMTER_TYPE_CONSTANT, type)) {
            validateCostantParameter(context, cube, value);
        }
        validateReturnType(context, cube, func);

        if (func.isCount())
            countFuncs.add(func);
    }

    if (countFuncs.size() != 1) {
        context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are " + countFuncs.size() + " -- " + countFuncs);
    }
}