Java Code Examples for org.apache.kylin.metadata.model.ParameterDesc#getNextParameter()

The following examples show how to use org.apache.kylin.metadata.model.ParameterDesc#getNextParameter() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SegmentMemoryStore.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private Object buildValueOf(int idxOfMeasure, List<String> row) {
    MeasureDesc measure = parsedStreamingCubeInfo.measureDescs[idxOfMeasure];
    FunctionDesc function = measure.getFunction();
    int[] colIdxOnFlatTable = parsedStreamingCubeInfo.intermediateTableDesc.getMeasureColumnIndexes()[idxOfMeasure];

    int paramCount = function.getParameterCount();
    String[] inputToMeasure = new String[paramCount];

    // pick up parameter values
    ParameterDesc param = function.getParameter();
    int paramColIdx = 0; // index among parameters of column type
    for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
        String value;
        if (function.isCount()) {
            value = "1";
        } else if (param.isColumnType()) {
            value = row.get(colIdxOnFlatTable[paramColIdx++]);
        } else {
            value = param.getValue();
        }
        inputToMeasure[i] = value;
    }
    return parsedStreamingCubeInfo.measureIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, dictionaryMap);
}
Example 2
Source File: SegmentMemoryStore.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private Object buildValueOf(int idxOfMeasure, List<String> row) {
    MeasureDesc measure = parsedStreamingCubeInfo.measureDescs[idxOfMeasure];
    FunctionDesc function = measure.getFunction();
    int[] colIdxOnFlatTable = parsedStreamingCubeInfo.intermediateTableDesc.getMeasureColumnIndexes()[idxOfMeasure];

    int paramCount = function.getParameterCount();
    String[] inputToMeasure = new String[paramCount];

    // pick up parameter values
    ParameterDesc param = function.getParameter();
    int paramColIdx = 0; // index among parameters of column type
    for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
        String value;
        if (function.isCount()) {
            value = "1";
        } else if (param.isColumnType()) {
            value = row.get(colIdxOnFlatTable[paramColIdx++]);
        } else {
            value = param.getValue();
        }
        inputToMeasure[i] = value;
    }
    return parsedStreamingCubeInfo.measureIngesters[idxOfMeasure].valueOf(inputToMeasure, measure, dictionaryMap);
}
Example 3
Source File: KeyValueBuilder.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public String[] buildValueOf(int idxOfMeasure, String[] row) {
    MeasureDesc measure = cubeDesc.getMeasures().get(idxOfMeasure);
    FunctionDesc function = measure.getFunction();
    int[] colIdxOnFlatTable = flatDesc.getMeasureColumnIndexes()[idxOfMeasure];

    int paramCount = function.getParameterCount();
    List<String> inputToMeasure = Lists.newArrayListWithExpectedSize(paramCount);

    // pick up parameter values
    ParameterDesc param = function.getParameter();
    int colParamIdx = 0; // index among parameters of column type
    for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
        String value;
        if (param.isColumnType()) {
            value = getCell(colIdxOnFlatTable[colParamIdx++], row);
            if (function.isCount() && value == null) {
                value = ZERO;
            } else if (function.isCount()) {
                value = ONE;
            }
        } else {
            value = param.getValue();
            if (function.isCount()) {
                value = ONE;
            }
        }
        inputToMeasure.add(value);
    }

    return inputToMeasure.toArray(new String[inputToMeasure.size()]);
}
Example 4
Source File: KeyValueBuilder.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public String[] buildValueOf(int idxOfMeasure, String[] row) {
    MeasureDesc measure = cubeDesc.getMeasures().get(idxOfMeasure);
    FunctionDesc function = measure.getFunction();
    int[] colIdxOnFlatTable = flatDesc.getMeasureColumnIndexes()[idxOfMeasure];

    int paramCount = function.getParameterCount();
    List<String> inputToMeasure = Lists.newArrayListWithExpectedSize(paramCount);

    // pick up parameter values
    ParameterDesc param = function.getParameter();
    int colParamIdx = 0; // index among parameters of column type
    for (int i = 0; i < paramCount; i++, param = param.getNextParameter()) {
        String value;
        if (param.isColumnType()) {
            value = getCell(colIdxOnFlatTable[colParamIdx++], row);
            if (function.isCount() && value == null) {
                value = ZERO;
            } else if (function.isCount()) {
                value = ONE;
            }
        } else {
            value = param.getValue();
            if (function.isCount()) {
                value = ONE;
            }
        }
        inputToMeasure.add(value);
    }
    if (BitmapMapMeasureType.DATATYPE_BITMAP_MAP.equalsIgnoreCase(function.getReturnType())) {
        inputToMeasure.add(segmentStartTime);
    }

    return inputToMeasure.toArray(new String[inputToMeasure.size()]);
}
Example 5
Source File: FunctionRule.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    List<MeasureDesc> measures = cube.getMeasures();

    if (validateMeasureNamesDuplicated(measures, context)) {
        return;
    }

    List<FunctionDesc> countStarFuncs = new ArrayList<FunctionDesc>();

    Iterator<MeasureDesc> it = measures.iterator();
    while (it.hasNext()) {
        MeasureDesc measure = it.next();
        FunctionDesc func = measure.getFunction();
        ParameterDesc parameter = func.getParameter();
        if (parameter == null) {
            context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        String type = func.getParameter().getType();
        String value = func.getParameter().getValue();
        if (StringUtils.isEmpty(type)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(value)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(func.getReturnType())) {
            context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        if (StringUtils.equalsIgnoreCase(FunctionDesc.PARAMETER_TYPE_COLUMN, type)) {
            validateColumnParameter(context, cube, value);
        } else if (StringUtils.equals(FunctionDesc.PARAMETER_TYPE_CONSTANT, type)) {
            validateCostantParameter(context, cube, value);
        }

        try {
            func.getMeasureType().validate(func);
        } catch (IllegalArgumentException ex) {
            context.addResult(ResultLevel.ERROR, ex.getMessage());
        }

        if (func.isCount() && func.getParameter().isConstant())
            countStarFuncs.add(func);

        if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(func.getExpression())) {
            if (parameter.getNextParameter() == null) {
                context.addResult(ResultLevel.ERROR, "Must define at least 2 parameters for function " + func.getExpression() + " in " + measure.getName());
                return;
            }

            ParameterDesc groupByCol = parameter.getNextParameter();
            List<String> duplicatedCol = Lists.newArrayList();
            while (groupByCol != null) {
                String embeded_groupby = groupByCol.getValue();
                for (DimensionDesc dimensionDesc : cube.getDimensions()) {
                    if (dimensionDesc.getColumn() != null && dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) {
                        duplicatedCol.add(embeded_groupby);
                    }
                }
                groupByCol = groupByCol.getNextParameter();
            }

        }
    }


    if (countStarFuncs.size() != 1) {
        context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are "
                + countStarFuncs.size() + " -- " + countStarFuncs);
    }
}
Example 6
Source File: CubeDescManager.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
/**
 * if there is some change need be applied after getting a cubeDesc from front-end, do it here
 * @param cubeDesc
 */
private void postProcessCubeDesc(CubeDesc cubeDesc) {
    for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
        if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(measureDesc.getFunction().getExpression())) {
            // update return type scale with the estimated key length
            Map<String, String> configuration = measureDesc.getFunction().getConfiguration();
            ParameterDesc parameter = measureDesc.getFunction().getParameter();
            parameter = parameter.getNextParameter();
            int keyLength = 0;
            while (parameter != null) {
                String encoding = configuration.get(TopNMeasureType.CONFIG_ENCODING_PREFIX + parameter.getValue());
                String encodingVersionStr = configuration
                        .get(TopNMeasureType.CONFIG_ENCODING_VERSION_PREFIX + parameter.getValue());
                if (StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) {
                    keyLength += DictionaryDimEnc.MAX_ENCODING_LENGTH; // estimation for dict encoding
                } else if (encoding.startsWith("dict")) {
                    throw new IllegalArgumentException(
                            "TOP_N's Encoding is " + encoding + ", please choose the correct one");
                } else {
                    // non-dict encoding
                    int encodingVersion = 1;
                    if (!StringUtils.isEmpty(encodingVersionStr)) {
                        try {
                            encodingVersion = Integer.parseInt(encodingVersionStr);
                        } catch (NumberFormatException e) {
                            throw new RuntimeException("invalid encoding version: " + encodingVersionStr);
                        }
                    }
                    Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding);
                    DimensionEncoding dimensionEncoding = DimensionEncodingFactory.create((String) encodingConf[0],
                            (String[]) encodingConf[1], encodingVersion);
                    keyLength += dimensionEncoding.getLengthOfEncoding();
                }

                parameter = parameter.getNextParameter();
            }

            DataType returnType = DataType.getType(measureDesc.getFunction().getReturnType());
            DataType newReturnType = new DataType(returnType.getName(), returnType.getPrecision(), keyLength);
            measureDesc.getFunction().setReturnType(newReturnType.toString());
        }
    }
}
Example 7
Source File: FunctionRule.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    List<MeasureDesc> measures = cube.getMeasures();

    if (validateMeasureNamesDuplicated(measures, context)) {
        return;
    }

    List<FunctionDesc> countStarFuncs = new ArrayList<FunctionDesc>();

    Iterator<MeasureDesc> it = measures.iterator();
    while (it.hasNext()) {
        MeasureDesc measure = it.next();
        FunctionDesc func = measure.getFunction();
        ParameterDesc parameter = func.getParameter();
        if (parameter == null) {
            context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        String type = func.getParameter().getType();
        String value = func.getParameter().getValue();
        if (StringUtils.isEmpty(type)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(value)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(func.getReturnType())) {
            context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        if (StringUtils.equalsIgnoreCase(FunctionDesc.PARAMETER_TYPE_COLUMN, type)) {
            validateColumnParameter(context, cube, value);
        } else if (StringUtils.equals(FunctionDesc.PARAMETER_TYPE_CONSTANT, type)) {
            validateCostantParameter(context, cube, value);
        }

        try {
            func.getMeasureType().validate(func);
        } catch (IllegalArgumentException ex) {
            context.addResult(ResultLevel.ERROR, ex.getMessage());
        }

        if (func.isCount() && func.getParameter().isConstant())
            countStarFuncs.add(func);

        if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(func.getExpression())) {
            if (parameter.getNextParameter() == null) {
                context.addResult(ResultLevel.ERROR, "Must define at least 2 parameters for function " + func.getExpression() + " in " + measure.getName());
                return;
            }

            ParameterDesc groupByCol = parameter.getNextParameter();
            List<String> duplicatedCol = Lists.newArrayList();
            while (groupByCol != null) {
                String embeded_groupby = groupByCol.getValue();
                for (DimensionDesc dimensionDesc : cube.getDimensions()) {
                    if (dimensionDesc.getColumn() != null && dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) {
                        duplicatedCol.add(embeded_groupby);
                    }
                }
                groupByCol = groupByCol.getNextParameter();
            }

        }
    }


    if (countStarFuncs.size() != 1) {
        context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are "
                + countStarFuncs.size() + " -- " + countStarFuncs);
    }
}
Example 8
Source File: CubeDescManager.java    From kylin with Apache License 2.0 4 votes vote down vote up 
/**
 * if there is some change need be applied after getting a cubeDesc from front-end, do it here
 * @param cubeDesc
 */
private void postProcessCubeDesc(CubeDesc cubeDesc) {
    for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
        if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(measureDesc.getFunction().getExpression())) {
            // update return type scale with the estimated key length
            Map<String, String> configuration = measureDesc.getFunction().getConfiguration();
            ParameterDesc parameter = measureDesc.getFunction().getParameter();
            parameter = parameter.getNextParameter();
            int keyLength = 0;
            while (parameter != null) {
                String encoding = configuration.get(TopNMeasureType.CONFIG_ENCODING_PREFIX + parameter.getValue());
                String encodingVersionStr = configuration
                        .get(TopNMeasureType.CONFIG_ENCODING_VERSION_PREFIX + parameter.getValue());
                if (StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding)) {
                    keyLength += DictionaryDimEnc.MAX_ENCODING_LENGTH; // estimation for dict encoding
                } else if (encoding.startsWith("dict")) {
                    throw new IllegalArgumentException(
                            "TOP_N's Encoding is " + encoding + ", please choose the correct one");
                } else {
                    // non-dict encoding
                    int encodingVersion = 1;
                    if (!StringUtils.isEmpty(encodingVersionStr)) {
                        try {
                            encodingVersion = Integer.parseInt(encodingVersionStr);
                        } catch (NumberFormatException e) {
                            throw new RuntimeException("invalid encoding version: " + encodingVersionStr);
                        }
                    }
                    Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding);
                    DimensionEncoding dimensionEncoding = DimensionEncodingFactory.create((String) encodingConf[0],
                            (String[]) encodingConf[1], encodingVersion);
                    keyLength += dimensionEncoding.getLengthOfEncoding();
                }

                parameter = parameter.getNextParameter();
            }

            DataType returnType = DataType.getType(measureDesc.getFunction().getReturnType());
            int precision = returnType.getPrecision() < 1 ? 100 : returnType.getPrecision();
            DataType newReturnType = new DataType(returnType.getName(), precision, keyLength);
            measureDesc.getFunction().setReturnType(newReturnType.toString());
        }
    }
}