org.apache.kylin.cube.model.validation.ResultLevel Java Examples

The following examples show how to use org.apache.kylin.cube.model.validation.ResultLevel. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RowKeyAttrRule.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    RowKeyDesc row = cube.getRowkey();
    if (row == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
        return;
    }

    RowKeyColDesc[] rcd = row.getRowKeyColumns();
    if (rcd == null || rcd.length == 0) {
        context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
        return;
    }

    for (int i = 0; i < rcd.length; i++) {
        RowKeyColDesc rd = rcd[i];
        if (rd.getColumn() == null || rd.getColumn().length() == 0) {
            context.addResult(ResultLevel.ERROR, "Rowkey column empty");
        }

    }

}
Example #2
Source File: FunctionRule.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private void validateReturnType(ValidateContext context, CubeDesc cube, FunctionDesc funcDesc) {

        String func = funcDesc.getExpression();
        DataType rtype = funcDesc.getReturnDataType();

        if (funcDesc.isCount()) {
            if (rtype.isIntegerFamily() == false) {
                context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be one of " + DataType.INTEGER_FAMILY);
            }
        } else if (funcDesc.isCountDistinct()) {
            if (rtype.isHLLC() == false && funcDesc.isHolisticCountDistinct() == false) {
                context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be hllc(10), hllc(12) etc.");
            }
        } else if (funcDesc.isMax() || funcDesc.isMin() || funcDesc.isSum()) {
            if (rtype.isNumberFamily() == false) {
                context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be one of " + DataType.NUMBER_FAMILY);
            }
        } else {
            if (StringUtils.equalsIgnoreCase(KylinConfig.getInstanceFromEnv().getProperty(KEY_IGNORE_UNKNOWN_FUNC, "false"), "false")) {
                context.addResult(ResultLevel.ERROR, "Unrecognized function: [" + func + "]");
            }
        }

    }
Example #3
Source File: RowKeyAttrRule.java    From kylin with Apache License 2.0 6 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    RowKeyDesc row = cube.getRowkey();
    if (row == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
        return;
    }

    RowKeyColDesc[] rcd = row.getRowKeyColumns();
    if (rcd == null || rcd.length == 0) {
        context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
        return;
    }

    for (int i = 0; i < rcd.length; i++) {
        RowKeyColDesc rd = rcd[i];
        if (rd.getColumn() == null || rd.getColumn().length() == 0) {
            context.addResult(ResultLevel.ERROR, "Rowkey column empty");
        }

    }

}
Example #4
Source File: FunctionRule.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
/**
 * @param context
 * @param cube
 * @param value
 */
private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) {
    String factTable = cube.getFactTable();
    if (StringUtils.isEmpty(factTable)) {
        context.addResult(ResultLevel.ERROR, "Fact table can not be null.");
        return;
    }
    TableDesc table = MetadataManager.getInstance(cube.getConfig()).getTableDesc(factTable);
    if (table == null) {
        context.addResult(ResultLevel.ERROR, "Fact table can not be found: " + cube);
        return;
    }
    // Prepare column set
    Set<String> set = new HashSet<String>();
    ColumnDesc[] cdesc = table.getColumns();
    for (int i = 0; i < cdesc.length; i++) {
        ColumnDesc columnDesc = cdesc[i];
        set.add(columnDesc.getName());
    }

    String[] items = value.split(",");
    for (int i = 0; i < items.length; i++) {
        String item = items[i].trim();
        if (StringUtils.isEmpty(item)) {
            continue;
        }
        if (!set.contains(item)) {
            context.addResult(ResultLevel.ERROR, "Column [" + item + "] does not exist in factable table" + factTable);
        }
    }

}
Example #5
Source File: StreamingCubeRule.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    DataModelDesc model = cube.getModel();
    
    if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING
            && !model.getRootFactTable().getTableDesc().isStreamingTable()) {
        return;
    }

    if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) {
        context.addResult(ResultLevel.ERROR, "Must define a partition column.");
        return;
    }

    final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef();
    boolean found = false;
    for (DimensionDesc dimensionDesc : cube.getDimensions()) {
        for (TblColRef dimCol : dimensionDesc.getColumnRefs()) {
            if (dimCol.equals(partitionCol)) {
                found = true;
                break;
            }
        }
    }

    if (found == false) {
        context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list.");
        return;
    }

}
Example #6
Source File: FunctionRule.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
/**
 * @param context
 * @param cube
 * @param value
 */
private void validateCostantParameter(ValidateContext context, CubeDesc cube, String value) {
    try {
        Integer.parseInt(value);
    } catch (Exception e) {
        context.addResult(ResultLevel.ERROR, "Parameter value must be number, but it is " + value);
    }
}
Example #7
Source File: MandatoryColumnRule.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    Set<String> mands = new HashSet<String>();
    RowKeyColDesc[] cols = cube.getRowkey().getRowKeyColumns();
    if (cols == null || cols.length == 0) {
        return;
    }
    for (int i = 0; i < cols.length; i++) {
        RowKeyColDesc rowKeyColDesc = cols[i];
        if (rowKeyColDesc.isMandatory()) {
            mands.add(rowKeyColDesc.getColumn());
        }
    }
    if (mands.isEmpty()) {
        return;
    }
    String[][] groups = cube.getRowkey().getAggregationGroups();
    for (int i = 0; i < groups.length; i++) {
        String[] group = groups[i];
        for (int j = 0; j < group.length; j++) {
            String col = group[j];
            if (mands.contains(col)) {
                context.addResult(ResultLevel.ERROR, "mandatory column " + col + " must not be in aggregation group [" + ArrayUtils.toString(group) + "]");
            }
        }
    }

}
Example #8
Source File: RowKeyAttrRule.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    RowKeyDesc row = cube.getRowkey();
    if (row == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
        return;
    }

    RowKeyColDesc[] rcd = row.getRowKeyColumns();
    if (rcd == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
        return;
    }
    if(rcd.length == 0){
        context.addResult(ResultLevel.ERROR, "Rowkey columns is empty");
        return;       	
    }

    for (int i = 0; i < rcd.length; i++) {
        RowKeyColDesc rd = rcd[i];
        if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary())&&!rd.getDictionary().equals("false"))) {
            context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute");
        }
        if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary())||rd.getDictionary().equals("false"))) {
            context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty");
        }
    }

}
Example #9
Source File: AggregationGroupSizeRule.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
/**
 * @param cube
 * @param context
 */
private void innerValidateMaxSize(CubeDesc cube, ValidateContext context) {
    int maxSize = getMaxAgrGroupSize();
    String[][] groups = cube.getRowkey().getAggregationGroups();
    for (int i = 0; i < groups.length; i++) {
        String[] group = groups[i];
        if (group.length >= maxSize) {
            context.addResult(ResultLevel.ERROR, "Length of the number " + i + " aggregation group's length should be less than " + maxSize);
        }
    }
}
Example #10
Source File: FunctionRule.java    From kylin with Apache License 2.0 5 votes vote down vote up 
/**
 * @param measures
 */
private boolean validateMeasureNamesDuplicated(List<MeasureDesc> measures, ValidateContext context) {
    Set<String> nameSet = new HashSet<>();
    for (MeasureDesc measure: measures){
        if (nameSet.contains(measure.getName())){
            context.addResult(ResultLevel.ERROR, "There is duplicated measure's name: " + measure.getName());
            return true;
        } else {
            nameSet.add(measure.getName());
        }
    }
    return false;
}
Example #11
Source File: FunctionRule.java    From kylin with Apache License 2.0 5 votes vote down vote up 
/**
 * @param context
 * @param cube
 * @param value
 */
private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) {
    DataModelDesc model = cube.getModel();
    try {
        model.findColumn(value);
    } catch (IllegalArgumentException e) {
        context.addResult(ResultLevel.ERROR, e.getMessage());
    }
}
Example #12
Source File: FunctionRule.java    From kylin with Apache License 2.0 5 votes vote down vote up 
/**
 * @param context
 * @param cube
 * @param value
 */
private void validateCostantParameter(ValidateContext context, CubeDesc cube, String value) {
    try {
        Integer.parseInt(value);
    } catch (Exception e) {
        context.addResult(ResultLevel.ERROR, "Parameter value must be number, but it is " + value);
    }
}
Example #13
Source File: StreamingCubeRule.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    DataModelDesc model = cube.getModel();
    
    if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING
            && !model.getRootFactTable().getTableDesc().isStreamingTable()) {
        return;
    }

    if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) {
        context.addResult(ResultLevel.ERROR, "Must define a partition column.");
        return;
    }

    final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef();
    boolean found = false;
    for (DimensionDesc dimensionDesc : cube.getDimensions()) {
        for (TblColRef dimCol : dimensionDesc.getColumnRefs()) {
            if (dimCol.equals(partitionCol)) {
                found = true;
                break;
            }
        }
    }

    if (found == false) {
        context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list.");
        return;
    }

}
Example #14
Source File: FunctionRule.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
/**
 * @param measures
 */
private boolean validateMeasureNamesDuplicated(List<MeasureDesc> measures, ValidateContext context) {
    Set<String> nameSet = new HashSet<>();
    for (MeasureDesc measure: measures){
        if (nameSet.contains(measure.getName())){
            context.addResult(ResultLevel.ERROR, "There is duplicated measure's name: " + measure.getName());
            return true;
        } else {
            nameSet.add(measure.getName());
        }
    }
    return false;
}
Example #15
Source File: FunctionRule.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
/**
 * @param context
 * @param cube
 * @param value
 */
private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) {
    DataModelDesc model = cube.getModel();
    try {
        model.findColumn(value);
    } catch (IllegalArgumentException e) {
        context.addResult(ResultLevel.ERROR, e.getMessage());
    }
}
Example #16
Source File: FunctionRule.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
/**
 * @param context
 * @param cube
 * @param value
 */
private void validateCostantParameter(ValidateContext context, CubeDesc cube, String value) {
    try {
        Integer.parseInt(value);
    } catch (Exception e) {
        context.addResult(ResultLevel.ERROR, "Parameter value must be number, but it is " + value);
    }
}
Example #17
Source File: FunctionRule.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    List<MeasureDesc> measures = cube.getMeasures();

    if (validateMeasureNamesDuplicated(measures, context)) {
        return;
    }

    List<FunctionDesc> countStarFuncs = new ArrayList<FunctionDesc>();

    Iterator<MeasureDesc> it = measures.iterator();
    while (it.hasNext()) {
        MeasureDesc measure = it.next();
        FunctionDesc func = measure.getFunction();
        ParameterDesc parameter = func.getParameter();
        if (parameter == null) {
            context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        String type = func.getParameter().getType();
        String value = func.getParameter().getValue();
        if (StringUtils.isEmpty(type)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(value)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(func.getReturnType())) {
            context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        if (StringUtils.equalsIgnoreCase(FunctionDesc.PARAMETER_TYPE_COLUMN, type)) {
            validateColumnParameter(context, cube, value);
        } else if (StringUtils.equals(FunctionDesc.PARAMETER_TYPE_CONSTANT, type)) {
            validateCostantParameter(context, cube, value);
        }

        try {
            func.getMeasureType().validate(func);
        } catch (IllegalArgumentException ex) {
            context.addResult(ResultLevel.ERROR, ex.getMessage());
        }

        if (func.isCount() && func.getParameter().isConstant())
            countStarFuncs.add(func);

        if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(func.getExpression())) {
            if (parameter.getNextParameter() == null) {
                context.addResult(ResultLevel.ERROR, "Must define at least 2 parameters for function " + func.getExpression() + " in " + measure.getName());
                return;
            }

            ParameterDesc groupByCol = parameter.getNextParameter();
            List<String> duplicatedCol = Lists.newArrayList();
            while (groupByCol != null) {
                String embeded_groupby = groupByCol.getValue();
                for (DimensionDesc dimensionDesc : cube.getDimensions()) {
                    if (dimensionDesc.getColumn() != null && dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) {
                        duplicatedCol.add(embeded_groupby);
                    }
                }
                groupByCol = groupByCol.getNextParameter();
            }

        }
    }


    if (countStarFuncs.size() != 1) {
        context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are "
                + countStarFuncs.size() + " -- " + countStarFuncs);
    }
}
Example #18
Source File: FunctionRule.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    List<MeasureDesc> measures = cube.getMeasures();

    List<FunctionDesc> countFuncs = new ArrayList<FunctionDesc>();

    Iterator<MeasureDesc> it = measures.iterator();
    while (it.hasNext()) {
        MeasureDesc measure = it.next();
        FunctionDesc func = measure.getFunction();
        ParameterDesc parameter = func.getParameter();
        if (parameter == null) {
            context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        String type = func.getParameter().getType();
        String value = func.getParameter().getValue();
        if (StringUtils.isEmpty(type)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(value)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(func.getReturnType())) {
            context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        if (StringUtils.equalsIgnoreCase(FunctionDesc.PARAMETER_TYPE_COLUMN, type)) {
            validateColumnParameter(context, cube, value);
        } else if (StringUtils.equals(FunctionDesc.PARAMTER_TYPE_CONSTANT, type)) {
            validateCostantParameter(context, cube, value);
        }
        validateReturnType(context, cube, func);

        if (func.isCount())
            countFuncs.add(func);
    }

    if (countFuncs.size() != 1) {
        context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are " + countFuncs.size() + " -- " + countFuncs);
    }
}
Example #19
Source File: FunctionRule.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    List<MeasureDesc> measures = cube.getMeasures();

    if (validateMeasureNamesDuplicated(measures, context)) {
        return;
    }

    List<FunctionDesc> countStarFuncs = new ArrayList<FunctionDesc>();

    Iterator<MeasureDesc> it = measures.iterator();
    while (it.hasNext()) {
        MeasureDesc measure = it.next();
        FunctionDesc func = measure.getFunction();
        ParameterDesc parameter = func.getParameter();
        if (parameter == null) {
            context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        String type = func.getParameter().getType();
        String value = func.getParameter().getValue();
        if (StringUtils.isEmpty(type)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(value)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(func.getReturnType())) {
            context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        if (StringUtils.equalsIgnoreCase(FunctionDesc.PARAMETER_TYPE_COLUMN, type)) {
            validateColumnParameter(context, cube, value);
        } else if (StringUtils.equals(FunctionDesc.PARAMETER_TYPE_CONSTANT, type)) {
            validateCostantParameter(context, cube, value);
        }

        try {
            func.getMeasureType().validate(func);
        } catch (IllegalArgumentException ex) {
            context.addResult(ResultLevel.ERROR, ex.getMessage());
        }

        if (func.isCount() && func.getParameter().isConstant())
            countStarFuncs.add(func);

        if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(func.getExpression())) {
            if (parameter.getNextParameter() == null) {
                context.addResult(ResultLevel.ERROR, "Must define at least 2 parameters for function " + func.getExpression() + " in " + measure.getName());
                return;
            }

            ParameterDesc groupByCol = parameter.getNextParameter();
            List<String> duplicatedCol = Lists.newArrayList();
            while (groupByCol != null) {
                String embeded_groupby = groupByCol.getValue();
                for (DimensionDesc dimensionDesc : cube.getDimensions()) {
                    if (dimensionDesc.getColumn() != null && dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) {
                        duplicatedCol.add(embeded_groupby);
                    }
                }
                groupByCol = groupByCol.getNextParameter();
            }

        }
    }


    if (countStarFuncs.size() != 1) {
        context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are "
                + countStarFuncs.size() + " -- " + countStarFuncs);
    }
}