Java Code Examples for org.apache.kylin.cube.model.CubeDesc#getDimensions()

The following examples show how to use org.apache.kylin.cube.model.CubeDesc#getDimensions() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HBaseLookupMRSteps.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) {
    CubeDesc cubeDesc = cube.getDescriptor();
    Set<String> allLookupTables = Sets.newHashSet();
    for (DimensionDesc dim : cubeDesc.getDimensions()) {
        TableRef table = dim.getTableRef();
        if (cubeDesc.getModel().isLookupTable(table)) {
            allLookupTables.add(table.getTableIdentity());
        }
    }
    List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList();
    for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) {
        if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType())
                && allLookupTables.contains(snapshotTableDesc.getTableName())) {
            addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc);
        }
    }
}
Example 2
Source File: HBaseLookupMRSteps.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) {
    CubeDesc cubeDesc = cube.getDescriptor();
    Set<String> allLookupTables = Sets.newHashSet();
    for (DimensionDesc dim : cubeDesc.getDimensions()) {
        TableRef table = dim.getTableRef();
        if (cubeDesc.getModel().isLookupTable(table)) {
            allLookupTables.add(table.getTableIdentity());
        }
    }
    List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList();
    for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) {
        if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType())
                && allLookupTables.contains(snapshotTableDesc.getTableName())) {
            addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc);
        }
    }
}
Example 3
Source File: MetadataUpgradeTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private void checkCubeDesc(String descName) {
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeDesc cubedesc1 = cubeDescMgr.getCubeDesc(descName);
    Assert.assertNotNull(cubedesc1);
    DataModelDesc model = cubedesc1.getModel();
    Assert.assertNotNull(model);
    Assert.assertTrue(model.getLookups().length > 0);

    List<DimensionDesc> dims = cubedesc1.getDimensions();

    Assert.assertTrue(dims.size() > 0);

    for (DimensionDesc dim : dims) {
        Assert.assertTrue(dim.getColumn().length > 0);
    }

    Assert.assertTrue(cubedesc1.getMeasures().size() > 0);

    CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    List<CubeInstance> cubes = cubeMgr.getCubesByDesc(descName);

    Assert.assertTrue(cubes.size() > 0);
}
Example 4
Source File: CubeControllerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGetSql() {
    GeneralResponse response = cubeController.getSql("test_kylin_cube_with_slr_ready");
    String sql = response.getProperty("sql");
    CubeDesc cubeDesc = cubeDescController.getDesc("test_kylin_cube_with_slr_ready");

    for (DimensionDesc dimensionDesc : cubeDesc.getDimensions()) {
        if (dimensionDesc.getDerived() != null) {
            for (String derivedDimension : dimensionDesc.getDerived()) {
                Assert.assertTrue(sql.contains(derivedDimension));
            }
        }
    }
}
Example 5
Source File: StreamingCubeRule.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    DataModelDesc model = cube.getModel();
    
    if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING
            && !model.getRootFactTable().getTableDesc().isStreamingTable()) {
        return;
    }

    if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) {
        context.addResult(ResultLevel.ERROR, "Must define a partition column.");
        return;
    }

    final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef();
    boolean found = false;
    for (DimensionDesc dimensionDesc : cube.getDimensions()) {
        for (TblColRef dimCol : dimensionDesc.getColumnRefs()) {
            if (dimCol.equals(partitionCol)) {
                found = true;
                break;
            }
        }
    }

    if (found == false) {
        context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list.");
        return;
    }

}
Example 6
Source File: CubeControllerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGetSql() {
    GeneralResponse response = cubeController.getSql("test_kylin_cube_with_slr_ready");
    String sql = response.getProperty("sql");
    CubeDesc cubeDesc = cubeDescController.getDesc("test_kylin_cube_with_slr_ready");

    for (DimensionDesc dimensionDesc : cubeDesc.getDimensions()) {
        if (dimensionDesc.getDerived() != null) {
            for (String derivedDimension : dimensionDesc.getDerived()) {
                Assert.assertTrue(sql.contains(derivedDimension));
            }
        }
    }
}
Example 7
Source File: StreamingCubeRule.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    DataModelDesc model = cube.getModel();
    
    if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING
            && !model.getRootFactTable().getTableDesc().isStreamingTable()) {
        return;
    }

    if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) {
        context.addResult(ResultLevel.ERROR, "Must define a partition column.");
        return;
    }

    final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef();
    boolean found = false;
    for (DimensionDesc dimensionDesc : cube.getDimensions()) {
        for (TblColRef dimCol : dimensionDesc.getColumnRefs()) {
            if (dimCol.equals(partitionCol)) {
                found = true;
                break;
            }
        }
    }

    if (found == false) {
        context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list.");
        return;
    }

}
Example 8
Source File: CubeCapabilityChecker.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private static boolean isMatchedWithJoins(Collection<JoinDesc> joins, CubeInstance cube) {
    CubeDesc cubeDesc = cube.getDescriptor();

    List<JoinDesc> cubeJoins = new ArrayList<JoinDesc>(cubeDesc.getDimensions().size());
    for (DimensionDesc d : cubeDesc.getDimensions()) {
        if (d.getJoin() != null) {
            cubeJoins.add(d.getJoin());
        }
    }
    for (JoinDesc j : joins) {
        // optiq engine can't decide which one is fk or pk
        String pTable = j.getPrimaryKeyColumns()[0].getTable();
        String factTable = cubeDesc.getFactTable();
        if (factTable.equals(pTable)) {
            j.swapPKFK();
        }

        // check primary key, all PK column should refer to same tale, the Fact Table of cube.
        // Using first column's table name to check.
        String fTable = j.getForeignKeyColumns()[0].getTable();
        if (!factTable.equals(fTable)) {
            logger.info("Fact Table" + factTable + " not matched in join: " + j + " on cube " + cube.getName());
            return false;
        }

        // The hashcode() function of JoinDesc has been overwritten,
        // which takes into consideration: pk,fk,jointype
        if (!cubeJoins.contains(j)) {
            logger.info("Query joins don't macth on cube " + cube.getName());
            return false;
        }
    }
    return true;
}
Example 9
Source File: CubeSizeEstimationCLI.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private static RowKeyColInfo extractRowKeyInfo(CubeDesc cubeDesc) {
    RowKeyDesc rowKeyDesc = cubeDesc.getRowkey();
    RowKeyColInfo info = new RowKeyColInfo();
    info.hierachyColBitIndice = new ArrayList<List<Integer>>();
    info.nonHierachyColBitIndice = new ArrayList<Integer>();
    HashSet<Integer> heirachyIndexSet = new HashSet<Integer>();

    for (DimensionDesc dim : cubeDesc.getDimensions()) {
        if (dim.getHierarchy() != null) {
            LinkedList<Integer> hlist = new LinkedList<Integer>();
            for (HierarchyDesc hierarchyDesc : dim.getHierarchy()) {
                int index = rowKeyDesc.getColumnBitIndex(hierarchyDesc.getColumnRef());
                hlist.add(index);
                heirachyIndexSet.add(index);
            }
            info.hierachyColBitIndice.add(hlist);
        }
    }

    for (int i = 0; i < rowKeyDesc.getRowKeyColumns().length; ++i) {
        if (!heirachyIndexSet.contains(i)) {
            info.nonHierachyColBitIndice.add(i);
        }
    }

    return info;
}
Example 10
Source File: FunctionRule.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    List<MeasureDesc> measures = cube.getMeasures();

    if (validateMeasureNamesDuplicated(measures, context)) {
        return;
    }

    List<FunctionDesc> countStarFuncs = new ArrayList<FunctionDesc>();

    Iterator<MeasureDesc> it = measures.iterator();
    while (it.hasNext()) {
        MeasureDesc measure = it.next();
        FunctionDesc func = measure.getFunction();
        ParameterDesc parameter = func.getParameter();
        if (parameter == null) {
            context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        String type = func.getParameter().getType();
        String value = func.getParameter().getValue();
        if (StringUtils.isEmpty(type)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(value)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(func.getReturnType())) {
            context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        if (StringUtils.equalsIgnoreCase(FunctionDesc.PARAMETER_TYPE_COLUMN, type)) {
            validateColumnParameter(context, cube, value);
        } else if (StringUtils.equals(FunctionDesc.PARAMETER_TYPE_CONSTANT, type)) {
            validateCostantParameter(context, cube, value);
        }

        try {
            func.getMeasureType().validate(func);
        } catch (IllegalArgumentException ex) {
            context.addResult(ResultLevel.ERROR, ex.getMessage());
        }

        if (func.isCount() && func.getParameter().isConstant())
            countStarFuncs.add(func);

        if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(func.getExpression())) {
            if (parameter.getNextParameter() == null) {
                context.addResult(ResultLevel.ERROR, "Must define at least 2 parameters for function " + func.getExpression() + " in " + measure.getName());
                return;
            }

            ParameterDesc groupByCol = parameter.getNextParameter();
            List<String> duplicatedCol = Lists.newArrayList();
            while (groupByCol != null) {
                String embeded_groupby = groupByCol.getValue();
                for (DimensionDesc dimensionDesc : cube.getDimensions()) {
                    if (dimensionDesc.getColumn() != null && dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) {
                        duplicatedCol.add(embeded_groupby);
                    }
                }
                groupByCol = groupByCol.getNextParameter();
            }

        }
    }


    if (countStarFuncs.size() != 1) {
        context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are "
                + countStarFuncs.size() + " -- " + countStarFuncs);
    }
}
Example 11
Source File: FunctionRule.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    List<MeasureDesc> measures = cube.getMeasures();

    if (validateMeasureNamesDuplicated(measures, context)) {
        return;
    }

    List<FunctionDesc> countStarFuncs = new ArrayList<FunctionDesc>();

    Iterator<MeasureDesc> it = measures.iterator();
    while (it.hasNext()) {
        MeasureDesc measure = it.next();
        FunctionDesc func = measure.getFunction();
        ParameterDesc parameter = func.getParameter();
        if (parameter == null) {
            context.addResult(ResultLevel.ERROR, "Must define parameter for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        String type = func.getParameter().getType();
        String value = func.getParameter().getValue();
        if (StringUtils.isEmpty(type)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter type " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(value)) {
            context.addResult(ResultLevel.ERROR, "Must define type for parameter value " + func.getExpression() + " in " + measure.getName());
            return;
        }
        if (StringUtils.isEmpty(func.getReturnType())) {
            context.addResult(ResultLevel.ERROR, "Must define return type for function " + func.getExpression() + " in " + measure.getName());
            return;
        }

        if (StringUtils.equalsIgnoreCase(FunctionDesc.PARAMETER_TYPE_COLUMN, type)) {
            validateColumnParameter(context, cube, value);
        } else if (StringUtils.equals(FunctionDesc.PARAMETER_TYPE_CONSTANT, type)) {
            validateCostantParameter(context, cube, value);
        }

        try {
            func.getMeasureType().validate(func);
        } catch (IllegalArgumentException ex) {
            context.addResult(ResultLevel.ERROR, ex.getMessage());
        }

        if (func.isCount() && func.getParameter().isConstant())
            countStarFuncs.add(func);

        if (TopNMeasureType.FUNC_TOP_N.equalsIgnoreCase(func.getExpression())) {
            if (parameter.getNextParameter() == null) {
                context.addResult(ResultLevel.ERROR, "Must define at least 2 parameters for function " + func.getExpression() + " in " + measure.getName());
                return;
            }

            ParameterDesc groupByCol = parameter.getNextParameter();
            List<String> duplicatedCol = Lists.newArrayList();
            while (groupByCol != null) {
                String embeded_groupby = groupByCol.getValue();
                for (DimensionDesc dimensionDesc : cube.getDimensions()) {
                    if (dimensionDesc.getColumn() != null && dimensionDesc.getColumn().equalsIgnoreCase(embeded_groupby)) {
                        duplicatedCol.add(embeded_groupby);
                    }
                }
                groupByCol = groupByCol.getNextParameter();
            }

        }
    }


    if (countStarFuncs.size() != 1) {
        context.addResult(ResultLevel.ERROR, "Must define one and only one count(1) function, but there are "
                + countStarFuncs.size() + " -- " + countStarFuncs);
    }
}