Java Code Examples for org.apache.kylin.cube.CubeInstance#getAllColumns()

The following examples show how to use org.apache.kylin.cube.CubeInstance#getAllColumns() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TableSchemaUpdateChecker.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
/**
 * check whether all columns used in `cube` has compatible schema in current hive schema denoted by `fieldsMap`.
 * @param cube cube to check, must use `table` in its model
 * @param origTable kylin's table metadata
 * @param newTable current hive schema of `table`
 * @return columns in origTable that can't be found in newTable
 */
private List<String> checkAllColumnsInCube(CubeInstance cube, TableDesc origTable, TableDesc newTable) {
    Set<ColumnDesc> usedColumns = Sets.newHashSet();
    for (TblColRef col : cube.getAllColumns()) {
        usedColumns.add(col.getColumnDesc());
    }

    List<String> violateColumns = Lists.newArrayList();
    for (ColumnDesc column : origTable.getColumns()) {
        if (!column.isComputedColumn() && usedColumns.contains(column)) {
            ColumnDesc newCol = newTable.findColumnByName(column.getName());
            if (newCol == null || !isColumnCompatible(column, newCol)) {
                violateColumns.add(column.getName());
            }
        }
    }
    return violateColumns;
}
Example 2
Source File: TableSchemaUpdateChecker.java    From kylin with Apache License 2.0 6 votes vote down vote up 
/**
 * check whether all columns used in `cube` has compatible schema in current hive schema denoted by `fieldsMap`.
 * @param cube cube to check, must use `table` in its model
 * @param origTable kylin's table metadata
 * @param newTable current hive schema of `table`
 * @return columns in origTable that can't be found in newTable
 */
private List<String> checkAllColumnsInCube(CubeInstance cube, TableDesc origTable, TableDesc newTable) {
    Set<ColumnDesc> usedColumns = Sets.newHashSet();
    for (TblColRef col : cube.getAllColumns()) {
        usedColumns.add(col.getColumnDesc());
    }

    List<String> violateColumns = Lists.newArrayList();
    for (ColumnDesc column : origTable.getColumns()) {
        if (!column.isComputedColumn() && usedColumns.contains(column)) {
            ColumnDesc newCol = newTable.findColumnByName(column.getName());
            if (newCol == null || !isColumnCompatible(column, newCol)) {
                violateColumns.add(column.getName());
            }
        }
    }
    return violateColumns;
}
Example 3
Source File: FactDistinctColumnsReducerMapping.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private FactDistinctColumnsReducerMapping(CubeInstance cube, int cuboidRowCounterReducerNum) {
    CubeDesc desc = cube.getDescriptor();
    Set<TblColRef> allCols = cube.getAllColumns();
    Set<TblColRef> dictCols = desc.getAllColumnsNeedDictionaryBuilt();
    List<TblColRef> dimCols = desc.listDimensionColumnsExcludingDerived(true);
    for (TblColRef colRef : allCols) {
        if (dictCols.contains(colRef)) {
            allDimDictCols.add(colRef);
        } else if (dimCols.indexOf(colRef) >= 0){
            allDimDictCols.add(colRef);
        }
    }

    colIdToReducerBeginId = new int[allDimDictCols.size() + 1];

    int uhcReducerCount = cube.getConfig().getUHCReducerCount();
    List<TblColRef> uhcList = desc.getAllUHCColumns();
    int counter = 0;
    for (int i = 0; i < allDimDictCols.size(); i++) {
        colIdToReducerBeginId[i] = counter;
        boolean isUHC = uhcList.contains(allDimDictCols.get(i));
        counter += (isUHC) ? uhcReducerCount : 1;
    }
    colIdToReducerBeginId[allDimDictCols.size()] = counter;
    nDimReducers = counter;

    nCuboidRowCounters = cuboidRowCounterReducerNum == 0 ? //
            MapReduceUtil.getCuboidHLLCounterReducerNum(cube) : cuboidRowCounterReducerNum;
    nTotalReducers = nDimReducers + nCuboidRowCounters;

    reducerRolePlay = new int[nTotalReducers];
    for (int i = 0, dictId = 0; i < nTotalReducers; i++) {
        if (i >= nDimReducers) {
            // cuboid HLL counter reducer
            reducerRolePlay[i] = MARK_FOR_HLL_COUNTER;
        } else {
            if (i == colIdToReducerBeginId[dictId + 1])
                dictId++;

            reducerRolePlay[i] = dictId;
        }
    }
}
Example 4
Source File: FactDistinctColumnsReducerMapping.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private FactDistinctColumnsReducerMapping(CubeInstance cube, int cuboidRowCounterReducerNum) {
    CubeDesc desc = cube.getDescriptor();
    Set<TblColRef> allCols = cube.getAllColumns();
    Set<TblColRef> dictCols = desc.getAllColumnsNeedDictionaryBuilt();
    List<TblColRef> dimCols = desc.listDimensionColumnsExcludingDerived(true);
    for (TblColRef colRef : allCols) {
        if (dictCols.contains(colRef)) {
            allDimDictCols.add(colRef);
        } else if (dimCols.indexOf(colRef) >= 0){
            allDimDictCols.add(colRef);
        }
    }

    colIdToReducerBeginId = new int[allDimDictCols.size() + 1];

    int uhcReducerCount = cube.getConfig().getUHCReducerCount();
    List<TblColRef> uhcList = desc.getAllUHCColumns();
    int counter = 0;
    for (int i = 0; i < allDimDictCols.size(); i++) {
        colIdToReducerBeginId[i] = counter;
        boolean isUHC = uhcList.contains(allDimDictCols.get(i));
        counter += (isUHC) ? uhcReducerCount : 1;
    }
    colIdToReducerBeginId[allDimDictCols.size()] = counter;
    nDimReducers = counter;

    nCuboidRowCounters = cuboidRowCounterReducerNum == 0 ? //
            MapReduceUtil.getCuboidHLLCounterReducerNum(cube) : cuboidRowCounterReducerNum;
    nTotalReducers = nDimReducers + nCuboidRowCounters;

    reducerRolePlay = new int[nTotalReducers];
    for (int i = 0, dictId = 0; i < nTotalReducers; i++) {
        if (i >= nDimReducers) {
            // cuboid HLL counter reducer
            reducerRolePlay[i] = MARK_FOR_HLL_COUNTER;
        } else {
            if (i == colIdToReducerBeginId[dictId + 1])
                dictId++;

            reducerRolePlay[i] = dictId;
        }
    }
}