Java Code Examples for org.apache.kylin.cube.cuboid.Cuboid#getBaseCuboid()

The following examples show how to use org.apache.kylin.cube.cuboid.Cuboid#getBaseCuboid() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CubeStatsReader.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private static Map<Long, Double> getCuboidSizeMapFromRowCount(CubeSegment cubeSegment, Map<Long, Long> rowCountMap,
                                                              long sourceRowCount, boolean origin) {
    final CubeDesc cubeDesc = cubeSegment.getCubeDesc();
    final List<Integer> rowkeyColumnSize = Lists.newArrayList();
    final Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc);
    final List<TblColRef> columnList = baseCuboid.getColumns();
    final CubeDimEncMap dimEncMap = cubeSegment.getDimensionEncodingMap();
    final Long baseCuboidRowCount = rowCountMap.get(baseCuboid.getId());

    for (int i = 0; i < columnList.size(); i++) {
        rowkeyColumnSize.add(dimEncMap.get(columnList.get(i)).getLengthOfEncoding());
    }

    Map<Long, Double> sizeMap = Maps.newHashMap();
    for (Map.Entry<Long, Long> entry : rowCountMap.entrySet()) {
        sizeMap.put(entry.getKey(), estimateCuboidStorageSize(cubeSegment, entry.getKey(), entry.getValue(),
                baseCuboid.getId(), baseCuboidRowCount, rowkeyColumnSize, sourceRowCount));
    }

    if (origin == false && cubeSegment.getConfig().enableJobCuboidSizeOptimize()) {
        optimizeSizeMap(sizeMap, cubeSegment);
    }

    return sizeMap;
}
Example 2
Source File: BaseCuboidBuilder.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public BaseCuboidBuilder(KylinConfig kylinConfig, CubeDesc cubeDesc, CubeSegment cubeSegment,
        CubeJoinedFlatTableEnrich intermediateTableDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
    this.kylinConfig = kylinConfig;
    this.cubeDesc = cubeDesc;
    this.cubeSegment = cubeSegment;
    this.intermediateTableDesc = intermediateTableDesc;
    this.dictionaryMap = dictionaryMap;

    Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc);
    rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);

    measureDescList = cubeDesc.getMeasures();
    aggrIngesters = MeasureIngester.create(measureDescList);
    measureCodec = new BufferedMeasureCodec(measureDescList);

    kvBuilder = new KeyValueBuilder(intermediateTableDesc);
    checkMrDictClolumn();
}
Example 3
Source File: CubeStatsReader.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private static Map<Long, Double> getCuboidSizeMapFromRowCount(CubeSegment cubeSegment, Map<Long, Long> rowCountMap,
                                                              long sourceRowCount, boolean origin) {
    final CubeDesc cubeDesc = cubeSegment.getCubeDesc();
    final List<Integer> rowkeyColumnSize = Lists.newArrayList();
    final Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc);
    final List<TblColRef> columnList = baseCuboid.getColumns();
    final CubeDimEncMap dimEncMap = cubeSegment.getDimensionEncodingMap();
    final Long baseCuboidRowCount = rowCountMap.get(baseCuboid.getId());

    for (int i = 0; i < columnList.size(); i++) {
        rowkeyColumnSize.add(dimEncMap.get(columnList.get(i)).getLengthOfEncoding());
    }

    Map<Long, Double> sizeMap = Maps.newHashMap();
    for (Map.Entry<Long, Long> entry : rowCountMap.entrySet()) {
        sizeMap.put(entry.getKey(), estimateCuboidStorageSize(cubeSegment, entry.getKey(), entry.getValue(),
                baseCuboid.getId(), baseCuboidRowCount, rowkeyColumnSize, sourceRowCount));
    }

    if (origin == false && cubeSegment.getConfig().enableJobCuboidSizeOptimize()) {
        optimizeSizeMap(sizeMap, cubeSegment);
    }

    return sizeMap;
}
Example 4
Source File: BaseCuboidBuilder.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public BaseCuboidBuilder(KylinConfig kylinConfig, CubeDesc cubeDesc, CubeSegment cubeSegment,
        CubeJoinedFlatTableEnrich intermediateTableDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
    this.kylinConfig = kylinConfig;
    this.cubeDesc = cubeDesc;
    this.cubeSegment = cubeSegment;
    this.intermediateTableDesc = intermediateTableDesc;
    this.dictionaryMap = dictionaryMap;

    Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc);
    rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);

    measureDescList = cubeDesc.getMeasures();
    aggrIngesters = MeasureIngester.create(measureDescList);
    measureCodec = new BufferedMeasureCodec(measureDescList);

    kvBuilder = new KeyValueBuilder(intermediateTableDesc);
    checkHiveGlobalDictionaryColumn();
}
Example 5
Source File: CubeStatsReader.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void printKVInfo(PrintWriter writer) {
    Cuboid cuboid = Cuboid.getBaseCuboid(seg.getCubeDesc());
    RowKeyEncoder encoder = new RowKeyEncoder(seg, cuboid);
    for (TblColRef col : cuboid.getColumns()) {
        writer.println("Length of dimension " + col + " is " + encoder.getColumnLength(col));
    }
}
Example 6
Source File: ColumnarSplitDataReader.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public void init(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    baseCuboid = Cuboid.getBaseCuboid(cubeDesc);
    rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);

    FileSystem fs = FileSystem.get(context.getConfiguration());
    FileSplit fSplit = (FileSplit) split;
    Path path = fSplit.getPath();
    rowRecordReader = new RowRecordReader(cubeDesc, path, fs);
    metricsValuesBuffer = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);

    rowCount = new AtomicInteger(0);
}
Example 7
Source File: CubeJoinedFlatTableEnrich.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void parseCubeDesc() {
    Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc);

    // build index for rowkey columns
    List<TblColRef> cuboidColumns = baseCuboid.getColumns();
    int rowkeyColCount = cubeDesc.getRowkey().getRowKeyColumns().length;
    rowKeyColumnIndexes = new int[rowkeyColCount];
    for (int i = 0; i < rowkeyColCount; i++) {
        TblColRef col = cuboidColumns.get(i);
        rowKeyColumnIndexes[i] = flatDesc.getColumnIndex(col);
    }

    List<MeasureDesc> measures = cubeDesc.getMeasures();
    int measureSize = measures.size();
    measureColumnIndexes = new int[measureSize][];
    for (int i = 0; i < measureSize; i++) {
        FunctionDesc func = measures.get(i).getFunction();
        List<TblColRef> colRefs = func.getParameter().getColRefs();
        if (colRefs == null) {
            measureColumnIndexes[i] = null;
        } else {
            measureColumnIndexes[i] = new int[colRefs.size()];
            for (int j = 0; j < colRefs.size(); j++) {
                TblColRef c = colRefs.get(j);
                measureColumnIndexes[i][j] = flatDesc.getColumnIndex(c);
            }
        }
    }
}
Example 8
Source File: GTScanReqSerDerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGTInfo() {
    CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("test_kylin_cube_with_slr_ready");
    CubeSegment segment = cube.getFirstSegment();

    Cuboid baseCuboid = Cuboid.getBaseCuboid(cube.getDescriptor());
    GTInfo info = CubeGridTable.newGTInfo(baseCuboid, new CubeDimEncMap(segment));
    GTInfo.serializer.serialize(info, buffer);
    buffer.flip();

    GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
    this.compareTwoGTInfo(info, sInfo);
}
Example 9
Source File: ParsedStreamingCubeInfo.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public ParsedStreamingCubeInfo(CubeInstance cubeInstance) {
    this.cubeInstance = cubeInstance;
    this.cubeDesc = cubeInstance.getDescriptor();
    this.basicCuboid = Cuboid.getBaseCuboid(cubeDesc);
    this.intermediateTableDesc = new CubeJoinedFlatTableEnrich(new CubeJoinedFlatTableDesc(cubeDesc), cubeDesc);
    this.measureCount = cubeDesc.getMeasures().size();
    this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);
    this.measureIngesters = MeasureIngester.create(cubeDesc.getMeasures());
    this.dimensions = basicCuboid.getColumns().toArray(new TblColRef[basicCuboid.getColumns().size()]);
    this.dimCount = dimensions.length;
    this.basicCuboidMapping = new CuboidToGridTableMapping(basicCuboid);

    boolean buildAdditionalCuboids = cubeDesc.getConfig().isStreamingBuildAdditionalCuboids();
    Set<Long> mandatoryCuboids = cubeDesc.getMandatoryCuboids();
    if (buildAdditionalCuboids) {
        additionalCuboidsToBuild = Lists.newArrayListWithCapacity(mandatoryCuboids.size());
        for (long cuboidID : mandatoryCuboids) {
            CuboidInfo cuboidInfo = new CuboidInfo(cuboidID);
            cuboidInfo.init(cubeDesc, intermediateTableDesc);
            additionalCuboidsToBuild.add(cuboidInfo);
        }
    }

    List<TblColRef> dimUseDictList = Lists.newArrayList();
    for (TblColRef column : dimensions) {
        dimensionsMap.put(column.getName(), column);
        if (cubeDesc.getRowkey().isUseDictionary(column)) {
            dimUseDictList.add(column);
        }
    }
    this.dimensionsUseDictEncoding = dimUseDictList.toArray(new TblColRef[dimUseDictList.size()]);

    List<String> metricsAggrFuncsList = Lists.newArrayListWithCapacity(measureCount);
    for (int i = 0; i < measureCount; i++) {
        MeasureDesc measureDesc = measureDescs[i];
        metricsAggrFuncsList.add(measureDesc.getFunction().getExpression());
    }
    this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]);
}
Example 10
Source File: CubeStatsReader.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void printKVInfo(PrintWriter writer) {
    Cuboid cuboid = Cuboid.getBaseCuboid(seg.getCubeDesc());
    RowKeyEncoder encoder = new RowKeyEncoder(seg, cuboid);
    for (TblColRef col : cuboid.getColumns()) {
        writer.println("Length of dimension " + col + " is " + encoder.getColumnLength(col));
    }
}
Example 11
Source File: ColumnarSplitDataReader.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public void init(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    baseCuboid = Cuboid.getBaseCuboid(cubeDesc);
    rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);

    FileSystem fs = FileSystem.get(context.getConfiguration());
    FileSplit fSplit = (FileSplit) split;
    Path path = fSplit.getPath();
    rowRecordReader = new RowRecordReader(cubeDesc, path, fs);
    metricsValuesBuffer = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE);

    rowCount = new AtomicInteger(0);
}
Example 12
Source File: CubeJoinedFlatTableEnrich.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void parseCubeDesc() {
    Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc);

    // build index for rowkey columns
    List<TblColRef> cuboidColumns = baseCuboid.getColumns();
    int rowkeyColCount = cubeDesc.getRowkey().getRowKeyColumns().length;
    rowKeyColumnIndexes = new int[rowkeyColCount];
    for (int i = 0; i < rowkeyColCount; i++) {
        TblColRef col = cuboidColumns.get(i);
        rowKeyColumnIndexes[i] = flatDesc.getColumnIndex(col);
    }

    List<MeasureDesc> measures = cubeDesc.getMeasures();
    int measureSize = measures.size();
    measureColumnIndexes = new int[measureSize][];
    for (int i = 0; i < measureSize; i++) {
        FunctionDesc func = measures.get(i).getFunction();
        List<TblColRef> colRefs = func.getParameter().getColRefs();
        if (colRefs == null) {
            measureColumnIndexes[i] = null;
        } else {
            measureColumnIndexes[i] = new int[colRefs.size()];
            for (int j = 0; j < colRefs.size(); j++) {
                TblColRef c = colRefs.get(j);
                measureColumnIndexes[i][j] = flatDesc.getColumnIndex(c);
            }
        }
    }
}
Example 13
Source File: GTScanReqSerDerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGTInfo() {
    CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("test_kylin_cube_with_slr_ready");
    CubeSegment segment = cube.getFirstSegment();

    Cuboid baseCuboid = Cuboid.getBaseCuboid(cube.getDescriptor());
    GTInfo info = CubeGridTable.newGTInfo(baseCuboid, new CubeDimEncMap(segment));
    GTInfo.serializer.serialize(info, buffer);
    buffer.flip();

    GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
    this.compareTwoGTInfo(info, sInfo);
}
Example 14
Source File: ParsedStreamingCubeInfo.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public ParsedStreamingCubeInfo(CubeInstance cubeInstance) {
    this.cubeInstance = cubeInstance;
    this.cubeDesc = cubeInstance.getDescriptor();
    this.basicCuboid = Cuboid.getBaseCuboid(cubeDesc);
    this.intermediateTableDesc = new CubeJoinedFlatTableEnrich(new CubeJoinedFlatTableDesc(cubeDesc), cubeDesc);
    this.measureCount = cubeDesc.getMeasures().size();
    this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);
    this.measureIngesters = MeasureIngester.create(cubeDesc.getMeasures());
    this.dimensions = basicCuboid.getColumns().toArray(new TblColRef[basicCuboid.getColumns().size()]);
    this.dimCount = dimensions.length;
    this.basicCuboidMapping = new CuboidToGridTableMapping(basicCuboid);

    boolean buildAdditionalCuboids = cubeDesc.getConfig().isStreamingBuildAdditionalCuboids();
    Set<Long> mandatoryCuboids = cubeDesc.getMandatoryCuboids();
    if (buildAdditionalCuboids) {
        additionalCuboidsToBuild = Lists.newArrayListWithCapacity(mandatoryCuboids.size());
        for (long cuboidID : mandatoryCuboids) {
            CuboidInfo cuboidInfo = new CuboidInfo(cuboidID);
            cuboidInfo.init(cubeDesc, intermediateTableDesc);
            additionalCuboidsToBuild.add(cuboidInfo);
        }
    }

    List<TblColRef> dimUseDictList = Lists.newArrayList();
    for (TblColRef column : dimensions) {
        dimensionsMap.put(column.getName(), column);
        if (cubeDesc.getRowkey().isUseDictionary(column)) {
            dimUseDictList.add(column);
        }
    }
    this.dimensionsUseDictEncoding = dimUseDictList.toArray(new TblColRef[dimUseDictList.size()]);

    List<String> metricsAggrFuncsList = Lists.newArrayListWithCapacity(measureCount);
    for (int i = 0; i < measureCount; i++) {
        MeasureDesc measureDesc = measureDescs[i];
        metricsAggrFuncsList.add(measureDesc.getFunction().getExpression());
    }
    this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]);
}