org.apache.kylin.cube.model.RowKeyColDesc Java Examples

The following examples show how to use org.apache.kylin.cube.model.RowKeyColDesc. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ParsedStreamingCubeInfo.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public static DimensionEncoding[] getDimensionEncodings(CubeDesc cubeDesc, TblColRef[] dimensions,
        Map<TblColRef, Dictionary<String>> dimDictMap) {
    DimensionEncoding[] result = new DimensionEncoding[dimensions.length];
    for (int i = 0; i < dimensions.length; i++) {
        TblColRef dimension = dimensions[i];
        RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(dimension);
        if (colDesc.isUsingDictionary()) {
            @SuppressWarnings({ "unchecked" })
            Dictionary<String> dict = dimDictMap.get(dimension);
            if (dict == null) {
                throw new RuntimeException("No dictionary found for dict-encoding column " + dimension);
            } else {
                result[i] = new DictionaryDimEnc(dict);
            }
        } else {
            result[i] = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(),
                    colDesc.getEncodingVersion());
        }
    }
    return result;
}
Example #2
Source File: CubeDimEncMap.java    From kylin with Apache License 2.0 6 votes vote down vote up 
@Override
public DimensionEncoding get(TblColRef col) {
    DimensionEncoding result = encMap.get(col);
    if (result == null) {
        RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col);
        if (colDesc.isUsingDictionary()) {
            // special dictionary encoding
            Dictionary<String> dict = getDictionary(col);
            if (dict == null) {
                logger.warn("No dictionary found for dict-encoding column " + col + ", segment " + seg);
                result = new FixedLenDimEnc(0);
            } else {
                result = new DictionaryDimEnc(dict);
            }
        } else {
            // normal case
            result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(), colDesc.getEncodingVersion());
        }
        encMap.put(col, result);
    }
    return result;
}
Example #3
Source File: RowKeyAttrRule.java    From kylin with Apache License 2.0 6 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    RowKeyDesc row = cube.getRowkey();
    if (row == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
        return;
    }

    RowKeyColDesc[] rcd = row.getRowKeyColumns();
    if (rcd == null || rcd.length == 0) {
        context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
        return;
    }

    for (int i = 0; i < rcd.length; i++) {
        RowKeyColDesc rd = rcd[i];
        if (rd.getColumn() == null || rd.getColumn().length() == 0) {
            context.addResult(ResultLevel.ERROR, "Rowkey column empty");
        }

    }

}
Example #4
Source File: RowRecordReader.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private Map<String, DimensionEncoding> getDimensionEncodings(FragmentMetaInfo fragmentMetaInfo,
        List<DimensionMetaInfo> allDimensions, FSDataInputStream dictInputStream) throws IOException {
    Map<String, Dictionary> dictionaryMap = readAllDimensionsDictionary(fragmentMetaInfo, dictInputStream);

    Map<String, DimensionEncoding> result = Maps.newHashMap();
    for (DimensionMetaInfo dimension : allDimensions) {
        TblColRef col = cubeDesc.getModel().findColumn(dimension.getName());
        RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col);
        if (colDesc.isUsingDictionary()) {
            @SuppressWarnings({ "unchecked" })
            Dictionary<String> dict = dictionaryMap.get(dimension.getName());
            if (dict == null) {
                logger.error("No dictionary found for dict-encoding column " + col);
                throw new RuntimeException("No dictionary found for dict-encoding column " + col);
            } else {
                result.put(dimension.getName(), new DictionaryDimEnc(dict));
            }
        } else {
            result.put(
                    dimension.getName(),
                    DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(),
                            colDesc.getEncodingVersion()));
        }
    }
    return result;
}
Example #5
Source File: CubeDescCreator.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public static int getTimeRowKeyColDesc(String tableName, RowKeyColDesc[] rowKeyColDescs) {
    int idx = 0;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.DAY_DATE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.WEEK_BEGIN_DATE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.MONTH.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.YEAR.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.TIME_HOUR.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.TIME_MINUTE.toString(), idx + 1);
    idx++;
    return idx;
}
Example #6
Source File: ParsedStreamingCubeInfo.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public static DimensionEncoding[] getDimensionEncodings(CubeDesc cubeDesc, TblColRef[] dimensions,
        Map<TblColRef, Dictionary<String>> dimDictMap) {
    DimensionEncoding[] result = new DimensionEncoding[dimensions.length];
    for (int i = 0; i < dimensions.length; i++) {
        TblColRef dimension = dimensions[i];
        RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(dimension);
        if (colDesc.isUsingDictionary()) {
            @SuppressWarnings({ "unchecked" })
            Dictionary<String> dict = dimDictMap.get(dimension);
            if (dict == null) {
                throw new RuntimeException("No dictionary found for dict-encoding column " + dimension);
            } else {
                result[i] = new DictionaryDimEnc(dict);
            }
        } else {
            result[i] = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(),
                    colDesc.getEncodingVersion());
        }
    }
    return result;
}
Example #7
Source File: ParsedStreamingCubeInfo.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public void init(CubeDesc cubeDesc, CubeJoinedFlatTableEnrich intermediateTableDesc) {
    dimensions = Lists.newArrayList();
    columnsIndex = new int[Long.bitCount(cuboidID)];

    int colIdx = 0;
    RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns();
    for (int i = 0; i < allColumns.length; i++) {
        // NOTE: the order of column in list!!!
        long bitmask = 1L << allColumns[i].getBitIndex();
        if ((cuboidID & bitmask) != 0) {
            TblColRef colRef = allColumns[i].getColRef();
            dimensions.add(colRef);
            columnsIndex[colIdx] = intermediateTableDesc.getColumnIndex(colRef);
            colIdx++;
        }
    }
}
Example #8
Source File: ParsedStreamingCubeInfo.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public void init(CubeDesc cubeDesc, CubeJoinedFlatTableEnrich intermediateTableDesc) {
    dimensions = Lists.newArrayList();
    columnsIndex = new int[Long.bitCount(cuboidID)];

    int colIdx = 0;
    RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns();
    for (int i = 0; i < allColumns.length; i++) {
        // NOTE: the order of column in list!!!
        long bitmask = 1L << allColumns[i].getBitIndex();
        if ((cuboidID & bitmask) != 0) {
            TblColRef colRef = allColumns[i].getColRef();
            dimensions.add(colRef);
            columnsIndex[colIdx] = intermediateTableDesc.getColumnIndex(colRef);
            colIdx++;
        }
    }
}
Example #9
Source File: CubeDimEncMap.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
@Override
public DimensionEncoding get(TblColRef col) {
    DimensionEncoding result = encMap.get(col);
    if (result == null) {
        RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col);
        if (colDesc.isUsingDictionary()) {
            // special dictionary encoding
            Dictionary<String> dict = getDictionary(col);
            if (dict == null) {
                logger.warn("No dictionary found for dict-encoding column " + col + ", segment " + seg);
                result = new FixedLenDimEnc(0);
            } else {
                result = new DictionaryDimEnc(dict);
            }
        } else {
            // normal case
            result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(), colDesc.getEncodingVersion());
        }
        encMap.put(col, result);
    }
    return result;
}
Example #10
Source File: RowKeyAttrRule.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    RowKeyDesc row = cube.getRowkey();
    if (row == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
        return;
    }

    RowKeyColDesc[] rcd = row.getRowKeyColumns();
    if (rcd == null || rcd.length == 0) {
        context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
        return;
    }

    for (int i = 0; i < rcd.length; i++) {
        RowKeyColDesc rd = rcd[i];
        if (rd.getColumn() == null || rd.getColumn().length() == 0) {
            context.addResult(ResultLevel.ERROR, "Rowkey column empty");
        }

    }

}
Example #11
Source File: RowRecordReader.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private Map<String, DimensionEncoding> getDimensionEncodings(FragmentMetaInfo fragmentMetaInfo,
        List<DimensionMetaInfo> allDimensions, FSDataInputStream dictInputStream) throws IOException {
    Map<String, Dictionary> dictionaryMap = readAllDimensionsDictionary(fragmentMetaInfo, dictInputStream);

    Map<String, DimensionEncoding> result = Maps.newHashMap();
    for (DimensionMetaInfo dimension : allDimensions) {
        TblColRef col = cubeDesc.getModel().findColumn(dimension.getName());
        RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col);
        if (colDesc.isUsingDictionary()) {
            @SuppressWarnings({ "unchecked" })
            Dictionary<String> dict = dictionaryMap.get(dimension.getName());
            if (dict == null) {
                logger.error("No dictionary found for dict-encoding column " + col);
                throw new RuntimeException("No dictionary found for dict-encoding column " + col);
            } else {
                result.put(dimension.getName(), new DictionaryDimEnc(dict));
            }
        } else {
            result.put(
                    dimension.getName(),
                    DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(),
                            colDesc.getEncodingVersion()));
        }
    }
    return result;
}
Example #12
Source File: CubeDescCreator.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public static int getTimeRowKeyColDesc(String tableName, RowKeyColDesc[] rowKeyColDescs) {
    int idx = 0;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.DAY_DATE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.WEEK_BEGIN_DATE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.MONTH.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.YEAR.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.TIME_HOUR.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.TIME_MINUTE.toString(), idx + 1);
    idx++;
    return idx;
}
Example #13
Source File: CubeSizeEstimationCLI.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private static int[] estimateRowKeyColSpace(RowKeyDesc rowKeyDesc, long[] cardinality) {
    RowKeyColDesc[] rowKeyColDescs = rowKeyDesc.getRowKeyColumns();
    int[] ret = new int[rowKeyColDescs.length];
    for (int i = 0; i < rowKeyColDescs.length; ++i) {
        RowKeyColDesc rowKeyColDesc = rowKeyColDescs[rowKeyColDescs.length - 1 - i];
        if (rowKeyColDesc.getDictionary() == null) {
            if (rowKeyColDesc.getLength() == 0)
                throw new IllegalStateException("The non-dictionary col " + rowKeyColDesc.getColumn() + " has length of 0");
            ret[i] = rowKeyColDesc.getLength();
        } else {
            ret[i] = estimateDictionaryColSpace(cardinality[i]);
        }
    }
    return ret;
}
Example #14
Source File: JoinedFlatTable.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public static String generateRedistributeFlatTableStatement(IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) {
    final String tableName = flatDesc.getTableName();
    StringBuilder sql = new StringBuilder();
    sql.append("INSERT OVERWRITE TABLE " + quoteIdentifier(tableName, null) + " SELECT * FROM " + quoteIdentifier(tableName, null));

    if (flatDesc.getClusterBy() != null) {
        appendClusterStatement(sql, flatDesc.getClusterBy());
    } else if (flatDesc.getDistributedBy() != null) {
        appendDistributeStatement(sql, Lists.newArrayList(flatDesc.getDistributedBy()));
    } else {
        int redistColumnCount = cubeDesc.getConfig().getHiveRedistributeColumnCount();

        RowKeyColDesc[] rowKeyColDescs = cubeDesc.getRowkey().getRowKeyColumns();

        if (rowKeyColDescs.length < redistColumnCount)
            redistColumnCount = rowKeyColDescs.length;

        List<TblColRef> redistColumns = Lists.newArrayListWithCapacity(redistColumnCount);

        for (int i = 0; i < redistColumnCount; i++) {
            redistColumns.add(rowKeyColDescs[i].getColRef());
        }

        appendDistributeStatement(sql, redistColumns);
    }

    return sql.toString();
}
Example #15
Source File: Cuboid.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private List<TblColRef> translateIdToColumns(long cuboidID) {
    List<TblColRef> dimesnions = new ArrayList<TblColRef>();
    RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns();
    for (int i = 0; i < allColumns.length; i++) {
        // NOTE: the order of column in list!!!
        long bitmask = 1L << allColumns[i].getBitIndex();
        if ((cuboidID & bitmask) != 0) {
            TblColRef colRef = allColumns[i].getColRef();
            dimesnions.add(colRef);
        }
    }
    return dimesnions;
}
Example #16
Source File: Cuboid.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private List<TblColRef> translateIdToColumns(long cuboidID) {
    List<TblColRef> dimesnions = new ArrayList<TblColRef>();
    RowKeyColDesc[] allColumns = cube.getRowkey().getRowKeyColumns();
    for (int i = 0; i < allColumns.length; i++) {
        // NOTE: the order of column in list!!!
        long bitmask = 1L << allColumns[i].getBitIndex();
        if ((cuboidID & bitmask) != 0) {
            TblColRef colRef = allColumns[i].getColRef();
            dimesnions.add(colRef);
        }
    }
    return dimesnions;
}
Example #17
Source File: JoinedFlatTable.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public static String generateRedistributeFlatTableStatement(IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) {
    final String tableName = flatDesc.getTableName();
    StringBuilder sql = new StringBuilder();
    sql.append("INSERT OVERWRITE TABLE " + quoteIdentifier(tableName, null) + " SELECT * FROM " + quoteIdentifier(tableName, null));

    if (flatDesc.getClusterBy() != null) {
        appendClusterStatement(sql, flatDesc.getClusterBy());
    } else if (flatDesc.getDistributedBy() != null) {
        appendDistributeStatement(sql, Lists.newArrayList(flatDesc.getDistributedBy()));
    } else {
        int redistColumnCount = cubeDesc.getConfig().getHiveRedistributeColumnCount();

        RowKeyColDesc[] rowKeyColDescs = cubeDesc.getRowkey().getRowKeyColumns();

        if (rowKeyColDescs.length < redistColumnCount)
            redistColumnCount = rowKeyColDescs.length;

        List<TblColRef> redistColumns = Lists.newArrayListWithCapacity(redistColumnCount);

        for (int i = 0; i < redistColumnCount; i++) {
            redistColumns.add(rowKeyColDescs[i].getColRef());
        }

        appendDistributeStatement(sql, redistColumns);
    }

    return sql.toString();
}
Example #18
Source File: MandatoryColumnRule.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    Set<String> mands = new HashSet<String>();
    RowKeyColDesc[] cols = cube.getRowkey().getRowKeyColumns();
    if (cols == null || cols.length == 0) {
        return;
    }
    for (int i = 0; i < cols.length; i++) {
        RowKeyColDesc rowKeyColDesc = cols[i];
        if (rowKeyColDesc.isMandatory()) {
            mands.add(rowKeyColDesc.getColumn());
        }
    }
    if (mands.isEmpty()) {
        return;
    }
    String[][] groups = cube.getRowkey().getAggregationGroups();
    for (int i = 0; i < groups.length; i++) {
        String[] group = groups[i];
        for (int j = 0; j < group.length; j++) {
            String col = group[j];
            if (mands.contains(col)) {
                context.addResult(ResultLevel.ERROR, "mandatory column " + col + " must not be in aggregation group [" + ArrayUtils.toString(group) + "]");
            }
        }
    }

}
Example #19
Source File: GTCubeStorageQueryBase.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private long getQueryFilterMask(Set<TblColRef> filterColumnD) {
    long filterMask = 0;

    logger.info("Filter column set for query: {}", filterColumnD);
    if (filterColumnD.isEmpty() == false) {
        RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns();
        for (int i = 0; i < allColumns.length; i++) {
            if (filterColumnD.contains(allColumns[i].getColRef())) {
                filterMask |= 1L << allColumns[i].getBitIndex();
            }
        }
    }
    logger.info("Filter mask is: {}", filterMask);
    return filterMask;
}
Example #20
Source File: CubeDescCreator.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public static RowKeyColDesc getRowKeyColDesc(String tableName, String column, int id) {
    RowKeyColDesc rowKeyColDesc = new RowKeyColDesc();
    rowKeyColDesc.setIndex(Integer.toString(id));
    rowKeyColDesc.setColumn(tableName.substring(tableName.lastIndexOf(".") + 1) + "." + column);
    rowKeyColDesc.setEncoding(DictionaryDimEnc.ENCODING_NAME);
    rowKeyColDesc.setShardBy(false);
    return rowKeyColDesc;
}
Example #21
Source File: RowKeyAttrRule.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    RowKeyDesc row = cube.getRowkey();
    if (row == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
        return;
    }

    RowKeyColDesc[] rcd = row.getRowKeyColumns();
    if (rcd == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
        return;
    }
    if(rcd.length == 0){
        context.addResult(ResultLevel.ERROR, "Rowkey columns is empty");
        return;       	
    }

    for (int i = 0; i < rcd.length; i++) {
        RowKeyColDesc rd = rcd[i];
        if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary())&&!rd.getDictionary().equals("false"))) {
            context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute");
        }
        if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary())||rd.getDictionary().equals("false"))) {
            context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty");
        }
    }

}
Example #22
Source File: RowKeySplitter.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
    this.enableSharding = cubeSeg.isEnableSharding();
    this.cubeDesc = cubeSeg.getCubeDesc();
    IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg);

    for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
        dimEncoding.get(rowKeyColDesc.getColRef());
    }

    this.colIO = new RowKeyColumnIO(dimEncoding);

    this.splitBuffers = new ByteArray[splitLen];
    this.splitOffsets = new int[splitLen];
    this.bufferSize = 0;
}
Example #23
Source File: ResponseResultSchema.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void init(Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics) {
    this.dimensions = new TblColRef[selectedDimensions.size()];
    this.metrics = new FunctionDesc[selectedMetrics.size()];
    this.measures = new MeasureDesc[selectedMetrics.size()];
    this.dimDataTypes = new DataType[dimensions.length];
    this.metricsDataTypes = new DataType[metrics.length];
    // sort dimensions according to the rowKey definition
    dimColIdxMap = Maps.newHashMap();
    RowKeyDesc rowKeyDesc = cubeDesc.getRowkey();
    int colIdx = 0;
    for (RowKeyColDesc rowKeyColDesc : rowKeyDesc.getRowKeyColumns()) {
        TblColRef dimension = rowKeyColDesc.getColRef();
        if (selectedDimensions.contains(dimension)) {
            dimensions[colIdx] = dimension;
            dimDataTypes[colIdx] = dimension.getType();
            dimColIdxMap.put(dimension, colIdx);
            colIdx++;
        }
    }

    nDimensions = colIdx;

    colIdx = 0;
    // metrics
    metricsColIdxMap = Maps.newHashMap();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc func = measure.getFunction();
        if (selectedMetrics.contains(func)) {
            metrics[colIdx] = func;
            measures[colIdx] = measure;
            metricsColIdxMap.put(func.getParameter().getColRef(), colIdx);
            metricsDataTypes[colIdx] = func.getReturnDataType();
            colIdx++;
        }
    }

    nMetrics = colIdx;
}
Example #24
Source File: CubeDescCreator.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public static RowKeyColDesc getRowKeyColDesc(String tableName, String column, int id) {
    RowKeyColDesc rowKeyColDesc = new RowKeyColDesc();
    rowKeyColDesc.setIndex(Integer.toString(id));
    rowKeyColDesc.setColumn(tableName.substring(tableName.lastIndexOf(".") + 1) + "." + column);
    rowKeyColDesc.setEncoding(DictionaryDimEnc.ENCODING_NAME);
    rowKeyColDesc.setShardBy(false);
    return rowKeyColDesc;
}
Example #25
Source File: CubeController.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void validateColumnFamily(CubeDesc cubeDesc) {
    Set<String> columnFamilyMetricsSet = Sets.newHashSet();
    for (HBaseColumnFamilyDesc hBaseColumnFamilyDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc hBaseColumnDesc : hBaseColumnFamilyDesc.getColumns()) {
            for (String columnName : hBaseColumnDesc.getMeasureRefs()) {
                columnFamilyMetricsSet.add(columnName);
            }
        }
    }
    for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
        if (!columnFamilyMetricsSet.contains(measureDesc.getName())) {
            throw new BadRequestException("column family lack measure:" + measureDesc.getName());
        }
    }
    if (cubeDesc.getMeasures().size() != columnFamilyMetricsSet.size()) {
        throw new BadRequestException(
                "the number of input measure and the number of measure defined in cubedesc are not consistent");
    }

    for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
        Object[] encodingConf = DimensionEncoding.parseEncodingConf(rowKeyColDesc.getEncoding());
        String encodingName = (String) encodingConf[0];
        String[] encodingArgs = (String[]) encodingConf[1];

        if (!DimensionEncodingFactory.isValidEncoding(encodingName, encodingArgs,
                rowKeyColDesc.getEncodingVersion())) {
            throw new BadRequestException("Illegal row key column desc: " + rowKeyColDesc);
        }
    }
}
Example #26
Source File: CubeController.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void validateColumnFamily(CubeDesc cubeDesc) {
    Set<String> columnFamilyMetricsSet = Sets.newHashSet();
    for (HBaseColumnFamilyDesc hBaseColumnFamilyDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc hBaseColumnDesc : hBaseColumnFamilyDesc.getColumns()) {
            for (String columnName : hBaseColumnDesc.getMeasureRefs()) {
                columnFamilyMetricsSet.add(columnName);
            }
        }
    }
    for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
        if (!columnFamilyMetricsSet.contains(measureDesc.getName())) {
            throw new BadRequestException("column family lack measure:" + measureDesc.getName());
        }
    }
    if (cubeDesc.getMeasures().size() != columnFamilyMetricsSet.size()) {
        throw new BadRequestException(
                "the number of input measure and the number of measure defined in cubedesc are not consistent");
    }

    for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
        Object[] encodingConf = DimensionEncoding.parseEncodingConf(rowKeyColDesc.getEncoding());
        String encodingName = (String) encodingConf[0];
        String[] encodingArgs = (String[]) encodingConf[1];

        if (!DimensionEncodingFactory.isValidEncoding(encodingName, encodingArgs,
                rowKeyColDesc.getEncodingVersion())) {
            throw new BadRequestException("Illegal row key column desc: " + rowKeyColDesc);
        }
    }
}
Example #27
Source File: GTCubeStorageQueryBase.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private long getQueryFilterMask(Set<TblColRef> filterColumnD) {
    long filterMask = 0;

    logger.info("Filter column set for query: {}", filterColumnD);
    if (filterColumnD.isEmpty() == false) {
        RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns();
        for (int i = 0; i < allColumns.length; i++) {
            if (filterColumnD.contains(allColumns[i].getColRef())) {
                filterMask |= 1L << allColumns[i].getBitIndex();
            }
        }
    }
    logger.info("Filter mask is: {}", filterMask);
    return filterMask;
}
Example #28
Source File: ResponseResultSchema.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void init(Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics) {
    this.dimensions = new TblColRef[selectedDimensions.size()];
    this.metrics = new FunctionDesc[selectedMetrics.size()];
    this.measures = new MeasureDesc[selectedMetrics.size()];
    this.dimDataTypes = new DataType[dimensions.length];
    this.metricsDataTypes = new DataType[metrics.length];
    // sort dimensions according to the rowKey definition
    dimColIdxMap = Maps.newHashMap();
    RowKeyDesc rowKeyDesc = cubeDesc.getRowkey();
    int colIdx = 0;
    for (RowKeyColDesc rowKeyColDesc : rowKeyDesc.getRowKeyColumns()) {
        TblColRef dimension = rowKeyColDesc.getColRef();
        if (selectedDimensions.contains(dimension)) {
            dimensions[colIdx] = dimension;
            dimDataTypes[colIdx] = dimension.getType();
            dimColIdxMap.put(dimension, colIdx);
            colIdx++;
        }
    }

    nDimensions = colIdx;

    colIdx = 0;
    // metrics
    metricsColIdxMap = Maps.newHashMap();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc func = measure.getFunction();
        if (selectedMetrics.contains(func)) {
            metrics[colIdx] = func;
            measures[colIdx] = measure;
            metricsColIdxMap.put(func.getParameter().getColRef(), colIdx);
            metricsDataTypes[colIdx] = func.getReturnDataType();
            colIdx++;
        }
    }

    nMetrics = colIdx;
}
Example #29
Source File: RowKeySplitter.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
    this.enableSharding = cubeSeg.isEnableSharding();
    this.cubeDesc = cubeSeg.getCubeDesc();
    IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg);

    for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
        dimEncoding.get(rowKeyColDesc.getColRef());
    }

    this.colIO = new RowKeyColumnIO(dimEncoding);

    this.splitBuffers = new ByteArray[splitLen];
    this.splitOffsets = new int[splitLen];
    this.bufferSize = 0;
}
Example #30
Source File: Cuboid.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private List<TblColRef> translateIdToColumns(long cuboidID) {
    List<TblColRef> dimesnions = new ArrayList<TblColRef>();
    RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns();
    for (int i = 0; i < allColumns.length; i++) {
        // NOTE: the order of column in list!!!
        long bitmask = 1L << allColumns[i].getBitIndex();
        if ((cuboidID & bitmask) != 0) {
            TblColRef colRef = allColumns[i].getColRef();
            dimesnions.add(colRef);
        }
    }
    return dimesnions;
}