Java Code Examples for org.apache.kylin.cube.cuboid.Cuboid#getBaseCuboidId()

@Override
public void open(Configuration parameters) throws Exception {
    KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl);
    try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig
            .setAndUnsetThreadLocalConfig(kConfig)) {
        CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName);
        CubeDesc cubeDesc = cubeInstance.getDescriptor();
        CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
        CubeJoinedFlatTableEnrich interDesc = new CubeJoinedFlatTableEnrich(
                EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
        long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
        Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
        baseCuboidBuilder = new BaseCuboidBuilder(kConfig, cubeDesc, cubeSegment, interDesc,
                AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid),
                MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap());
    }
}
 

@Override
protected void doSetup(Context context) throws IOException {
    Configuration conf = context.getConfiguration();
    bindCurrentConfiguration(conf);
    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

    cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
    cube = CubeManager.getInstance(config).getCube(cubeName);
    cubeSeg = cube.getSegmentById(conf.get(BatchConstants.CFG_CUBE_SEGMENT_ID));
    cubeDesc = cube.getDescriptor();
    baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    reducerMapping = new FactDistinctColumnsReducerMapping(cube);
    allCols = reducerMapping.getAllDimDictCols();

    flatTableInputFormat = MRUtil.getBatchCubingInputSide(cubeSeg).getFlatTableInputFormat();

    intermediateTableDesc = new CubeJoinedFlatTableEnrich(EngineFactory.getJoinedFlatTableDesc(cubeSeg), cubeDesc);
    columnIndex = new int[allCols.size()];
    for (int i = 0; i < allCols.size(); i++) {
        TblColRef colRef = allCols.get(i);
        int columnIndexOnFlatTbl = intermediateTableDesc.getColumnIndex(colRef);
        columnIndex[i] = columnIndexOnFlatTbl;
    }
}
 

public InMemCubeBuilder(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
        Map<TblColRef, Dictionary<String>> dictionaryMap) {
    super(cuboidScheduler, flatDesc, dictionaryMap);
    this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    this.totalCuboidCount = cuboidScheduler.getCuboidCount();

    this.measureCount = cubeDesc.getMeasures().size();
    this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);

    List<String> metricsAggrFuncsList = Lists.newArrayList();

    for (int i = 0; i < measureCount; i++) {
        MeasureDesc measureDesc = measureDescs[i];
        metricsAggrFuncsList.add(measureDesc.getFunction().getExpression());
    }
    this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]);
}
 

public int getBuildLevel() {
    int ret = 1;//base cuboid => partial cube root
    if (this.getPartialCubeFullMask() == Cuboid.getBaseCuboidId(cubeDesc)) {
        ret -= 1;//if partial cube's root is base cuboid, then one round less agg
    }

    ret += getNormalDims().size();
    for (HierarchyMask hierarchyMask : this.hierarchyMasks) {
        ret += hierarchyMask.allMasks.length;
    }
    for (Long joint : joints) {
        if ((joint & this.getHierarchyDimsMask()) == 0) {
            ret += 1;
        }
    }

    return ret;
}
 

@Override
public void open(Configuration parameters) throws Exception {
    KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl);
    try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig
            .setAndUnsetThreadLocalConfig(kConfig)) {
        CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName);
        CubeDesc cubeDesc = cubeInstance.getDescriptor();
        CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
        CubeJoinedFlatTableEnrich interDesc = new CubeJoinedFlatTableEnrich(
                EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
        long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
        Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
        baseCuboidBuilder = new BaseCuboidBuilder(kConfig, cubeDesc, cubeSegment, interDesc,
                AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid),
                MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap());
    }
}
 

private void calculateHitCuboid() {
    long basicCuboid = Cuboid.getBaseCuboidId(cubeDesc);
    this.setBasicCuboid(basicCuboid);
    if (!cubeDesc.getConfig().isStreamingBuildAdditionalCuboids()) {
        this.setHitCuboid(basicCuboid);
        return;
    }
    long targetCuboidID = identifyCuboid(dimensions);
    Set<Long> mandatoryCuboids = getMandatoryCuboids();
    for (long cuboidID : mandatoryCuboids) {
        if ((targetCuboidID & ~cuboidID) == 0) {
            this.setHitCuboid(cuboidID);
            return;
        }
    }
    this.setHitCuboid(basicCuboid);
}
 

@Test
public void testEncodeWithSlr() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
    // CubeSegment seg = cube.getTheOnlySegment();
    CubeDesc cubeDesc = cube.getDescriptor();
    // String data =
    // "1234567892013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
    byte[][] data = new byte[9][];
    data[0] = Bytes.toBytes("123456789");
    data[1] = Bytes.toBytes("2012-12-15");
    data[2] = Bytes.toBytes("11848");
    data[3] = Bytes.toBytes("Health & Beauty");
    data[4] = Bytes.toBytes("Fragrances");
    data[5] = Bytes.toBytes("Women");
    data[6] = Bytes.toBytes("FP-GTC");
    data[7] = Bytes.toBytes("0");
    data[8] = Bytes.toBytes("15");

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
    AbstractRowKeyEncoder rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(48, encodedKey.length);
    byte[] sellerId = Arrays.copyOfRange(encodedKey, 8, 26);
    byte[] cuboidId = Arrays.copyOfRange(encodedKey, 0, 8);
    byte[] rest = Arrays.copyOfRange(encodedKey, 26, encodedKey.length);
    assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
    assertEquals(511, Bytes.toLong(cuboidId));
    assertArrayEquals(new byte[] { 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest);
}
 

private boolean checkMandatoryColumns(long cuboidID) {
    if ((cuboidID & mandatoryColumnMask) != mandatoryColumnMask) {
        return false;
    } else {
        //base cuboid is always valid
        if (cuboidID == Cuboid.getBaseCuboidId(cubeDesc)) {
            return true;
        }

        //cuboid with only mandatory columns maybe valid
        return isMandatoryOnlyValid || (cuboidID & ~mandatoryColumnMask) != 0;
    }
}
 

@Override
protected void doSetup(Mapper.Context context) throws IOException {
    super.doSetup(context);

    long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
    GTInfo gtInfo = CubeGridTable.newGTInfo(Cuboid.findForMandatory(cubeDesc, baseCuboid),
            new CubeDimEncMap(cubeDesc, dictionaryMap));
    keyValueBuffer = ByteBuffer.allocate(gtInfo.getMaxRecordLength());
    keyOffset = cubeSegment.getRowKeyPreambleSize();
}
 

private void printCuboidInfoTreeEntry(Map<Long, Long> cuboidRows, Map<Long, Double> cuboidSizes, PrintWriter out) {
    if (cuboidScheduler == null) {
        throw new UnsupportedOperationException("cuboid scheduler is null");
    }
    long baseCuboid = Cuboid.getBaseCuboidId(seg.getCubeDesc());
    int dimensionCount = Long.bitCount(baseCuboid);
    printCuboidInfoTree(-1L, baseCuboid, cuboidScheduler, cuboidRows, cuboidSizes, dimensionCount, 0, out);
}
 

private boolean isSelectAllQuery(Cuboid cuboid, Set<TblColRef> groupsD, TupleFilter filterD) {
    if (Cuboid.getBaseCuboidId(cubeDesc) == cuboid.getId() && filterD == null
            && cuboid.getColumns().size() == groupsD.size()) {
        return true;
    }
    return false;
}
 

@Override
protected void setup(Context context) throws IOException {
    super.publishConfiguration(context.getConfiguration());

    cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
    segmentName = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_NAME);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration());

    metadataManager = MetadataManager.getInstance(config);
    cube = CubeManager.getInstance(config).getCube(cubeName);
    cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
    cubeDesc = cube.getDescriptor();
    factTableDesc = metadataManager.getTableDesc(cubeDesc.getFactTable());

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);

    // intermediateTableDesc = new
    // JoinedFlatTableDesc(cube.getDescriptor());

    rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);

    measureCodec = new MeasureCodec(cubeDesc.getMeasures());
    measures = new Object[cubeDesc.getMeasures().size()];

    int colCount = cubeDesc.getRowkey().getRowKeyColumns().length;
    keyBytesBuf = new byte[colCount][];

    bytesSplitter = new BytesSplitter(factTableDesc.getColumns().length, 4096);

    nullValue = new byte[] { (byte) '\\', (byte) 'N' };// As in Hive, null
    // value is
    // represented by \N

    prepareJoins();
    prepareMetrics();
}
 

@Ignore
@Test
public void testEncodeWithSlr() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
    // CubeSegment seg = cube.getTheOnlySegment();
    CubeDesc cubeDesc = cube.getDescriptor();
    // String data =
    // "1234567892013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
    String[] data = new String[9];
    data[0] = "123456789";
    data[1] = "2012-12-15";
    data[2] = "11848";
    data[3] = "Health & Beauty";
    data[4] = "Fragrances";
    data[5] = "Women";
    data[6] = "FP-GTC";
    data[7] = "0";
    data[8] = "15";

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
    RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(43 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
    byte[] shard = Arrays.copyOfRange(encodedKey, 0, RowConstants.ROWKEY_SHARDID_LEN);
    @SuppressWarnings("unused")
    byte[] sellerId = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), 4 + rowKeyEncoder.getHeaderLength());
    byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength());
    byte[] rest = Arrays.copyOfRange(encodedKey, 4 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
    assertEquals(0, Bytes.toShort(shard));
    //        assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
    assertEquals(511, Bytes.toLong(cuboidId));
    assertArrayEquals(new byte[] { 11, 55, -13, 49, 49, 56, 52, 56, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest);
}
 

@Override
protected void setup(Context context) throws IOException {
    super.publishConfiguration(context.getConfiguration());

    cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
    segmentName = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_NAME);
    intermediateTableRowDelimiter = context.getConfiguration().get(BatchConstants.CFG_CUBE_INTERMEDIATE_TABLE_ROW_DELIMITER, Character.toString(BatchConstants.INTERMEDIATE_TABLE_ROW_DELIMITER));
    if (Bytes.toBytes(intermediateTableRowDelimiter).length > 1) {
        throw new RuntimeException("Expected delimiter byte length is 1, but got " + Bytes.toBytes(intermediateTableRowDelimiter).length);
    }

    byteRowDelimiter = Bytes.toBytes(intermediateTableRowDelimiter)[0];

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration());

    cube = CubeManager.getInstance(config).getCube(cubeName);
    cubeDesc = cube.getDescriptor();
    cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);

    intermediateTableDesc = new CubeJoinedFlatTableDesc(cube.getDescriptor(), cubeSegment);

    bytesSplitter = new BytesSplitter(200, 4096);
    rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);

    measureCodec = new MeasureCodec(cubeDesc.getMeasures());
    measures = new Object[cubeDesc.getMeasures().size()];

    int colCount = cubeDesc.getRowkey().getRowKeyColumns().length;
    keyBytesBuf = new byte[colCount][];

    initNullBytes();
}
 

@Test
public void testEncodeAndDecodeWithUtf8() throws IOException {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
    CubeDesc cubeDesc = cube.getDescriptor();

    byte[][] data = new byte[8][];
    data[0] = Bytes.toBytes("2012-12-15");
    data[1] = Bytes.toBytes("11848");
    data[2] = Bytes.toBytes("Health & Beauty");
    data[3] = Bytes.toBytes("Fragrances");
    data[4] = Bytes.toBytes("Women");
    data[5] = Bytes.toBytes("刊登格式测试");// UTF-8
    data[6] = Bytes.toBytes("0");
    data[7] = Bytes.toBytes("15");

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
    AbstractRowKeyEncoder rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(30, encodedKey.length);

    RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
    rowKeyDecoder.decode(encodedKey);
    List<String> names = rowKeyDecoder.getNames(null);
    List<String> values = rowKeyDecoder.getValues();
    assertEquals("[CAL_DT, LEAF_CATEG_ID, META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME, LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD]", names.toString());
    assertEquals("[2012-12-15, 11848, Health & Beauty, Fragrances, Women, 刊登格式, 0, 15]", values.toString());
}
 

@Override
public Tuple2<ByteArray, Object[]> call(String[] rowArray) throws Exception {
    if (initialized == false) {
        synchronized (SparkCubingByLayer.class) {
            if (initialized == false) {
                KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl);
                try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig
                        .setAndUnsetThreadLocalConfig(kConfig)) {
                    CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName);
                    CubeDesc cubeDesc = cubeInstance.getDescriptor();
                    CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
                    CubeJoinedFlatTableEnrich interDesc = new CubeJoinedFlatTableEnrich(
                            EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
                    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
                    Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
                    baseCuboidBuilder = new BaseCuboidBuilder(kConfig, cubeDesc, cubeSegment, interDesc,
                            AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid),
                            MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap());
                    initialized = true;
                }
            }
        }
    }
    baseCuboidBuilder.resetAggrs();
    byte[] rowKey = baseCuboidBuilder.buildKey(rowArray);
    Object[] result = baseCuboidBuilder.buildValueObjects(rowArray);
    return new Tuple2<>(new ByteArray(rowKey), result);
}
 

private boolean checkMandatoryColumns(long cuboidID) {
    if ((cuboidID & mandatoryColumnMask) != mandatoryColumnMask) {
        return false;
    } else {
        //base cuboid is always valid
        if (cuboidID == Cuboid.getBaseCuboidId(cubeDesc)) {
            return true;
        }

        //cuboid with only mandatory columns maybe valid
        return isMandatoryOnlyValid || (cuboidID & ~mandatoryColumnMask) != 0;
    }
}
 

public InMemCubeBuilder2(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
        Map<TblColRef, Dictionary<String>> dictionaryMap) {
    super(cuboidScheduler, flatDesc, dictionaryMap);
    this.measureCount = cubeDesc.getMeasures().size();
    this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);
    List<String> metricsAggrFuncsList = Lists.newArrayList();

    for (int i = 0; i < measureCount; i++) {
        MeasureDesc measureDesc = measureDescs[i];
        metricsAggrFuncsList.add(measureDesc.getFunction().getExpression());
    }
    this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]);
    this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
}
 

public InMemCubeBuilder2(CuboidScheduler cuboidScheduler, IJoinedFlatTableDesc flatDesc,
        Map<TblColRef, Dictionary<String>> dictionaryMap) {
    super(cuboidScheduler, flatDesc, dictionaryMap);
    this.measureCount = cubeDesc.getMeasures().size();
    this.measureDescs = cubeDesc.getMeasures().toArray(new MeasureDesc[measureCount]);
    List<String> metricsAggrFuncsList = Lists.newArrayList();

    for (int i = 0; i < measureCount; i++) {
        MeasureDesc measureDesc = measureDescs[i];
        metricsAggrFuncsList.add(measureDesc.getFunction().getExpression());
    }
    this.metricsAggrFuncs = metricsAggrFuncsList.toArray(new String[metricsAggrFuncsList.size()]);
    this.baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
}
 

@Override
protected void setup(Context context) throws IOException {
    super.publishConfiguration(context.getConfiguration());

    Configuration conf = context.getConfiguration();

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
    cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
    cube = CubeManager.getInstance(config).getCube(cubeName);
    cubeDesc = cube.getDescriptor();
    intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
    List<TblColRef> columns = baseCuboid.getColumns();

    ArrayList<Integer> factDictCols = new ArrayList<Integer>();
    RowKeyDesc rowkey = cubeDesc.getRowkey();
    DictionaryManager dictMgr = DictionaryManager.getInstance(config);
    for (int i = 0; i < columns.size(); i++) {
        TblColRef col = columns.get(i);
        if (rowkey.isUseDictionary(col) == false)
            continue;

        String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
        if (cubeDesc.getModel().isFactTable(scanTable)) {
            factDictCols.add(i);
        }
    }
    this.factDictCols = new int[factDictCols.size()];
    for (int i = 0; i < factDictCols.size(); i++)
        this.factDictCols[i] = factDictCols.get(i);

    schema = HCatInputFormat.getTableSchema(context.getConfiguration());
}