Java Code Examples for org.apache.kylin.cube.CubeInstance#getDescriptor()

public Map<TblColRef, Set<CubeInstance>> getUsedNonDimCols(String modelName, String project) {
    Map<TblColRef, Set<CubeInstance>> ret = Maps.newHashMap();
    List<CubeInstance> cubeInstances = cubeService.listAllCubes(null, project, modelName, true);
    for (CubeInstance cubeInstance : cubeInstances) {
        CubeDesc cubeDesc = cubeInstance.getDescriptor();
        Set<TblColRef> tblColRefs = Sets.newHashSet(cubeDesc.listAllColumns());//make a copy
        tblColRefs.removeAll(cubeDesc.listDimensionColumnsIncludingDerived());
        for (TblColRef tblColRef : tblColRefs) {
            if (ret.containsKey(tblColRef)) {
                ret.get(tblColRef).add(cubeInstance);
            } else {
                Set<CubeInstance> set = Sets.newHashSet(cubeInstance);
                ret.put(tblColRef, set);
            }
        }
    }
    return ret;
}
 

public LookupTableEnumerator(OLAPContext olapContext) {

        //TODO: assuming LookupTableEnumerator is handled by a cube
        CubeInstance cube = (CubeInstance) olapContext.realization;

        String lookupTableName = olapContext.firstTableScan.getTableName();
        DimensionDesc dim = cube.getDescriptor().findDimensionByTable(lookupTableName);
        if (dim == null)
            throw new IllegalStateException("No dimension with derived columns found for lookup table " + lookupTableName + ", cube desc " + cube.getDescriptor());

        CubeManager cubeMgr = CubeManager.getInstance(cube.getConfig());
        LookupStringTable table = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim);
        this.allRows = table.getAllRows();

        OLAPTable olapTable = (OLAPTable) olapContext.firstTableScan.getOlapTable();
        this.colDescs = olapTable.getExposedColumns();
        this.current = new Object[colDescs.size()];

        reset();
    }
 

/**
 * For the new segment, we need to create new dimension dictionaries by merging underlying
 * dictionaries. (https://issues.apache.org/jira/browse/KYLIN-2457, https://issues.apache.org/jira/browse/KYLIN-2800)
 * @param cube
 * @param newSeg
 * @throws IOException
 */
private void makeDictForNewSegment(KylinConfig conf, CubeInstance cube, CubeSegment newSeg, List<CubeSegment> mergingSegments) throws IOException {
    DictionaryManager dictMgr = DictionaryManager.getInstance(conf);
    CubeDesc cubeDesc = cube.getDescriptor();

    for (TblColRef col : cubeDesc.getAllColumnsNeedDictionaryBuilt()) {
        logger.info("Merging fact table dictionary on : " + col);
        List<DictionaryInfo> dictInfos = new ArrayList<DictionaryInfo>();
        for (CubeSegment segment : mergingSegments) {
            logger.info("Including fact table dictionary of segment : " + segment);
            if (segment.getDictResPath(col) != null) {
                DictionaryInfo dictInfo = dictMgr.getDictionaryInfo(segment.getDictResPath(col));
                if (dictInfo != null && !dictInfos.contains(dictInfo)) {
                    dictInfos.add(dictInfo);
                } else {
                    logger.warn("Failed to load DictionaryInfo from " + segment.getDictResPath(col));
                }
            }
        }
        mergeDictionaries(dictMgr, newSeg, dictInfos, col);
    }
}
 

private void checkCubeDescParams(List<CubeInstance> cubes) {
    for (CubeInstance cube : cubes) {
        CubeDesc desc = cube.getDescriptor();
        long[] autoMergeTS = desc.getAutoMergeTimeRanges();
        if (autoMergeTS == null || autoMergeTS.length == 0) {
            logger.info("Cube: {} in project: {} with no auto merge params", cube.getName(), cube.getProject());
        }
        // long volatileRange = desc.getVolatileRange();
        long retentionRange = desc.getRetentionRange();
        if (retentionRange == 0) {
            logger.info("Cube: {} in project: {} with no retention params", cube.getName(), cube.getProject());
        }
        // queue params
    }
}
 

private void init() {
    KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl);
    try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig
            .setAndUnsetThreadLocalConfig(kConfig)) {
        CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName);
        CubeDesc cubeDesc = cubeInstance.getDescriptor();
        CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
        CubeJoinedFlatTableEnrich intermediateTableDesc = new CubeJoinedFlatTableEnrich(
                EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);

        keyValueBuilder = new KeyValueBuilder(intermediateTableDesc);
        reducerMapping = new FactDistinctColumnsReducerMapping(cubeInstance);
        tmpbuf = ByteBuffer.allocate(4096);

        int[] rokeyColumnIndexes = intermediateTableDesc.getRowKeyColumnIndexes();

        Long[] cuboidIds = getCuboidIds(cubeSegment);

        Integer[][] cuboidsBitSet = CuboidUtil.getCuboidBitSet(cuboidIds, rokeyColumnIndexes.length);

        boolean isNewAlgorithm = isUsePutRowKeyToHllNewAlgorithm(cubeDesc);

        HLLCounter[] cuboidsHLL = getInitCuboidsHLL(cuboidIds.length,
                cubeDesc.getConfig().getCubeStatsHLLPrecision());

        cuboidStatCalculator = new CuboidStatCalculator(rokeyColumnIndexes, cuboidIds, cuboidsBitSet,
                isNewAlgorithm, cuboidsHLL);
        allCols = reducerMapping.getAllDimDictCols();

        initDictColDeduper(cubeDesc);
        initColumnIndex(intermediateTableDesc);

        initialized = true;
    }
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig kylinConfig = context.getConfig();
    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);

    CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams()));
    List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams());
    String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams());

    String extLookupSnapshotStr = this.getParam(BatchConstants.ARG_EXT_LOOKUP_SNAPSHOTS_INFO);
    if (extLookupSnapshotStr == null || extLookupSnapshotStr.isEmpty()) {
        return new ExecuteResult();
    }

    Map<String, String> extLookupSnapshotMap = LookupMaterializeContext.parseLookupSnapshots(extLookupSnapshotStr);
    String snapshotResPath = extLookupSnapshotMap.get(lookupTableName);
    if (snapshotResPath == null) {
        logger.info("no snapshot path exist in the context, so no need to update snapshot path");
        return new ExecuteResult();
    }
    CubeDesc cubeDesc = cube.getDescriptor();
    try {
        logger.info("update snapshot path:{} to cube:{}", snapshotResPath, cube.getName());
        if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) {
            if (!snapshotResPath.equals(cube.getSnapshotResPath(lookupTableName))) {
                LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName,
                        snapshotResPath);
            }
        } else {
            LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName,
                    snapshotResPath);
        }
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to save cuboid statistics", e);
        return ExecuteResult.createError(e);
    }
}
 

@Override
public void open(Configuration parameters) throws Exception {
    KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl);
    try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig
            .setAndUnsetThreadLocalConfig(kConfig)) {
        CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName);
        cubeDesc = cubeInstance.getDescriptor();
        aggregators = new MeasureAggregators(cubeDesc.getMeasures());
        measureNum = cubeDesc.getMeasures().size();
    }
}
 

/**
 * Get detail information of the "Cube ID"
 * 
 * @param cubeName
 *            Cube Name
 * @return
 * @throws IOException
 */
@RequestMapping(value = "/{cubeName}", method = { RequestMethod.GET }, produces = { "application/json" })
@ResponseBody
public CubeDesc[] getCube(@PathVariable String cubeName) {
    CubeInstance cubeInstance = cubeService.getCubeManager().getCube(cubeName);
    if (cubeInstance == null) {
        return null;
    }
    CubeDesc cSchema = cubeInstance.getDescriptor();
    if (cSchema != null) {
        return new CubeDesc[] { cSchema };
    } else {
        return null;
    }
}
 

@Ignore
@Test
public void testEncodeWithSlr2() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
    // CubeSegment seg = cube.getTheOnlySegment();
    CubeDesc cubeDesc = cube.getDescriptor();
    // String data =
    // "1234567892013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
    String[] data = new String[9];
    data[0] = "123456789";
    data[1] = null;
    data[2] = null;
    data[3] = null;
    data[4] = null;
    data[5] = null;
    data[6] = null;
    data[7] = null;
    data[8] = null;

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
    RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(43 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
    byte[] shard = Arrays.copyOfRange(encodedKey, 0, RowConstants.ROWKEY_SHARDID_LEN);
    byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength());
    @SuppressWarnings("unused")
    byte[] sellerId = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), 18 + rowKeyEncoder.getHeaderLength());
    byte[] rest = Arrays.copyOfRange(encodedKey, 4 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
    assertEquals(0, Bytes.toShort(shard));
    //assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
    assertEquals(511, Bytes.toLong(cuboidId));
    assertArrayEquals(new byte[] { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, rest);
}
 

@Test
public void testEncodeWithoutSlr() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
    // CubeSegment seg = cube.getTheOnlySegment();
    CubeDesc cubeDesc = cube.getDescriptor();
    // String data =
    // "2013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
    String[] data = new String[8];
    data[0] = "2012-12-15";
    data[1] = "11848";
    data[2] = "Health & Beauty";
    data[3] = "Fragrances";
    data[4] = "Women";
    data[5] = "FP-GTC";
    data[6] = "0";
    data[7] = "15";

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
    RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
    byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength());
    byte[] rest = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), encodedKey.length);
    assertEquals(255, Bytes.toLong(cuboidId));
    assertArrayEquals(new byte[] { 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest);
}
 

/**
 * Get detail information of the "Cube ID"
 * 
 * @param cubeName
 *            Cube Name
 * @return
 * @throws IOException
 */
@RequestMapping(value = "/{cubeName}", method = { RequestMethod.GET }, produces = { "application/json" })
@ResponseBody
public CubeDesc[] getCube(@PathVariable String cubeName) {
    CubeInstance cubeInstance = cubeService.getCubeManager().getCube(cubeName);
    if (cubeInstance == null) {
        return null;
    }
    CubeDesc cSchema = cubeInstance.getDescriptor();
    if (cSchema != null) {
        return new CubeDesc[] { cSchema };
    } else {
        return null;
    }
}
 

@Override
public void open(Configuration parameters) throws Exception {
    KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl);
    try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig
            .setAndUnsetThreadLocalConfig(kConfig)) {
        CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName);
        this.cubeSegment = cubeInstance.getSegmentById(segmentId);
        this.cubeDesc = cubeInstance.getDescriptor();
        this.ndCuboidBuilder = new NDCuboidBuilder(cubeSegment, new RowKeyEncoderProvider(cubeSegment));
        this.rowKeySplitter = new RowKeySplitter(cubeSegment);
    }
}
 

protected void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, Set<String> dictAndSnapshot)
        throws IOException {

    CubeDesc cubeDesc = cube.getDescriptor();
    String prj = cubeDesc.getProject();

    metaResource.add(cube.getResourcePath());
    metaResource.add(cubeDesc.getResourcePath());
    metaResource.add(DataModelDesc.concatResourcePath(cubeDesc.getModelName()));

    Set<TableRef> tblRefs = cubeDesc.getModel().getAllTables();
    metaResource.addAll(getCompatibleTablePath(tblRefs, prj, ResourceStore.TABLE_RESOURCE_ROOT));
    metaResource.addAll(getCompatibleTablePath(tblRefs, prj, ResourceStore.TABLE_EXD_RESOURCE_ROOT));

    if (doMigrateSegment) {
        for (CubeSegment segment : cube.getSegments()) {
            metaResource.add(segment.getStatisticsResourcePath());
            dictAndSnapshot.addAll(segment.getSnapshotPaths());
            dictAndSnapshot.addAll(segment.getDictionaryPaths());
        }
    }

    if (doAclCopy) {
        metaResource.add(ACL_PREFIX + cube.getUuid());
        metaResource.add(ACL_PREFIX + cube.getModel().getUuid());
    }

    if (cubeDesc.isStreamingCube()) {
        // add streaming source config info for streaming cube
        metaResource.add(StreamingSourceConfig.concatResourcePath(cubeDesc.getModel().getRootFactTableName()));
    }
}
 

@Override
public Tuple2<ByteArray, Object[]> call(String[] rowArray) throws Exception {
    if (initialized == false) {
        synchronized (SparkCubingByLayer.class) {
            if (initialized == false) {
                KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl);
                try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig
                        .setAndUnsetThreadLocalConfig(kConfig)) {
                    CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName);
                    CubeDesc cubeDesc = cubeInstance.getDescriptor();
                    CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
                    CubeJoinedFlatTableEnrich interDesc = new CubeJoinedFlatTableEnrich(
                            EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
                    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
                    Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
                    baseCuboidBuilder = new BaseCuboidBuilder(kConfig, cubeDesc, cubeSegment, interDesc,
                            AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid),
                            MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap());
                    initialized = true;
                }
            }
        }
    }
    baseCuboidBuilder.resetAggrs();
    byte[] rowKey = baseCuboidBuilder.buildKey(rowArray);
    Object[] result = baseCuboidBuilder.buildValueObjects(rowArray);
    return new Tuple2<>(new ByteArray(rowKey), result);
}
 

@Test
public void testEncodeAndDecodeWithUtf8() throws IOException {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
    CubeDesc cubeDesc = cube.getDescriptor();

    String[] data = new String[8];
    data[0] = "2012-12-15";
    data[1] = "11848";
    data[2] = "Health & Beauty";
    data[3] = "Fragrances";
    data[4] = "Women";
    data[5] = "刊登格式测试";// UTF-8
    data[6] = "0";
    data[7] = "15";

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
    RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length);

    RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
    rowKeyDecoder.decode(encodedKey);
    List<String> values = rowKeyDecoder.getValues();
    assertEquals("[" + millis("2012-12-15") + ", 11848, Health & Beauty, Fragrances, Women, 刊登格式, 0, 15]", values.toString());
}
 

public StreamingCubeDataSearcher(StreamingSegmentManager streamingSegmentManager) {
    this.streamingSegmentManager = streamingSegmentManager;
    this.cubeName = streamingSegmentManager.getCubeInstance().getName();
    CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    this.cubeDesc = cubeInstance.getDescriptor();
}
 

@RequestMapping(value = "/{cubeName}/clone", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public CubeInstance cloneCube(@PathVariable String cubeName, @RequestBody CubeRequest cubeRequest) {
    String newCubeName = cubeRequest.getCubeName();
    String projectName = cubeRequest.getProject();

    checkCubeExists(cubeName);
    CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
    if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
        throw new BadRequestException("Broken cube can't be cloned");
    }
    if (!ValidateUtil.isAlphanumericUnderscore(newCubeName)) {
        throw new BadRequestException("Invalid Cube name, only letters, numbers and underscore supported.");
    }

    ProjectInstance project = cubeService.getProjectManager().getProject(projectName);
    if (project == null) {
        throw new NotFoundException("Project " + projectName + " doesn't exist");
    }
    // KYLIN-1925, forbid cloning cross projects
    if (!project.getName().equals(cube.getProject())) {
        throw new BadRequestException("Cloning cubes across projects is not supported.");
    }

    CubeDesc cubeDesc = cube.getDescriptor();
    CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);

    newCubeDesc.setName(newCubeName);

    CubeInstance newCube;
    try {
        newCube = cubeService.createCubeAndDesc(project, newCubeDesc);

        //reload to avoid shallow clone
        cubeService.getCubeDescManager().reloadCubeDescLocal(newCubeName);
    } catch (IOException e) {
        throw new InternalErrorException("Failed to clone cube ", e);
    }

    return newCube;

}
 

public GTCubeStorageQueryBase(CubeInstance cube) {
    this.cubeInstance = cube;
    this.cubeDesc = cube.getDescriptor();
}
 

@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION')  or hasPermission(#cube, 'MANAGEMENT')")
public void updateCubeNotifyList(CubeInstance cube, List<String> notifyList) throws IOException {
    CubeDesc desc = cube.getDescriptor();
    desc.setNotifyList(notifyList);
    getCubeDescManager().updateCubeDesc(desc);
}
 

@Override
protected void doSetup(Context context) throws IOException {
    super.bindCurrentConfiguration(context.getConfiguration());
    Configuration conf = context.getConfiguration();
    mos = new MultipleOutputs(context);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
    String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
    CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
    cubeConfig = cube.getConfig();
    cubeDesc = cube.getDescriptor();

    taskId = context.getTaskAttemptID().getTaskID().getId();

    reducerMapping = new FactDistinctColumnsReducerMapping(cube);

    logger.info("reducer no " + taskId + ", role play " + reducerMapping.getRolePlayOfReducer(taskId));

    if (reducerMapping.isCuboidRowCounterReducer(taskId)) {
        // hll
        isStatistics = true;
        baseCuboidId = cube.getCuboidScheduler().getBaseCuboidId();
        baseCuboidRowCountInMappers = Lists.newArrayList();
        cuboidHLLMap = Maps.newHashMap();
        samplingPercentage = Integer
                .parseInt(context.getConfiguration().get(BatchConstants.CFG_STATISTICS_SAMPLING_PERCENT));
        logger.info("Reducer " + taskId + " handling stats");
    } else {
        // normal col
        col = reducerMapping.getColForReducer(taskId);
        Preconditions.checkNotNull(col);

        // local build dict
        buildDictInReducer = config.isBuildDictInReducerEnabled();
        if (cubeDesc.getDictionaryBuilderClass(col) != null) { // only works with default dictionary builder
            buildDictInReducer = false;
        }
        if (reducerMapping.getReducerNumForDimCol(col) > 1) {
            buildDictInReducer = false; // only works if this is the only reducer of a dictionary column
        }
        if (buildDictInReducer) {
            builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
            builder.init(null, 0, null);
        }
        logger.info("Reducer " + taskId + " handling column " + col + ", buildDictInReducer=" + buildDictInReducer);
    }
}