Java Code Examples for org.apache.kylin.cube.CubeInstance#getSegments()

private boolean isInOptimize(CubeInstance cube) {
    Segments<CubeSegment> readyPendingSegments = cube.getSegments(SegmentStatusEnum.READY_PENDING);
    if (readyPendingSegments.size() > 0) {
        logger.info("The cube {} has READY_PENDING segments {}. It's not allowed for building", cube.getName(),
                readyPendingSegments);
        return true;
    }
    Segments<CubeSegment> newSegments = cube.getSegments(SegmentStatusEnum.NEW);
    for (CubeSegment newSegment : newSegments) {
        String jobId = newSegment.getLastBuildJobID();
        if (jobId == null) {
            continue;
        }
        AbstractExecutable job = getExecutableManager().getJob(jobId);
        if (job != null && job instanceof CubingJob) {
            CubingJob cubingJob = (CubingJob) job;
            if (CubingJob.CubingJobTypeEnum.OPTIMIZE.toString().equals(cubingJob.getJobType())) {
                logger.info(
                        "The cube {} is in optimization. It's not allowed to build new segments during optimization.",
                        cube.getName());
                return true;
            }
        }
    }
    return false;
}
 

private static List<String> getHTableNames(KylinConfig config) {
    CubeManager cubeMgr = CubeManager.getInstance(config);

    ArrayList<String> result = new ArrayList<String>();
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
        if (cube.getStorageType() == IStorageAware.ID_HBASE
                || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE
                || cube.getStorageType() == IStorageAware.ID_REALTIME_AND_HBASE) {
            for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) {
                String tableName = seg.getStorageLocationIdentifier();
                if (StringUtils.isBlank(tableName) == false) {
                    result.add(tableName);
                    System.out.println("added new table: " + tableName);
                }
            }
        }
    }

    return result;
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
    final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));

    Set<Long> recommendCuboids = cube.getCuboidsRecommend();
    try {
        List<CubeSegment> newSegments = cube.getSegments(SegmentStatusEnum.READY_PENDING);
        Map<Long, Long> recommendCuboidsWithStats = CuboidStatsReaderUtil
                .readCuboidStatsFromSegments(recommendCuboids, newSegments);
        if (recommendCuboidsWithStats == null) {
            throw new RuntimeException("Fail to get statistics info for recommended cuboids after optimization!!!");
        }
        cubeManager.promoteCheckpointOptimizeSegments(cube, recommendCuboidsWithStats,
                newSegments.toArray(new CubeSegment[newSegments.size()]));
        return new ExecuteResult();
    } catch (Exception e) {
        logger.error("fail to update cube after build", e);
        return ExecuteResult.createError(e);
    }
}
 

private static List<String> filterByProjects(List<String> allTableNames, List<String> projectNames) {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());

    List<String> result = Lists.newArrayList();
    for (String p : projectNames) {
        p = p.trim();
        if (p.endsWith(",")) {
            p = p.substring(0, p.length() - 1);
        }

        ProjectInstance projectInstance = projectManager.getProject(p);
        List<RealizationEntry> cubeList = projectInstance.getRealizationEntries(RealizationType.CUBE);
        for (RealizationEntry cube : cubeList) {
            CubeInstance cubeInstance = cubeManager.getCube(cube.getRealization());
            for (CubeSegment segment : cubeInstance.getSegments()) {
                String tableName = segment.getStorageLocationIdentifier();
                if (allTableNames.contains(tableName)) {
                    result.add(tableName);
                }
            }
        }
    }
    return result;
}
 

private void checkSegmentHDFSPath(List<CubeInstance> cubes) throws IOException {
    reporter.log("## Fix missing HDFS path of segments");
    FileSystem defaultFs = HadoopUtil.getWorkingFileSystem();
    for (CubeInstance cube : cubes) {
        for (CubeSegment segment : cube.getSegments()) {
            String jobUuid = segment.getLastBuildJobID();
            if (jobUuid != null && jobUuid.equals("") == false) {
                String path = JobBuilderSupport.getJobWorkingDir(config.getHdfsWorkingDirectory(), jobUuid);
                if (!defaultFs.exists(new Path(path))) {
                    reporter.log(
                            "Project: {} cube: {} segment: {} cube id data: {} don't exist and need to rebuild it",
                            cube.getProject(), cube.getName(), segment, path);
                    reporter.log(
                            "The rebuild url: -d '{\"startTime\":{}, \"endTime\":{}, \"buildType\":\"REFRESH\"}' /kylin/api/cubes/{}/build",
                            segment.getTSRange().start, segment.getTSRange().end, cube.getName());
                }
            }
        }
    }
}
 

private static void listCubeRelatedResources(CubeInstance cube, List<String> metaResource,
        Set<String> dictAndSnapshot) throws IOException {

    CubeDesc cubeDesc = cube.getDescriptor();
    metaResource.add(cube.getResourcePath());
    metaResource.add(cubeDesc.getResourcePath());
    metaResource.add(DataModelDesc.concatResourcePath(cubeDesc.getModelName()));

    for (TableRef table : cubeDesc.getModel().getAllTables()) {
        metaResource.add(table.getTableDesc().getResourcePath());
    }

    for (CubeSegment segment : cube.getSegments()) {
        metaResource.add(segment.getStatisticsResourcePath());
        dictAndSnapshot.addAll(segment.getSnapshotPaths());
        dictAndSnapshot.addAll(segment.getDictionaryPaths());
    }
}
 

private static void renameFoldersInHdfs(CubeInstance cube) {
    for (CubeSegment segment : cube.getSegments()) {

        String jobUuid = segment.getLastBuildJobID();
        String src = JobInstance.getJobWorkingDir(jobUuid, srcConfig.getHdfsWorkingDirectory());
        String tgt = JobInstance.getJobWorkingDir(jobUuid, dstConfig.getHdfsWorkingDirectory());

        operations.add(new Opt(OptType.RENAME_FOLDER_IN_HDFS, new Object[] { src, tgt }));
    }

}
 

@Override
protected void updateMetaAfterBuilding(KylinConfig config) throws IOException {
    CubeManager cubeManager = CubeManager.getInstance(config);
    CubeInstance currentInstanceCopy = cubeManager.getCube(getCubeName()).latestCopyForWrite();
    KylinConfig kylinDistConfig = MetaDumpUtil.loadKylinConfigFromHdfs(getDistMetaUrl());
    CubeInstance distCube = CubeManager.getInstance(kylinDistConfig).reloadCube(getCubeName());
    CubeUpdate update = new CubeUpdate(currentInstanceCopy);
    Set<String> segmentIds = Sets.newHashSet(org.apache.hadoop.util.StringUtils.split(getParam(MetadataConstants.P_SEGMENT_IDS)));
    CubeSegment toUpdateSegs = distCube.getSegmentById(segmentIds.iterator().next());

    List<CubeSegment> tobe = currentInstanceCopy.calculateToBeSegments(toUpdateSegs);

    if (tobe.contains(toUpdateSegs) == false)
        throw new IllegalStateException(
                String.format(Locale.ROOT, "For cube %s, segment %s is expected but not in the tobe %s",
                        currentInstanceCopy.toString(), toUpdateSegs.toString(), tobe.toString()));

    toUpdateSegs.setStatus(SegmentStatusEnum.READY);

    List<CubeSegment> toRemoveSegs = Lists.newArrayList();
    for (CubeSegment segment : currentInstanceCopy.getSegments()) {
        if (!tobe.contains(segment))
            toRemoveSegs.add(segment);
    }

    logger.info("Promoting cube {}, new segment {}, to remove segments {}", currentInstanceCopy, toUpdateSegs, toRemoveSegs);

    update.setToRemoveSegs(toRemoveSegs.toArray(new CubeSegment[toRemoveSegs.size()]))
            .setToUpdateSegs(toUpdateSegs);
    if (currentInstanceCopy.getConfig().isJobAutoReadyCubeEnabled()) {
        update.setStatus(RealizationStatusEnum.READY);
    }
    cubeManager.updateCube(update);
}
 

public List<CubeSegment> listSegmentsForQuery(CubeInstance cube) {
    List<CubeSegment> r = new ArrayList<>();
    for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) {
        if (check(seg))
            r.add(seg);
    }
    return r;
}
 

private static void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, List<String> dictAndSnapshot) throws IOException {

        CubeDesc cubeDesc = cube.getDescriptor();
        metaResource.add(cube.getResourcePath());
        metaResource.add(cubeDesc.getResourcePath());

        for (String table : cubeDesc.getModel().getAllTables()) {
            metaResource.add(TableDesc.concatResourcePath(table.toUpperCase()));
        }

        for (CubeSegment segment : cube.getSegments()) {
            dictAndSnapshot.addAll(segment.getSnapshotPaths());
            dictAndSnapshot.addAll(segment.getDictionaryPaths());
        }
    }
 

protected void listCubeRelatedResources(CubeInstance cube, List<String> metaResource, Set<String> dictAndSnapshot)
        throws IOException {

    CubeDesc cubeDesc = cube.getDescriptor();
    String prj = cubeDesc.getProject();

    metaResource.add(cube.getResourcePath());
    metaResource.add(cubeDesc.getResourcePath());
    metaResource.add(DataModelDesc.concatResourcePath(cubeDesc.getModelName()));

    Set<TableRef> tblRefs = cubeDesc.getModel().getAllTables();
    metaResource.addAll(getCompatibleTablePath(tblRefs, prj, ResourceStore.TABLE_RESOURCE_ROOT));
    metaResource.addAll(getCompatibleTablePath(tblRefs, prj, ResourceStore.TABLE_EXD_RESOURCE_ROOT));

    if (doMigrateSegment) {
        for (CubeSegment segment : cube.getSegments()) {
            metaResource.add(segment.getStatisticsResourcePath());
            dictAndSnapshot.addAll(segment.getSnapshotPaths());
            dictAndSnapshot.addAll(segment.getDictionaryPaths());
        }
    }

    if (doAclCopy) {
        metaResource.add(ACL_PREFIX + cube.getUuid());
        metaResource.add(ACL_PREFIX + cube.getModel().getUuid());
    }

    if (cubeDesc.isStreamingCube()) {
        // add streaming source config info for streaming cube
        metaResource.add(StreamingSourceConfig.concatResourcePath(cubeDesc.getModel().getRootFactTableName()));
    }
}
 

@Test(expected = InternalErrorException.class)
public void testDeleteSegmentNew() throws IOException {
    String cubeName = "test_kylin_cube_with_slr_ready_3_segments";
    CubeDesc[] cubes = cubeDescController.getCube(cubeName);
    Assert.assertNotNull(cubes);

    String segmentName = "20131212000000_20140112000000";

    CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
    cubeController.disableCube(cubeName);

    CubeSegment toDelete = null;
    for (CubeSegment seg : cube.getSegments()) {
        if (seg.getName().equals(segmentName)) {
            toDelete = seg;
            break;
        }
    }

    Assert.assertNotNull(toDelete);
    String segId = toDelete.getUuid();

    cubeController.deleteSegment(cubeName, segmentName);

    // delete success, no related job 'NEW' segment can be delete
    if (cubeService.isOrphonSegment(cube, segId)){
        throw new InternalErrorException();
    }
    cubeController.enableCube(cubeName);
}
 

private void checkAllowBuilding(CubeInstance cube) {
    if (cube.getConfig().isCubePlannerEnabled()) {
        Segments<CubeSegment> readyPendingSegments = cube.getSegments(SegmentStatusEnum.READY_PENDING);
        if (readyPendingSegments.size() > 0) {
            throw new BadRequestException("The cube " + cube.getName() + " has READY_PENDING segments "
                    + readyPendingSegments + ". It's not allowed for building");
        }
    }
}
 

/**
 * purge the cube
 *
 * @throws IOException
 * @throws JobException
 */
private void releaseAllSegments(CubeInstance cube) throws IOException {
    releaseAllJobs(cube);

    List<CubeSegment> toRemoveSegs = cube.getSegments();

    // remove from metadata
    getCubeManager().clearSegments(cube);

    cleanSegmentStorage(toRemoveSegs);
}
 

private CubeSegment findSegmentWithUuid(String jobID, CubeInstance cubeInstance) {
    for (CubeSegment segment : cubeInstance.getSegments()) {
        String lastBuildJobID = segment.getLastBuildJobID();
        if (lastBuildJobID != null && lastBuildJobID.equalsIgnoreCase(jobID)) {
            return segment;
        }
    }

    throw new IllegalStateException("No merging segment's last build job ID equals " + jobID);

}
 

@Test
public void testGetHoles() throws IOException {
    String cubeName = "test_kylin_cube_with_slr_ready_3_segments";
    CubeDesc[] cubes = cubeDescController.getCube(cubeName);
    Assert.assertNotNull(cubes);

    CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
    List<CubeSegment> segments = cube.getSegments();

    cubeController.disableCube(cubeName);

    final long dateEnd = segments.get(segments.size() - 1).getTSRange().end.v;

    final long ONEDAY = 24 * 60 * 60000;
    cubeService.getCubeManager().appendSegment(cube, new TSRange(dateEnd + ONEDAY, dateEnd + ONEDAY * 2));

    List<CubeSegment> holes = cubeController.getHoles(cubeName);

    Assert.assertTrue(holes.size() == 1);

    CubeSegment hole = holes.get(0);

    Assert.assertTrue(hole.getTSRange().equals(new TSRange(dateEnd, dateEnd + ONEDAY)));
}
 

private static List<Double> getHistoricalRating(CubeSegment cubeSegment,
                                                CubeInstance cubeInstance,
                                                int totalLevels) {
    boolean isMerged = cubeSegment.isMerged();

    Map<Integer, List<Double>> layerRatio = Maps.newHashMap();
    List<Double> result = Lists.newArrayList();

    for (CubeSegment seg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
        if (seg.isMerged() != isMerged || seg.getEstimateRatio() == null) {
            continue;
        }

        logger.info("get ratio from {} with: {}", seg.getName(), StringUtils.join(seg.getEstimateRatio(), ","));

        for(int level = 0; level <= totalLevels; level++) {
            if (seg.getEstimateRatio().get(level) <= 0) {
                continue;
            }

            List<Double> temp = layerRatio.get(level) == null ? Lists.newArrayList() : layerRatio.get(level);

            temp.add(seg.getEstimateRatio().get(level));
            layerRatio.put(level, temp);
        }
    }

    if (layerRatio.size() == 0) {
        logger.info("Fail to get historical rating.");
        return null;
    } else {
        for(int level = 0; level <= totalLevels; level++) {
            logger.debug("level {}: {}", level, StringUtils.join(layerRatio.get(level), ","));
            result.add(level, harmonicMean(layerRatio.get(level)));
        }

        logger.info("Finally estimate ratio is {}", StringUtils.join(result, ","));

        return result;
    }
}
 

private static void changeHtableHost(CubeInstance cube) {
    for (CubeSegment segment : cube.getSegments()) {
        operations
                .add(new Opt(OptType.CHANGE_HTABLE_HOST, new Object[] { segment.getStorageLocationIdentifier() }));
    }
}
 

private static List<Double> getHistoricalRating(CubeSegment cubeSegment,
                                                CubeInstance cubeInstance,
                                                int totalLevels) {
    boolean isMerged = cubeSegment.isMerged();

    Map<Integer, List<Double>> layerRatio = Maps.newHashMap();
    List<Double> result = Lists.newArrayList();

    for (CubeSegment seg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
        if (seg.isMerged() != isMerged || seg.getEstimateRatio() == null) {
            continue;
        }

        logger.info("get ratio from {} with: {}", seg.getName(), StringUtils.join(seg.getEstimateRatio(), ","));

        for(int level = 0; level <= totalLevels; level++) {
            if (seg.getEstimateRatio().get(level) <= 0) {
                continue;
            }

            List<Double> temp = layerRatio.get(level) == null ? Lists.newArrayList() : layerRatio.get(level);

            temp.add(seg.getEstimateRatio().get(level));
            layerRatio.put(level, temp);
        }
    }

    if (layerRatio.size() == 0) {
        logger.info("Fail to get historical rating.");
        return null;
    } else {
        for(int level = 0; level <= totalLevels; level++) {
            logger.debug("level {}: {}", level, StringUtils.join(layerRatio.get(level), ","));
            result.add(level, harmonicMean(layerRatio.get(level)));
        }

        logger.info("Finally estimate ratio is {}", StringUtils.join(result, ","));

        return result;
    }
}
 

@Test
public void testBuildTwoSegmentsAndMerge() throws Exception {
    String cubeName = "ci_inner_join_cube";
    cleanupSegments(cubeName);
    CubeInstance cubeInstance = cubeMgr.getCube(cubeName);
    ExecutableManager jobService = ExecutableManager.getInstance(kylinConfig);

    /**
     * Round1. Build 2 segment
     */
    long date1 = dateToLong("2010-01-01");
    long date2 = dateToLong("2012-01-01");
    long date3 = dateToLong("2014-01-01");

    CubeSegment segment = cubeMgr.appendSegment(cubeInstance, new SegmentRange.TSRange(date1, date2));
    NSparkCubingJob job = NSparkCubingJob.create(Sets.newHashSet(segment), "ADMIN");
    jobService.addJob(job);
    // wait job done
    ExecutableState state = wait(job);
    Assert.assertEquals(ExecutableState.SUCCEED, state);

    CubeSegment segment2 = cubeMgr.appendSegment(cubeInstance, new SegmentRange.TSRange(date2, date3));
    NSparkCubingJob job2 = NSparkCubingJob.create(Sets.newHashSet(segment2), "ADMIN");
    jobService.addJob(job2);
    // wait job done
    ExecutableState state2 = wait(job2);
    Assert.assertEquals(ExecutableState.SUCCEED, state2);

    cubeInstance = cubeMgr.reloadCube(cubeName);

    /**
     * Round2. Merge two segments
     */
    CubeSegment firstMergeSeg = cubeMgr.mergeSegments(cubeInstance, new SegmentRange.TSRange(date1, date3),
            null, false);
    NSparkMergingJob firstMergeJob = NSparkMergingJob.merge(firstMergeSeg, "ADMIN");
    jobService.addJob(firstMergeJob);
    // wait job done
    Assert.assertEquals(ExecutableState.SUCCEED, wait(firstMergeJob));

    // Parquet cuboids vs Spark SQL
    cubeInstance = cubeMgr.reloadCube(cubeName);
    for (CubeSegment seg : cubeInstance.getSegments()) {
        queryTest(seg);
    }
}