Java Code Examples for org.apache.kylin.cube.CubeSegment#getName()

The following examples show how to use org.apache.kylin.cube.CubeSegment#getName() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CuboidRecommenderUtil.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
/** For future segment level recommend */
public static Map<Long, Long> getRecommendCuboidList(CubeSegment segment, Map<Long, Long> hitFrequencyMap,
        Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap, boolean ifForceRecommend)
        throws IOException {
    if (segment == null) {
        return null;
    }

    CubeStatsReader cubeStatsReader = new CubeStatsReader(segment, null, segment.getConfig());
    if (cubeStatsReader.getCuboidRowEstimatesHLL() == null
            || cubeStatsReader.getCuboidRowEstimatesHLL().isEmpty()) {
        logger.info("Cuboid Statistics is not enabled.");
        return null;
    }
    CubeInstance cube = segment.getCubeInstance();
    long baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
    if (cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == null
            || cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == 0L) {
        logger.info(BASE_CUBOID_COUNT_IN_CUBOID_STATISTICS_IS_ZERO);
        return null;
    }

    String key = cube.getName() + "-" + segment.getName();
    CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, cubeStatsReader.getCuboidRowEstimatesHLL(),
            cubeStatsReader.getCuboidSizeMap()).setHitFrequencyMap(hitFrequencyMap)
                    .setRollingUpCountSourceMap(rollingUpCountSourceMap).build();
    return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, segment.getConfig(),
            ifForceRecommend);
}
Example 2
Source File: CuboidRecommenderUtil.java    From kylin with Apache License 2.0 5 votes vote down vote up 
/** For future segment level recommend */
public static Map<Long, Long> getRecommendCuboidList(CubeSegment segment, Map<Long, Long> hitFrequencyMap,
        Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap, boolean ifForceRecommend)
        throws IOException {
    if (segment == null) {
        return null;
    }

    CubeStatsReader cubeStatsReader = new CubeStatsReader(segment, null, segment.getConfig());
    if (cubeStatsReader.getCuboidRowEstimatesHLL() == null
            || cubeStatsReader.getCuboidRowEstimatesHLL().isEmpty()) {
        logger.info("Cuboid Statistics is not enabled.");
        return null;
    }
    CubeInstance cube = segment.getCubeInstance();
    long baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
    if (cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == null
            || cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == 0L) {
        logger.info(BASE_CUBOID_COUNT_IN_CUBOID_STATISTICS_IS_ZERO);
        return null;
    }

    String key = cube.getName() + "-" + segment.getName();
    CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, cubeStatsReader.getCuboidRowEstimatesHLL(),
            cubeStatsReader.getCuboidSizeMap()).setHitFrequencyMap(hitFrequencyMap)
                    .setRollingUpCountSourceMap(rollingUpCountSourceMap).build();
    return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, segment.getConfig(),
            ifForceRecommend);
}
Example 3
Source File: MergeCuboidMapperTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Test
    public void test() throws IOException, ParseException {

        String cubeName = "test_kylin_cube_without_slr_left_join_ready_2_segments";

        CubeSegment newSeg = cubeManager.mergeSegments(cube, 0L, 1386835200000L);
        String segmentName = newSeg.getName();

        final Dictionary<?> dictionary = cubeManager.getDictionary(newSeg, lfn);
        assertTrue(dictionary == null);
//        ((TrieDictionary) dictionary).dump(System.out);

        // hack for distributed cache
//        File metaDir = new File("../job/meta");
//        FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl()), metaDir);
//
//        mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
//        mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
//        // mapDriver.getConfiguration().set(KylinConfig.KYLIN_METADATA_URL,
//        // "../job/meta");
//
//        byte[] key = new byte[] { 0, 0, 0, 0, 0, 0, 0, -92, 1, 1, 1 };
//        byte[] value = new byte[] { 1, 2, 3 };
//        byte[] newkey = new byte[] { 0, 0, 0, 0, 0, 0, 0, -92, 1, 1, 2 };
//        byte[] newvalue = new byte[] { 1, 2, 3 };
//
//        mapDriver.withInput(new Text(key), new Text(value));
//        mapDriver.withOutput(new Text(newkey), new Text(newvalue));
//        mapDriver.setMapInputPath(new Path("/apps/hdmi-prod/b_kylin/prod/kylin-f24668f6-dcff-4cb6-a89b-77f1119df8fa/vac_sw_cube_v4/cuboid/15d_cuboid"));
//
//        mapDriver.runTest();
    }
Example 4
Source File: JobService.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public JobInstance submitRecoverSegmentOptimizeJob(CubeSegment segment, String submitter)
        throws IOException, JobException {
    CubeInstance cubeInstance = segment.getCubeInstance();

    checkCubeDescSignature(cubeInstance);

    String cubeName = cubeInstance.getName();
    List<JobInstance> jobInstanceList = searchJobsByCubeName(cubeName, null,
            Lists.newArrayList(JobStatusEnum.NEW, JobStatusEnum.PENDING, JobStatusEnum.ERROR),
            JobTimeFilterEnum.ALL, JobSearchMode.CHECKPOINT_ONLY);
    if (jobInstanceList.size() > 1) {
        throw new IllegalStateException("Exist more than one CheckpointExecutable for cube " + cubeName);
    } else if (jobInstanceList.size() == 0) {
        throw new IllegalStateException("There's no CheckpointExecutable for cube " + cubeName);
    }
    CheckpointExecutable checkpointExecutable = (CheckpointExecutable) getExecutableManager()
            .getJob(jobInstanceList.get(0).getId());

    AbstractExecutable toBeReplaced = null;
    for (AbstractExecutable taskForCheck : checkpointExecutable.getSubTasksForCheck()) {
        if (taskForCheck instanceof CubingJob) {
            CubingJob subCubingJob = (CubingJob) taskForCheck;
            String segmentName = CubingExecutableUtil.getSegmentName(subCubingJob.getParams());
            if (segmentName != null && segmentName.equals(segment.getName())) {
                String segmentID = CubingExecutableUtil.getSegmentId(subCubingJob.getParams());
                CubeSegment beingOptimizedSegment = cubeInstance.getSegmentById(segmentID);
                if (beingOptimizedSegment != null) { // beingOptimizedSegment exists & should not be recovered
                    throw new IllegalStateException("Segment " + beingOptimizedSegment.getName() + "-"
                            + beingOptimizedSegment.getUuid()
                            + " still exists. Please delete it or discard the related optimize job first!!!");
                }
                toBeReplaced = taskForCheck;
                break;
            }
        }
    }
    if (toBeReplaced == null) {
        throw new IllegalStateException("There's no CubingJob for segment " + segment.getName()
                + " in CheckpointExecutable " + checkpointExecutable.getName());
    }

    /** Add CubingJob for the related segment **/
    CubeSegment optimizeSegment = getCubeManager().appendSegment(cubeInstance, segment.getTSRange());

    DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, submitter);

    getExecutableManager().addJob(optimizeJob);

    JobInstance optimizeJobInstance = getSingleJobInstance(optimizeJob);

    /** Update the checkpoint job */
    checkpointExecutable.getSubTasksForCheck().set(checkpointExecutable.getSubTasksForCheck().indexOf(toBeReplaced),
            optimizeJob);

    getExecutableManager().updateCheckpointJob(checkpointExecutable.getId(),
            checkpointExecutable.getSubTasksForCheck());

    return optimizeJobInstance;
}
Example 5
Source File: JobService.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public JobInstance submitRecoverSegmentOptimizeJob(CubeSegment segment, String submitter)
        throws IOException, JobException {
    CubeInstance cubeInstance = segment.getCubeInstance();

    checkCubeDescSignature(cubeInstance);

    String cubeName = cubeInstance.getName();
    List<JobInstance> jobInstanceList = searchJobsByCubeName(cubeName, null,
            Lists.newArrayList(JobStatusEnum.NEW, JobStatusEnum.PENDING, JobStatusEnum.ERROR),
            JobTimeFilterEnum.ALL, JobSearchMode.CHECKPOINT_ONLY);
    if (jobInstanceList.size() > 1) {
        throw new IllegalStateException("Exist more than one CheckpointExecutable for cube " + cubeName);
    } else if (jobInstanceList.size() == 0) {
        throw new IllegalStateException("There's no CheckpointExecutable for cube " + cubeName);
    }
    CheckpointExecutable checkpointExecutable = (CheckpointExecutable) getExecutableManager()
            .getJob(jobInstanceList.get(0).getId());

    AbstractExecutable toBeReplaced = null;
    for (AbstractExecutable taskForCheck : checkpointExecutable.getSubTasksForCheck()) {
        if (taskForCheck instanceof CubingJob) {
            CubingJob subCubingJob = (CubingJob) taskForCheck;
            String segmentName = CubingExecutableUtil.getSegmentName(subCubingJob.getParams());
            if (segmentName != null && segmentName.equals(segment.getName())) {
                String segmentID = CubingExecutableUtil.getSegmentId(subCubingJob.getParams());
                CubeSegment beingOptimizedSegment = cubeInstance.getSegmentById(segmentID);
                if (beingOptimizedSegment != null) { // beingOptimizedSegment exists & should not be recovered
                    throw new IllegalStateException("Segment " + beingOptimizedSegment.getName() + "-"
                            + beingOptimizedSegment.getUuid()
                            + " still exists. Please delete it or discard the related optimize job first!!!");
                }
                toBeReplaced = taskForCheck;
                break;
            }
        }
    }
    if (toBeReplaced == null) {
        throw new IllegalStateException("There's no CubingJob for segment " + segment.getName()
                + " in CheckpointExecutable " + checkpointExecutable.getName());
    }

    /** Add CubingJob for the related segment **/
    CubeSegment optimizeSegment = getCubeManager().appendSegment(cubeInstance, segment.getTSRange());

    DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, submitter);

    getExecutableManager().addJob(optimizeJob);

    JobInstance optimizeJobInstance = getSingleJobInstance(optimizeJob);

    /** Update the checkpoint job */
    checkpointExecutable.getSubTasksForCheck().set(checkpointExecutable.getSubTasksForCheck().indexOf(toBeReplaced),
            optimizeJob);

    getExecutableManager().updateCheckpointJob(checkpointExecutable.getId(),
            checkpointExecutable.getSubTasksForCheck());

    return optimizeJobInstance;
}