Java Code Examples for org.apache.kylin.cube.CubeInstance#isReady()

The following examples show how to use org.apache.kylin.cube.CubeInstance#isReady() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KylinHealthCheckJob.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private void checkOutOfDateCube(List<CubeInstance> cubes) {
    reporter.log("## Checking out-of-date Cubes");
    int staleCubeThresholdInDays = config.getStaleCubeThresholdInDays();
    long outdatedCubeTimeCut = System.currentTimeMillis() - 1L * staleCubeThresholdInDays * 24 * 60 * 60 * 1000;
    for (CubeInstance cube : cubes) {
        long lastTime = cube.getLastModified();
        logger.info("Cube {} last modified time: {}, {}", cube.getName(), new Date(lastTime),
                cube.getDescriptor().getNotifyList());
        if (lastTime < outdatedCubeTimeCut) {
            if (cube.isReady()) {
                reporter.log(
                        "Ready Cube: {} in project: {} is not built more then {} days, maybe it can be disabled",
                        cube.getName(), cube.getProject(), staleCubeThresholdInDays);
            } else {
                reporter.log(
                        "Disabled Cube: {} in project: {} is not built more then {} days, maybe it can be deleted",
                        cube.getName(), cube.getProject(), staleCubeThresholdInDays);
            }
        }
    }
}
Example 2
Source File: KylinHealthCheckJob.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private void checkOutOfDateCube(List<CubeInstance> cubes) {
    reporter.log("## Checking out-of-date Cubes");
    int staleCubeThresholdInDays = config.getStaleCubeThresholdInDays();
    long outdatedCubeTimeCut = System.currentTimeMillis() - 1L * staleCubeThresholdInDays * 24 * 60 * 60 * 1000;
    for (CubeInstance cube : cubes) {
        long lastTime = cube.getLastModified();
        logger.info("Cube {} last modified time: {}, {}", cube.getName(), new Date(lastTime),
                cube.getDescriptor().getNotifyList());
        if (lastTime < outdatedCubeTimeCut) {
            if (cube.isReady()) {
                reporter.log(
                        "Ready Cube: {} in project: {} is not built more then {} days, maybe it can be disabled",
                        cube.getName(), cube.getProject(), staleCubeThresholdInDays);
            } else {
                reporter.log(
                        "Disabled Cube: {} in project: {} is not built more then {} days, maybe it can be deleted",
                        cube.getName(), cube.getProject(), staleCubeThresholdInDays);
            }
        }
    }
}
Example 3
Source File: KylinHealthCheckJob.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void checkCubeHoles(List<CubeInstance> cubes) {
    reporter.log("## Checking holes of Cubes");
    for (CubeInstance cube : cubes) {
        if (cube.isReady()) {
            List<CubeSegment> holes = cubeManager.calculateHoles(cube.getName());
            if (holes.size() > 0) {
                reporter.log("{} holes in cube: {}, project: {}", holes.size(), cube.getName(), cube.getProject());
            }
        }
    }
}
Example 4
Source File: CubeService.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public String mergeCubeSegment(String cubeName) {
    CubeInstance cube = getCubeManager().getCube(cubeName);
    if (!cube.needAutoMerge())
        return null;

    if (!cube.isReady()) {
        logger.info("The cube: {} is disabled", cubeName);
        return null;
    }

    synchronized (CubeService.class) {
        try {
            cube = getCubeManager().getCube(cubeName);
            SegmentRange offsets = cube.autoMergeCubeSegments();
            if (offsets != null && !isMergingJobBeenDiscarded(cube, cubeName, cube.getProject(), offsets)) {
                CubeSegment newSeg = getCubeManager().mergeSegments(cube, null, offsets, true);
                logger.info("Will submit merge job on " + newSeg);
                DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(newSeg, "SYSTEM");
                getExecutableManager().addJob(job);
                return job.getId();
            } else {
                logger.info("Not ready for merge on cube " + cubeName);
            }
        } catch (IOException e) {
            logger.error("Failed to auto merge cube " + cubeName, e);
        }
    }
    return null;
}
Example 5
Source File: RealizationSignature.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
static CubeSignature getCubeSignature(KylinConfig config, String realizationName) {
    CubeInstance cubeInstance = CubeManager.getInstance(config).getCube(realizationName);
    if (cubeInstance == null) {
        return null;
    }
    if (!cubeInstance.isReady()) {
        return new CubeSignature(realizationName, RealizationStatusEnum.DISABLED, null);
    }
    List<CubeSegment> readySegments = cubeInstance.getSegments(SegmentStatusEnum.READY);
    Set<SegmentSignature> segmentSignatureSet = Sets.newHashSetWithExpectedSize(readySegments.size());
    for (CubeSegment cubeSeg : readySegments) {
        segmentSignatureSet.add(new SegmentSignature(cubeSeg.getName(), cubeSeg.getLastBuildTime()));
    }
    return new CubeSignature(realizationName, RealizationStatusEnum.READY, segmentSignatureSet);
}
Example 6
Source File: KylinHealthCheckJob.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void checkCubeHoles(List<CubeInstance> cubes) {
    reporter.log("## Checking holes of Cubes");
    for (CubeInstance cube : cubes) {
        if (cube.isReady()) {
            List<CubeSegment> holes = cubeManager.calculateHoles(cube.getName());
            if (holes.size() > 0) {
                reporter.log("{} holes in cube: {}, project: {}", holes.size(), cube.getName(), cube.getProject());
            }
        }
    }
}
Example 7
Source File: CubeService.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public String mergeCubeSegment(String cubeName) {
    CubeInstance cube = getCubeManager().getCube(cubeName);
    if (!cube.needAutoMerge())
        return null;

    if (!cube.isReady()) {
        logger.info("The cube: {} is disabled", cubeName);
        return null;
    }

    synchronized (CubeService.class) {
        try {
            cube = getCubeManager().getCube(cubeName);
            SegmentRange offsets = cube.autoMergeCubeSegments();
            if (offsets != null && !isMergingJobBeenDiscarded(cube, cubeName, cube.getProject(), offsets)) {
                CubeSegment newSeg = getCubeManager().mergeSegments(cube, null, offsets, true);
                logger.info("Will submit merge job on " + newSeg);
                DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(newSeg, "SYSTEM");
                getExecutableManager().addJob(job);
                return job.getId();
            } else {
                logger.info("Not ready for merge on cube " + cubeName);
            }
        } catch (IOException e) {
            logger.error("Failed to auto merge cube " + cubeName, e);
        }
    }
    return null;
}
Example 8
Source File: RealizationSignature.java    From kylin with Apache License 2.0 5 votes vote down vote up 
static CubeSignature getCubeSignature(KylinConfig config, String realizationName) {
    CubeInstance cubeInstance = CubeManager.getInstance(config).getCube(realizationName);
    if (cubeInstance == null) {
        return null;
    }
    if (!cubeInstance.isReady()) {
        return new CubeSignature(realizationName, RealizationStatusEnum.DISABLED, null);
    }
    List<CubeSegment> readySegments = cubeInstance.getSegments(SegmentStatusEnum.READY);
    Set<SegmentSignature> segmentSignatureSet = Sets.newHashSetWithExpectedSize(readySegments.size());
    for (CubeSegment cubeSeg : readySegments) {
        segmentSignatureSet.add(new SegmentSignature(cubeSeg.getName(), cubeSeg.getLastBuildTime()));
    }
    return new CubeSignature(realizationName, RealizationStatusEnum.READY, segmentSignatureSet);
}