Java Code Examples for org.apache.kylin.metadata.model.SegmentStatusEnum#NEW

The following examples show how to use org.apache.kylin.metadata.model.SegmentStatusEnum#NEW . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KylinHealthCheckJob.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private void checkHBaseTables(List<CubeInstance> cubes) throws IOException {
    reporter.log("## Checking HBase Table of segments");
    HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create());
    try {
        for (CubeInstance cube : cubes) {
            for (CubeSegment segment : cube.getSegments()) {
                if (segment.getStatus() != SegmentStatusEnum.NEW) {
                    String tableName = segment.getStorageLocationIdentifier();
                    if ((!hbaseAdmin.tableExists(tableName)) || (!hbaseAdmin.isTableEnabled(tableName))) {
                        reporter.log("HBase table: {} not exist for segment: {}, project: {}", tableName, segment,
                                cube.getProject());
                        reporter.log(
                                "The rebuild url: -d '{\"startTime\":{}, \"endTime\":{}, \"buildType\":\"REFRESH\"}' /kylin/api/cubes/{}/build",
                                segment.getTSRange().start, segment.getTSRange().end, cube.getName());
                    }
                }
            }
        }
    } finally {
        if (null != hbaseAdmin) {
            hbaseAdmin.close();
        }
    }

}
Example 2
Source File: JobService.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private void cancelCubingJobInner(CubingJob cubingJob) throws IOException {
    CubeInstance cubeInstance = getCubeManager().getCube(CubingExecutableUtil.getCubeName(cubingJob.getParams()));
    // might not a cube job
    final String segmentIds = CubingExecutableUtil.getSegmentId(cubingJob.getParams());
    if (!StringUtils.isEmpty(segmentIds)) {
        for (String segmentId : StringUtils.split(segmentIds)) {
            final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
            if (segment != null
                    && (segment.getStatus() == SegmentStatusEnum.NEW || segment.getTSRange().end.v == 0)) {
                // Remove this segment
                getCubeManager().updateCubeDropSegments(cubeInstance, segment);
            }
        }
    }
    getExecutableManager().discardJob(cubingJob.getId());
}
Example 3
Source File: AfterMergeOrRefreshResourceMerger.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
List<CubeSegment> getToRemoveSegs(CubeInstance cube, CubeSegment mergedSegment) {
    Segments tobe = cube.calculateToBeSegments(mergedSegment);

    if (!tobe.contains(mergedSegment))
        throw new IllegalStateException(
                "For Cube " + cube + ", segment " + mergedSegment + " is expected but not in the tobe " + tobe);

    if (mergedSegment.getStatus() == SegmentStatusEnum.NEW)
        mergedSegment.setStatus(SegmentStatusEnum.READY);

    List<CubeSegment> toRemoveSegs = Lists.newArrayList();
    for (CubeSegment s : cube.getSegments()) {
        if (!tobe.contains(s))
            toRemoveSegs.add(s);
    }

    return toRemoveSegs;
}
Example 4
Source File: KylinHealthCheckJob.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private void checkHBaseTables(List<CubeInstance> cubes) throws IOException {
    reporter.log("## Checking HBase Table of segments");
    HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create());
    try {
        for (CubeInstance cube : cubes) {
            for (CubeSegment segment : cube.getSegments()) {
                if (segment.getStatus() != SegmentStatusEnum.NEW) {
                    String tableName = segment.getStorageLocationIdentifier();
                    if ((!hbaseAdmin.tableExists(tableName)) || (!hbaseAdmin.isTableEnabled(tableName))) {
                        reporter.log("HBase table: {} not exist for segment: {}, project: {}", tableName, segment,
                                cube.getProject());
                        reporter.log(
                                "The rebuild url: -d '{\"startTime\":{}, \"endTime\":{}, \"buildType\":\"REFRESH\"}' /kylin/api/cubes/{}/build",
                                segment.getTSRange().start, segment.getTSRange().end, cube.getName());
                    }
                }
            }
        }
    } finally {
        if (null != hbaseAdmin) {
            hbaseAdmin.close();
        }
    }

}
Example 5
Source File: JobService.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private void cancelCubingJobInner(CubingJob cubingJob) throws IOException {
    CubeInstance cubeInstance = getCubeManager().getCube(CubingExecutableUtil.getCubeName(cubingJob.getParams()));
    // might not a cube job
    final String segmentIds = CubingExecutableUtil.getSegmentId(cubingJob.getParams());
    if (!StringUtils.isEmpty(segmentIds)) {
        for (String segmentId : StringUtils.split(segmentIds)) {
            final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
            if (segment != null
                    && (segment.getStatus() == SegmentStatusEnum.NEW || segment.getTSRange().end.v == 0)) {
                // Remove this segment
                getCubeManager().updateCubeDropSegments(cubeInstance, segment);
            }
        }
    }
    getExecutableManager().discardJob(cubingJob.getId());
}
Example 6
Source File: JobService.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#job, 'ADMINISTRATION') or hasPermission(#job, 'OPERATION') or hasPermission(#job, 'MANAGEMENT')")
public JobInstance cancelJob(String jobId) throws IOException, JobException {
    //        CubeInstance cube = this.getCubeManager().getCube(job.getRelatedCube());
    //        for (BuildCubeJob cubeJob: listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING))) {
    //            getExecutableManager().stopJob(cubeJob.getId());
    //        }
    final JobInstance jobInstance = getJobInstance(jobId);
    final String segmentId = jobInstance.getRelatedSegment();
    CubeInstance cubeInstance = getCubeManager().getCube(jobInstance.getRelatedCube());
    final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
    if (segment.getStatus() == SegmentStatusEnum.NEW) {
        cubeInstance.getSegments().remove(segment);
        getCubeManager().updateCube(cubeInstance);
    }
    getExecutableManager().discardJob(jobId);
    return jobInstance;
}
Example 7
Source File: CubeInstance.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
public List<CubeSegment> getBuildingSegments() {
    List<CubeSegment> buildingSegments = new ArrayList<CubeSegment>();
    if (null != segments) {
        for (CubeSegment segment : segments) {
            if (SegmentStatusEnum.NEW == segment.getStatus() || SegmentStatusEnum.READY_PENDING == segment.getStatus()) {
                buildingSegments.add(segment);
            }
        }
    }

    return buildingSegments;
}
Example 8
Source File: CubeService.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public CubeInstance deleteSegment(CubeInstance cube, String segmentName) throws IOException {
    aclEvaluate.checkProjectOperationPermission(cube);
    Message msg = MsgPicker.getMsg();

    if (cube.getStatus() == RealizationStatusEnum.READY) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getDELETE_SEG_FROM_READY_CUBE(), segmentName, cube.getName()));
    }

    CubeSegment toDelete = null;
    for (CubeSegment seg : cube.getSegments()) {
        if (seg.getName().equals(segmentName)) {
            toDelete = seg;
            break;
        }
    }

    if (toDelete == null) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getSEG_NOT_FOUND(), segmentName));
    }

    if (toDelete.getStatus() != SegmentStatusEnum.READY) {
        if (toDelete.getStatus() == SegmentStatusEnum.NEW) {
            if (!isOrphonSegment(cube, toDelete.getUuid())) {
                throw new BadRequestException(
                        String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
            }
        } else {
            throw new BadRequestException(String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
        }
    }

    if (!segmentName.equals(cube.getSegments().get(0).getName())
            && !segmentName.equals(cube.getSegments().get(cube.getSegments().size() - 1).getName())) {
        logger.warn(String.format(Locale.ROOT, msg.getDELETE_SEGMENT_CAUSE_GAPS(), cube.getName(), segmentName));
    }

    CubeInstance cubeInstance = CubeManager.getInstance(getConfig()).updateCubeDropSegments(cube, toDelete);

    cleanSegmentStorage(Collections.singletonList(toDelete));

    return cubeInstance;
}
Example 9
Source File: CubeService.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public CubeInstance deleteSegment(CubeInstance cube, String segmentName) throws IOException {
    aclEvaluate.checkProjectOperationPermission(cube);
    Message msg = MsgPicker.getMsg();

    if (cube.getStatus() == RealizationStatusEnum.READY) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getDELETE_SEG_FROM_READY_CUBE(), segmentName, cube.getName()));
    }

    CubeSegment toDelete = null;
    for (CubeSegment seg : cube.getSegments()) {
        if (seg.getName().equals(segmentName)) {
            toDelete = seg;
            break;
        }
    }

    if (toDelete == null) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getSEG_NOT_FOUND(), segmentName));
    }

    if (toDelete.getStatus() != SegmentStatusEnum.READY) {
        if (toDelete.getStatus() == SegmentStatusEnum.NEW) {
            if (!isOrphonSegment(cube, toDelete.getUuid())) {
                throw new BadRequestException(
                        String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
            }
        } else {
            throw new BadRequestException(String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
        }
    }

    if (!segmentName.equals(cube.getSegments().get(0).getName())
            && !segmentName.equals(cube.getSegments().get(cube.getSegments().size() - 1).getName())) {
        logger.warn(String.format(Locale.ROOT, msg.getDELETE_SEGMENT_CAUSE_GAPS(), cube.getName(), segmentName));
    }

    CubeInstance cubeInstance = CubeManager.getInstance(getConfig()).updateCubeDropSegments(cube, toDelete);

    cleanSegmentStorage(Collections.singletonList(toDelete));

    return cubeInstance;
}
Example 10
Source File: CubeManager.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
private boolean isNew(CubeSegment seg) {
    return seg.getStatus() == SegmentStatusEnum.NEW || seg.getStatus() == SegmentStatusEnum.READY_PENDING;
}