Java Code Examples for org.apache.kylin.cube.CubeInstance#getStatus()

/**
 * Stop all jobs belonging to this cube and clean out all segments
 *
 * @param cube
 * @return
 * @throws IOException
 * @throws CubeIntegrityException
 * @throws JobException
 */
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
@Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
public CubeInstance purgeCube(CubeInstance cube) throws IOException, JobException {
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();
    if (null != ostatus && !RealizationStatusEnum.DISABLED.equals(ostatus)) {
        throw new InternalErrorException("Only disabled cube can be purged, status of " + cubeName + " is " + ostatus);
    }

    try {
        this.releaseAllSegments(cube);
        return cube;
    } catch (IOException e) {
        throw e;
    }

}
 

public void checkEnableCubeCondition(CubeInstance cube) {
    aclEvaluate.checkProjectWritePermission(cube);
    Message msg = MsgPicker.getMsg();
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();

    if (!cube.getStatus().equals(RealizationStatusEnum.DISABLED)) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getENABLE_NOT_DISABLED_CUBE(), cubeName, ostatus));
    }

    if (cube.getSegments(SegmentStatusEnum.READY).size() == 0 && !cube.getDescriptor().isStreamingCube()) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getNO_READY_SEGMENT(), cubeName));
    }

    if (!cube.getDescriptor().checkSignature()) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getINCONSISTENT_CUBE_DESC_SIGNATURE(), cube.getDescriptor()));
    }
}
 

/**
 * Update a cube status from disable to ready.
 *
 * @return
 * @throws IOException
 */
public CubeInstance enableCube(CubeInstance cube) throws IOException {
    boolean cubeStatusUpdated = false;
    RealizationStatusEnum ostatus = cube.getStatus();
    try {
        CubeInstance cubeInstance = getCubeManager().updateCubeStatus(cube, RealizationStatusEnum.READY);
        cubeStatusUpdated = true;
        // for streaming cube.
        if (cube.getDescriptor().isStreamingCube()) {
            getStreamingCoordinator().assignCube(cube.getName());
        }
        return cubeInstance;
    } catch (Exception e) {
        cube.setStatus(ostatus);
        // roll back if cube status updated
        if (cubeStatusUpdated) {
            logger.info("roll back cube status to:{}", ostatus);
            getCubeManager().updateCubeStatus(cube, ostatus);
        }
        throw e;
    }
}
 

/**
 * Stop all jobs belonging to this cube and clean out all segments
 *
 * @param cube
 * @return
 * @throws IOException
 * @throws JobException
 */
public CubeInstance purgeCube(CubeInstance cube) throws IOException {
    aclEvaluate.checkProjectOperationPermission(cube);
    Message msg = MsgPicker.getMsg();

    String cubeName = cube.getName();

    final List<CubingJob> cubingJobs = jobService.listJobsByRealizationName(cubeName, null, EnumSet
            .of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR, ExecutableState.STOPPED));
    if (!cubingJobs.isEmpty()) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getDISCARD_JOB_FIRST(), cubeName));
    }

    RealizationStatusEnum ostatus = cube.getStatus();
    if (null != ostatus && !RealizationStatusEnum.DISABLED.equals(ostatus)) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getPURGE_NOT_DISABLED_CUBE(), cubeName, ostatus));
    }

    this.releaseAllSegments(cube);
    return cube;
}
 

/**
 * Update a cube status from ready to disabled.
 *
 * @return
 * @throws CubeIntegrityException
 * @throws IOException
 * @throws JobException
 */
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
@Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
public CubeInstance disableCube(CubeInstance cube) throws IOException, JobException {
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();
    if (null != ostatus && !RealizationStatusEnum.READY.equals(ostatus)) {
        throw new InternalErrorException("Only ready cube can be disabled, status of " + cubeName + " is " + ostatus);
    }

    cube.setStatus(RealizationStatusEnum.DISABLED);

    try {
        return getCubeManager().updateCube(cube);
    } catch (IOException e) {
        cube.setStatus(ostatus);
        throw e;
    }
}
 

/**
 * Update a cube status from disable to ready.
 *
 * @return
 * @throws IOException
 */
public CubeInstance enableCube(CubeInstance cube) throws IOException {
    boolean cubeStatusUpdated = false;
    RealizationStatusEnum ostatus = cube.getStatus();
    try {
        CubeInstance cubeInstance = getCubeManager().updateCubeStatus(cube, RealizationStatusEnum.READY);
        cubeStatusUpdated = true;
        // for streaming cube.
        if (cube.getDescriptor().isStreamingCube()) {
            getStreamingCoordinator().assignCube(cube.getName());
        }
        return cubeInstance;
    } catch (Exception e) {
        cube.setStatus(ostatus);
        // roll back if cube status updated
        if (cubeStatusUpdated) {
            logger.info("roll back cube status to:{}", ostatus);
            getCubeManager().updateCubeStatus(cube, ostatus);
        }
        throw e;
    }
}
 

List<StreamingCubeInfo> getEnableStreamingCubes() {
    List<StreamingCubeInfo> allCubes = getStreamingCubes();
    List<StreamingCubeInfo> result = Lists.newArrayList();
    for (StreamingCubeInfo cube : allCubes) {
        CubeInstance cubeInstance = getCubeManager().getCube(cube.getCubeName());
        if (cubeInstance.getStatus() == RealizationStatusEnum.READY) {
            result.add(cube);
        }
    }
    return result;
}
 

@Override
public void apply(Context ctx) throws RuleValidationException {
    CubeInstance cube = ctx.getCubeInstance();
    RealizationStatusEnum status = cube.getStatus();
    if (status != RealizationStatusEnum.READY) {
        throw new RuleValidationException("The cube named " + cube.getName() + " is not in READY state.");
    }
}
 

protected void checkCubeState(CubeInstance cube) {
    if (cube.getStatus() != RealizationStatusEnum.READY)
        throw new IllegalStateException("Cannot migrate cube that is not in READY state.");

    for (CubeSegment segment : cube.getSegments()) {
        if (segment.getStatus() != SegmentStatusEnum.READY) {
            throw new IllegalStateException("At least one segment is not in READY state");
        }
    }
}
 

private List<StreamingCubeInfo> getEnableStreamingCubes() {
    List<StreamingCubeInfo> allCubes = getStreamingCubes();
    List<StreamingCubeInfo> result = Lists.newArrayList();
    for (StreamingCubeInfo cube : allCubes) {
        CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
                .getCube(cube.getCubeName());
        if (cubeInstance.getStatus() == RealizationStatusEnum.READY) {
            result.add(cube);
        }
    }
    return result;
}
 

public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {

        srcConfig = srcCfg;
        srcStore = ResourceStore.getStore(srcConfig);
        dstConfig = dstCfg;
        dstStore = ResourceStore.getStore(dstConfig);

        CubeManager cubeManager = CubeManager.getInstance(srcConfig);
        CubeInstance cube = cubeManager.getCube(cubeName);
        logger.info("cube to be moved is : " + cubeName);

        if (cube.getStatus() != RealizationStatusEnum.READY)
            throw new IllegalStateException("Cannot migrate cube that is not in READY state.");

        for (CubeSegment segment : cube.getSegments()) {
            if (segment.getStatus() != SegmentStatusEnum.READY) {
                throw new IllegalStateException("At least one segment is not in READY state");
            }
        }

        checkAndGetHbaseUrl();

        Configuration conf = HBaseConfiguration.create();
        hbaseAdmin = new HBaseAdmin(conf);

        hdfsFS = FileSystem.get(new Configuration());

        operations = new ArrayList<Opt>();

        copyFilesInMetaStore(cube, overwriteIfExists);
        renameFoldersInHdfs(cube);
        changeHtableHost(cube);
        addCubeIntoProject(cubeName, projectName);

        if (realExecute.equalsIgnoreCase("true")) {
            doOpts();
        } else {
            showOpts();
        }
    }
 

/**
 * Update a cube status from disable to ready.
 *
 * @return
 * @throws CubeIntegrityException
 * @throws IOException
 * @throws JobException
 */
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION')  or hasPermission(#cube, 'MANAGEMENT')")
public CubeInstance enableCube(CubeInstance cube) throws IOException, JobException {
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();
    if (!cube.getStatus().equals(RealizationStatusEnum.DISABLED)) {
        throw new InternalErrorException("Only disabled cube can be enabled, status of " + cubeName + " is " + ostatus);
    }

    if (cube.getSegments(SegmentStatusEnum.READY).size() == 0) {
        throw new InternalErrorException("Cube " + cubeName + " dosen't contain any READY segment");
    }

    final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
    if (!cubingJobs.isEmpty()) {
        throw new JobException("Enable is not allowed with a running job.");
    }
    if (!cube.getDescriptor().calculateSignature().equals(cube.getDescriptor().getSignature())) {
        this.releaseAllSegments(cube);
    }

    cube.setStatus(RealizationStatusEnum.READY);
    try {
        return getCubeManager().updateCube(cube);
    } catch (IOException e) {
        cube.setStatus(ostatus);
        throw e;
    }
}
 

protected void checkCubeState(CubeInstance cube) {
    if (cube.getStatus() != RealizationStatusEnum.READY)
        throw new IllegalStateException("Cannot migrate cube that is not in READY state.");

    for (CubeSegment segment : cube.getSegments()) {
        if (segment.getStatus() != SegmentStatusEnum.READY) {
            throw new IllegalStateException("At least one segment is not in READY state");
        }
    }
}
 

private Pair<JobInstance, List<JobInstance>> submitOptimizeJobInternal(CubeInstance cube,
        Set<Long> cuboidsRecommend, String submitter) throws IOException {
    Message msg = MsgPicker.getMsg();

    if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getBUILD_BROKEN_CUBE(), cube.getName()));
    }

    checkCubeDescSignature(cube);
    checkAllowOptimization(cube, cuboidsRecommend);

    CubeSegment[] optimizeSegments = null;
    try {
        /** Add optimize segments */
        optimizeSegments = getCubeManager().optimizeSegments(cube, cuboidsRecommend);
        List<JobInstance> optimizeJobInstances = Lists.newLinkedList();

        /** Add optimize jobs */
        List<AbstractExecutable> optimizeJobList = Lists.newArrayListWithExpectedSize(optimizeSegments.length);
        for (CubeSegment optimizeSegment : optimizeSegments) {
            DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, submitter);
            getExecutableManager().addJob(optimizeJob);

            optimizeJobList.add(optimizeJob);
            optimizeJobInstances.add(getSingleJobInstance(optimizeJob));
        }

        /** Add checkpoint job for batch jobs */
        CheckpointExecutable checkpointJob = new BatchOptimizeJobCheckpointBuilder(cube, submitter).build();
        checkpointJob.addTaskListForCheck(optimizeJobList);

        getExecutableManager().addJob(checkpointJob);

        return new Pair(getCheckpointJobInstance(checkpointJob), optimizeJobInstances);
    } catch (Exception e) {
        if (optimizeSegments != null) {
            logger.error("Job submission might failed for NEW segments {}, will clean the NEW segments from cube",
                    optimizeSegments);
            try {
                // Remove this segments
                getCubeManager().updateCubeDropSegments(cube, optimizeSegments);
            } catch (Exception ee) {
                // swallow the exception
                logger.error("Clean New segments failed, ignoring it", e);
            }
        }
        throw e;
    }
}
 

public CubeInstance deleteSegment(CubeInstance cube, String segmentName) throws IOException {
    aclEvaluate.checkProjectOperationPermission(cube);
    Message msg = MsgPicker.getMsg();

    if (cube.getStatus() == RealizationStatusEnum.READY) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getDELETE_SEG_FROM_READY_CUBE(), segmentName, cube.getName()));
    }

    CubeSegment toDelete = null;
    for (CubeSegment seg : cube.getSegments()) {
        if (seg.getName().equals(segmentName)) {
            toDelete = seg;
            break;
        }
    }

    if (toDelete == null) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getSEG_NOT_FOUND(), segmentName));
    }

    if (toDelete.getStatus() != SegmentStatusEnum.READY) {
        if (toDelete.getStatus() == SegmentStatusEnum.NEW) {
            if (!isOrphonSegment(cube, toDelete.getUuid())) {
                throw new BadRequestException(
                        String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
            }
        } else {
            throw new BadRequestException(String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
        }
    }

    if (!segmentName.equals(cube.getSegments().get(0).getName())
            && !segmentName.equals(cube.getSegments().get(cube.getSegments().size() - 1).getName())) {
        logger.warn(String.format(Locale.ROOT, msg.getDELETE_SEGMENT_CAUSE_GAPS(), cube.getName(), segmentName));
    }

    CubeInstance cubeInstance = CubeManager.getInstance(getConfig()).updateCubeDropSegments(cube, toDelete);

    cleanSegmentStorage(Collections.singletonList(toDelete));

    return cubeInstance;
}
 

/**
 * Update a cube status from ready to disabled.
 *
 * @return
 * @throws IOException
 * @throws JobException
 */
public CubeInstance disableCube(CubeInstance cube) throws IOException {
    aclEvaluate.checkProjectWritePermission(cube);
    Message msg = MsgPicker.getMsg();

    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();
    if (null != ostatus && !RealizationStatusEnum.READY.equals(ostatus)) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getDISABLE_NOT_READY_CUBE(), cubeName, ostatus));
    }

    boolean isStreamingCube = cube.getDescriptor().isStreamingCube();

    boolean cubeStatusUpdated = false;
    try {
        CubeInstance cubeInstance = getCubeManager().updateCubeStatus(cube, RealizationStatusEnum.DISABLED);
        cubeStatusUpdated = true;
        // for streaming cube.
        if (isStreamingCube) {
            //drop not ready segments
            CubeSegment[] buildingSegments = new CubeSegment[cubeInstance.getBuildingSegments().size()];
            Segments segments = cubeInstance.getBuildingSegments();
            if (!CollectionUtils.isEmpty(segments)) {
                for (int i = 0; i < segments.size(); i++) {
                    buildingSegments[i] = (CubeSegment) segments.get(i);
                }
                getCubeManager().dropOptmizingSegments(cubeInstance, buildingSegments);
            }
            //unAssign cube
            getStreamingCoordinator().unAssignCube(cubeName);

            //discard jobs
            releaseAllJobs(cubeInstance);

        }
        return cubeInstance;
    } catch (Exception e) {
        cube.setStatus(ostatus);
        // roll back if cube status updated
        if (cubeStatusUpdated) {
            logger.info("roll back cube status to:{}", ostatus);
            getCubeManager().updateCubeStatus(cube, ostatus);
        }
        throw e;
    }
}
 

public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName,
        String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute)
        throws IOException, InterruptedException {

    srcConfig = srcCfg;
    srcStore = ResourceStore.getStore(srcConfig);
    dstConfig = dstCfg;
    dstStore = ResourceStore.getStore(dstConfig);

    CubeManager cubeManager = CubeManager.getInstance(srcConfig);
    CubeInstance cube = cubeManager.getCube(cubeName);
    logger.info("cube to be moved is : " + cubeName);

    if (cube.getStatus() != RealizationStatusEnum.READY)
        throw new IllegalStateException("Cannot migrate cube that is not in READY state.");

    for (CubeSegment segment : cube.getSegments()) {
        if (segment.getStatus() != SegmentStatusEnum.READY) {
            throw new IllegalStateException("At least one segment is not in READY state");
        }
    }

    checkAndGetHbaseUrl();

    Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
    hbaseAdmin = conn.getAdmin();

    hdfsFS = HadoopUtil.getWorkingFileSystem();

    operations = new ArrayList<Opt>();

    copyFilesInMetaStore(cube, overwriteIfExists);
    renameFoldersInHdfs(cube);
    changeHtableHost(cube);
    addCubeAndModelIntoProject(cube, cubeName, projectName);
    if (Boolean.parseBoolean(copyAcl) == true) {
        copyACL(cube, projectName);
    }

    if (Boolean.parseBoolean(purgeAndDisable) == true) {
        purgeAndDisable(cubeName); // this should be the last action
    }

    if (realExecute.equalsIgnoreCase("true")) {
        doOpts();
        checkMigrationSuccess(dstConfig, cubeName, true);
        updateMeta(dstConfig);
    } else {
        showOpts();
    }
}
 

@RequestMapping(value = "/{cubeName}/clone", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public CubeInstance cloneCube(@PathVariable String cubeName, @RequestBody CubeRequest cubeRequest) {
    String newCubeName = cubeRequest.getCubeName();
    String projectName = cubeRequest.getProject();

    checkCubeExists(cubeName);
    CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
    if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
        throw new BadRequestException("Broken cube can't be cloned");
    }
    if (!ValidateUtil.isAlphanumericUnderscore(newCubeName)) {
        throw new BadRequestException("Invalid Cube name, only letters, numbers and underscore supported.");
    }

    ProjectInstance project = cubeService.getProjectManager().getProject(projectName);
    if (project == null) {
        throw new NotFoundException("Project " + projectName + " doesn't exist");
    }
    // KYLIN-1925, forbid cloning cross projects
    if (!project.getName().equals(cube.getProject())) {
        throw new BadRequestException("Cloning cubes across projects is not supported.");
    }

    CubeDesc cubeDesc = cube.getDescriptor();
    CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);

    newCubeDesc.setName(newCubeName);

    CubeInstance newCube;
    try {
        newCube = cubeService.createCubeAndDesc(project, newCubeDesc);

        //reload to avoid shallow clone
        cubeService.getCubeDescManager().reloadCubeDescLocal(newCubeName);
    } catch (IOException e) {
        throw new InternalErrorException("Failed to clone cube ", e);
    }

    return newCube;

}
 

@RequestMapping(value = "/{cubeName}/clone", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public CubeInstance cloneCube(@PathVariable String cubeName, @RequestBody CubeRequest cubeRequest) {
    String newCubeName = cubeRequest.getCubeName();
    String projectName = cubeRequest.getProject();

    checkCubeExists(cubeName);
    CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
    if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
        throw new BadRequestException("Broken cube can't be cloned");
    }
    if (!ValidateUtil.isAlphanumericUnderscore(newCubeName)) {
        throw new BadRequestException("Invalid Cube name, only letters, numbers and underscore supported.");
    }

    ProjectInstance project = cubeService.getProjectManager().getProject(projectName);
    if (project == null) {
        throw new NotFoundException("Project " + projectName + " doesn't exist");
    }
    // KYLIN-1925, forbid cloning cross projects
    if (!project.getName().equals(cube.getProject())) {
        throw new BadRequestException("Cloning cubes across projects is not supported.");
    }

    CubeDesc cubeDesc = cube.getDescriptor();
    CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);

    newCubeDesc.setName(newCubeName);

    CubeInstance newCube;
    try {
        newCube = cubeService.createCubeAndDesc(project, newCubeDesc);

        //reload to avoid shallow clone
        cubeService.getCubeDescManager().reloadCubeDescLocal(newCubeName);
    } catch (IOException e) {
        throw new InternalErrorException("Failed to clone cube ", e);
    }

    return newCube;

}
 

private Pair<JobInstance, List<JobInstance>> submitOptimizeJobInternal(CubeInstance cube,
        Set<Long> cuboidsRecommend, String submitter) throws IOException {
    Message msg = MsgPicker.getMsg();

    if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getBUILD_BROKEN_CUBE(), cube.getName()));
    }

    checkCubeDescSignature(cube);
    checkAllowOptimization(cube, cuboidsRecommend);

    CubeSegment[] optimizeSegments = null;
    try {
        /** Add optimize segments */
        optimizeSegments = getCubeManager().optimizeSegments(cube, cuboidsRecommend);
        List<JobInstance> optimizeJobInstances = Lists.newLinkedList();

        /** Add optimize jobs */
        List<AbstractExecutable> optimizeJobList = Lists.newArrayListWithExpectedSize(optimizeSegments.length);
        for (CubeSegment optimizeSegment : optimizeSegments) {
            DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, submitter);
            getExecutableManager().addJob(optimizeJob);

            optimizeJobList.add(optimizeJob);
            optimizeJobInstances.add(getSingleJobInstance(optimizeJob));
        }

        /** Add checkpoint job for batch jobs */
        CheckpointExecutable checkpointJob = new BatchOptimizeJobCheckpointBuilder(cube, submitter).build();
        checkpointJob.addTaskListForCheck(optimizeJobList);

        getExecutableManager().addJob(checkpointJob);

        return new Pair(getCheckpointJobInstance(checkpointJob), optimizeJobInstances);
    } catch (Exception e) {
        if (optimizeSegments != null) {
            logger.error("Job submission might failed for NEW segments {}, will clean the NEW segments from cube",
                    optimizeSegments);
            try {
                // Remove this segments
                getCubeManager().updateCubeDropSegments(cube, optimizeSegments);
            } catch (Exception ee) {
                // swallow the exception
                logger.error("Clean New segments failed, ignoring it", e);
            }
        }
        throw e;
    }
}