Java Code Examples for org.apache.kylin.metadata.realization.RealizationStatusEnum#DESCBROKEN

The following examples show how to use org.apache.kylin.metadata.realization.RealizationStatusEnum#DESCBROKEN . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CubeInstance.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
void init(KylinConfig config) {
    CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(descName);
    checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", descName, name);

    if (cubeDesc.isBroken()) {
        setStatus(RealizationStatusEnum.DESCBROKEN);
        logger.error("cube descriptor {} (for cube '{}') is broken", cubeDesc.getResourcePath(), name);
        logger.error("Errors: {}", cubeDesc.getErrorsAsString());
    } else if (getStatus() == RealizationStatusEnum.DESCBROKEN) {
        setStatus(RealizationStatusEnum.DISABLED);
        logger.info("cube {} changed from DESCBROKEN to DISABLED", name);
    }

    setConfig((KylinConfigExt) cubeDesc.getConfig());
}
Example 2
Source File: CubeInstance.java    From kylin with Apache License 2.0 5 votes vote down vote up 
void init(KylinConfig config) {
    CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(descName);
    checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", descName, name);

    if (cubeDesc.isBroken()) {
        setStatus(RealizationStatusEnum.DESCBROKEN);
        logger.error("cube descriptor {} (for cube '{}') is broken", cubeDesc.getResourcePath(), name);
        logger.error("Errors: {}", cubeDesc.getErrorsAsString());
    } else if (getStatus() == RealizationStatusEnum.DESCBROKEN) {
        setStatus(RealizationStatusEnum.DISABLED);
        logger.info("cube {} changed from DESCBROKEN to DISABLED", name);
    }

    setConfig((KylinConfigExt) cubeDesc.getConfig());
}
Example 3
Source File: CubeInstance.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public boolean allowBrokenDescriptor() {
    return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() == RealizationStatusEnum.DESCBROKEN)
            && segments.isEmpty();
}
Example 4
Source File: CubeController.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@RequestMapping(value = "/{cubeName}/clone", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public CubeInstance cloneCube(@PathVariable String cubeName, @RequestBody CubeRequest cubeRequest) {
    String newCubeName = cubeRequest.getCubeName();
    String projectName = cubeRequest.getProject();

    checkCubeExists(cubeName);
    CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
    if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
        throw new BadRequestException("Broken cube can't be cloned");
    }
    if (!ValidateUtil.isAlphanumericUnderscore(newCubeName)) {
        throw new BadRequestException("Invalid Cube name, only letters, numbers and underscore supported.");
    }

    ProjectInstance project = cubeService.getProjectManager().getProject(projectName);
    if (project == null) {
        throw new NotFoundException("Project " + projectName + " doesn't exist");
    }
    // KYLIN-1925, forbid cloning cross projects
    if (!project.getName().equals(cube.getProject())) {
        throw new BadRequestException("Cloning cubes across projects is not supported.");
    }

    CubeDesc cubeDesc = cube.getDescriptor();
    CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);

    newCubeDesc.setName(newCubeName);

    CubeInstance newCube;
    try {
        newCube = cubeService.createCubeAndDesc(project, newCubeDesc);

        //reload to avoid shallow clone
        cubeService.getCubeDescManager().reloadCubeDescLocal(newCubeName);
    } catch (IOException e) {
        throw new InternalErrorException("Failed to clone cube ", e);
    }

    return newCube;

}
Example 5
Source File: JobService.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private Pair<JobInstance, List<JobInstance>> submitOptimizeJobInternal(CubeInstance cube,
        Set<Long> cuboidsRecommend, String submitter) throws IOException {
    Message msg = MsgPicker.getMsg();

    if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getBUILD_BROKEN_CUBE(), cube.getName()));
    }

    checkCubeDescSignature(cube);
    checkAllowOptimization(cube, cuboidsRecommend);

    CubeSegment[] optimizeSegments = null;
    try {
        /** Add optimize segments */
        optimizeSegments = getCubeManager().optimizeSegments(cube, cuboidsRecommend);
        List<JobInstance> optimizeJobInstances = Lists.newLinkedList();

        /** Add optimize jobs */
        List<AbstractExecutable> optimizeJobList = Lists.newArrayListWithExpectedSize(optimizeSegments.length);
        for (CubeSegment optimizeSegment : optimizeSegments) {
            DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, submitter);
            getExecutableManager().addJob(optimizeJob);

            optimizeJobList.add(optimizeJob);
            optimizeJobInstances.add(getSingleJobInstance(optimizeJob));
        }

        /** Add checkpoint job for batch jobs */
        CheckpointExecutable checkpointJob = new BatchOptimizeJobCheckpointBuilder(cube, submitter).build();
        checkpointJob.addTaskListForCheck(optimizeJobList);

        getExecutableManager().addJob(checkpointJob);

        return new Pair(getCheckpointJobInstance(checkpointJob), optimizeJobInstances);
    } catch (Exception e) {
        if (optimizeSegments != null) {
            logger.error("Job submission might failed for NEW segments {}, will clean the NEW segments from cube",
                    optimizeSegments);
            try {
                // Remove this segments
                getCubeManager().updateCubeDropSegments(cube, optimizeSegments);
            } catch (Exception ee) {
                // swallow the exception
                logger.error("Clean New segments failed, ignoring it", e);
            }
        }
        throw e;
    }
}
Example 6
Source File: CubeInstance.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public boolean allowBrokenDescriptor() {
    return (getStatus() == RealizationStatusEnum.DISABLED || getStatus() == RealizationStatusEnum.DESCBROKEN)
            && segments.isEmpty();
}
Example 7
Source File: CubeController.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@RequestMapping(value = "/{cubeName}/clone", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public CubeInstance cloneCube(@PathVariable String cubeName, @RequestBody CubeRequest cubeRequest) {
    String newCubeName = cubeRequest.getCubeName();
    String projectName = cubeRequest.getProject();

    checkCubeExists(cubeName);
    CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
    if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
        throw new BadRequestException("Broken cube can't be cloned");
    }
    if (!ValidateUtil.isAlphanumericUnderscore(newCubeName)) {
        throw new BadRequestException("Invalid Cube name, only letters, numbers and underscore supported.");
    }

    ProjectInstance project = cubeService.getProjectManager().getProject(projectName);
    if (project == null) {
        throw new NotFoundException("Project " + projectName + " doesn't exist");
    }
    // KYLIN-1925, forbid cloning cross projects
    if (!project.getName().equals(cube.getProject())) {
        throw new BadRequestException("Cloning cubes across projects is not supported.");
    }

    CubeDesc cubeDesc = cube.getDescriptor();
    CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);

    newCubeDesc.setName(newCubeName);

    CubeInstance newCube;
    try {
        newCube = cubeService.createCubeAndDesc(project, newCubeDesc);

        //reload to avoid shallow clone
        cubeService.getCubeDescManager().reloadCubeDescLocal(newCubeName);
    } catch (IOException e) {
        throw new InternalErrorException("Failed to clone cube ", e);
    }

    return newCube;

}
Example 8
Source File: JobService.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public JobInstance submitJobInternal(CubeInstance cube, TSRange tsRange, SegmentRange segRange, //
        Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd, //
        CubeBuildTypeEnum buildType, boolean force, String submitter, Integer priorityOffset) throws IOException {
    Message msg = MsgPicker.getMsg();

    if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getBUILD_BROKEN_CUBE(), cube.getName()));
    }

    checkCubeDescSignature(cube);
    checkAllowBuilding(cube);

    if (buildType == CubeBuildTypeEnum.BUILD || buildType == CubeBuildTypeEnum.REFRESH) {
        checkAllowParallelBuilding(cube);
    }

    DefaultChainedExecutable job;

    CubeSegment newSeg = null;
    try {
        if (buildType == CubeBuildTypeEnum.BUILD) {
            ISource source = SourceManager.getSource(cube);
            SourcePartition src = new SourcePartition(tsRange, segRange, sourcePartitionOffsetStart,
                    sourcePartitionOffsetEnd);
            src = source.enrichSourcePartitionBeforeBuild(cube, src);
            newSeg = getCubeManager().appendSegment(cube, src);
            job = EngineFactory.createBatchCubingJob(newSeg, submitter, priorityOffset);
        } else if (buildType == CubeBuildTypeEnum.MERGE) {
            newSeg = getCubeManager().mergeSegments(cube, tsRange, segRange, force);
            job = EngineFactory.createBatchMergeJob(newSeg, submitter);
        } else if (buildType == CubeBuildTypeEnum.REFRESH) {
            newSeg = getCubeManager().refreshSegment(cube, tsRange, segRange);
            job = EngineFactory.createBatchCubingJob(newSeg, submitter, priorityOffset);
        } else {
            throw new BadRequestException(String.format(Locale.ROOT, msg.getINVALID_BUILD_TYPE(), buildType));
        }

        getExecutableManager().addJob(job);

    } catch (Exception e) {
        if (newSeg != null) {
            logger.error("Job submission might failed for NEW segment {}, will clean the NEW segment from cube",
                    newSeg.getName());
            try {
                // Remove this segment
                getCubeManager().updateCubeDropSegments(cube, newSeg);
            } catch (Exception ee) {
                // swallow the exception
                logger.error("Clean New segment failed, ignoring it", e);
            }
        }
        throw e;
    }

    JobInstance jobInstance = getSingleJobInstance(job);

    return jobInstance;
}
Example 9
Source File: JobService.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private Pair<JobInstance, List<JobInstance>> submitOptimizeJobInternal(CubeInstance cube,
        Set<Long> cuboidsRecommend, String submitter) throws IOException {
    Message msg = MsgPicker.getMsg();

    if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getBUILD_BROKEN_CUBE(), cube.getName()));
    }

    checkCubeDescSignature(cube);
    checkAllowOptimization(cube, cuboidsRecommend);

    CubeSegment[] optimizeSegments = null;
    try {
        /** Add optimize segments */
        optimizeSegments = getCubeManager().optimizeSegments(cube, cuboidsRecommend);
        List<JobInstance> optimizeJobInstances = Lists.newLinkedList();

        /** Add optimize jobs */
        List<AbstractExecutable> optimizeJobList = Lists.newArrayListWithExpectedSize(optimizeSegments.length);
        for (CubeSegment optimizeSegment : optimizeSegments) {
            DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, submitter);
            getExecutableManager().addJob(optimizeJob);

            optimizeJobList.add(optimizeJob);
            optimizeJobInstances.add(getSingleJobInstance(optimizeJob));
        }

        /** Add checkpoint job for batch jobs */
        CheckpointExecutable checkpointJob = new BatchOptimizeJobCheckpointBuilder(cube, submitter).build();
        checkpointJob.addTaskListForCheck(optimizeJobList);

        getExecutableManager().addJob(checkpointJob);

        return new Pair(getCheckpointJobInstance(checkpointJob), optimizeJobInstances);
    } catch (Exception e) {
        if (optimizeSegments != null) {
            logger.error("Job submission might failed for NEW segments {}, will clean the NEW segments from cube",
                    optimizeSegments);
            try {
                // Remove this segments
                getCubeManager().updateCubeDropSegments(cube, optimizeSegments);
            } catch (Exception ee) {
                // swallow the exception
                logger.error("Clean New segments failed, ignoring it", e);
            }
        }
        throw e;
    }
}