Java Code Examples for org.apache.kylin.cube.CubeInstance#getName()

/**
 * Stop all jobs belonging to this cube and clean out all segments
 *
 * @param cube
 * @return
 * @throws IOException
 * @throws JobException
 */
public CubeInstance purgeCube(CubeInstance cube) throws IOException {
    aclEvaluate.checkProjectOperationPermission(cube);
    Message msg = MsgPicker.getMsg();

    String cubeName = cube.getName();

    final List<CubingJob> cubingJobs = jobService.listJobsByRealizationName(cubeName, null, EnumSet
            .of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR, ExecutableState.STOPPED));
    if (!cubingJobs.isEmpty()) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getDISCARD_JOB_FIRST(), cubeName));
    }

    RealizationStatusEnum ostatus = cube.getStatus();
    if (null != ostatus && !RealizationStatusEnum.DISABLED.equals(ostatus)) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getPURGE_NOT_DISABLED_CUBE(), cubeName, ostatus));
    }

    this.releaseAllSegments(cube);
    return cube;
}
 

public void checkEnableCubeCondition(CubeInstance cube) {
    aclEvaluate.checkProjectWritePermission(cube);
    Message msg = MsgPicker.getMsg();
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();

    if (!cube.getStatus().equals(RealizationStatusEnum.DISABLED)) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getENABLE_NOT_DISABLED_CUBE(), cubeName, ostatus));
    }

    if (cube.getSegments(SegmentStatusEnum.READY).size() == 0 && !cube.getDescriptor().isStreamingCube()) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getNO_READY_SEGMENT(), cubeName));
    }

    if (!cube.getDescriptor().checkSignature()) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getINCONSISTENT_CUBE_DESC_SIGNATURE(), cube.getDescriptor()));
    }
}
 

private static LookupSnapshotBuildJob initJob(CubeInstance cube, String tableName, String submitter,
        KylinConfig kylinConfig) {
    List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(),
            cube.getName());
    if (projList == null || projList.size() == 0) {
        throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!");
    } else if (projList.size() >= 2) {
        String msg = "Find more than one project containing the cube " + cube.getName()
                + ". It does't meet the uniqueness requirement!!! ";
        throw new RuntimeException(msg);
    }

    LookupSnapshotBuildJob result = new LookupSnapshotBuildJob();
    SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss", Locale.ROOT);
    format.setTimeZone(TimeZone.getTimeZone(kylinConfig.getTimeZone()));
    result.setDeployEnvName(kylinConfig.getDeployEnv());
    result.setProjectName(projList.get(0).getName());
    CubingExecutableUtil.setCubeName(cube.getName(), result.getParams());
    result.setName(JOB_TYPE + " CUBE - " + cube.getName() + " - " + " TABLE - " + tableName + " - "
            + format.format(new Date(System.currentTimeMillis())));
    result.setSubmitter(submitter);
    result.setNotifyList(cube.getDescriptor().getNotifyList());
    return result;
}
 

private void checkAllowParallelBuilding(CubeInstance cube) {
    if (cube.getConfig().isCubePlannerEnabled()) {
        if (cube.getCuboids() == null) {
            Segments<CubeSegment> cubeSegments = cube.getSegments();
            if (cubeSegments.size() > 0 && cubeSegments.getSegments(SegmentStatusEnum.READY).size() <= 0) {
                throw new BadRequestException("The cube " + cube.getName() + " has segments " + cubeSegments
                        + ", but none of them is READY. It's not allowed for parallel building");
            }
        }
    }
}
 

public RocksDBSegmentStore(String baseStorePath, CubeInstance cubeInstance, String segmentName) {
    this.cubeInstance = cubeInstance;
    this.cubeName = cubeInstance.getName();
    this.segmentName = segmentName;

    this.dataSegmentFolder = new File(baseStorePath + File.separator + cubeName + File.separator + segmentName);
    if (!dataSegmentFolder.exists()) {
        dataSegmentFolder.mkdirs();
    }
}
 

/** For future segment level recommend */
public static Map<Long, Long> getRecommendCuboidList(CubeSegment segment, Map<Long, Long> hitFrequencyMap,
        Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap, boolean ifForceRecommend)
        throws IOException {
    if (segment == null) {
        return null;
    }

    CubeStatsReader cubeStatsReader = new CubeStatsReader(segment, null, segment.getConfig());
    if (cubeStatsReader.getCuboidRowEstimatesHLL() == null
            || cubeStatsReader.getCuboidRowEstimatesHLL().isEmpty()) {
        logger.info("Cuboid Statistics is not enabled.");
        return null;
    }
    CubeInstance cube = segment.getCubeInstance();
    long baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
    if (cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == null
            || cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == 0L) {
        logger.info(BASE_CUBOID_COUNT_IN_CUBOID_STATISTICS_IS_ZERO);
        return null;
    }

    String key = cube.getName() + "-" + segment.getName();
    CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, cubeStatsReader.getCuboidRowEstimatesHLL(),
            cubeStatsReader.getCuboidSizeMap()).setHitFrequencyMap(hitFrequencyMap)
                    .setRollingUpCountSourceMap(rollingUpCountSourceMap).build();
    return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, segment.getConfig(),
            ifForceRecommend);
}
 

public ColumnarSegmentStore(String baseStorePath, CubeInstance cubeInstance, String segmentName) {
    this.maxRowsInMemory = cubeInstance.getConfig().getStreamingIndexMaxRows();
    this.baseStorePath = baseStorePath;
    this.parsedStreamingCubeInfo = new ParsedStreamingCubeInfo(cubeInstance);
    this.cubeName = cubeInstance.getName();
    this.segmentName = segmentName;

    this.dataSegmentFolder = new File(baseStorePath + File.separator + cubeName + File.separator + segmentName);
    if (!dataSegmentFolder.exists()) {
        dataSegmentFolder.mkdirs();
    }
    this.activeMemoryStore = new SegmentMemoryStore(parsedStreamingCubeInfo, segmentName);
    this.memoryStorePersister = new ColumnarMemoryStorePersister(parsedStreamingCubeInfo, segmentName);
    this.autoMergeEnabled = cubeInstance.getConfig().isStreamingFragmentsAutoMergeEnabled();
    try {
        StreamingMetrics
                .getInstance()
                .getMetricRegistry()
                .register(MetricRegistry.name("streaming.inMem.row.cnt", cubeInstance.getName(), segmentName),
                        new Gauge<Integer>() {
                            @Override
                            public Integer getValue() {
                                return activeMemoryStore.getRowCount();
                            }
                        });
    } catch (Exception e) {
        logger.warn("metrics register failed", e);
    }
}
 

private void checkAllowParallelBuilding(CubeInstance cube) {
    if (cube.getConfig().isCubePlannerEnabled()) {
        if (cube.getCuboids() == null) {
            Segments<CubeSegment> cubeSegments = cube.getSegments();
            if (cubeSegments.size() > 0 && cubeSegments.getSegments(SegmentStatusEnum.READY).size() <= 0) {
                throw new BadRequestException("The cube " + cube.getName() + " has segments " + cubeSegments
                        + ", but none of them is READY. It's not allowed for parallel building");
            }
        }
    }
}
 

@Override
public void apply(Context ctx) throws RuleValidationException {
    CubeInstance cube = ctx.getCubeInstance();
    RealizationStatusEnum status = cube.getStatus();
    if (status != RealizationStatusEnum.READY) {
        throw new RuleValidationException("The cube named " + cube.getName() + " is not in READY state.");
    }
}
 

@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public void deleteCube(CubeInstance cube) throws IOException, JobException {
    final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
    if (!cubingJobs.isEmpty()) {
        throw new JobException("The cube " + cube.getName() + " has running job, please discard it and try again.");
    }

    this.releaseAllSegments(cube);
    getCubeManager().dropCube(cube.getName(), true);
    accessService.clean(cube, true);
}
 

public StreamingCubeSegment(CubeInstance cubeInstance, IStreamingSegmentStore segmentStore, long segmentStart,
        long segmentEnd) {
    this.cubeInstance = cubeInstance;

    this.cubeName = cubeInstance.getName();
    this.dateRangeStart = segmentStart;
    this.dateRangeEnd = segmentEnd;
    this.createTime = System.currentTimeMillis();
    this.lastUpdateTime = System.currentTimeMillis();
    this.state = State.ACTIVE;
    this.segmentStore = segmentStore;
    this.segmentName = CubeSegment.makeSegmentName(new TSRange(segmentStart, segmentEnd), null, cubeInstance.getModel());
}
 

private static CubingJob initCubingJob(CubeSegment seg, String jobType, String submitter, JobEngineConfig config) {
    KylinConfig kylinConfig = config.getConfig();
    CubeInstance cube = seg.getCubeInstance();
    List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(),
            cube.getName());
    if (projList == null || projList.size() == 0) {
        throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!");
    } else if (projList.size() >= 2) {
        String msg = "Find more than one project containing the cube " + cube.getName()
                + ". It does't meet the uniqueness requirement!!! ";
        if (!config.getConfig().allowCubeAppearInMultipleProjects()) {
            throw new RuntimeException(msg);
        } else {
            logger.warn(msg);
        }
    }

    CubingJob result = new CubingJob();
    SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss", Locale.ROOT);
    format.setTimeZone(TimeZone.getTimeZone(config.getTimeZone()));
    result.setDeployEnvName(kylinConfig.getDeployEnv());
    result.setProjectName(projList.get(0).getName());
    result.setJobType(jobType);
    CubingExecutableUtil.setCubeName(seg.getCubeInstance().getName(), result.getParams());
    CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
    CubingExecutableUtil.setSegmentName(seg.getName(), result.getParams());
    result.setName(jobType + " CUBE - " + seg.getCubeInstance().getDisplayName() + " - " + seg.getName() + " - "
            + format.format(new Date(System.currentTimeMillis())));
    result.setSubmitter(submitter);
    result.setNotifyList(seg.getCubeInstance().getDescriptor().getNotifyList());
    return result;
}
 

/** For future segment level recommend */
public static Map<Long, Long> getRecommendCuboidList(CubeSegment segment, Map<Long, Long> hitFrequencyMap,
        Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap, boolean ifForceRecommend)
        throws IOException {
    if (segment == null) {
        return null;
    }

    CubeStatsReader cubeStatsReader = new CubeStatsReader(segment, null, segment.getConfig());
    if (cubeStatsReader.getCuboidRowEstimatesHLL() == null
            || cubeStatsReader.getCuboidRowEstimatesHLL().isEmpty()) {
        logger.info("Cuboid Statistics is not enabled.");
        return null;
    }
    CubeInstance cube = segment.getCubeInstance();
    long baseCuboid = cube.getCuboidScheduler().getBaseCuboidId();
    if (cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == null
            || cubeStatsReader.getCuboidRowEstimatesHLL().get(baseCuboid) == 0L) {
        logger.info(BASE_CUBOID_COUNT_IN_CUBOID_STATISTICS_IS_ZERO);
        return null;
    }

    String key = cube.getName() + "-" + segment.getName();
    CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, cubeStatsReader.getCuboidRowEstimatesHLL(),
            cubeStatsReader.getCuboidSizeMap()).setHitFrequencyMap(hitFrequencyMap)
                    .setRollingUpCountSourceMap(rollingUpCountSourceMap).build();
    return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, segment.getConfig(),
            ifForceRecommend);
}
 

/** Trigger cube planner phase two for optimization */
public static Map<Long, Long> getRecommendCuboidList(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
        Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap) throws IOException {

    CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
    Set<Long> currentCuboids = cuboidScheduler.getAllCuboidIds();
    Pair<Map<Long, Long>, Map<Long, Double>> statsPair = CuboidStatsReaderUtil
            .readCuboidStatsAndSizeFromCube(currentCuboids, cube);
    long baseCuboid = cuboidScheduler.getBaseCuboidId();
    if (statsPair.getFirst().get(baseCuboid) == null || statsPair.getFirst().get(baseCuboid) == 0L) {
        logger.info(BASE_CUBOID_COUNT_IN_CUBOID_STATISTICS_IS_ZERO);
        return null;
    }

    KylinConfig config = cube.getConfig();
    String key = cube.getName();
    double queryUncertaintyRatio = config.getCubePlannerQueryUncertaintyRatio();
    double bpusMinBenefitRatio = config.getCubePlannerBPUSMinBenefitRatio();
    CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, statsPair.getFirst(),
            statsPair.getSecond()) {
        @Override
        public Map<Long, Double> estimateCuboidsSize(Map<Long, Long> statistics) {
            try {
                return CuboidStatsReaderUtil.readCuboidSizeFromCube(statistics, cube);
            } catch (IOException e) {
                logger.warn("Fail to get cuboid size from cube due to ", e);
                return null;
            }
        }
    }.setQueryUncertaintyRatio(queryUncertaintyRatio) //
            .setBPUSMinBenefitRatio(bpusMinBenefitRatio) //
            .setHitFrequencyMap(hitFrequencyMap) //
            .setRollingUpCountSourceMap(rollingUpCountSourceMap) //
            .build();
    return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, config);
}
 

/**
 * Update a cube status from disable to ready.
 *
 * @return
 * @throws CubeIntegrityException
 * @throws IOException
 * @throws JobException
 */
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION')  or hasPermission(#cube, 'MANAGEMENT')")
public CubeInstance enableCube(CubeInstance cube) throws IOException, JobException {
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();
    if (!cube.getStatus().equals(RealizationStatusEnum.DISABLED)) {
        throw new InternalErrorException("Only disabled cube can be enabled, status of " + cubeName + " is " + ostatus);
    }

    if (cube.getSegments(SegmentStatusEnum.READY).size() == 0) {
        throw new InternalErrorException("Cube " + cubeName + " dosen't contain any READY segment");
    }

    final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
    if (!cubingJobs.isEmpty()) {
        throw new JobException("Enable is not allowed with a running job.");
    }
    if (!cube.getDescriptor().calculateSignature().equals(cube.getDescriptor().getSignature())) {
        this.releaseAllSegments(cube);
    }

    cube.setStatus(RealizationStatusEnum.READY);
    try {
        return getCubeManager().updateCube(cube);
    } catch (IOException e) {
        cube.setStatus(ostatus);
        throw e;
    }
}
 

public StreamingSegmentManager(String baseStorePath, CubeInstance cubeInstance, ISourcePositionHandler sourcePosHandler, IConsumerProvider consumerProvider) {
    this.baseStorePath = baseStorePath;
    this.cubeName = cubeInstance.getName();
    this.cubeInstance = cubeInstance;
    this.cubeWindow = cubeInstance.getConfig().getStreamingCubeWindowInSecs() * 1000L;
    this.cubeDuration = cubeInstance.getConfig().getStreamingCubeDurationInSecs() * 1000L;
    this.maxCubeDuration = cubeInstance.getConfig().getStreamingCubeMaxDurationInSecs() * 1000L;
    this.checkPointIntervals = cubeInstance.getConfig().getStreamingCheckPointIntervalsInSecs() * 1000;
    this.maxImmutableSegments = cubeInstance.getConfig().getStreamingMaxImmutableSegments();
    this.consumerProvider = consumerProvider;
    this.cubeDataFolder = new File(baseStorePath, cubeName);
    this.sourcePositionHandler = sourcePosHandler;
    this.consumePosition = sourcePositionHandler.createEmptyPosition();
    if (!cubeDataFolder.exists()) {
        cubeDataFolder.mkdirs();
    }
    this.longLatencyInfo = new LongLatencyInfo();
    this.checkPointStore = new CheckPointStore(cubeName, cubeDataFolder, cubeInstance.getConfig()
            .getStreamingCheckPointFileMaxNum());

    // Prepare for realtime dictionary encoder.
    CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    List<MeasureDesc> bitmapMeasureList = cube.getDescriptor().getMeasures().stream()
            .filter(measureDesc -> measureDesc.getFunction().getMeasureType() instanceof BitmapMeasureType)
            .collect(Collectors.toList());
    if (!bitmapMeasureList.isEmpty()) {
        List<String> realtimeDictColumn = bitmapMeasureList.stream()
                .map(measureDesc -> measureDesc.getFunction().getParameter().getColRef().getIdentity())
                .collect(Collectors.toList());
        String str = String.join(", ", realtimeDictColumn);
        logger.info("Find these columns {} need to be encoded realtime.", str);
        List<TblColRef> tblColRefs = bitmapMeasureList.stream()
                .map(measureDesc -> measureDesc.getFunction().getParameter().getColRef())
                .collect(Collectors.toList());

        streamingDictionaryClient = new StreamingDictionaryClient(cubeName,
                realtimeDictColumn.toArray(new String[0]));
        tblColRefs.forEach(col -> dictionaryMap.put(col,
                new StreamingDistributedDictionary(col.getIdentity(), streamingDictionaryClient)));
    }
}
 

public StreamingSegmentManager(String baseStorePath, CubeInstance cubeInstance, ISourcePositionHandler sourcePosHandler, IConsumerProvider consumerProvider) {
    this.baseStorePath = baseStorePath;
    this.cubeName = cubeInstance.getName();
    this.cubeInstance = cubeInstance;
    this.cubeWindow = cubeInstance.getConfig().getStreamingCubeWindowInSecs() * 1000L;
    this.cubeDuration = cubeInstance.getConfig().getStreamingCubeDurationInSecs() * 1000L;
    this.maxCubeDuration = cubeInstance.getConfig().getStreamingCubeMaxDurationInSecs() * 1000L;
    this.checkPointIntervals = cubeInstance.getConfig().getStreamingCheckPointIntervalsInSecs() * 1000;
    this.maxImmutableSegments = cubeInstance.getConfig().getStreamingMaxImmutableSegments();
    this.consumerProvider = consumerProvider;
    this.cubeDataFolder = new File(baseStorePath, cubeName);
    this.sourcePositionHandler = sourcePosHandler;
    this.consumePosition = sourcePositionHandler.createEmptyPosition();
    if (!cubeDataFolder.exists()) {
        cubeDataFolder.mkdirs();
    }
    this.longLatencyInfo = new LongLatencyInfo();
    this.checkPointStore = new CheckPointStore(cubeName, cubeDataFolder, cubeInstance.getConfig()
            .getStreamingCheckPointFileMaxNum());

    // Prepare for realtime dictionary encoder.
    CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    List<MeasureDesc> bitmapMeasureList = cube.getDescriptor().getMeasures().stream()
            .filter(measureDesc -> measureDesc.getFunction().getMeasureType() instanceof BitmapMeasureType)
            .collect(Collectors.toList());
    if (!bitmapMeasureList.isEmpty()) {
        List<String> realtimeDictColumn = bitmapMeasureList.stream()
                .map(measureDesc -> measureDesc.getFunction().getParameter().getColRef().getIdentity())
                .collect(Collectors.toList());
        String str = String.join(", ", realtimeDictColumn);
        logger.info("Find these columns {} need to be encoded realtime.", str);
        List<TblColRef> tblColRefs = bitmapMeasureList.stream()
                .map(measureDesc -> measureDesc.getFunction().getParameter().getColRef())
                .collect(Collectors.toList());

        streamingDictionaryClient = new StreamingDictionaryClient(cubeName,
                realtimeDictColumn.toArray(new String[0]));
        tblColRefs.forEach(col -> dictionaryMap.put(col,
                new StreamingDistributedDictionary(col.getIdentity(), streamingDictionaryClient)));
    }
}
 

public JobInstance submitRecoverSegmentOptimizeJob(CubeSegment segment, String submitter)
        throws IOException, JobException {
    CubeInstance cubeInstance = segment.getCubeInstance();

    checkCubeDescSignature(cubeInstance);

    String cubeName = cubeInstance.getName();
    List<JobInstance> jobInstanceList = searchJobsByCubeName(cubeName, null,
            Lists.newArrayList(JobStatusEnum.NEW, JobStatusEnum.PENDING, JobStatusEnum.ERROR),
            JobTimeFilterEnum.ALL, JobSearchMode.CHECKPOINT_ONLY);
    if (jobInstanceList.size() > 1) {
        throw new IllegalStateException("Exist more than one CheckpointExecutable for cube " + cubeName);
    } else if (jobInstanceList.size() == 0) {
        throw new IllegalStateException("There's no CheckpointExecutable for cube " + cubeName);
    }
    CheckpointExecutable checkpointExecutable = (CheckpointExecutable) getExecutableManager()
            .getJob(jobInstanceList.get(0).getId());

    AbstractExecutable toBeReplaced = null;
    for (AbstractExecutable taskForCheck : checkpointExecutable.getSubTasksForCheck()) {
        if (taskForCheck instanceof CubingJob) {
            CubingJob subCubingJob = (CubingJob) taskForCheck;
            String segmentName = CubingExecutableUtil.getSegmentName(subCubingJob.getParams());
            if (segmentName != null && segmentName.equals(segment.getName())) {
                String segmentID = CubingExecutableUtil.getSegmentId(subCubingJob.getParams());
                CubeSegment beingOptimizedSegment = cubeInstance.getSegmentById(segmentID);
                if (beingOptimizedSegment != null) { // beingOptimizedSegment exists & should not be recovered
                    throw new IllegalStateException("Segment " + beingOptimizedSegment.getName() + "-"
                            + beingOptimizedSegment.getUuid()
                            + " still exists. Please delete it or discard the related optimize job first!!!");
                }
                toBeReplaced = taskForCheck;
                break;
            }
        }
    }
    if (toBeReplaced == null) {
        throw new IllegalStateException("There's no CubingJob for segment " + segment.getName()
                + " in CheckpointExecutable " + checkpointExecutable.getName());
    }

    /** Add CubingJob for the related segment **/
    CubeSegment optimizeSegment = getCubeManager().appendSegment(cubeInstance, segment.getTSRange());

    DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, submitter);

    getExecutableManager().addJob(optimizeJob);

    JobInstance optimizeJobInstance = getSingleJobInstance(optimizeJob);

    /** Update the checkpoint job */
    checkpointExecutable.getSubTasksForCheck().set(checkpointExecutable.getSubTasksForCheck().indexOf(toBeReplaced),
            optimizeJob);

    getExecutableManager().updateCheckpointJob(checkpointExecutable.getId(),
            checkpointExecutable.getSubTasksForCheck());

    return optimizeJobInstance;
}
 

public JobInstance submitRecoverSegmentOptimizeJob(CubeSegment segment, String submitter)
        throws IOException, JobException {
    CubeInstance cubeInstance = segment.getCubeInstance();

    checkCubeDescSignature(cubeInstance);

    String cubeName = cubeInstance.getName();
    List<JobInstance> jobInstanceList = searchJobsByCubeName(cubeName, null,
            Lists.newArrayList(JobStatusEnum.NEW, JobStatusEnum.PENDING, JobStatusEnum.ERROR),
            JobTimeFilterEnum.ALL, JobSearchMode.CHECKPOINT_ONLY);
    if (jobInstanceList.size() > 1) {
        throw new IllegalStateException("Exist more than one CheckpointExecutable for cube " + cubeName);
    } else if (jobInstanceList.size() == 0) {
        throw new IllegalStateException("There's no CheckpointExecutable for cube " + cubeName);
    }
    CheckpointExecutable checkpointExecutable = (CheckpointExecutable) getExecutableManager()
            .getJob(jobInstanceList.get(0).getId());

    AbstractExecutable toBeReplaced = null;
    for (AbstractExecutable taskForCheck : checkpointExecutable.getSubTasksForCheck()) {
        if (taskForCheck instanceof CubingJob) {
            CubingJob subCubingJob = (CubingJob) taskForCheck;
            String segmentName = CubingExecutableUtil.getSegmentName(subCubingJob.getParams());
            if (segmentName != null && segmentName.equals(segment.getName())) {
                String segmentID = CubingExecutableUtil.getSegmentId(subCubingJob.getParams());
                CubeSegment beingOptimizedSegment = cubeInstance.getSegmentById(segmentID);
                if (beingOptimizedSegment != null) { // beingOptimizedSegment exists & should not be recovered
                    throw new IllegalStateException("Segment " + beingOptimizedSegment.getName() + "-"
                            + beingOptimizedSegment.getUuid()
                            + " still exists. Please delete it or discard the related optimize job first!!!");
                }
                toBeReplaced = taskForCheck;
                break;
            }
        }
    }
    if (toBeReplaced == null) {
        throw new IllegalStateException("There's no CubingJob for segment " + segment.getName()
                + " in CheckpointExecutable " + checkpointExecutable.getName());
    }

    /** Add CubingJob for the related segment **/
    CubeSegment optimizeSegment = getCubeManager().appendSegment(cubeInstance, segment.getTSRange());

    DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, submitter);

    getExecutableManager().addJob(optimizeJob);

    JobInstance optimizeJobInstance = getSingleJobInstance(optimizeJob);

    /** Update the checkpoint job */
    checkpointExecutable.getSubTasksForCheck().set(checkpointExecutable.getSubTasksForCheck().indexOf(toBeReplaced),
            optimizeJob);

    getExecutableManager().updateCheckpointJob(checkpointExecutable.getId(),
            checkpointExecutable.getSubTasksForCheck());

    return optimizeJobInstance;
}
 

public static String getParquetStoragePath(CubeInstance cube, String segName, String identifier, Long cuboidId) {
    String hdfsWorkDir = cube.getConfig().getHdfsWorkingDirectory(cube.getProject());
    return hdfsWorkDir + "parquet" + File.separator + cube.getName() + File.separator + segName + "_" + identifier + File.separator + cuboidId;
}