Java Code Examples for org.apache.kylin.metadata.realization.RealizationStatusEnum#READY

The following examples show how to use org.apache.kylin.metadata.realization.RealizationStatusEnum#READY . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StreamingCoordinator.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
List<StreamingCubeInfo> getEnableStreamingCubes() {
    List<StreamingCubeInfo> allCubes = getStreamingCubes();
    List<StreamingCubeInfo> result = Lists.newArrayList();
    for (StreamingCubeInfo cube : allCubes) {
        CubeInstance cubeInstance = getCubeManager().getCube(cube.getCubeName());
        if (cubeInstance.getStatus() == RealizationStatusEnum.READY) {
            result.add(cube);
        }
    }
    return result;
}
Example 2
Source File: BuildIIWithEngineTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@After
public void after() throws Exception {
    IIInstance ii = iiManager.getII(TEST_II_NAME);
    if (ii.getStatus() != RealizationStatusEnum.READY) {
        ii.setStatus(RealizationStatusEnum.READY);
        iiManager.updateII(ii);
    }
    backup();
}
Example 3
Source File: CubeMigrationCLI.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {

        srcConfig = srcCfg;
        srcStore = ResourceStore.getStore(srcConfig);
        dstConfig = dstCfg;
        dstStore = ResourceStore.getStore(dstConfig);

        CubeManager cubeManager = CubeManager.getInstance(srcConfig);
        CubeInstance cube = cubeManager.getCube(cubeName);
        logger.info("cube to be moved is : " + cubeName);

        if (cube.getStatus() != RealizationStatusEnum.READY)
            throw new IllegalStateException("Cannot migrate cube that is not in READY state.");

        for (CubeSegment segment : cube.getSegments()) {
            if (segment.getStatus() != SegmentStatusEnum.READY) {
                throw new IllegalStateException("At least one segment is not in READY state");
            }
        }

        checkAndGetHbaseUrl();

        Configuration conf = HBaseConfiguration.create();
        hbaseAdmin = new HBaseAdmin(conf);

        hdfsFS = FileSystem.get(new Configuration());

        operations = new ArrayList<Opt>();

        copyFilesInMetaStore(cube, overwriteIfExists);
        renameFoldersInHdfs(cube);
        changeHtableHost(cube);
        addCubeIntoProject(cubeName, projectName);

        if (realExecute.equalsIgnoreCase("true")) {
            doOpts();
        } else {
            showOpts();
        }
    }
Example 4
Source File: RealizationSignature.java    From kylin with Apache License 2.0 5 votes vote down vote up 
static CubeSignature getCubeSignature(KylinConfig config, String realizationName) {
    CubeInstance cubeInstance = CubeManager.getInstance(config).getCube(realizationName);
    if (cubeInstance == null) {
        return null;
    }
    if (!cubeInstance.isReady()) {
        return new CubeSignature(realizationName, RealizationStatusEnum.DISABLED, null);
    }
    List<CubeSegment> readySegments = cubeInstance.getSegments(SegmentStatusEnum.READY);
    Set<SegmentSignature> segmentSignatureSet = Sets.newHashSetWithExpectedSize(readySegments.size());
    for (CubeSegment cubeSeg : readySegments) {
        segmentSignatureSet.add(new SegmentSignature(cubeSeg.getName(), cubeSeg.getLastBuildTime()));
    }
    return new CubeSignature(realizationName, RealizationStatusEnum.READY, segmentSignatureSet);
}
Example 5
Source File: CubeInstance.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public boolean needAutoMerge() {
    if (!this.getDescriptor().getModel().getPartitionDesc().isPartitioned())
        return false;

    return this.getConfig().isAutoMergeEnabled() && this.getDescriptor().getAutoMergeTimeRanges() != null
            && this.getDescriptor().getAutoMergeTimeRanges().length > 0 && this.getStatus() == RealizationStatusEnum.READY;
}
Example 6
Source File: MigrationRuleSet.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void apply(Context ctx) throws RuleValidationException {
    CubeInstance cube = ctx.getCubeInstance();
    RealizationStatusEnum status = cube.getStatus();
    if (status != RealizationStatusEnum.READY) {
        throw new RuleValidationException("The cube named " + cube.getName() + " is not in READY state.");
    }
}
Example 7
Source File: CubeMigrationCLI.java    From kylin with Apache License 2.0 5 votes vote down vote up 
protected void checkCubeState(CubeInstance cube) {
    if (cube.getStatus() != RealizationStatusEnum.READY)
        throw new IllegalStateException("Cannot migrate cube that is not in READY state.");

    for (CubeSegment segment : cube.getSegments()) {
        if (segment.getStatus() != SegmentStatusEnum.READY) {
            throw new IllegalStateException("At least one segment is not in READY state");
        }
    }
}
Example 8
Source File: StreamingCoordinator.java    From kylin with Apache License 2.0 5 votes vote down vote up 
List<StreamingCubeInfo> getEnableStreamingCubes() {
    List<StreamingCubeInfo> allCubes = getStreamingCubes();
    List<StreamingCubeInfo> result = Lists.newArrayList();
    for (StreamingCubeInfo cube : allCubes) {
        CubeInstance cubeInstance = getCubeManager().getCube(cube.getCubeName());
        if (cubeInstance.getStatus() == RealizationStatusEnum.READY) {
            result.add(cube);
        }
    }
    return result;
}
Example 9
Source File: RealizationSignature.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
static CubeSignature getCubeSignature(KylinConfig config, String realizationName) {
    CubeInstance cubeInstance = CubeManager.getInstance(config).getCube(realizationName);
    if (cubeInstance == null) {
        return null;
    }
    if (!cubeInstance.isReady()) {
        return new CubeSignature(realizationName, RealizationStatusEnum.DISABLED, null);
    }
    List<CubeSegment> readySegments = cubeInstance.getSegments(SegmentStatusEnum.READY);
    Set<SegmentSignature> segmentSignatureSet = Sets.newHashSetWithExpectedSize(readySegments.size());
    for (CubeSegment cubeSeg : readySegments) {
        segmentSignatureSet.add(new SegmentSignature(cubeSeg.getName(), cubeSeg.getLastBuildTime()));
    }
    return new CubeSignature(realizationName, RealizationStatusEnum.READY, segmentSignatureSet);
}
Example 10
Source File: CubeInstance.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public boolean needAutoMerge() {
    if (!this.getDescriptor().getModel().getPartitionDesc().isPartitioned())
        return false;

    return this.getConfig().isAutoMergeEnabled() && this.getDescriptor().getAutoMergeTimeRanges() != null
            && this.getDescriptor().getAutoMergeTimeRanges().length > 0 && this.getStatus() == RealizationStatusEnum.READY;
}
Example 11
Source File: CubeMigrationCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
protected void checkCubeState(CubeInstance cube) {
    if (cube.getStatus() != RealizationStatusEnum.READY)
        throw new IllegalStateException("Cannot migrate cube that is not in READY state.");

    for (CubeSegment segment : cube.getSegments()) {
        if (segment.getStatus() != SegmentStatusEnum.READY) {
            throw new IllegalStateException("At least one segment is not in READY state");
        }
    }
}
Example 12
Source File: Coordinator.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private List<StreamingCubeInfo> getEnableStreamingCubes() {
    List<StreamingCubeInfo> allCubes = getStreamingCubes();
    List<StreamingCubeInfo> result = Lists.newArrayList();
    for (StreamingCubeInfo cube : allCubes) {
        CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
                .getCube(cube.getCubeName());
        if (cubeInstance.getStatus() == RealizationStatusEnum.READY) {
            result.add(cube);
        }
    }
    return result;
}
Example 13
Source File: IIInstance.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
public boolean isReady() {
    return getStatus() == RealizationStatusEnum.READY;
}
Example 14
Source File: CubeService.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public CubeInstance deleteSegment(CubeInstance cube, String segmentName) throws IOException {
    aclEvaluate.checkProjectOperationPermission(cube);
    Message msg = MsgPicker.getMsg();

    if (cube.getStatus() == RealizationStatusEnum.READY) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getDELETE_SEG_FROM_READY_CUBE(), segmentName, cube.getName()));
    }

    CubeSegment toDelete = null;
    for (CubeSegment seg : cube.getSegments()) {
        if (seg.getName().equals(segmentName)) {
            toDelete = seg;
            break;
        }
    }

    if (toDelete == null) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getSEG_NOT_FOUND(), segmentName));
    }

    if (toDelete.getStatus() != SegmentStatusEnum.READY) {
        if (toDelete.getStatus() == SegmentStatusEnum.NEW) {
            if (!isOrphonSegment(cube, toDelete.getUuid())) {
                throw new BadRequestException(
                        String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
            }
        } else {
            throw new BadRequestException(String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
        }
    }

    if (!segmentName.equals(cube.getSegments().get(0).getName())
            && !segmentName.equals(cube.getSegments().get(cube.getSegments().size() - 1).getName())) {
        logger.warn(String.format(Locale.ROOT, msg.getDELETE_SEGMENT_CAUSE_GAPS(), cube.getName(), segmentName));
    }

    CubeInstance cubeInstance = CubeManager.getInstance(getConfig()).updateCubeDropSegments(cube, toDelete);

    cleanSegmentStorage(Collections.singletonList(toDelete));

    return cubeInstance;
}
Example 15
Source File: CubeMigrationCLI.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName,
        String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute)
        throws IOException, InterruptedException {

    srcConfig = srcCfg;
    srcStore = ResourceStore.getStore(srcConfig);
    dstConfig = dstCfg;
    dstStore = ResourceStore.getStore(dstConfig);

    CubeManager cubeManager = CubeManager.getInstance(srcConfig);
    CubeInstance cube = cubeManager.getCube(cubeName);
    logger.info("cube to be moved is : " + cubeName);

    if (cube.getStatus() != RealizationStatusEnum.READY)
        throw new IllegalStateException("Cannot migrate cube that is not in READY state.");

    for (CubeSegment segment : cube.getSegments()) {
        if (segment.getStatus() != SegmentStatusEnum.READY) {
            throw new IllegalStateException("At least one segment is not in READY state");
        }
    }

    checkAndGetHbaseUrl();

    Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
    hbaseAdmin = conn.getAdmin();

    hdfsFS = HadoopUtil.getWorkingFileSystem();

    operations = new ArrayList<Opt>();

    copyFilesInMetaStore(cube, overwriteIfExists);
    renameFoldersInHdfs(cube);
    changeHtableHost(cube);
    addCubeAndModelIntoProject(cube, cubeName, projectName);
    if (Boolean.parseBoolean(copyAcl) == true) {
        copyACL(cube, projectName);
    }

    if (Boolean.parseBoolean(purgeAndDisable) == true) {
        purgeAndDisable(cubeName); // this should be the last action
    }

    if (realExecute.equalsIgnoreCase("true")) {
        doOpts();
        checkMigrationSuccess(dstConfig, cubeName, true);
        updateMeta(dstConfig);
    } else {
        showOpts();
    }
}
Example 16
Source File: CubeInstance.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public boolean isReady() {
    return getStatus() == RealizationStatusEnum.READY;
}
Example 17
Source File: CubeInstance.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public boolean isReady() {
    return getStatus() == RealizationStatusEnum.READY;
}
Example 18
Source File: CubeService.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public CubeInstance deleteSegment(CubeInstance cube, String segmentName) throws IOException {
    aclEvaluate.checkProjectOperationPermission(cube);
    Message msg = MsgPicker.getMsg();

    if (cube.getStatus() == RealizationStatusEnum.READY) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getDELETE_SEG_FROM_READY_CUBE(), segmentName, cube.getName()));
    }

    CubeSegment toDelete = null;
    for (CubeSegment seg : cube.getSegments()) {
        if (seg.getName().equals(segmentName)) {
            toDelete = seg;
            break;
        }
    }

    if (toDelete == null) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getSEG_NOT_FOUND(), segmentName));
    }

    if (toDelete.getStatus() != SegmentStatusEnum.READY) {
        if (toDelete.getStatus() == SegmentStatusEnum.NEW) {
            if (!isOrphonSegment(cube, toDelete.getUuid())) {
                throw new BadRequestException(
                        String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
            }
        } else {
            throw new BadRequestException(String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
        }
    }

    if (!segmentName.equals(cube.getSegments().get(0).getName())
            && !segmentName.equals(cube.getSegments().get(cube.getSegments().size() - 1).getName())) {
        logger.warn(String.format(Locale.ROOT, msg.getDELETE_SEGMENT_CAUSE_GAPS(), cube.getName(), segmentName));
    }

    CubeInstance cubeInstance = CubeManager.getInstance(getConfig()).updateCubeDropSegments(cube, toDelete);

    cleanSegmentStorage(Collections.singletonList(toDelete));

    return cubeInstance;
}
Example 19
Source File: CubeMigrationCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName,
        String copyAcl, String purgeAndDisable, String overwriteIfExists, String realExecute)
        throws IOException, InterruptedException {

    srcConfig = srcCfg;
    srcStore = ResourceStore.getStore(srcConfig);
    dstConfig = dstCfg;
    dstStore = ResourceStore.getStore(dstConfig);

    CubeManager cubeManager = CubeManager.getInstance(srcConfig);
    CubeInstance cube = cubeManager.getCube(cubeName);
    logger.info("cube to be moved is : " + cubeName);

    if (cube.getStatus() != RealizationStatusEnum.READY)
        throw new IllegalStateException("Cannot migrate cube that is not in READY state.");

    for (CubeSegment segment : cube.getSegments()) {
        if (segment.getStatus() != SegmentStatusEnum.READY) {
            throw new IllegalStateException("At least one segment is not in READY state");
        }
    }

    checkAndGetHbaseUrl();

    Connection conn = HBaseConnection.get(srcConfig.getStorageUrl());
    hbaseAdmin = conn.getAdmin();

    hdfsFS = HadoopUtil.getWorkingFileSystem();

    operations = new ArrayList<Opt>();

    copyFilesInMetaStore(cube, overwriteIfExists);
    renameFoldersInHdfs(cube);
    changeHtableHost(cube);
    addCubeAndModelIntoProject(cube, cubeName, projectName);
    if (Boolean.parseBoolean(copyAcl) == true) {
        copyACL(cube, projectName);
    }

    if (Boolean.parseBoolean(purgeAndDisable) == true) {
        purgeAndDisable(cubeName); // this should be the last action
    }

    if (realExecute.equalsIgnoreCase("true")) {
        doOpts();
        checkMigrationSuccess(dstConfig, cubeName, true);
        updateMeta(dstConfig);
    } else {
        showOpts();
    }
}
Example 20
Source File: CubeInstance.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
public boolean isReady() {
    return getStatus() == RealizationStatusEnum.READY;
}