org.apache.kylin.metadata.realization.RealizationStatusEnum Java Examples

The following examples show how to use org.apache.kylin.metadata.realization.RealizationStatusEnum. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CubeService.java    From kylin with Apache License 2.0 6 votes vote down vote up
/**
 * Stop all jobs belonging to this cube and clean out all segments
 *
 * @param cube
 * @return
 * @throws IOException
 * @throws JobException
 */
public CubeInstance purgeCube(CubeInstance cube) throws IOException {
    aclEvaluate.checkProjectOperationPermission(cube);
    Message msg = MsgPicker.getMsg();

    String cubeName = cube.getName();

    final List<CubingJob> cubingJobs = jobService.listJobsByRealizationName(cubeName, null, EnumSet
            .of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR, ExecutableState.STOPPED));
    if (!cubingJobs.isEmpty()) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getDISCARD_JOB_FIRST(), cubeName));
    }

    RealizationStatusEnum ostatus = cube.getStatus();
    if (null != ostatus && !RealizationStatusEnum.DISABLED.equals(ostatus)) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getPURGE_NOT_DISABLED_CUBE(), cubeName, ostatus));
    }

    this.releaseAllSegments(cube);
    return cube;
}
 
Example #2
Source File: CubeService.java    From Kylin with Apache License 2.0 6 votes vote down vote up
/**
 * Stop all jobs belonging to this cube and clean out all segments
 *
 * @param cube
 * @return
 * @throws IOException
 * @throws CubeIntegrityException
 * @throws JobException
 */
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
@Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
public CubeInstance purgeCube(CubeInstance cube) throws IOException, JobException {
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();
    if (null != ostatus && !RealizationStatusEnum.DISABLED.equals(ostatus)) {
        throw new InternalErrorException("Only disabled cube can be purged, status of " + cubeName + " is " + ostatus);
    }

    try {
        this.releaseAllSegments(cube);
        return cube;
    } catch (IOException e) {
        throw e;
    }

}
 
Example #3
Source File: SCCreatorTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Test
public void testExecute() throws Exception {
    String metadataPath = tempMetadataDir.getPath();

    String inputPath = "src/main/resources/SCSinkTools.json";

    SCCreator cli = new SCCreator();
    cli.execute("ADMIN", metadataPath, inputPath);
    Assert.assertTrue(tempMetadataDir.isDirectory());

    KylinConfig local = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv());
    local.setMetadataUrl(metadataPath);

    CubeManager cubeManager = CubeManager.getInstance(local);
    List<CubeInstance> cubeList = cubeManager.listAllCubes();
    System.out.println("System cubes: " + cubeList);
    assertEquals(cubeList.size(), 10);

    for (CubeInstance cube : cubeList) {
        Assert.assertTrue(cube.getStatus() != RealizationStatusEnum.DESCBROKEN);
    }
}
 
Example #4
Source File: CubeService.java    From kylin with Apache License 2.0 6 votes vote down vote up
/**
 * Update a cube status from disable to ready.
 *
 * @return
 * @throws IOException
 */
public CubeInstance enableCube(CubeInstance cube) throws IOException {
    boolean cubeStatusUpdated = false;
    RealizationStatusEnum ostatus = cube.getStatus();
    try {
        CubeInstance cubeInstance = getCubeManager().updateCubeStatus(cube, RealizationStatusEnum.READY);
        cubeStatusUpdated = true;
        // for streaming cube.
        if (cube.getDescriptor().isStreamingCube()) {
            getStreamingCoordinator().assignCube(cube.getName());
        }
        return cubeInstance;
    } catch (Exception e) {
        cube.setStatus(ostatus);
        // roll back if cube status updated
        if (cubeStatusUpdated) {
            logger.info("roll back cube status to:{}", ostatus);
            getCubeManager().updateCubeStatus(cube, ostatus);
        }
        throw e;
    }
}
 
Example #5
Source File: CubeManagerCacheTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up
@Test
public void testReloadCache() throws Exception {
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/a_whole_new_cube.json");
    CubeDescManager cubeDescMgr = getCubeDescManager();
    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    cubeManager.createCube("a_whole_new_cube", "default", desc, null);

    CubeInstance createdCube = cubeManager.getCube("a_whole_new_cube");
    assertEquals(0, createdCube.getSegments().size());
    assertEquals(RealizationStatusEnum.DISABLED, createdCube.getStatus());
    createdCube.setStatus(RealizationStatusEnum.DESCBROKEN);

    cubeManager.updateCube(createdCube);
    assertEquals(RealizationStatusEnum.DESCBROKEN, cubeManager.getCube("a_whole_new_cube").getStatus());
}
 
Example #6
Source File: SCCreatorTest.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Test
public void testExecute() throws Exception {
    String metadataPath = tempMetadataDir.getPath();

    String inputPath = "src/main/resources/SCSinkTools.json";

    SCCreator cli = new SCCreator();
    cli.execute("ADMIN", metadataPath, inputPath);
    Assert.assertTrue(tempMetadataDir.isDirectory());

    KylinConfig local = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv());
    local.setMetadataUrl(metadataPath);

    CubeManager cubeManager = CubeManager.getInstance(local);
    List<CubeInstance> cubeList = cubeManager.listAllCubes();
    System.out.println("System cubes: " + cubeList);
    assertEquals(cubeList.size(), 10);

    for (CubeInstance cube : cubeList) {
        Assert.assertTrue(cube.getStatus() != RealizationStatusEnum.DESCBROKEN);
    }
}
 
Example #7
Source File: CubeService.java    From Kylin with Apache License 2.0 6 votes vote down vote up
/**
 * Update a cube status from ready to disabled.
 *
 * @return
 * @throws CubeIntegrityException
 * @throws IOException
 * @throws JobException
 */
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
@Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
public CubeInstance disableCube(CubeInstance cube) throws IOException, JobException {
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();
    if (null != ostatus && !RealizationStatusEnum.READY.equals(ostatus)) {
        throw new InternalErrorException("Only ready cube can be disabled, status of " + cubeName + " is " + ostatus);
    }

    cube.setStatus(RealizationStatusEnum.DISABLED);

    try {
        return getCubeManager().updateCube(cube);
    } catch (IOException e) {
        cube.setStatus(ostatus);
        throw e;
    }
}
 
Example #8
Source File: CubeManagerCacheTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Test
public void testReloadCache() throws Exception {
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/a_whole_new_cube.json");
    CubeDescManager cubeDescMgr = getCubeDescManager();
    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    cubeManager.createCube("a_whole_new_cube", "default", desc, null);

    CubeInstance createdCube = cubeManager.getCube("a_whole_new_cube");
    assertEquals(0, createdCube.getSegments().size());
    assertEquals(RealizationStatusEnum.DISABLED, createdCube.getStatus());

    cubeManager.updateCubeStatus(createdCube, RealizationStatusEnum.READY);

    assertEquals(RealizationStatusEnum.READY, cubeManager.getCube("a_whole_new_cube").getStatus());
}
 
Example #9
Source File: CubeService.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
/**
 * Update a cube status from disable to ready.
 *
 * @return
 * @throws IOException
 */
public CubeInstance enableCube(CubeInstance cube) throws IOException {
    boolean cubeStatusUpdated = false;
    RealizationStatusEnum ostatus = cube.getStatus();
    try {
        CubeInstance cubeInstance = getCubeManager().updateCubeStatus(cube, RealizationStatusEnum.READY);
        cubeStatusUpdated = true;
        // for streaming cube.
        if (cube.getDescriptor().isStreamingCube()) {
            getStreamingCoordinator().assignCube(cube.getName());
        }
        return cubeInstance;
    } catch (Exception e) {
        cube.setStatus(ostatus);
        // roll back if cube status updated
        if (cubeStatusUpdated) {
            logger.info("roll back cube status to:{}", ostatus);
            getCubeManager().updateCubeStatus(cube, ostatus);
        }
        throw e;
    }
}
 
Example #10
Source File: CubeService.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
/**
 * Stop all jobs belonging to this cube and clean out all segments
 *
 * @param cube
 * @return
 * @throws IOException
 * @throws JobException
 */
public CubeInstance purgeCube(CubeInstance cube) throws IOException {
    aclEvaluate.checkProjectOperationPermission(cube);
    Message msg = MsgPicker.getMsg();

    String cubeName = cube.getName();

    final List<CubingJob> cubingJobs = jobService.listJobsByRealizationName(cubeName, null, EnumSet
            .of(ExecutableState.READY, ExecutableState.RUNNING, ExecutableState.ERROR, ExecutableState.STOPPED));
    if (!cubingJobs.isEmpty()) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getDISCARD_JOB_FIRST(), cubeName));
    }

    RealizationStatusEnum ostatus = cube.getStatus();
    if (null != ostatus && !RealizationStatusEnum.DISABLED.equals(ostatus)) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getPURGE_NOT_DISABLED_CUBE(), cubeName, ostatus));
    }

    this.releaseAllSegments(cube);
    return cube;
}
 
Example #11
Source File: CubeService.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public void checkEnableCubeCondition(CubeInstance cube) {
    aclEvaluate.checkProjectWritePermission(cube);
    Message msg = MsgPicker.getMsg();
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();

    if (!cube.getStatus().equals(RealizationStatusEnum.DISABLED)) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getENABLE_NOT_DISABLED_CUBE(), cubeName, ostatus));
    }

    if (cube.getSegments(SegmentStatusEnum.READY).size() == 0 && !cube.getDescriptor().isStreamingCube()) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getNO_READY_SEGMENT(), cubeName));
    }

    if (!cube.getDescriptor().checkSignature()) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getINCONSISTENT_CUBE_DESC_SIGNATURE(), cube.getDescriptor()));
    }
}
 
Example #12
Source File: CubeControllerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void tesDeleteDescBrokenCube() throws Exception {
    final String cubeName = "ci_left_join_cube";
    CubeInstance cubeInstance = cubeService.getCubeManager().getCube(cubeName);
    CubeDesc cubeDesc = cubeInstance.getDescriptor();
    cubeDesc.setModel(null);
    cubeInstance.setStatus(RealizationStatusEnum.DESCBROKEN);
    cubeController.deleteCube(cubeName);
    Assert.assertNull(cubeService.getCubeManager().getCube(cubeName));
}
 
Example #13
Source File: Coordinator.java    From kylin with Apache License 2.0 5 votes vote down vote up
private List<StreamingCubeInfo> getEnableStreamingCubes() {
    List<StreamingCubeInfo> allCubes = getStreamingCubes();
    List<StreamingCubeInfo> result = Lists.newArrayList();
    for (StreamingCubeInfo cube : allCubes) {
        CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
                .getCube(cube.getCubeName());
        if (cubeInstance.getStatus() == RealizationStatusEnum.READY) {
            result.add(cube);
        }
    }
    return result;
}
 
Example #14
Source File: CubeMigrationCLI.java    From kylin with Apache License 2.0 5 votes vote down vote up
protected void checkCubeState(CubeInstance cube) {
    if (cube.getStatus() != RealizationStatusEnum.READY)
        throw new IllegalStateException("Cannot migrate cube that is not in READY state.");

    for (CubeSegment segment : cube.getSegments()) {
        if (segment.getStatus() != SegmentStatusEnum.READY) {
            throw new IllegalStateException("At least one segment is not in READY state");
        }
    }
}
 
Example #15
Source File: QueryInfoCollectorTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
private void enableCube(String... cubes) throws IOException {
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());

    for (String cube : cubes) {
        CubeInstance cubeInstance = cubeManager.getCube(cube);
        cubeManager.updateCubeStatus(cubeInstance, RealizationStatusEnum.READY);
    }
}
 
Example #16
Source File: CubeInstance.java    From kylin with Apache License 2.0 5 votes vote down vote up
public boolean needAutoMerge() {
    if (!this.getDescriptor().getModel().getPartitionDesc().isPartitioned())
        return false;

    return this.getConfig().isAutoMergeEnabled() && this.getDescriptor().getAutoMergeTimeRanges() != null
            && this.getDescriptor().getAutoMergeTimeRanges().length > 0 && this.getStatus() == RealizationStatusEnum.READY;
}
 
Example #17
Source File: CubeMigrationCLI.java    From Kylin with Apache License 2.0 5 votes vote down vote up
public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {

        srcConfig = srcCfg;
        srcStore = ResourceStore.getStore(srcConfig);
        dstConfig = dstCfg;
        dstStore = ResourceStore.getStore(dstConfig);

        CubeManager cubeManager = CubeManager.getInstance(srcConfig);
        CubeInstance cube = cubeManager.getCube(cubeName);
        logger.info("cube to be moved is : " + cubeName);

        if (cube.getStatus() != RealizationStatusEnum.READY)
            throw new IllegalStateException("Cannot migrate cube that is not in READY state.");

        for (CubeSegment segment : cube.getSegments()) {
            if (segment.getStatus() != SegmentStatusEnum.READY) {
                throw new IllegalStateException("At least one segment is not in READY state");
            }
        }

        checkAndGetHbaseUrl();

        Configuration conf = HBaseConfiguration.create();
        hbaseAdmin = new HBaseAdmin(conf);

        hdfsFS = FileSystem.get(new Configuration());

        operations = new ArrayList<Opt>();

        copyFilesInMetaStore(cube, overwriteIfExists);
        renameFoldersInHdfs(cube);
        changeHtableHost(cube);
        addCubeIntoProject(cubeName, projectName);

        if (realExecute.equalsIgnoreCase("true")) {
            doOpts();
        } else {
            showOpts();
        }
    }
 
Example #18
Source File: CubeService.java    From Kylin with Apache License 2.0 5 votes vote down vote up
/**
 * Update a cube status from disable to ready.
 *
 * @return
 * @throws CubeIntegrityException
 * @throws IOException
 * @throws JobException
 */
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION')  or hasPermission(#cube, 'MANAGEMENT')")
public CubeInstance enableCube(CubeInstance cube) throws IOException, JobException {
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();
    if (!cube.getStatus().equals(RealizationStatusEnum.DISABLED)) {
        throw new InternalErrorException("Only disabled cube can be enabled, status of " + cubeName + " is " + ostatus);
    }

    if (cube.getSegments(SegmentStatusEnum.READY).size() == 0) {
        throw new InternalErrorException("Cube " + cubeName + " dosen't contain any READY segment");
    }

    final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
    if (!cubingJobs.isEmpty()) {
        throw new JobException("Enable is not allowed with a running job.");
    }
    if (!cube.getDescriptor().calculateSignature().equals(cube.getDescriptor().getSignature())) {
        this.releaseAllSegments(cube);
    }

    cube.setStatus(RealizationStatusEnum.READY);
    try {
        return getCubeManager().updateCube(cube);
    } catch (IOException e) {
        cube.setStatus(ostatus);
        throw e;
    }
}
 
Example #19
Source File: CubeInstance.java    From kylin with Apache License 2.0 5 votes vote down vote up
void init(KylinConfig config) {
    CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(descName);
    checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", descName, name);

    if (cubeDesc.isBroken()) {
        setStatus(RealizationStatusEnum.DESCBROKEN);
        logger.error("cube descriptor {} (for cube '{}') is broken", cubeDesc.getResourcePath(), name);
        logger.error("Errors: {}", cubeDesc.getErrorsAsString());
    } else if (getStatus() == RealizationStatusEnum.DESCBROKEN) {
        setStatus(RealizationStatusEnum.DISABLED);
        logger.info("cube {} changed from DESCBROKEN to DISABLED", name);
    }

    setConfig((KylinConfigExt) cubeDesc.getConfig());
}
 
Example #20
Source File: MigrationRuleSet.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Override
public void apply(Context ctx) throws RuleValidationException {
    CubeInstance cube = ctx.getCubeInstance();
    RealizationStatusEnum status = cube.getStatus();
    if (status != RealizationStatusEnum.READY) {
        throw new RuleValidationException("The cube named " + cube.getName() + " is not in READY state.");
    }
}
 
Example #21
Source File: QueryInfoCollectorTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private void enableCube(String... cubes) throws IOException {
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());

    for (String cube : cubes) {
        CubeInstance cubeInstance = cubeManager.getCube(cube);
        cubeManager.updateCubeStatus(cubeInstance, RealizationStatusEnum.READY);
    }
}
 
Example #22
Source File: NSparkCubingStep.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Override
protected void updateMetaAfterBuilding(KylinConfig config) throws IOException {
    CubeManager cubeManager = CubeManager.getInstance(config);
    CubeInstance currentInstanceCopy = cubeManager.getCube(getCubeName()).latestCopyForWrite();
    KylinConfig kylinDistConfig = MetaDumpUtil.loadKylinConfigFromHdfs(getDistMetaUrl());
    CubeInstance distCube = CubeManager.getInstance(kylinDistConfig).reloadCube(getCubeName());
    CubeUpdate update = new CubeUpdate(currentInstanceCopy);
    Set<String> segmentIds = Sets.newHashSet(org.apache.hadoop.util.StringUtils.split(getParam(MetadataConstants.P_SEGMENT_IDS)));
    CubeSegment toUpdateSegs = distCube.getSegmentById(segmentIds.iterator().next());

    List<CubeSegment> tobe = currentInstanceCopy.calculateToBeSegments(toUpdateSegs);

    if (tobe.contains(toUpdateSegs) == false)
        throw new IllegalStateException(
                String.format(Locale.ROOT, "For cube %s, segment %s is expected but not in the tobe %s",
                        currentInstanceCopy.toString(), toUpdateSegs.toString(), tobe.toString()));

    toUpdateSegs.setStatus(SegmentStatusEnum.READY);

    List<CubeSegment> toRemoveSegs = Lists.newArrayList();
    for (CubeSegment segment : currentInstanceCopy.getSegments()) {
        if (!tobe.contains(segment))
            toRemoveSegs.add(segment);
    }

    logger.info("Promoting cube {}, new segment {}, to remove segments {}", currentInstanceCopy, toUpdateSegs, toRemoveSegs);

    update.setToRemoveSegs(toRemoveSegs.toArray(new CubeSegment[toRemoveSegs.size()]))
            .setToUpdateSegs(toUpdateSegs);
    if (currentInstanceCopy.getConfig().isJobAutoReadyCubeEnabled()) {
        update.setStatus(RealizationStatusEnum.READY);
    }
    cubeManager.updateCube(update);
}
 
Example #23
Source File: SignatureCalculatorTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private CubeInstance cloneCubeInstance(CubeManager cubeManager, CubeInstance cube, String name) throws IOException {
    CubeInstance cubeClone = cubeManager.createCube(name, projectName, cube.getDescriptor(), cube.getOwner());
    CubeUpdate cubeUpdate = new CubeUpdate(cubeClone.latestCopyForWrite());
    cubeUpdate.setToAddSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()]));
    cubeUpdate.setStatus(RealizationStatusEnum.READY);
    return cubeManager.updateCube(cubeUpdate);
}
 
Example #24
Source File: SignatureCalculatorTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetRealizationSignature() {
    RealizationSignature signature1 = RealizationSetCalculator.getRealizationSignature(this.config,
            "Test" + System.currentTimeMillis());
    Assert.assertNull(signature1);

    CubeSignature signature2 = (CubeSignature) RealizationSetCalculator.getRealizationSignature(this.config, "ssb");
    Assert.assertEquals(RealizationStatusEnum.DISABLED, signature2.status);
    Assert.assertNull(signature2.segmentSignatureSet);

    CubeSignature signature3 = (CubeSignature) RealizationSetCalculator.getRealizationSignature(this.config,
            "test_kylin_cube_with_slr_left_join_ready");
    Assert.assertNotNull(signature3.segmentSignatureSet);
}
 
Example #25
Source File: CubeManager.java    From kylin with Apache License 2.0 5 votes vote down vote up
public CubeInstance updateCubeStatus(CubeInstance cube, RealizationStatusEnum newStatus) throws IOException {
    try (AutoLock lock = cubeMapLock.lockForWrite()) {
        cube = cube.latestCopyForWrite(); // get a latest copy
        CubeUpdate update = new CubeUpdate(cube);
        update.setStatus(newStatus);
        ProjectManager.getInstance(config).touchProject(cube.getProject());
        return updateCube(update);
    }
}
 
Example #26
Source File: RealizationSignature.java    From kylin with Apache License 2.0 5 votes vote down vote up
static CubeSignature getCubeSignature(KylinConfig config, String realizationName) {
    CubeInstance cubeInstance = CubeManager.getInstance(config).getCube(realizationName);
    if (cubeInstance == null) {
        return null;
    }
    if (!cubeInstance.isReady()) {
        return new CubeSignature(realizationName, RealizationStatusEnum.DISABLED, null);
    }
    List<CubeSegment> readySegments = cubeInstance.getSegments(SegmentStatusEnum.READY);
    Set<SegmentSignature> segmentSignatureSet = Sets.newHashSetWithExpectedSize(readySegments.size());
    for (CubeSegment cubeSeg : readySegments) {
        segmentSignatureSet.add(new SegmentSignature(cubeSeg.getName(), cubeSeg.getLastBuildTime()));
    }
    return new CubeSignature(realizationName, RealizationStatusEnum.READY, segmentSignatureSet);
}
 
Example #27
Source File: CubeManagerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testAutoMergeNormal() throws Exception {
    CubeManager mgr = CubeManager.getInstance(getTestConfig());
    CubeInstance cube = mgr.getCube("test_kylin_cube_with_slr_empty").latestCopyForWrite();

    cube.getDescriptor().setAutoMergeTimeRanges(new long[] { 2000, 6000 });

    mgr.updateCube(new CubeUpdate(cube).setStatus(RealizationStatusEnum.READY));
    assertEquals(RealizationStatusEnum.READY, cube.getStatus());
    assertTrue(cube.needAutoMerge());

    // no segment at first
    assertEquals(0, cube.getSegments().size());

    // append first
    CubeSegment seg1 = mgr.appendSegment(cube, new TSRange(0L, 1000L), null, null, null);
    mgr.updateCubeSegStatus(seg1, SegmentStatusEnum.READY);

    CubeSegment seg2 = mgr.appendSegment(cube, new TSRange(1000L, 2000L), null, null, null);
    mgr.updateCubeSegStatus(seg2, SegmentStatusEnum.READY);

    cube = mgr.getCube(cube.getName());
    assertEquals(2, cube.getSegments().size());

    SegmentRange mergedSeg = cube.autoMergeCubeSegments();

    assertTrue(mergedSeg != null);
}
 
Example #28
Source File: BuildIIWithEngineTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Before
public void before() throws Exception {
    HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);

    DeployUtil.initCliWorkDir();
    //        DeployUtil.deployMetadata();
    DeployUtil.overrideJobJarLocations();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(kylinConfig));
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    iiManager = IIManager.getInstance(kylinConfig);
    jobEngineConfig = new JobEngineConfig(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        if(jobService.getJob(jobId) instanceof IIJob){
            jobService.deleteJob(jobId);
        }
    }

    IIInstance ii = iiManager.getII(TEST_II_NAME);
    if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
        ii.setStatus(RealizationStatusEnum.DISABLED);
        iiManager.updateII(ii);
    }
}
 
Example #29
Source File: SignatureCalculatorTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
private CubeInstance cloneCubeInstance(CubeManager cubeManager, CubeInstance cube, String name) throws IOException {
    CubeInstance cubeClone = cubeManager.createCube(name, projectName, cube.getDescriptor(), cube.getOwner());
    CubeUpdate cubeUpdate = new CubeUpdate(cubeClone.latestCopyForWrite());
    cubeUpdate.setToAddSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()]));
    cubeUpdate.setStatus(RealizationStatusEnum.READY);
    return cubeManager.updateCube(cubeUpdate);
}
 
Example #30
Source File: StreamingCoordinator.java    From kylin with Apache License 2.0 5 votes vote down vote up
List<StreamingCubeInfo> getEnableStreamingCubes() {
    List<StreamingCubeInfo> allCubes = getStreamingCubes();
    List<StreamingCubeInfo> result = Lists.newArrayList();
    for (StreamingCubeInfo cube : allCubes) {
        CubeInstance cubeInstance = getCubeManager().getCube(cube.getCubeName());
        if (cubeInstance.getStatus() == RealizationStatusEnum.READY) {
            result.add(cube);
        }
    }
    return result;
}