Java Code Examples for org.apache.kylin.cube.CubeDescManager#getInstance()

public void update() {
    logger.info("Reloading Cube Metadata from store: " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
    CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
    List<CubeDesc> cubeDescs;
    if (ArrayUtils.isEmpty(cubeNames)) {
        cubeDescs = cubeDescManager.listAllDesc();
    } else {
        String[] names = StringUtil.splitByComma(cubeNames[0]);
        if (ArrayUtils.isEmpty(names))
            return;
        cubeDescs = Lists.newArrayListWithCapacity(names.length);
        for (String name : names) {
            cubeDescs.add(cubeDescManager.getCubeDesc(name));
        }
    }
    for (CubeDesc cubeDesc : cubeDescs) {
        updateCubeDesc(cubeDesc);
    }

    verify();
}
 

public void before() throws Exception {
    deployEnv();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        AbstractExecutable executable = jobService.getJob(jobId);
        if (executable instanceof CubingJob || executable instanceof CheckpointExecutable) {
            jobService.deleteJob(jobId);
        }
    }

    cubeDescManager = CubeDescManager.getInstance(kylinConfig);

    // update enginType
    updateCubeEngineType(Lists.newArrayList("ci_inner_join_cube", "ci_left_join_cube"));
}
 

private void injest(File metaRoot) throws IOException {
    KylinConfig srcConfig = KylinConfig.createInstanceFromUri(metaRoot.getAbsolutePath());
    TableMetadataManager srcMetadataManager = TableMetadataManager.getInstance(srcConfig);
    DataModelManager srcModelManager = DataModelManager.getInstance(srcConfig);
    HybridManager srcHybridManager = HybridManager.getInstance(srcConfig);
    CubeManager srcCubeManager = CubeManager.getInstance(srcConfig);
    CubeDescManager srcCubeDescManager = CubeDescManager.getInstance(srcConfig);

    checkAndMark(srcMetadataManager, srcModelManager, srcHybridManager, srcCubeManager, srcCubeDescManager);
    new ResourceTool().copy(srcConfig, kylinConfig, Lists.newArrayList(requiredResources));

    // clear the cache
    Broadcaster.getInstance(kylinConfig).notifyClearAll();
    
    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        logger.info("add " + tableDesc + " to " + targetProjectName);
        projectManager.addTableDescToProject(Lists.newArrayList(tableDesc.getIdentity()).toArray(new String[0]), targetProjectName);
    }

    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        logger.info("add " + cube + " to " + targetProjectName);
        projectManager.addModelToProject(cube.getModel().getName(), targetProjectName);
        projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), targetProjectName, null);
    }

}
 

@Test
public void testExistingProject() throws Exception {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());

    int originalProjectCount = prjMgr.listAllProjects().size();
    int originalCubeCount = cubeMgr.listAllCubes().size();
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/new_cube_in_default.json");

    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube = cubeMgr.createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null);
    assertTrue(createdCube.equals(cubeMgr.getCube("new_cube_in_default")));

    //System.out.println(JsonUtil.writeValueAsIndentString(createdCube));

    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);

    CubeInstance droppedCube = cubeMgr.dropCube("new_cube_in_default", true);

    assertTrue(createdCube.equals(droppedCube));
    assertNull(cubeMgr.getCube("new_cube_in_default"));
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);
}
 

public ExtendCubeToHybridCLI() {
    this.kylinConfig = KylinConfig.getInstanceFromEnv();
    this.store = ResourceStore.getStore(kylinConfig);
    this.cubeManager = CubeManager.getInstance(kylinConfig);
    this.cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    this.metadataManager = DataModelManager.getInstance(kylinConfig);
}
 

@Test
public void testClearCacheForCubeMigration() throws IOException {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    String CUBENAME = "test_kylin_cube_without_slr_desc";

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
    CubeDesc cubeDesc = cubeDescManager.getCubeDesc(CUBENAME);
    DataModelDesc modelDesc = cubeDesc.getModel();
    Map<String, String> tableToProjects = new HashMap<>();
    for (TableRef tableRef : modelDesc.getAllTables()) {
        tableToProjects.put(tableRef.getTableIdentity(), tableRef.getTableDesc().getProject());
    }

    String uuid = cubeDesc.getUuid();
    String signature = cubeDesc.getSignature();

    assertEquals(cubeDesc.getRetentionRange(), 0);

    //update cubeDesc
    cubeDesc.setRetentionRange(2018);
    cubeDesc.updateRandomUuid();

    //directly update metadata in store to simulate cube migration
    Serializer<CubeDesc> cubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class);
    getStore().checkAndPutResource(cubeDesc.getResourcePath(), cubeDesc, cubeDescSerializer);

    CubeMigrationRequest request = new CubeMigrationRequest();
    request.setCube(cubeDesc.getName());
    request.setModel(modelDesc.getName());
    request.setProject(modelDesc.getProject());
    request.setTableToProjects(tableToProjects);

    cacheController.clearCacheForCubeMigration(request);

    assertEquals(2018, cubeDescManager.getCubeDesc(CUBENAME).getRetentionRange());
    assertEquals(signature, cubeDescManager.getCubeDesc(CUBENAME).getSignature());
    assertNotEquals(uuid, cubeDescManager.getCubeDesc(CUBENAME).getUuid());
}
 

public ExtendCubeToHybridCLI() {
    this.kylinConfig = KylinConfig.getInstanceFromEnv();
    this.store = ResourceStore.getStore(kylinConfig);
    this.cubeManager = CubeManager.getInstance(kylinConfig);
    this.cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    this.metadataManager = DataModelManager.getInstance(kylinConfig);
}
 

public void verify() {
    logger.info("=================================================================");
    logger.info("The changes are applied, now it's time to verify the new metadata store by reloading all metadata:");
    logger.info("=================================================================");
    config.clearManagers();
    DataModelManager.getInstance(config);
    CubeDescManager.getInstance(config);
    CubeManager.getInstance(config);
    ProjectManager.getInstance(config);
    //cleanup();
}
 

private static CubeDescManager getCubeDescManager(KylinConfig config) throws Exception {
    return CubeDescManager.getInstance(config);
}
 

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getTestConfig());
}
 

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getTestConfig());
}
 

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getConfig());
}
 

public final CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getConfig());
}
 

@Test
public void testNewProject() throws Exception {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());
    
    int originalProjectCount = prjMgr.listAllProjects().size();
    int originalCubeCount = cubeMgr.listAllCubes().size();
    int originalCubeCountInDefault = prjMgr.listAllRealizations("default").size();
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/cube_in_alien_project.json");

    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube = cubeMgr.createCube("cube_in_alien_project", "alien", desc, null);
    assertTrue(createdCube == cubeMgr.getCube("cube_in_alien_project"));
    ProjectManager proMgr = ProjectManager.getInstance(getTestConfig());
    Set<IRealization> realizations = proMgr.listAllRealizations("alien");
    assertTrue(realizations.contains(createdCube));

    System.out.println(JsonUtil.writeValueAsIndentString(createdCube));

    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1);
    assertTrue(prjMgr.listAllRealizations("ALIEN").iterator().next().getName().equalsIgnoreCase("CUBE_IN_ALIEN_PROJECT"));
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);

    prjMgr.moveRealizationToProject(RealizationType.CUBE, "cube_in_alien_project", "default", null);
    assertTrue(prjMgr.listAllRealizations("ALIEN").size() == 0);
    assertTrue(prjMgr.listAllRealizations("default").size() == originalCubeCountInDefault + 1);
    assertTrue(ProjectManager.getInstance(getTestConfig()).listAllRealizations("default").contains(createdCube));

    prjMgr.moveRealizationToProject(RealizationType.CUBE, "cube_in_alien_project", "alien", null);
    assertTrue(prjMgr.listAllRealizations("ALIEN").size() == 1);
    assertTrue(prjMgr.listAllRealizations("default").size() == originalCubeCountInDefault);
    assertTrue(ProjectManager.getInstance(getTestConfig()).listAllRealizations("alien").contains(createdCube));

    CubeInstance droppedCube = cubeMgr.dropCube("cube_in_alien_project", true);

    assertTrue(createdCube == droppedCube);
    assertNull(cubeMgr.getCube("cube_in_alien_project"));
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);

    prjMgr.dropProject("alien");
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
}
 

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getTestConfig());
}
 

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getConfig());
}
 

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getTestConfig());
}
 

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getTestConfig());
}
 

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getConfig());
}
 

private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) {
    if (srcHybridManager.listHybridInstances().size() > 0) {
        throw new IllegalStateException("Does not support ingest hybrid yet");
    }

    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    ProjectInstance targetProject = projectManager.getProject(targetProjectName);
    if (targetProject == null) {
        throw new IllegalStateException("Target project does not exist in target metadata: " + targetProjectName);
    }

    TableMetadataManager metadataManager = TableMetadataManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        TableDesc existing = metadataManager.getTableDesc(tableDesc.getIdentity(), targetProjectName);
        if (existing != null && !existing.equals(tableDesc)) {
            logger.info("Table {} already has a different version in target metadata store", tableDesc.getIdentity());
            logger.info("Existing version: {}", existing);
            logger.info("New version: {}", tableDesc);

            if (!forceIngest && !overwriteTables) {
                throw new IllegalStateException("table already exists with a different version: " + tableDesc.getIdentity() + ". Consider adding -overwriteTables option to force overwriting (with caution)");
            } else {
                logger.warn("Overwriting the old table desc: {}", tableDesc.getIdentity());
            }
        }
        requiredResources.add(tableDesc.getResourcePath());
    }

    DataModelManager modelManager = DataModelManager.getInstance(kylinConfig);
    for (DataModelDesc dataModelDesc : srcModelManager.listDataModels()) {
        checkExesting(modelManager.getDataModelDesc(dataModelDesc.getName()), "model", dataModelDesc.getName());
        requiredResources.add(DataModelDesc.concatResourcePath(dataModelDesc.getName()));
    }

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    for (CubeDesc cubeDesc : srcCubeDescManager.listAllDesc()) {
        checkExesting(cubeDescManager.getCubeDesc(cubeDesc.getName()), "cube desc", cubeDesc.getName());
        requiredResources.add(CubeDesc.concatResourcePath(cubeDesc.getName()));
    }

    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        checkExesting(cubeManager.getCube(cube.getName()), "cube", cube.getName());
        requiredResources.add(CubeInstance.concatResourcePath(cube.getName()));
    }

}