Java Code Examples for org.apache.kylin.cube.CubeManager#listAllCubes()
The following examples show how to use
org.apache.kylin.cube.CubeManager#listAllCubes() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SampleCubeSetupTest.java From Kylin with Apache License 2.0 | 6 votes |
@Test public void testCubes() throws Exception { DeployUtil.initCliWorkDir(); DeployUtil.deployMetadata(); DeployUtil.deployJobJars(); deployJobConfToEtc(); DeployUtil.prepareTestData("inner", "test_kylin_cube_with_slr_empty"); // remove all other cubes to keep it clean CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); for (CubeInstance cubeInstance : cubeManager.listAllCubes()) { if (!cubeInstance.getName().equalsIgnoreCase("test_kylin_cube_without_slr_empty") && !cubeInstance.getName().equalsIgnoreCase("test_kylin_cube_with_slr_empty")) cubeManager.dropCube(cubeInstance.getName(), false); } }
Example 2
Source File: SCCreatorTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
@Test public void testExecute() throws Exception { String metadataPath = tempMetadataDir.getPath(); String inputPath = "src/main/resources/SCSinkTools.json"; SCCreator cli = new SCCreator(); cli.execute("ADMIN", metadataPath, inputPath); Assert.assertTrue(tempMetadataDir.isDirectory()); KylinConfig local = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv()); local.setMetadataUrl(metadataPath); CubeManager cubeManager = CubeManager.getInstance(local); List<CubeInstance> cubeList = cubeManager.listAllCubes(); System.out.println("System cubes: " + cubeList); assertEquals(cubeList.size(), 10); for (CubeInstance cube : cubeList) { Assert.assertTrue(cube.getStatus() != RealizationStatusEnum.DESCBROKEN); } }
Example 3
Source File: UpdateHTableHostCLI.java From kylin with Apache License 2.0 | 6 votes |
private static List<String> getHTableNames(KylinConfig config) { CubeManager cubeMgr = CubeManager.getInstance(config); ArrayList<String> result = new ArrayList<>(); for (CubeInstance cube : cubeMgr.listAllCubes()) { for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) { String tableName = seg.getStorageLocationIdentifier(); if (!StringUtils.isBlank(tableName)) { result.add(tableName); logger.info("added new table: {}", tableName); } } } return result; }
Example 4
Source File: DeployCoprocessorCLI.java From kylin with Apache License 2.0 | 6 votes |
private static List<String> getHTableNames(KylinConfig config) { CubeManager cubeMgr = CubeManager.getInstance(config); ArrayList<String> result = new ArrayList<String>(); for (CubeInstance cube : cubeMgr.listAllCubes()) { if (cube.getStorageType() == IStorageAware.ID_HBASE || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE || cube.getStorageType() == IStorageAware.ID_REALTIME_AND_HBASE) { for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) { String tableName = seg.getStorageLocationIdentifier(); if (StringUtils.isBlank(tableName) == false) { result.add(tableName); System.out.println("added new table: " + tableName); } } } } return result; }
Example 5
Source File: DeployCoprocessorCLI.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
private static List<String> getHTableNames(KylinConfig config) { CubeManager cubeMgr = CubeManager.getInstance(config); ArrayList<String> result = new ArrayList<String>(); for (CubeInstance cube : cubeMgr.listAllCubes()) { if (cube.getStorageType() == IStorageAware.ID_HBASE || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE || cube.getStorageType() == IStorageAware.ID_REALTIME_AND_HBASE) { for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) { String tableName = seg.getStorageLocationIdentifier(); if (StringUtils.isBlank(tableName) == false) { result.add(tableName); System.out.println("added new table: " + tableName); } } } } return result; }
Example 6
Source File: UpdateHTableHostCLI.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
private static List<String> getHTableNames(KylinConfig config) { CubeManager cubeMgr = CubeManager.getInstance(config); ArrayList<String> result = new ArrayList<>(); for (CubeInstance cube : cubeMgr.listAllCubes()) { for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) { String tableName = seg.getStorageLocationIdentifier(); if (!StringUtils.isBlank(tableName)) { result.add(tableName); logger.info("added new table: {}", tableName); } } } return result; }
Example 7
Source File: SCCreatorTest.java From kylin with Apache License 2.0 | 6 votes |
@Test public void testExecute() throws Exception { String metadataPath = tempMetadataDir.getPath(); String inputPath = "src/main/resources/SCSinkTools.json"; SCCreator cli = new SCCreator(); cli.execute("ADMIN", metadataPath, inputPath); Assert.assertTrue(tempMetadataDir.isDirectory()); KylinConfig local = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv()); local.setMetadataUrl(metadataPath); CubeManager cubeManager = CubeManager.getInstance(local); List<CubeInstance> cubeList = cubeManager.listAllCubes(); System.out.println("System cubes: " + cubeList); assertEquals(cubeList.size(), 10); for (CubeInstance cube : cubeList) { Assert.assertTrue(cube.getStatus() != RealizationStatusEnum.DESCBROKEN); } }
Example 8
Source File: CubeMigrationCheckCLI.java From kylin with Apache License 2.0 | 5 votes |
public void checkAll() { List<String> segFullNameList = Lists.newArrayList(); CubeManager cubeMgr = CubeManager.getInstance(dstCfg); for (CubeInstance cube : cubeMgr.listAllCubes()) { addHTableNamesForCube(cube, segFullNameList); } check(segFullNameList); }
Example 9
Source File: KylinHealthCheckJob.java From kylin with Apache License 2.0 | 5 votes |
private void checkCubeHealth() throws Exception { CubeManager cubeManager = CubeManager.getInstance(config); List<CubeInstance> cubes = cubeManager.listAllCubes(); checkErrorMeta(); // Check if the cubeid data exist for later cube merge checkSegmentHDFSPath(cubes); // Check if the hbase table exits or online checkHBaseTables(cubes); // Check if there are holes in cube // TODO: check if there are overlaps in segments of cube checkCubeHoles(cubes); // Check if there are too many segments checkTooManySegments(cubes); // Check if there are stale metadata checkStaleSegments(cubes); // Disable/Delete the out-of-date cube checkOutOfDateCube(cubes); // Check data expand rate checkDataExpansionRate(cubes); // Check auto merge param checkCubeDescParams(cubes); // ERROR history stopped build job checkStoppedJob(); sendMail(reporter.getBufferedLog()); }
Example 10
Source File: CubeMigrationCheckCLI.java From kylin with Apache License 2.0 | 5 votes |
public void checkAll() { List<String> segFullNameList = Lists.newArrayList(); CubeManager cubeMgr = CubeManager.getInstance(dstCfg); for (CubeInstance cube : cubeMgr.listAllCubes()) { addHTableNamesForCube(cube, segFullNameList); } check(segFullNameList); }
Example 11
Source File: CubeMetaIngester.java From kylin with Apache License 2.0 | 5 votes |
private void injest(File metaRoot) throws IOException { KylinConfig srcConfig = KylinConfig.createInstanceFromUri(metaRoot.getAbsolutePath()); TableMetadataManager srcMetadataManager = TableMetadataManager.getInstance(srcConfig); DataModelManager srcModelManager = DataModelManager.getInstance(srcConfig); HybridManager srcHybridManager = HybridManager.getInstance(srcConfig); CubeManager srcCubeManager = CubeManager.getInstance(srcConfig); CubeDescManager srcCubeDescManager = CubeDescManager.getInstance(srcConfig); checkAndMark(srcMetadataManager, srcModelManager, srcHybridManager, srcCubeManager, srcCubeDescManager); new ResourceTool().copy(srcConfig, kylinConfig, Lists.newArrayList(requiredResources)); // clear the cache Broadcaster.getInstance(kylinConfig).notifyClearAll(); ProjectManager projectManager = ProjectManager.getInstance(kylinConfig); for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) { logger.info("add " + tableDesc + " to " + targetProjectName); projectManager.addTableDescToProject(Lists.newArrayList(tableDesc.getIdentity()).toArray(new String[0]), targetProjectName); } for (CubeInstance cube : srcCubeManager.listAllCubes()) { logger.info("add " + cube + " to " + targetProjectName); projectManager.addModelToProject(cube.getModel().getName(), targetProjectName); projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), targetProjectName, null); } }
Example 12
Source File: CubeMigrationCheckCLI.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
public void checkAll() { List<String> segFullNameList = Lists.newArrayList(); CubeManager cubeMgr = CubeManager.getInstance(dstCfg); for (CubeInstance cube : cubeMgr.listAllCubes()) { addHTableNamesForCube(cube, segFullNameList); } check(segFullNameList); }
Example 13
Source File: KylinHealthCheckJob.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
private void checkCubeHealth() throws Exception { CubeManager cubeManager = CubeManager.getInstance(config); List<CubeInstance> cubes = cubeManager.listAllCubes(); checkErrorMeta(); // Check if the cubeid data exist for later cube merge checkSegmentHDFSPath(cubes); // Check if the hbase table exits or online checkHBaseTables(cubes); // Check if there are holes in cube // TODO: check if there are overlaps in segments of cube checkCubeHoles(cubes); // Check if there are too many segments checkTooManySegments(cubes); // Check if there are stale metadata checkStaleSegments(cubes); // Disable/Delete the out-of-date cube checkOutOfDateCube(cubes); // Check data expand rate checkDataExpansionRate(cubes); // Check auto merge param checkCubeDescParams(cubes); // ERROR history stopped build job checkStoppedJob(); sendMail(reporter.getBufferedLog()); }
Example 14
Source File: CubeMigrationCheckCLI.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
public void checkAll() { List<String> segFullNameList = Lists.newArrayList(); CubeManager cubeMgr = CubeManager.getInstance(dstCfg); for (CubeInstance cube : cubeMgr.listAllCubes()) { addHTableNamesForCube(cube, segFullNameList); } check(segFullNameList); }
Example 15
Source File: CubeMetaIngester.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
private void injest(File metaRoot) throws IOException { KylinConfig srcConfig = KylinConfig.createInstanceFromUri(metaRoot.getAbsolutePath()); TableMetadataManager srcMetadataManager = TableMetadataManager.getInstance(srcConfig); DataModelManager srcModelManager = DataModelManager.getInstance(srcConfig); HybridManager srcHybridManager = HybridManager.getInstance(srcConfig); CubeManager srcCubeManager = CubeManager.getInstance(srcConfig); CubeDescManager srcCubeDescManager = CubeDescManager.getInstance(srcConfig); checkAndMark(srcMetadataManager, srcModelManager, srcHybridManager, srcCubeManager, srcCubeDescManager); new ResourceTool().copy(srcConfig, kylinConfig, Lists.newArrayList(requiredResources)); // clear the cache Broadcaster.getInstance(kylinConfig).notifyClearAll(); ProjectManager projectManager = ProjectManager.getInstance(kylinConfig); for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) { logger.info("add " + tableDesc + " to " + targetProjectName); projectManager.addTableDescToProject(Lists.newArrayList(tableDesc.getIdentity()).toArray(new String[0]), targetProjectName); } for (CubeInstance cube : srcCubeManager.listAllCubes()) { logger.info("add " + cube + " to " + targetProjectName); projectManager.addModelToProject(cube.getModel().getName(), targetProjectName); projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), targetProjectName, null); } }
Example 16
Source File: CubeMetaIngester.java From kylin with Apache License 2.0 | 4 votes |
private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) { if (srcHybridManager.listHybridInstances().size() > 0) { throw new IllegalStateException("Does not support ingest hybrid yet"); } ProjectManager projectManager = ProjectManager.getInstance(kylinConfig); ProjectInstance targetProject = projectManager.getProject(targetProjectName); if (targetProject == null) { throw new IllegalStateException("Target project does not exist in target metadata: " + targetProjectName); } TableMetadataManager metadataManager = TableMetadataManager.getInstance(kylinConfig); for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) { TableDesc existing = metadataManager.getTableDesc(tableDesc.getIdentity(), targetProjectName); if (existing != null && !existing.equals(tableDesc)) { logger.info("Table {} already has a different version in target metadata store", tableDesc.getIdentity()); logger.info("Existing version: {}", existing); logger.info("New version: {}", tableDesc); if (!forceIngest && !overwriteTables) { throw new IllegalStateException("table already exists with a different version: " + tableDesc.getIdentity() + ". Consider adding -overwriteTables option to force overwriting (with caution)"); } else { logger.warn("Overwriting the old table desc: {}", tableDesc.getIdentity()); } } requiredResources.add(tableDesc.getResourcePath()); } DataModelManager modelManager = DataModelManager.getInstance(kylinConfig); for (DataModelDesc dataModelDesc : srcModelManager.listDataModels()) { checkExesting(modelManager.getDataModelDesc(dataModelDesc.getName()), "model", dataModelDesc.getName()); requiredResources.add(DataModelDesc.concatResourcePath(dataModelDesc.getName())); } CubeDescManager cubeDescManager = CubeDescManager.getInstance(kylinConfig); for (CubeDesc cubeDesc : srcCubeDescManager.listAllDesc()) { checkExesting(cubeDescManager.getCubeDesc(cubeDesc.getName()), "cube desc", cubeDesc.getName()); requiredResources.add(CubeDesc.concatResourcePath(cubeDesc.getName())); } CubeManager cubeManager = CubeManager.getInstance(kylinConfig); for (CubeInstance cube : srcCubeManager.listAllCubes()) { checkExesting(cubeManager.getCube(cube.getName()), "cube", cube.getName()); requiredResources.add(CubeInstance.concatResourcePath(cube.getName())); } }
Example 17
Source File: MetadataCleanupJob.java From kylin with Apache License 2.0 | 4 votes |
public void cleanup() throws Exception { CubeManager cubeManager = CubeManager.getInstance(config); List<String> toDeleteResource = Lists.newArrayList(); // two level resources, snapshot tables and cube statistics for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) { NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot); if (snapshotTables != null) { for (String snapshotTable : snapshotTables) { NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable); if (snapshotNames != null) for (String snapshot : snapshotNames) { if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot))) toDeleteResource.add(snapshot); } } } } // three level resources, only dictionaries NavigableSet<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT); if (dictTables != null) { for (String table : dictTables) { NavigableSet<String> tableColNames = getStore().listResources(table); if (tableColNames != null) for (String tableCol : tableColNames) { NavigableSet<String> dictionaries = getStore().listResources(tableCol); if (dictionaries != null) for (String dict : dictionaries) if (isOlderThanThreshold(getStore().getResourceTimestamp(dict))) toDeleteResource.add(dict); } } } Set<String> activeResourceList = Sets.newHashSet(); for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) { for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) { activeResourceList.addAll(segment.getSnapshotPaths()); activeResourceList.addAll(segment.getDictionaryPaths()); activeResourceList.add(segment.getStatisticsResourcePath()); } } toDeleteResource.removeAll(activeResourceList); // delete old and completed jobs ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv()); List<ExecutablePO> allExecutable = executableDao.getJobs(); for (ExecutablePO executable : allExecutable) { long lastModified = executable.getLastModified(); ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid()); if (System.currentTimeMillis() - lastModified > TIME_THREADSHOLD_FOR_JOB && (ExecutableState.SUCCEED.toString().equals(output.getStatus()) || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) { toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid()); toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid()); for (ExecutablePO task : executable.getTasks()) { toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid()); } } } if (toDeleteResource.size() > 0) { logger.info("The following resources have no reference or is too old, will be cleaned from metadata store: \n"); for (String s : toDeleteResource) { logger.info(s); if (delete == true) { getStore().deleteResource(s); } } } else { logger.info("No resource to be cleaned up from metadata store;"); } }
Example 18
Source File: MetadataCleanupJob.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
public void cleanup() throws Exception { CubeManager cubeManager = CubeManager.getInstance(config); List<String> toDeleteResource = Lists.newArrayList(); // two level resources, snapshot tables and cube statistics for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) { NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot); if (snapshotTables != null) { for (String snapshotTable : snapshotTables) { NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable); if (snapshotNames != null) for (String snapshot : snapshotNames) { if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot))) toDeleteResource.add(snapshot); } } } } // three level resources, only dictionaries NavigableSet<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT); if (dictTables != null) { for (String table : dictTables) { NavigableSet<String> tableColNames = getStore().listResources(table); if (tableColNames != null) for (String tableCol : tableColNames) { NavigableSet<String> dictionaries = getStore().listResources(tableCol); if (dictionaries != null) for (String dict : dictionaries) if (isOlderThanThreshold(getStore().getResourceTimestamp(dict))) toDeleteResource.add(dict); } } } Set<String> activeResourceList = Sets.newHashSet(); for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) { for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) { activeResourceList.addAll(segment.getSnapshotPaths()); activeResourceList.addAll(segment.getDictionaryPaths()); activeResourceList.add(segment.getStatisticsResourcePath()); } } toDeleteResource.removeAll(activeResourceList); // delete old and completed jobs ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv()); List<ExecutablePO> allExecutable = executableDao.getJobs(); for (ExecutablePO executable : allExecutable) { long lastModified = executable.getLastModified(); ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid()); if (System.currentTimeMillis() - lastModified > TIME_THREADSHOLD_FOR_JOB && (ExecutableState.SUCCEED.toString().equals(output.getStatus()) || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) { toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid()); toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid()); for (ExecutablePO task : executable.getTasks()) { toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid()); } } } if (toDeleteResource.size() > 0) { logger.info("The following resources have no reference or is too old, will be cleaned from metadata store: \n"); for (String s : toDeleteResource) { logger.info(s); if (delete == true) { getStore().deleteResource(s); } } } else { logger.info("No resource to be cleaned up from metadata store;"); } }
Example 19
Source File: CubeMetaIngester.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) { if (srcHybridManager.listHybridInstances().size() > 0) { throw new IllegalStateException("Does not support ingest hybrid yet"); } ProjectManager projectManager = ProjectManager.getInstance(kylinConfig); ProjectInstance targetProject = projectManager.getProject(targetProjectName); if (targetProject == null) { throw new IllegalStateException("Target project does not exist in target metadata: " + targetProjectName); } TableMetadataManager metadataManager = TableMetadataManager.getInstance(kylinConfig); for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) { TableDesc existing = metadataManager.getTableDesc(tableDesc.getIdentity(), targetProjectName); if (existing != null && !existing.equals(tableDesc)) { logger.info("Table {} already has a different version in target metadata store", tableDesc.getIdentity()); logger.info("Existing version: {}", existing); logger.info("New version: {}", tableDesc); if (!forceIngest && !overwriteTables) { throw new IllegalStateException("table already exists with a different version: " + tableDesc.getIdentity() + ". Consider adding -overwriteTables option to force overwriting (with caution)"); } else { logger.warn("Overwriting the old table desc: {}", tableDesc.getIdentity()); } } requiredResources.add(tableDesc.getResourcePath()); } DataModelManager modelManager = DataModelManager.getInstance(kylinConfig); for (DataModelDesc dataModelDesc : srcModelManager.listDataModels()) { checkExesting(modelManager.getDataModelDesc(dataModelDesc.getName()), "model", dataModelDesc.getName()); requiredResources.add(DataModelDesc.concatResourcePath(dataModelDesc.getName())); } CubeDescManager cubeDescManager = CubeDescManager.getInstance(kylinConfig); for (CubeDesc cubeDesc : srcCubeDescManager.listAllDesc()) { checkExesting(cubeDescManager.getCubeDesc(cubeDesc.getName()), "cube desc", cubeDesc.getName()); requiredResources.add(CubeDesc.concatResourcePath(cubeDesc.getName())); } CubeManager cubeManager = CubeManager.getInstance(kylinConfig); for (CubeInstance cube : srcCubeManager.listAllCubes()) { checkExesting(cubeManager.getCube(cube.getName()), "cube", cube.getName()); requiredResources.add(CubeInstance.concatResourcePath(cube.getName())); } }