Java Code Examples for org.apache.kylin.metadata.project.ProjectManager#getInstance()

private List<ColumnDesc> listSourceColumns() {
    ProjectManager mgr = ProjectManager.getInstance(olapSchema.getConfig());
    List<ColumnDesc> exposedColumns = Lists.newArrayList(mgr.listExposedColumns(olapSchema.getProjectName(), sourceTable.getIdentity()));

    List<MeasureDesc> countMeasures = mgr.listEffectiveRewriteMeasures(olapSchema.getProjectName(), sourceTable.getIdentity());
    HashSet<String> metFields = new HashSet<String>();
    for (MeasureDesc m : countMeasures) {
        FunctionDesc func = m.getFunction();
        String fieldName = func.getRewriteFieldName();
        if (!metFields.contains(fieldName)) {
            metFields.add(fieldName);
            ColumnDesc fakeCountCol = new ColumnDesc();
            fakeCountCol.setName(fieldName);
            fakeCountCol.setDatatype(func.getSQLType());
            fakeCountCol.setNullable(false);
            fakeCountCol.init(sourceTable);
            exposedColumns.add(fakeCountCol);
        }
    }

    return exposedColumns;
}
 

private <T> List<T> filterByAcl(List<T> tables, String project, AclFilter filter) throws IOException {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());

    if (aclEvaluate.hasProjectAdminPermission(projectManager.getProject(project))) {
        return tables;
    }

    String username = SecurityContextHolder.getContext().getAuthentication().getName();
    Set<String> blockedTables = getBlockedTablesByUser(project, username, TYPE_USER);
    List<T> result = Lists.newArrayList();
    for (T table : tables) {
        if (filter.filter(table, blockedTables)) {
            result.add(table);
        }
    }
    return result;
}
 

private static List<String> filterByProjects(List<String> allTableNames, List<String> projectNames) {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());

    List<String> result = Lists.newArrayList();
    for (String p : projectNames) {
        p = p.trim();
        if (p.endsWith(",")) {
            p = p.substring(0, p.length() - 1);
        }

        ProjectInstance projectInstance = projectManager.getProject(p);
        List<RealizationEntry> cubeList = projectInstance.getRealizationEntries(RealizationType.CUBE);
        for (RealizationEntry cube : cubeList) {
            CubeInstance cubeInstance = cubeManager.getCube(cube.getRealization());
            for (CubeSegment segment : cubeInstance.getSegments()) {
                String tableName = segment.getStorageLocationIdentifier();
                if (allTableNames.contains(tableName)) {
                    result.add(tableName);
                }
            }
        }
    }
    return result;
}
 

public JobInstanceExtractor() {
    config = KylinConfig.getInstanceFromEnv();
    executableManager = ExecutableManager.getInstance(config);
    projectManager = ProjectManager.getInstance(config);

    packageType = "jobInstances";

    options.addOption(OPTION_PROJECT);
    options.addOption(OPTION_CUBE);
    options.addOption(OPTION_PERIOD);
}
 

@Test
public void testExistingProject() throws Exception {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());
    
    int originalProjectCount = prjMgr.listAllProjects().size();
    int originalCubeCount = cubeMgr.listAllCubes().size();
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/new_cube_in_default.json");

    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube = cubeMgr.createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null);
    assertTrue(createdCube == cubeMgr.getCube("new_cube_in_default"));

    System.out.println(JsonUtil.writeValueAsIndentString(createdCube));

    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);

    CubeInstance droppedCube = cubeMgr.dropCube("new_cube_in_default", true);

    assertTrue(createdCube == droppedCube);
    assertNull(cubeMgr.getCube("new_cube_in_default"));
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);
}
 

private void beforeExtract() {
    // reload metadata before extract diagnosis info
    logger.info("Start to reload metadata from diagnosis.");
    
    config.clearManagers();

    CubeManager.getInstance(config);
    CubeDescManager.getInstance(config);
    DataModelManager.getInstance(config);
    ProjectManager.getInstance(config);
}
 

private void beforeExtract() {
    // reload metadata before extract diagnosis info
    logger.info("Start to reload metadata from diagnosis.");
    
    config.clearManagers();

    CubeManager.getInstance(config);
    CubeDescManager.getInstance(config);
    DataModelManager.getInstance(config);
    ProjectManager.getInstance(config);
}
 

@Test
public void testProjectsDrop() throws IOException {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());

    CubeInstance cube = cubeMgr.getCube("ci_left_join_cube");
    assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
    assertTrue(prjMgr.listAllRealizations("default").contains(cube));

    cubeMgr.dropCube(cube.getName(), false);

    assertTrue(!prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
    assertTrue(!prjMgr.listAllRealizations("default").contains(cube));
}
 

@Override
protected void executeExtract(OptionsHelper optionsHelper, File exportDir) throws Exception {
    if (cachedHMasterUrl == null) {
        return;
    }
    kylinConfig = KylinConfig.getInstanceFromEnv();
    cubeManager = CubeManager.getInstance(kylinConfig);
    realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
    projectManager = ProjectManager.getInstance(kylinConfig);

    if (optionsHelper.hasOption(OPTION_PROJECT)) {
        String projectNames = optionsHelper.getOptionValue(OPTION_PROJECT);
        for (String projectName : StringUtil.splitByComma(projectNames)) {
            ProjectInstance projectInstance = projectManager.getProject(projectName);
            if (projectInstance == null) {
                throw new IllegalArgumentException("Project " + projectName + " does not exist");
            }
            List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
            for (RealizationEntry realizationEntry : realizationEntries) {
                retrieveResourcePath(getRealization(realizationEntry));
            }
        }
    } else if (optionsHelper.hasOption(OPTION_CUBE)) {
        String cubeNames = optionsHelper.getOptionValue(OPTION_CUBE);
        for (String cubeName : StringUtil.splitByComma(cubeNames)) {
            IRealization realization = cubeManager.getRealization(cubeName);
            if (realization != null) {
                retrieveResourcePath(realization);
            } else {
                throw new IllegalArgumentException("No cube found with name of " + cubeName);
            }
        }
    }

    extractCommonInfo(exportDir, kylinConfig);
    extractHTables(exportDir);
}
 

public SrcClusterUtil(String configURI, boolean ifJobFSHAEnabled, boolean ifHBaseFSHAEnabled) throws IOException {
    super(configURI, ifJobFSHAEnabled, ifHBaseFSHAEnabled);

    this.hbaseDataDir = hbaseConf.get(hbaseRootDirConfKey) + "/data/default/";
    metadataManager = TableMetadataManager.getInstance(kylinConfig);
    modelManager = DataModelManager.getInstance(kylinConfig);
    projectManager = ProjectManager.getInstance(kylinConfig);
    hybridManager = HybridManager.getInstance(kylinConfig);
    cubeManager = CubeManager.getInstance(kylinConfig);
    cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
    dictionaryManager = DictionaryManager.getInstance(kylinConfig);
    snapshotManager = SnapshotManager.getInstance(kylinConfig);
    extSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig);
}
 

public QueryRequestLimits(String project) {
    this.project = project;
    
    ProjectManager mgr = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    ProjectInstance prj = mgr.getProject(project);
    this.maxConcurrentQuery = prj.getConfig().getQueryConcurrentRunningThresholdForProject();
    
    boolean ok = openQueryRequest(project, maxConcurrentQuery);
    if (!ok) {
        Message msg = MsgPicker.getMsg();
        logger.warn("Directly return exception as too many concurrent query requests for project: {}", project);
        throw new BadRequestException(msg.getQUERY_TOO_MANY_RUNNING());
    }
}
 

private void verify() {
    try {
        Broadcaster.getInstance(config).notifyClearAll();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    DataModelManager.getInstance(config);
    CubeDescManager.getInstance(config);
    CubeManager.getInstance(config);
    ProjectManager.getInstance(config);
}
 

private List<String> getProjects(String projectSeed) {
    List<String> result = Lists.newLinkedList();
    if (projectSeed.equalsIgnoreCase("-all")) {
        ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
        for (ProjectInstance projectInstance : projectManager.listAllProjects()) {
            result.add(projectInstance.getName());
        }
    } else {
        result.add(projectSeed);
    }
    if (result.isEmpty()) {
        throw new RuntimeException("No project to extract.");
    }
    return result;
}
 

private void injest(File metaRoot) throws IOException {
    KylinConfig srcConfig = KylinConfig.createInstanceFromUri(metaRoot.getAbsolutePath());
    TableMetadataManager srcMetadataManager = TableMetadataManager.getInstance(srcConfig);
    DataModelManager srcModelManager = DataModelManager.getInstance(srcConfig);
    HybridManager srcHybridManager = HybridManager.getInstance(srcConfig);
    CubeManager srcCubeManager = CubeManager.getInstance(srcConfig);
    CubeDescManager srcCubeDescManager = CubeDescManager.getInstance(srcConfig);

    checkAndMark(srcMetadataManager, srcModelManager, srcHybridManager, srcCubeManager, srcCubeDescManager);
    new ResourceTool().copy(srcConfig, kylinConfig, Lists.newArrayList(requiredResources));

    // clear the cache
    Broadcaster.getInstance(kylinConfig).notifyClearAll();
    
    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        logger.info("add " + tableDesc + " to " + targetProjectName);
        projectManager.addTableDescToProject(Lists.newArrayList(tableDesc.getIdentity()).toArray(new String[0]), targetProjectName);
    }

    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        logger.info("add " + cube + " to " + targetProjectName);
        projectManager.addModelToProject(cube.getModel().getName(), targetProjectName);
        projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), targetProjectName, null);
    }

}
 

public ProjectManager getProjectManager() {
    return ProjectManager.getInstance(getConfig());
}
 

public ProjectManager getProjectManager() {
    return ProjectManager.getInstance(getConfig());
}
 

private static ProjectManager getProjectManager(KylinConfig config) throws Exception {
    return ProjectManager.getInstance(config);
}
 

public ProjectManager getProjectManager() {
    return ProjectManager.getInstance(getConfig());
}
 

private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) {
    if (srcHybridManager.listHybridInstances().size() > 0) {
        throw new IllegalStateException("Does not support ingest hybrid yet");
    }

    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    ProjectInstance targetProject = projectManager.getProject(targetProjectName);
    if (targetProject == null) {
        throw new IllegalStateException("Target project does not exist in target metadata: " + targetProjectName);
    }

    TableMetadataManager metadataManager = TableMetadataManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        TableDesc existing = metadataManager.getTableDesc(tableDesc.getIdentity(), targetProjectName);
        if (existing != null && !existing.equals(tableDesc)) {
            logger.info("Table {} already has a different version in target metadata store", tableDesc.getIdentity());
            logger.info("Existing version: {}", existing);
            logger.info("New version: {}", tableDesc);

            if (!forceIngest && !overwriteTables) {
                throw new IllegalStateException("table already exists with a different version: " + tableDesc.getIdentity() + ". Consider adding -overwriteTables option to force overwriting (with caution)");
            } else {
                logger.warn("Overwriting the old table desc: {}", tableDesc.getIdentity());
            }
        }
        requiredResources.add(tableDesc.getResourcePath());
    }

    DataModelManager modelManager = DataModelManager.getInstance(kylinConfig);
    for (DataModelDesc dataModelDesc : srcModelManager.listDataModels()) {
        checkExesting(modelManager.getDataModelDesc(dataModelDesc.getName()), "model", dataModelDesc.getName());
        requiredResources.add(DataModelDesc.concatResourcePath(dataModelDesc.getName()));
    }

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    for (CubeDesc cubeDesc : srcCubeDescManager.listAllDesc()) {
        checkExesting(cubeDescManager.getCubeDesc(cubeDesc.getName()), "cube desc", cubeDesc.getName());
        requiredResources.add(CubeDesc.concatResourcePath(cubeDesc.getName()));
    }

    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        checkExesting(cubeManager.getCube(cube.getName()), "cube", cube.getName());
        requiredResources.add(CubeInstance.concatResourcePath(cube.getName()));
    }

}
 

public ProjectManager getProjectManager() {
    return ProjectManager.getInstance(getConfig());
}