Java Code Examples for org.apache.kylin.metadata.project.ProjectManager#getProject()

The following examples show how to use org.apache.kylin.metadata.project.ProjectManager#getProject() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DeployCoprocessorCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private static List<String> filterByProjects(List<String> allTableNames, List<String> projectNames) {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());

    List<String> result = Lists.newArrayList();
    for (String p : projectNames) {
        p = p.trim();
        if (p.endsWith(",")) {
            p = p.substring(0, p.length() - 1);
        }

        ProjectInstance projectInstance = projectManager.getProject(p);
        List<RealizationEntry> cubeList = projectInstance.getRealizationEntries(RealizationType.CUBE);
        for (RealizationEntry cube : cubeList) {
            CubeInstance cubeInstance = cubeManager.getCube(cube.getRealization());
            for (CubeSegment segment : cubeInstance.getSegments()) {
                String tableName = segment.getStorageLocationIdentifier();
                if (allTableNames.contains(tableName)) {
                    result.add(tableName);
                }
            }
        }
    }
    return result;
}
Example 2
Source File: CubeService.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public List<CubeInstance> listAllCubes(String projectName) {
    ProjectManager projectManager = getProjectManager();
    ProjectInstance project = projectManager.getProject(projectName);
    if (project == null) {
        return Collections.emptyList();
    }
    ArrayList<CubeInstance> result = new ArrayList<CubeInstance>();
    for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
        if (projectDataModel.getType() == RealizationType.CUBE) {
            CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
            if (cube != null)
                result.add(cube);
            else
                logger.error("Cube instance " + projectDataModel.getRealization() + " is failed to load");
        }
    }
    return result;
}
Example 3
Source File: CubeService.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
protected boolean isCubeInProject(String projectName, CubeInstance target) {
    ProjectManager projectManager = getProjectManager();
    ProjectInstance project = projectManager.getProject(projectName);
    if (project == null) {
        return false;
    }
    for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
        if (projectDataModel.getType() == RealizationType.CUBE) {
            CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
            if (cube == null) {
                logger.error("Project " + projectName + " contains realization " + projectDataModel.getRealization()
                        + " which is not found by CubeManager");
                continue;
            }
            if (cube.equals(target)) {
                return true;
            }
        }
    }
    return false;
}
Example 4
Source File: DeployCoprocessorCLI.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private static List<String> filterByProjects(List<String> allTableNames, List<String> projectNames) {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());

    List<String> result = Lists.newArrayList();
    for (String p : projectNames) {
        p = p.trim();
        if (p.endsWith(",")) {
            p = p.substring(0, p.length() - 1);
        }

        ProjectInstance projectInstance = projectManager.getProject(p);
        List<RealizationEntry> cubeList = projectInstance.getRealizationEntries(RealizationType.CUBE);
        for (RealizationEntry cube : cubeList) {
            CubeInstance cubeInstance = cubeManager.getCube(cube.getRealization());
            for (CubeSegment segment : cubeInstance.getSegments()) {
                String tableName = segment.getStorageLocationIdentifier();
                if (allTableNames.contains(tableName)) {
                    result.add(tableName);
                }
            }
        }
    }
    return result;
}
Example 5
Source File: CubeService.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public List<CubeInstance> listAllCubes(String projectName) {
    ProjectManager projectManager = getProjectManager();
    ProjectInstance project = projectManager.getProject(projectName);
    if (project == null) {
        return Collections.emptyList();
    }
    ArrayList<CubeInstance> result = new ArrayList<CubeInstance>();
    for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
        if (projectDataModel.getType() == RealizationType.CUBE) {
            CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
            if (cube != null)
                result.add(cube);
            else
                logger.error("Cube instance " + projectDataModel.getRealization() + " is failed to load");
        }
    }
    return result;
}
Example 6
Source File: CubeService.java    From kylin with Apache License 2.0 6 votes vote down vote up 
protected boolean isCubeInProject(String projectName, CubeInstance target) {
    ProjectManager projectManager = getProjectManager();
    ProjectInstance project = projectManager.getProject(projectName);
    if (project == null) {
        return false;
    }
    for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
        if (projectDataModel.getType() == RealizationType.CUBE) {
            CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
            if (cube == null) {
                logger.error("Project " + projectName + " contains realization " + projectDataModel.getRealization()
                        + " which is not found by CubeManager");
                continue;
            }
            if (cube.equals(target)) {
                return true;
            }
        }
    }
    return false;
}
Example 7
Source File: DataModelManager.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public DataModelDesc createDataModelDesc(DataModelDesc desc, String projectName, String owner) throws IOException {
    try (AutoLock lock = modelMapLock.lockForWrite()) {
        String name = desc.getName();
        if (dataModelDescMap.containsKey(name))
            throw new IllegalArgumentException("DataModelDesc '" + name + "' already exists");

        ProjectManager prjMgr = ProjectManager.getInstance(config);
        ProjectInstance prj = prjMgr.getProject(projectName);
        if (prj.containsModel(name)) {
            throw new IllegalStateException("project " + projectName + " already contains model " + name);
        }
        desc.setOwner(owner);
        logger.info("Saving Model {} to Project {} with {} as owner", desc.getName(), projectName, owner);
        desc = saveDataModelDesc(desc, projectName);

        // now that model is saved, update project formally
        try {
            prjMgr.addModelToProject(name, projectName);
        } catch (WriteConflictException e) {
            logger.warn("Add model: {} to project: {} failed for write conflicts, rollback", name, projectName, e);
            crud.delete(desc);
            throw e;
        }

        return desc;
    }
}
Example 8
Source File: QueryRequestLimits.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public QueryRequestLimits(String project) {
    this.project = project;
    
    ProjectManager mgr = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    ProjectInstance prj = mgr.getProject(project);
    this.maxConcurrentQuery = prj.getConfig().getQueryConcurrentRunningThresholdForProject();
    
    boolean ok = openQueryRequest(project, maxConcurrentQuery);
    if (!ok) {
        Message msg = MsgPicker.getMsg();
        logger.warn("Directly return exception as too many concurrent query requests for project: {}", project);
        throw new BadRequestException(msg.getQUERY_TOO_MANY_RUNNING());
    }
}
Example 9
Source File: QueryUtil.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public static String massagePushDownSql(String sql, String project, String defaultSchema, boolean isPrepare) {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    ProjectInstance projectInstance = projectManager.getProject(project);
    KylinConfig kylinConfig = projectInstance.getConfig();
    while (sql.endsWith(";"))
        sql = sql.substring(0, sql.length() - 1);

    return massagePushDownSql(kylinConfig, sql, project, defaultSchema, isPrepare);
}
Example 10
Source File: DataModelManager.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public DataModelDesc createDataModelDesc(DataModelDesc desc, String projectName, String owner) throws IOException {
    try (AutoLock lock = modelMapLock.lockForWrite()) {
        String name = desc.getName();
        if (dataModelDescMap.containsKey(name))
            throw new IllegalArgumentException("DataModelDesc '" + name + "' already exists");

        ProjectManager prjMgr = ProjectManager.getInstance(config);
        ProjectInstance prj = prjMgr.getProject(projectName);
        if (prj.containsModel(name)) {
            throw new IllegalStateException("project " + projectName + " already contains model " + name);
        }
        desc.setOwner(owner);
        logger.info("Saving Model {} to Project {} with {} as owner", desc.getName(), projectName, owner);
        desc = saveDataModelDesc(desc, projectName);

        // now that model is saved, update project formally
        try {
            prjMgr.addModelToProject(name, projectName);
        } catch (WriteConflictException e) {
            logger.warn("Add model: {} to project: {} failed for write conflicts, rollback", name, projectName, e);
            crud.delete(desc);
            throw e;
        }

        return desc;
    }
}
Example 11
Source File: QueryRequestLimits.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public QueryRequestLimits(String project) {
    this.project = project;
    
    ProjectManager mgr = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    ProjectInstance prj = mgr.getProject(project);
    this.maxConcurrentQuery = prj.getConfig().getQueryConcurrentRunningThresholdForProject();
    
    boolean ok = openQueryRequest(project, maxConcurrentQuery);
    if (!ok) {
        Message msg = MsgPicker.getMsg();
        logger.warn("Directly return exception as too many concurrent query requests for project: {}", project);
        throw new BadRequestException(msg.getQUERY_TOO_MANY_RUNNING());
    }
}
Example 12
Source File: QueryUtil.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
/**
 * @deprecated Deprecated because of KYLIN-3594
 */
@Deprecated
public static String massageSql(String sql, String project, int limit, int offset, String defaultSchema) {
    sql = sql.trim();
    sql = sql.replace("\r", " ").replace("\n", System.getProperty("line.separator"));

    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    ProjectInstance projectInstance = projectManager.getProject(project);
    KylinConfig kylinConfig = projectInstance.getConfig();
    sql = removeCommentInSql(sql);
    while (sql.endsWith(";"))
        sql = sql.substring(0, sql.length() - 1);

    String sql1 = sql;
    final String suffixPattern = "^.+?\\s(limit\\s\\d+)?\\s(offset\\s\\d+)?\\s*$";
    sql = sql.replaceAll("\\s+", " ");
    Pattern pattern = Pattern.compile(suffixPattern);
    Matcher matcher = pattern.matcher(sql.toLowerCase(Locale.ROOT) + "  ");

    int toAppendLimit = 0;
    int toAppendOffset = 0;
    if (matcher.find()) {
        if (limit > 0 && matcher.group(1) == null) {
            toAppendLimit = limit;
        }
        if (offset > 0 && matcher.group(2) == null) {
            toAppendOffset = offset;
        }
    }

    // https://issues.apache.org/jira/browse/KYLIN-2649
    if (kylinConfig.getForceLimit() > 0 && limit <= 0 && matcher.group(1) == null
            && sql1.toLowerCase(Locale.ROOT).matches("^select\\s+\\*\\p{all}*")) {
        toAppendLimit = kylinConfig.getForceLimit();
    }

    sql1 = appendLimitOffsetToSql(sql1, toAppendLimit, toAppendOffset);

    // customizable SQL transformation
    if (queryTransformers == null) {
        initQueryTransformers();
    }
    for (IQueryTransformer t : queryTransformers) {
        sql1 = t.transform(sql1, project, defaultSchema);
    }
    return sql1;
}
Example 13
Source File: CubeMetaIngester.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) {
    if (srcHybridManager.listHybridInstances().size() > 0) {
        throw new IllegalStateException("Does not support ingest hybrid yet");
    }

    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    ProjectInstance targetProject = projectManager.getProject(targetProjectName);
    if (targetProject == null) {
        throw new IllegalStateException("Target project does not exist in target metadata: " + targetProjectName);
    }

    TableMetadataManager metadataManager = TableMetadataManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        TableDesc existing = metadataManager.getTableDesc(tableDesc.getIdentity(), targetProjectName);
        if (existing != null && !existing.equals(tableDesc)) {
            logger.info("Table {} already has a different version in target metadata store", tableDesc.getIdentity());
            logger.info("Existing version: {}", existing);
            logger.info("New version: {}", tableDesc);

            if (!forceIngest && !overwriteTables) {
                throw new IllegalStateException("table already exists with a different version: " + tableDesc.getIdentity() + ". Consider adding -overwriteTables option to force overwriting (with caution)");
            } else {
                logger.warn("Overwriting the old table desc: {}", tableDesc.getIdentity());
            }
        }
        requiredResources.add(tableDesc.getResourcePath());
    }

    DataModelManager modelManager = DataModelManager.getInstance(kylinConfig);
    for (DataModelDesc dataModelDesc : srcModelManager.listDataModels()) {
        checkExesting(modelManager.getDataModelDesc(dataModelDesc.getName()), "model", dataModelDesc.getName());
        requiredResources.add(DataModelDesc.concatResourcePath(dataModelDesc.getName()));
    }

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    for (CubeDesc cubeDesc : srcCubeDescManager.listAllDesc()) {
        checkExesting(cubeDescManager.getCubeDesc(cubeDesc.getName()), "cube desc", cubeDesc.getName());
        requiredResources.add(CubeDesc.concatResourcePath(cubeDesc.getName()));
    }

    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        checkExesting(cubeManager.getCube(cube.getName()), "cube", cube.getName());
        requiredResources.add(CubeInstance.concatResourcePath(cube.getName()));
    }

}
Example 14
Source File: QueryUtil.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public static String massageSql(String sql, String project, int limit, int offset, String defaultSchema, boolean isCCNeeded) {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    ProjectInstance projectInstance = projectManager.getProject(project);
    KylinConfig kylinConfig = projectInstance.getConfig();
    return massageSql(kylinConfig, sql, project, limit, offset, defaultSchema, isCCNeeded);
}
Example 15
Source File: QueryUtil.java    From kylin with Apache License 2.0 4 votes vote down vote up 
/**
 * @deprecated Deprecated because of KYLIN-3594
 */
@Deprecated
public static String massageSql(String sql, String project, int limit, int offset, String defaultSchema) {
    sql = sql.trim();
    sql = sql.replace("\r", " ").replace("\n", System.getProperty("line.separator"));

    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    ProjectInstance projectInstance = projectManager.getProject(project);
    KylinConfig kylinConfig = projectInstance.getConfig();
    sql = removeCommentInSql(sql);
    while (sql.endsWith(";"))
        sql = sql.substring(0, sql.length() - 1);

    String sql1 = sql;
    final String suffixPattern = "^.+?\\s(limit\\s\\d+)?\\s(offset\\s\\d+)?\\s*$";
    sql = sql.replaceAll("\\s+", " ");
    Pattern pattern = Pattern.compile(suffixPattern);
    Matcher matcher = pattern.matcher(sql.toLowerCase(Locale.ROOT) + "  ");

    int toAppendLimit = 0;
    int toAppendOffset = 0;
    if (matcher.find()) {
        if (limit > 0 && matcher.group(1) == null) {
            toAppendLimit = limit;
        }
        if (offset > 0 && matcher.group(2) == null) {
            toAppendOffset = offset;
        }
    }

    // https://issues.apache.org/jira/browse/KYLIN-2649
    if (kylinConfig.getForceLimit() > 0 && limit <= 0 && matcher.group(1) == null
            && sql1.toLowerCase(Locale.ROOT).matches("^select\\s+\\*\\p{all}*")) {
        toAppendLimit = kylinConfig.getForceLimit();
    }

    sql1 = appendLimitOffsetToSql(sql1, toAppendLimit, toAppendOffset);

    // customizable SQL transformation
    if (queryTransformers == null) {
        initQueryTransformers();
    }
    for (IQueryTransformer t : queryTransformers) {
        sql1 = t.transform(sql1, project, defaultSchema);
    }
    return sql1;
}
Example 16
Source File: CubeMetaIngester.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) {
    if (srcHybridManager.listHybridInstances().size() > 0) {
        throw new IllegalStateException("Does not support ingest hybrid yet");
    }

    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    ProjectInstance targetProject = projectManager.getProject(targetProjectName);
    if (targetProject == null) {
        throw new IllegalStateException("Target project does not exist in target metadata: " + targetProjectName);
    }

    TableMetadataManager metadataManager = TableMetadataManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        TableDesc existing = metadataManager.getTableDesc(tableDesc.getIdentity(), targetProjectName);
        if (existing != null && !existing.equals(tableDesc)) {
            logger.info("Table {} already has a different version in target metadata store", tableDesc.getIdentity());
            logger.info("Existing version: {}", existing);
            logger.info("New version: {}", tableDesc);

            if (!forceIngest && !overwriteTables) {
                throw new IllegalStateException("table already exists with a different version: " + tableDesc.getIdentity() + ". Consider adding -overwriteTables option to force overwriting (with caution)");
            } else {
                logger.warn("Overwriting the old table desc: {}", tableDesc.getIdentity());
            }
        }
        requiredResources.add(tableDesc.getResourcePath());
    }

    DataModelManager modelManager = DataModelManager.getInstance(kylinConfig);
    for (DataModelDesc dataModelDesc : srcModelManager.listDataModels()) {
        checkExesting(modelManager.getDataModelDesc(dataModelDesc.getName()), "model", dataModelDesc.getName());
        requiredResources.add(DataModelDesc.concatResourcePath(dataModelDesc.getName()));
    }

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    for (CubeDesc cubeDesc : srcCubeDescManager.listAllDesc()) {
        checkExesting(cubeDescManager.getCubeDesc(cubeDesc.getName()), "cube desc", cubeDesc.getName());
        requiredResources.add(CubeDesc.concatResourcePath(cubeDesc.getName()));
    }

    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        checkExesting(cubeManager.getCube(cube.getName()), "cube", cube.getName());
        requiredResources.add(CubeInstance.concatResourcePath(cube.getName()));
    }

}