org.apache.kylin.metadata.project.ProjectInstance Java Examples

protected static void populateSSWithCSVData(KylinConfig kylinConfig, String project, SparkSession sparkSession) {

        ProjectInstance projectInstance = ProjectManager.getInstance(kylinConfig).getProject(project);
        Preconditions.checkArgument(projectInstance != null);
        for (String table : projectInstance.getTables()) {

            if ("DEFAULT.STREAMING_TABLE".equals(table)) {
                continue;
            }
            if (!new File(String.format(Locale.ROOT, CSV_TABLE_DIR, table)).exists()) {
                continue;
            }
            TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(table, project);
            ColumnDesc[] columns = tableDesc.getColumns();
            StructType schema = new StructType();
            for (ColumnDesc column : columns) {
                schema = schema.add(column.getName(), convertType(column.getType()), false);
            }
            Dataset<Row> ret = sparkSession.read().schema(schema).csv(String.format(Locale.ROOT, CSV_TABLE_DIR, table));
            ret.createOrReplaceTempView(tableDesc.getName());
        }

    }
 

protected static void setupAll() throws Exception {
    //setup env
    HBaseMetadataTestCase.staticCreateTestMetadata();
    config = KylinConfig.getInstanceFromEnv();

    //setup cube conn
    String project = ProjectInstance.DEFAULT_PROJECT_NAME;
    cubeConnection = QueryConnection.getConnection(project);

    //setup h2
    h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + (h2InstanceCount++) + ";CACHE_SIZE=32072", "sa",
            "");
    // Load H2 Tables (inner join)
    H2Database h2DB = new H2Database(h2Connection, config, project);
    h2DB.loadAllTables();
}
 

@Test
public void testAddUpdateProject() throws IOException {
    int originalProjectCount = projectController.getProjects(null, null).size();

    //test add project
    ProjectInstance project = new ProjectInstance();
    project.setName("new_project");
    ProjectInstance ret = projectController.saveProject(getProjectRequest(project, null));

    Assert.assertEquals(ret.getOwner(), "ADMIN");
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);

    //test update project description only
    ProjectInstance newProject2 = new ProjectInstance();
    newProject2.setName("new_project");
    newProject2.setDescription("hello world");
    projectController.updateProject(getProjectRequest(newProject2, "new_project"));

    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);
    Assert.assertNotEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project"), null);
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project").getDescription(), "hello world");
}
 

public List<CubeInstance> listAllCubes(String projectName) {
    ProjectManager projectManager = getProjectManager();
    ProjectInstance project = projectManager.getProject(projectName);
    if (project == null) {
        return Collections.emptyList();
    }
    ArrayList<CubeInstance> result = new ArrayList<CubeInstance>();
    for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
        if (projectDataModel.getType() == RealizationType.CUBE) {
            CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
            if (cube != null)
                result.add(cube);
            else
                logger.error("Cube instance " + projectDataModel.getRealization() + " is failed to load");
        }
    }
    return result;
}
 

@Deprecated
public List<ProjectInstance> listAllProjects(final Integer limit, final Integer offset) {
    List<ProjectInstance> projects = getProjectManager().listAllProjects();

    int climit = (null == limit) ? Integer.MAX_VALUE : limit;
    int coffset = (null == offset) ? 0 : offset;

    if (projects.size() <= coffset) {
        return Collections.emptyList();
    }

    if ((projects.size() - coffset) < climit) {
        return projects.subList(coffset, projects.size());
    }

    return projects.subList(coffset, coffset + climit);
}
 

private static List<String> filterByProjects(List<String> allTableNames, List<String> projectNames) {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());

    List<String> result = Lists.newArrayList();
    for (String p : projectNames) {
        p = p.trim();
        if (p.endsWith(",")) {
            p = p.substring(0, p.length() - 1);
        }

        ProjectInstance projectInstance = projectManager.getProject(p);
        List<RealizationEntry> cubeList = projectInstance.getRealizationEntries(RealizationType.CUBE);
        for (RealizationEntry cube : cubeList) {
            CubeInstance cubeInstance = cubeManager.getCube(cube.getRealization());
            for (CubeSegment segment : cubeInstance.getSegments()) {
                String tableName = segment.getStorageLocationIdentifier();
                if (allTableNames.contains(tableName)) {
                    result.add(tableName);
                }
            }
        }
    }
    return result;
}
 

public void checkHiveTableCompatibility(String prj, TableDesc tableDesc) throws Exception {
    Preconditions.checkNotNull(tableDesc.getDatabase());
    Preconditions.checkNotNull(tableDesc.getName());

    String database = tableDesc.getDatabase().toUpperCase(Locale.ROOT);
    String tableName = tableDesc.getName().toUpperCase(Locale.ROOT);
    ProjectInstance projectInstance = getProjectManager().getProject(prj);
    ISourceMetadataExplorer explr = SourceManager.getSource(projectInstance).getSourceMetadataExplorer();

    TableDesc hiveTableDesc;
    try {
        Pair<TableDesc, TableExtDesc> pair = explr.loadTableMetadata(database, tableName, prj);
        hiveTableDesc = pair.getFirst();
    } catch (Exception e) {
        logger.error("Fail to get metadata for hive table {} due to ", tableDesc.getIdentity(), e);
        throw new RuntimeException("Fail to get metadata for hive table " + tableDesc.getIdentity());
    }

    TableSchemaUpdateChecker.CheckResult result = getSchemaUpdateChecker().allowMigrate(tableDesc, hiveTableDesc);
    result.raiseExceptionWhenInvalid();
}
 

/**
 * the project-specific table desc will be expand by computed columns from the projects' models
 * when the projects' model list changed, project-specific table should be reset and get expanded
 * again
 */
public void resetProjectSpecificTableDesc(String prj) throws IOException {
    // avoid cyclic locks
    ProjectInstance project = ProjectManager.getInstance(config).getProject(prj);
    
    try (AutoLock lock = srcTableMapLock.lockForWrite()) {
        for (String tableName : project.getTables()) {
            String tableIdentity = getTableIdentity(tableName);
            String key = mapKey(tableIdentity, prj);
            TableDesc originTableDesc = srcTableMap.get(key);
            if (originTableDesc == null) {
                continue;
            }

            if (originTableDesc.isBorrowedFromGlobal()) {
                srcTableMap.removeLocal(key);//delete it so that getProjectSpecificTableDesc will create again
            } else {
                srcTableCrud.reload(key);
            }
        }
    }
}
 

/**
 * Help function: mock kylin projects, to match projectUuid and projectName
 */
private static void mockKylinProjects() {
	KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
	ProjectManager projectManager = mock(ProjectManager.class);

	@SuppressWarnings({ "rawtypes", "unchecked" })
	Map<Class, Object> managersCache = (Map<Class, Object>) ReflectionTestUtils.getField(kylinConfig,
			"managersCache");
	managersCache.put(ProjectManager.class, projectManager);

	Answer<ProjectInstance> answer = new Answer<ProjectInstance>() {
		@Override
		public ProjectInstance answer(InvocationOnMock invocation) throws Throwable {
			Object[] args = invocation.getArguments();
			if (args == null || args.length == 0) {
				return null;
			}
			String uuid = (String) args[0];
			return uuid2Projects.get(uuid);
		}
	};
	when(projectManager.getPrjByUuid(anyString())).thenAnswer(answer);
}
 

@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
        throws IOException {
    if ("hybrid".equals(entity)) {
        String hybridName = cacheKey;

        try (AutoLock l = lock.lockForWrite()) {
            if (event == Event.DROP)
                hybridMap.removeLocal(hybridName);
            else
                crud.reloadQuietly(hybridName);
        }

        for (ProjectInstance prj : ProjectManager.getInstance(config).findProjects(RealizationType.HYBRID,
                hybridName)) {
            broadcaster.notifyProjectSchemaUpdate(prj.getName());
        }
    } else if ("cube".equals(entity)) {
        String cubeName = cacheKey;
        try (AutoLock l = lock.lockForWrite()) {
            for (HybridInstance hybrid : getHybridInstancesByChild(RealizationType.CUBE, cubeName)) {
                crud.reloadQuietly(hybrid.getName());
            }
        }
    }
}
 

@Override
public String calculateSignature(KylinConfig config, SQLResponse sqlResponse, ProjectInstance project) {
    Set<String> realizations = getRealizations(config, sqlResponse.getCube(), project);
    if (realizations == null) {
        return null;
    }
    Set<RealizationSignature> signatureSet = Sets.newTreeSet();
    for (String realization : realizations) {
        RealizationSignature realizationSignature = getRealizationSignature(config, realization);
        if (realizationSignature != null) {
            signatureSet.add(realizationSignature);
        }
    }
    if (signatureSet.isEmpty()) {
        return null;
    }
    try {
        MessageDigest md = MessageDigest.getInstance("MD5");
        byte[] signature = md.digest(signatureSet.toString().getBytes("UTF-8"));
        return new String(Base64.encodeBase64(signature), "UTF-8");
    } catch (Exception e) {
        logger.warn("Failed to calculate signature due to " + e);
        return null;
    }
}
 

@Test
public void testAuthInProjectLevel() throws Exception {
    List<AccessEntryResponse> aes = null;
    swichToAdmin();
    List<ProjectInstance> projects = projectController.getProjects(10000, 0);
    assertTrue(projects.size() > 0);
    ProjectInstance project = projects.get(0);
    swichToAnalyst();
    projects = projectController.getProjects(10000, 0);
    assertEquals(0, projects.size());
    //grant auth in project level
    swichToAdmin();
    aes = accessController.grant(PROJECT_INSTANCE, project.getUuid(), getAccessRequest(ANALYST, READ, true));
    swichToAnalyst();
    projects = projectController.getProjects(10000, 0);
    assertEquals(1, projects.size());

    //revoke auth
    swichToAdmin();
    AccessRequest request = getAccessRequest(ANALYST, READ, true);
    request.setAccessEntryId((Integer) aes.get(0).getId());
    accessController.revoke(PROJECT_INSTANCE, project.getUuid(), request);
    swichToAnalyst();
    projects = projectController.getProjects(10000, 0);
    assertEquals(0, projects.size());
}
 

public static RootPersistentEntity createAclEntity(String entityType, String uuid) {
    if ("CubeInstance".equals(entityType)) {
        CubeInstance cubeInstance = new CubeInstance();
        cubeInstance.setUuid(uuid);

        return cubeInstance;
    }

    if ("JobInstance".equals(entityType)) {
        JobInstance jobInstance = new JobInstance();
        jobInstance.setUuid(uuid);

        return jobInstance;
    }

    if ("ProjectInstance".equals(entityType)) {
        ProjectInstance projectInstance = new ProjectInstance();
        projectInstance.setUuid(uuid);

        return projectInstance;
    }

    throw new RuntimeException("Unsupported entity type!");
}
 

@Test
public void testCreateAndDrop() throws Exception {

    KylinConfig config = getTestConfig();
    CubeManager cubeMgr = CubeManager.getInstance(config);
    ProjectManager prjMgr = ProjectManager.getInstance(config);
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/a_whole_new_cube.json");

    CubeDescManager cubeDescMgr = getCubeDescManager();
    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube = cubeMgr.createCube("a_whole_new_cube", ProjectInstance.DEFAULT_PROJECT_NAME, desc,
            null);
    assertTrue(createdCube.equals(cubeMgr.getCube("a_whole_new_cube")));

    assertTrue(prjMgr.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME).contains(createdCube));

    CubeInstance droppedCube = CubeManager.getInstance(getTestConfig()).dropCube("a_whole_new_cube", false);
    assertTrue(createdCube.equals(droppedCube));

    assertTrue(!prjMgr.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME).contains(droppedCube));

    assertNull(CubeManager.getInstance(getTestConfig()).getCube("a_whole_new_cube"));
}
 

@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
        throws IOException {
    String cubeDescName = cacheKey;
    CubeDesc cubeDesc = getCubeDesc(cubeDescName);
    String modelName = cubeDesc == null ? null : cubeDesc.getModelName();

    if (event == Event.DROP)
        removeLocalCubeDesc(cubeDescName);
    else
        reloadCubeDescQuietly(cubeDescName);

    for (ProjectInstance prj : ProjectManager.getInstance(config).findProjectsByModel(modelName)) {
        broadcaster.notifyProjectSchemaUpdate(prj.getName());
    }
}
 

public CubeDesc updateCube(CubeInstance cube, CubeDesc desc, ProjectInstance project) throws IOException {
    aclEvaluate.checkProjectWritePermission(cube);
    Message msg = MsgPicker.getMsg();
    String projectName = project.getName();

    desc.setDraft(false);

    try {
        if (cube.getSegments().size() != 0 && !cube.getDescriptor().consistentWith(desc)) {
            throw new BadRequestException(
                    String.format(Locale.ROOT, msg.getINCONSISTENT_CUBE_DESC(), desc.getName()));
        }

        desc = updateCubeAndDesc(cube, desc, projectName, true);
    } catch (AccessDeniedException accessDeniedException) {
        throw new ForbiddenException(msg.getUPDATE_CUBE_NO_RIGHT());
    }

    if (desc.isBroken()) {
        throw new BadRequestException(desc.getErrorsAsString());
    }

    return desc;
}
 

@Test
public void testAddUpdateProject() throws IOException {
    int originalProjectCount = projectController.getProjects(null, null).size();

    //test add project
    ProjectInstance project = new ProjectInstance();
    project.setName("new_project");
    ProjectInstance ret = projectController.saveProject(getProjectRequest(project, null));

    Assert.assertEquals(ret.getOwner(), "ADMIN");
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);

    //test update project description only
    ProjectInstance newProject2 = new ProjectInstance();
    newProject2.setName("new_project");
    newProject2.setDescription("hello world");
    projectController.updateProject(getProjectRequest(newProject2, "new_project"));

    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);
    Assert.assertNotEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project"), null);
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project").getDescription(), "hello world");
}
 

@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
        throws IOException {
    try (AutoLock lock = srcTableMapLock.lockForWrite()) {
        if (event == Event.DROP)
            srcTableMap.removeLocal(cacheKey);
        else
            srcTableCrud.reloadQuietly(cacheKey);
    }

    TableProject tableProject = TableDesc.parseResourcePath(cacheKey);
    String table = tableProject.getTable();
    String prj = tableProject.getProject();

    if (prj == null) {
        for (ProjectInstance p : ProjectManager.getInstance(config).findProjectsByTable(table)) {
            broadcaster.notifyProjectSchemaUpdate(p.getName());
        }
    } else {
        broadcaster.notifyProjectSchemaUpdate(prj);
    }
}
 

public void calculateCardinalityIfNotPresent(String[] tables, String submitter, String prj) throws Exception {
    // calculate cardinality for Hive source
    ProjectInstance projectInstance = getProjectManager().getProject(prj);
    if (projectInstance == null || projectInstance.getSourceType() != ISourceAware.ID_HIVE) {
        return;
    }
    TableMetadataManager metaMgr = getTableManager();
    ExecutableManager exeMgt = ExecutableManager.getInstance(getConfig());
    for (String table : tables) {
        TableExtDesc tableExtDesc = metaMgr.getTableExt(table, prj);
        String jobID = tableExtDesc.getJodID();
        if (null == jobID || ExecutableState.RUNNING != exeMgt.getOutput(jobID).getState()) {
            calculateCardinality(table, submitter, prj);
        }
    }
}
 

private void cleanProjectCacheByRealization(RealizationType type, String realizationName) throws IOException {
    List<ProjectInstance> projectInstances = getProjectManager().findProjects(type, realizationName);
    for (ProjectInstance pi : projectInstances) {
        getProjectManager().reloadProject(pi.getName());
        removeOLAPDataSource(pi.getName());
    }
}
 

@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN +
        " or hasPermission(#project, 'ADMINISTRATION')" +
        " or hasPermission(#project, 'MANAGEMENT')" +
        " or hasPermission(#project, 'OPERATION')" +
        " or hasPermission(#project, 'READ')")
public boolean hasProjectReadPermission(ProjectInstance project) {
    return true;
}
 

/**
 * admin operation all projects sueecss
 */
@Test
@WithMockUser(username = ADMIN, roles = { ROLE_ADMIN })
public void operationProjectAllAsRoleAdmin() {
	for (ProjectInstance project : uuid2Projects.values()) {
		boolean result = aclEvaluate.hasProjectOperationPermission(project);
		Assert.assertTrue(result);
	}
}
 

@Override
public ExecutorService load(String projName) throws Exception {
    ExecutorService sharedPool = HBaseConnection.getCoprocessorPool();
    ProjectInstance projInst = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getProject(projName);
    return new SubThreadPoolExecutor(sharedPool, "PROJECT",
            projInst.getConfig().getHBaseMaxConnectionThreadsPerProject());
}
 

private static void addCubeAndModelIntoProject(CubeInstance srcCube, String cubeName, String projectName)
        throws IOException {
    String projectResPath = ProjectInstance.concatResourcePath(projectName);
    if (!dstStore.exists(projectResPath))
        throw new IllegalStateException("The target project " + projectName + "does not exist");

    operations.add(new Opt(OptType.ADD_INTO_PROJECT, new Object[] { srcCube, cubeName, projectName }));
}
 

public static String createSignature(KylinConfig config, SQLResponse sqlResponse, String projectName) {
    ProjectInstance project = ProjectManager.getInstance(config).getProject(projectName);
    Preconditions.checkNotNull(project);

    SignatureCalculator signatureCalculator;
    try {
        Class signatureClass = getSignatureClass(project.getConfig());
        signatureCalculator = (SignatureCalculator) signatureClass.getConstructor().newInstance();
    } catch (Exception e) {
        logger.warn("Will use default signature since fail to construct signature due to " + e);
        signatureCalculator = new FactTableRealizationSetCalculator();
    }
    return signatureCalculator.calculateSignature(config, sqlResponse, project);
}
 

public boolean hasProjectWritePermission(ProjectInstance project) {
    boolean _hasProjectWritePermission = false;
    try {
        _hasProjectWritePermission = aclUtil.hasProjectWritePermission(project);
    } catch (AccessDeniedException e) {
        //ignore to continue
    }
    return _hasProjectWritePermission;
}
 

@Test
public void testBasics() throws JobException, IOException, SQLException {
    Assert.assertNotNull(jobService.getConfig());
    Assert.assertNotNull(jobService.getConfig());
    Assert.assertNotNull(jobService.getDataModelManager());
    Assert.assertNotNull(QueryConnection.getConnection(ProjectInstance.DEFAULT_PROJECT_NAME));
    Assert.assertNull(jobService.getJobInstance("job_not_exist"));
    Assert.assertNotNull(jobService.searchJobs(null, null, null, 0, 0, JobTimeFilterEnum.ALL, JobService.JobSearchMode.ALL));
}
 

@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
        throws IOException {
    String cubeName = cacheKey;

    if (event == Event.DROP)
        removeCubeLocal(cubeName);
    else
        reloadCubeQuietly(cubeName);

    for (ProjectInstance prj : ProjectManager.getInstance(config).findProjects(RealizationType.CUBE,
            cubeName)) {
        broadcaster.notifyProjectDataUpdate(prj.getName());
    }
}
 

@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
        + " or hasPermission(#project, 'ADMINISTRATION') or hasPermission(#project, 'MANAGEMENT')")
public CubeInstance createCubeAndDesc(ProjectInstance project, CubeDesc desc) throws IOException {
    Message msg = MsgPicker.getMsg();
    String cubeName = desc.getName();

    if (getCubeManager().getCube(cubeName) != null) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getCUBE_ALREADY_EXIST(), cubeName));
    }

    if (getCubeDescManager().getCubeDesc(desc.getName()) != null) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getCUBE_DESC_ALREADY_EXIST(), desc.getName()));
    }

    String owner = SecurityContextHolder.getContext().getAuthentication().getName();
    CubeDesc createdDesc;
    CubeInstance createdCube;

    createdDesc = getCubeDescManager().createCubeDesc(desc);

    if (createdDesc.isBroken()) {
        throw new BadRequestException(createdDesc.getErrorsAsString());
    }

    int cuboidCount = CuboidCLI.simulateCuboidGeneration(createdDesc, false);
    logger.info("New cube " + cubeName + " has " + cuboidCount + " cuboids");

    createdCube = getCubeManager().createCube(cubeName, project.getName(), createdDesc, owner);
    return createdCube;
}
 

@Test
public void testExistingProject() throws Exception {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());

    int originalProjectCount = prjMgr.listAllProjects().size();
    int originalCubeCount = cubeMgr.listAllCubes().size();
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/new_cube_in_default.json");

    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube = cubeMgr.createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null);
    assertTrue(createdCube.equals(cubeMgr.getCube("new_cube_in_default")));

    //System.out.println(JsonUtil.writeValueAsIndentString(createdCube));

    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);

    CubeInstance droppedCube = cubeMgr.dropCube("new_cube_in_default", true);

    assertTrue(createdCube.equals(droppedCube));
    assertNull(cubeMgr.getCube("new_cube_in_default"));
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);
}