org.apache.kylin.common.persistence.JsonSerializer Java Examples

The following examples show how to use org.apache.kylin.common.persistence.JsonSerializer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ExtendCubeToHybridCLI.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void copyAcl(String origCubeId, String newCubeId, String projectName) throws Exception {
    String projectResPath = ProjectInstance.concatResourcePath(projectName);
    Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
    ProjectInstance project = store.getResource(projectResPath, projectSerializer);
    String projUUID = project.getUuid();
    Table aclHtable = null;
    try {
        aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));

        // cube acl
        Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
        if (result.listCells() != null) {
            for (Cell cell : result.listCells()) {
                byte[] family = CellUtil.cloneFamily(cell);
                byte[] column = CellUtil.cloneQualifier(cell);
                byte[] value = CellUtil.cloneValue(cell);

                // use the target project uuid as the parent
                if (Bytes.toString(family).equals(ACL_INFO_FAMILY) && Bytes.toString(column).equals(ACL_INFO_FAMILY_PARENT_COLUMN)) {
                    String valueString = "{\"id\":\"" + projUUID + "\",\"type\":\"org.apache.kylin.metadata.project.ProjectInstance\"}";
                    value = Bytes.toBytes(valueString);
                }
                Put put = new Put(Bytes.toBytes(newCubeId));
                put.add(family, column, value);
                aclHtable.put(put);
            }
        }
    } finally {
        IOUtils.closeQuietly(aclHtable);
    }
}
Example #2
Source File: CubeMetadataUpgrade.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void upgradeJobInstance(String path) throws IOException {
    JobInstance job = getStore().getResource(path, JobInstance.class, new JsonSerializer<JobInstance>(JobInstance.class));
    CubingJob cubingJob = new CubingJob();
    cubingJob.setId(job.getId());
    cubingJob.setName(job.getName());
    cubingJob.setSubmitter(job.getSubmitter());
    for (JobInstance.JobStep step : job.getSteps()) {
        final AbstractExecutable executable = parseToExecutable(step);
        cubingJob.addTask(executable);
    }
    getExecutableManager().addJob(cubingJob);

    cubingJob.setStartTime(job.getExecStartTime());
    cubingJob.setEndTime(job.getExecEndTime());
    cubingJob.setMapReduceWaitTime(job.getMrWaiting());
    getExecutableManager().resetJobOutput(cubingJob.getId(), parseState(job.getStatus()), job.getStatus().toString());

    for (int i = 0, size = job.getSteps().size(); i < size; ++i) {
        final JobInstance.JobStep jobStep = job.getSteps().get(i);
        final InputStream inputStream = getStore().getResource(ResourceStore.JOB_OUTPUT_PATH_ROOT + "/" + job.getId() + "." + i);

        String output = null;
        if (inputStream != null) {
            HashMap<String, String> job_output = JsonUtil.readValue(inputStream, HashMap.class);

            if (job_output != null) {
                output = job_output.get("output");
            }
        }
        updateJobStepOutput(jobStep, output, cubingJob.getTasks().get(i));
    }
}
Example #3
Source File: ExtendCubeToHybridCLI.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void copyAcl(String origCubeId, String newCubeId, String projectName) throws Exception {
    String projectResPath = ProjectInstance.concatResourcePath(projectName);
    Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
    ProjectInstance project = store.getResource(projectResPath, projectSerializer);
    String projUUID = project.getUuid();
    Table aclHtable = null;
    try {
        aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));

        // cube acl
        Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
        if (result.listCells() != null) {
            for (Cell cell : result.listCells()) {
                byte[] family = CellUtil.cloneFamily(cell);
                byte[] column = CellUtil.cloneQualifier(cell);
                byte[] value = CellUtil.cloneValue(cell);

                // use the target project uuid as the parent
                if (Bytes.toString(family).equals(ACL_INFO_FAMILY) && Bytes.toString(column).equals(ACL_INFO_FAMILY_PARENT_COLUMN)) {
                    String valueString = "{\"id\":\"" + projUUID + "\",\"type\":\"org.apache.kylin.metadata.project.ProjectInstance\"}";
                    value = Bytes.toBytes(valueString);
                }
                Put put = new Put(Bytes.toBytes(newCubeId));
                put.add(family, column, value);
                aclHtable.put(put);
            }
        }
    } finally {
        IOUtils.closeQuietly(aclHtable);
    }
}
Example #4
Source File: CacheControllerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testClearCacheForCubeMigration() throws IOException {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    String CUBENAME = "test_kylin_cube_without_slr_desc";

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
    CubeDesc cubeDesc = cubeDescManager.getCubeDesc(CUBENAME);
    DataModelDesc modelDesc = cubeDesc.getModel();
    Map<String, String> tableToProjects = new HashMap<>();
    for (TableRef tableRef : modelDesc.getAllTables()) {
        tableToProjects.put(tableRef.getTableIdentity(), tableRef.getTableDesc().getProject());
    }

    String uuid = cubeDesc.getUuid();
    String signature = cubeDesc.getSignature();

    assertEquals(cubeDesc.getRetentionRange(), 0);

    //update cubeDesc
    cubeDesc.setRetentionRange(2018);
    cubeDesc.updateRandomUuid();

    //directly update metadata in store to simulate cube migration
    Serializer<CubeDesc> cubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class);
    getStore().checkAndPutResource(cubeDesc.getResourcePath(), cubeDesc, cubeDescSerializer);

    CubeMigrationRequest request = new CubeMigrationRequest();
    request.setCube(cubeDesc.getName());
    request.setModel(modelDesc.getName());
    request.setProject(modelDesc.getProject());
    request.setTableToProjects(tableToProjects);

    cacheController.clearCacheForCubeMigration(request);

    assertEquals(2018, cubeDescManager.getCubeDesc(CUBENAME).getRetentionRange());
    assertEquals(signature, cubeDescManager.getCubeDesc(CUBENAME).getSignature());
    assertNotEquals(uuid, cubeDescManager.getCubeDesc(CUBENAME).getUuid());
}
Example #5
Source File: TableACLManagerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCaseInsensitiveFromDeserializer() throws IOException {
    final TableACLManager manager = new TableACLManager(configA);
    manager.addTableACL(PROJECT, USER, "TABLE1", MetadataConstants.TYPE_USER);
    TableACL tableACL = Preconditions.checkNotNull(getStore().getResource("/table_acl/" + PROJECT, new JsonSerializer<>(TableACL.class)));
    Assert.assertEquals(1, tableACL.getNoAccessList("table1", MetadataConstants.TYPE_USER).size());
}
Example #6
Source File: StreamingServer.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void reloadCubeMetadata(String cubeName) throws IOException {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    ResourceStore resourceStore = ResourceStore.getStore(kylinConfig);
    CubeInstance rawCubeInstance = resourceStore.getResource(CubeInstance.concatResourcePath(cubeName),
             CubeManager.CUBE_SERIALIZER);
    CubeDesc rawCubeDesc = resourceStore.getResource(CubeDesc.concatResourcePath(rawCubeInstance.getDescName()),
            CubeDescManager.CUBE_DESC_SERIALIZER);
    DataModelDesc rawModel = resourceStore.getResource(
            DataModelDesc.concatResourcePath(rawCubeDesc.getModelName()),
            new JsonSerializer<>(DataModelDesc.class));
    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    List<ProjectInstance> projects = projectManager.findProjectsByModel(rawModel.getName());
    if (projects.isEmpty()) {
        projectManager.reloadAll();
        projects = projectManager.findProjectsByModel(rawModel.getName());
    }
    if (projects.size() != 1) {
        throw new IllegalArgumentException("the cube:" + cubeName + " is not in any project");
    }

    TableMetadataManager.getInstance(kylinConfig).reloadSourceTableQuietly(rawModel.getRootFactTableName(),
            projects.get(0).getName());
    DataModelManager.getInstance(kylinConfig).reloadDataModel(rawModel.getName());
    CubeDescManager.getInstance(kylinConfig).reloadCubeDescLocal(cubeName);
    CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).reloadCubeQuietly(cubeName);
    StreamingSourceConfigManager.getInstance(kylinConfig).reloadStreamingConfigLocal(
            cubeInstance.getRootFactTable());
}
Example #7
Source File: CachedCrudAssist.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public CachedCrudAssist(ResourceStore store, String resourceRootPath, String resourcePathSuffix,
        Class<T> entityType, SingleValueCache<String, T> cache, boolean compact) {
    this.store = store;
    this.entityType = entityType;
    this.resRootPath = resourceRootPath;
    this.resPathSuffix = resourcePathSuffix;
    this.serializer = new JsonSerializer<T>(entityType, compact);
    this.cache = cache;

    this.checkCopyOnWrite = store.getConfig().isCheckCopyOnWrite();

    Preconditions.checkArgument(resRootPath.startsWith("/"));
    Preconditions.checkArgument(resRootPath.endsWith("/") == false);
}
Example #8
Source File: ExtendCubeToHybridCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void copyAcl(String origCubeId, String newCubeId, String projectName) throws Exception {
    String projectResPath = ProjectInstance.concatResourcePath(projectName);
    Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
    ProjectInstance project = store.getResource(projectResPath, projectSerializer);
    String projUUID = project.getUuid();
    Table aclHtable = null;
    try {
        aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));

        // cube acl
        Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
        if (result.listCells() != null) {
            for (Cell cell : result.listCells()) {
                byte[] family = CellUtil.cloneFamily(cell);
                byte[] column = CellUtil.cloneQualifier(cell);
                byte[] value = CellUtil.cloneValue(cell);

                // use the target project uuid as the parent
                if (Bytes.toString(family).equals(ACL_INFO_FAMILY) && Bytes.toString(column).equals(ACL_INFO_FAMILY_PARENT_COLUMN)) {
                    String valueString = "{\"id\":\"" + projUUID + "\",\"type\":\"org.apache.kylin.metadata.project.ProjectInstance\"}";
                    value = Bytes.toBytes(valueString);
                }
                Put put = new Put(Bytes.toBytes(newCubeId));
                put.add(family, column, value);
                aclHtable.put(put);
            }
        }
    } finally {
        IOUtils.closeQuietly(aclHtable);
    }
}
Example #9
Source File: ExtendCubeToHybridCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void copyAcl(String origCubeId, String newCubeId, String projectName) throws Exception {
    String projectResPath = ProjectInstance.concatResourcePath(projectName);
    Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
    ProjectInstance project = store.getResource(projectResPath, projectSerializer);
    String projUUID = project.getUuid();
    Table aclHtable = null;
    try {
        aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));

        // cube acl
        Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
        if (result.listCells() != null) {
            for (Cell cell : result.listCells()) {
                byte[] family = CellUtil.cloneFamily(cell);
                byte[] column = CellUtil.cloneQualifier(cell);
                byte[] value = CellUtil.cloneValue(cell);

                // use the target project uuid as the parent
                if (Bytes.toString(family).equals(ACL_INFO_FAMILY) && Bytes.toString(column).equals(ACL_INFO_FAMILY_PARENT_COLUMN)) {
                    String valueString = "{\"id\":\"" + projUUID + "\",\"type\":\"org.apache.kylin.metadata.project.ProjectInstance\"}";
                    value = Bytes.toBytes(valueString);
                }
                Put put = new Put(Bytes.toBytes(newCubeId));
                put.add(family, column, value);
                aclHtable.put(put);
            }
        }
    } finally {
        IOUtils.closeQuietly(aclHtable);
    }
}
Example #10
Source File: CacheControllerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testClearCacheForCubeMigration() throws IOException {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    String CUBENAME = "test_kylin_cube_without_slr_desc";

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
    CubeDesc cubeDesc = cubeDescManager.getCubeDesc(CUBENAME);
    DataModelDesc modelDesc = cubeDesc.getModel();
    Map<String, String> tableToProjects = new HashMap<>();
    for (TableRef tableRef : modelDesc.getAllTables()) {
        tableToProjects.put(tableRef.getTableIdentity(), tableRef.getTableDesc().getProject());
    }

    String uuid = cubeDesc.getUuid();
    String signature = cubeDesc.getSignature();

    assertEquals(cubeDesc.getRetentionRange(), 0);

    //update cubeDesc
    cubeDesc.setRetentionRange(2018);
    cubeDesc.updateRandomUuid();

    //directly update metadata in store to simulate cube migration
    Serializer<CubeDesc> cubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class);
    getStore().checkAndPutResource(cubeDesc.getResourcePath(), cubeDesc, cubeDescSerializer);

    CubeMigrationRequest request = new CubeMigrationRequest();
    request.setCube(cubeDesc.getName());
    request.setModel(modelDesc.getName());
    request.setProject(modelDesc.getProject());
    request.setTableToProjects(tableToProjects);

    cacheController.clearCacheForCubeMigration(request);

    assertEquals(2018, cubeDescManager.getCubeDesc(CUBENAME).getRetentionRange());
    assertEquals(signature, cubeDescManager.getCubeDesc(CUBENAME).getSignature());
    assertNotEquals(uuid, cubeDescManager.getCubeDesc(CUBENAME).getUuid());
}
Example #11
Source File: TableACLManagerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCaseInsensitiveFromDeserializer() throws IOException {
    final TableACLManager manager = new TableACLManager(configA);
    manager.addTableACL(PROJECT, USER, "TABLE1", MetadataConstants.TYPE_USER);
    TableACL tableACL = Preconditions.checkNotNull(getStore().getResource("/table_acl/" + PROJECT, new JsonSerializer<>(TableACL.class)));
    Assert.assertEquals(1, tableACL.getNoAccessList("table1", MetadataConstants.TYPE_USER).size());
}
Example #12
Source File: StreamingServer.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void reloadCubeMetadata(String cubeName) throws IOException {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    ResourceStore resourceStore = ResourceStore.getStore(kylinConfig);
    CubeInstance rawCubeInstance = resourceStore.getResource(CubeInstance.concatResourcePath(cubeName),
             CubeManager.CUBE_SERIALIZER);
    CubeDesc rawCubeDesc = resourceStore.getResource(CubeDesc.concatResourcePath(rawCubeInstance.getDescName()),
            CubeDescManager.CUBE_DESC_SERIALIZER);
    DataModelDesc rawModel = resourceStore.getResource(
            DataModelDesc.concatResourcePath(rawCubeDesc.getModelName()),
            new JsonSerializer<>(DataModelDesc.class));
    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    List<ProjectInstance> projects = projectManager.findProjectsByModel(rawModel.getName());
    if (projects.isEmpty()) {
        projectManager.reloadAll();
        projects = projectManager.findProjectsByModel(rawModel.getName());
    }
    if (projects.size() != 1) {
        throw new IllegalArgumentException("the cube:" + cubeName + " is not in any project");
    }

    TableMetadataManager.getInstance(kylinConfig).reloadSourceTableQuietly(rawModel.getRootFactTableName(),
            projects.get(0).getName());
    DataModelManager.getInstance(kylinConfig).reloadDataModel(rawModel.getName());
    CubeDescManager.getInstance(kylinConfig).reloadCubeDescLocal(cubeName);
    CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).reloadCubeQuietly(cubeName);
    StreamingSourceConfigManager.getInstance(kylinConfig).reloadStreamingConfigLocal(
            cubeInstance.getRootFactTable());
}
Example #13
Source File: CachedCrudAssist.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public CachedCrudAssist(ResourceStore store, String resourceRootPath, String resourcePathSuffix,
        Class<T> entityType, SingleValueCache<String, T> cache, boolean compact) {
    this.store = store;
    this.entityType = entityType;
    this.resRootPath = resourceRootPath;
    this.resPathSuffix = resourcePathSuffix;
    this.serializer = new JsonSerializer<T>(entityType, compact);
    this.cache = cache;

    this.checkCopyOnWrite = store.getConfig().isCheckCopyOnWrite();

    Preconditions.checkArgument(resRootPath.startsWith("/"));
    Preconditions.checkArgument(resRootPath.endsWith("/") == false);
}
Example #14
Source File: CubeInstanceTest.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() throws Exception {
    InputStream fileInputStream = new FileInputStream("src/test/resources/learn_kylin_cube2.json");
    JsonSerializer<CubeInstance> jsonSerializer = new JsonSerializer<>(CubeInstance.class);
    cubeInstance = jsonSerializer.deserialize(new DataInputStream(fileInputStream));
}
Example #15
Source File: CubeDesc.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public static JsonSerializer<CubeDesc> newSerializerForLowLevelAccess() {
    return new JsonSerializer<>(CubeDesc.class);
}
Example #16
Source File: CubeDesc.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public static JsonSerializer<CubeDesc> newSerializerForLowLevelAccess() {
    return new JsonSerializer<>(CubeDesc.class);
}
Example #17
Source File: CubeInstanceTest.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() throws Exception {
    InputStream fileInputStream = new FileInputStream("src/test/resources/learn_kylin_cube2.json");
    JsonSerializer<CubeInstance> jsonSerializer = new JsonSerializer<>(CubeInstance.class);
    cubeInstance = jsonSerializer.deserialize(new DataInputStream(fileInputStream));
}