Java Code Examples for org.apache.kylin.cube.model.CubeDesc#setLastModified()

The following examples show how to use org.apache.kylin.cube.model.CubeDesc#setLastModified() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public static CubeDesc generateKylinCubeDesc(String tableName, int storageType,
        List<DimensionDesc> dimensionDescList, List<MeasureDesc> measureDescList, RowKeyDesc rowKeyDesc,
        AggregationGroup aggGroup, HBaseMappingDesc hBaseMapping, Map<String, String> overrideProperties) {
    CubeDesc desc = new CubeDesc();
    desc.setName(tableName.replace('.', '_'));
    desc.setModelName(tableName.replace('.', '_'));
    desc.setDescription("");
    desc.setLastModified(0L);
    desc.setDimensions(dimensionDescList);
    desc.setMeasures(measureDescList);
    desc.setRowkey(rowKeyDesc);
    desc.setHbaseMapping(hBaseMapping);
    desc.setNotifyList(Lists.<String> newArrayList());
    desc.setStatusNeedNotify(Lists.newArrayList(JobStatusEnum.ERROR.toString()));
    desc.setAutoMergeTimeRanges(new long[] { 86400000L, 604800000L, 2419200000L });
    desc.setEngineType(IEngineAware.ID_MR_V2);
    desc.setStorageType(storageType);
    desc.setAggregationGroups(Lists.newArrayList(aggGroup));
    desc.getOverrideKylinProps().putAll(overrideProperties);
    desc.updateRandomUuid();
    return desc;
}

Example 2

Source File: CubeDescCreator.java From kylin with Apache License 2.0

6 votes

public static CubeDesc generateKylinCubeDesc(String tableName, int storageType,
        List<DimensionDesc> dimensionDescList, List<MeasureDesc> measureDescList, RowKeyDesc rowKeyDesc,
        AggregationGroup aggGroup, HBaseMappingDesc hBaseMapping, Map<String, String> overrideProperties) {
    CubeDesc desc = new CubeDesc();
    desc.setName(tableName.replace('.', '_'));
    desc.setModelName(tableName.replace('.', '_'));
    desc.setDescription("");
    desc.setLastModified(0L);
    desc.setDimensions(dimensionDescList);
    desc.setMeasures(measureDescList);
    desc.setRowkey(rowKeyDesc);
    desc.setHbaseMapping(hBaseMapping);
    desc.setNotifyList(Lists.<String> newArrayList());
    desc.setStatusNeedNotify(Lists.newArrayList(JobStatusEnum.ERROR.toString()));
    desc.setAutoMergeTimeRanges(new long[] { 86400000L, 604800000L, 2419200000L, 7776000000L, 31104000000L });
    desc.setEngineType(IEngineAware.ID_MR_V2);
    desc.setStorageType(storageType);
    desc.setAggregationGroups(Lists.newArrayList(aggGroup));
    desc.getOverrideKylinProps().putAll(overrideProperties);
    desc.updateRandomUuid();
    return desc;
}

Example 3

Source File: TableSchemaUpdater.java From kylin with Apache License 2.0

6 votes

public static CubeDesc dealWithMappingForCubeDesc(CubeDesc other, Map<String, TableSchemaUpdateMapping> mappings) {
    CubeDesc copy = CubeDesc.getCopyOf(other);
    copy.setLastModified(other.getLastModified());

    // mapping for cube-level snapshot tables
    if (other.getSnapshotTableDescList() != null && !other.getSnapshotTableDescList().isEmpty()) {
        List<SnapshotTableDesc> snapshotTableDescListCopy = Lists
                .newArrayListWithExpectedSize(other.getSnapshotTableDescList().size());
        for (SnapshotTableDesc snapshotDesc : other.getSnapshotTableDescList()) {
            TableSchemaUpdateMapping mapping = getTableSchemaUpdateMapping(mappings, snapshotDesc.getTableName());
            if (mapping != null && mapping.isTableIdentityChanged()) {
                snapshotDesc = SnapshotTableDesc.getCopyOf(snapshotDesc);
                snapshotDesc.setTableName(mapping.getTableIdentity(snapshotDesc.getTableName()));
            }
            snapshotTableDescListCopy.add(snapshotDesc);
        }
        copy.setSnapshotTableDescList(snapshotTableDescListCopy);
    }

    return copy;
}

Example 4

Source File: CubeDescTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Test
public void testGetCopyOf() throws Exception {
    CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    CubeDesc copyDesc = CubeDesc.getCopyOf(desc);

    // uuid is different, set to equals for json comparison
    copyDesc.setUuid(desc.getUuid());
    copyDesc.setLastModified(desc.getLastModified());

    String descStr = JsonUtil.writeValueAsIndentString(desc);
    String copyStr = JsonUtil.writeValueAsIndentString(copyDesc);

    assertEquals(descStr, copyStr);
}

Example 5

Source File: CubeDescTest.java From kylin with Apache License 2.0

5 votes

@Test
public void testGetCopyOf() throws Exception {
    CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    CubeDesc copyDesc = CubeDesc.getCopyOf(desc);

    // uuid is different, set to equals for json comparison
    copyDesc.setUuid(desc.getUuid());
    copyDesc.setLastModified(desc.getLastModified());

    String descStr = JsonUtil.writeValueAsIndentString(desc);
    String copyStr = JsonUtil.writeValueAsIndentString(copyDesc);

    assertEquals(descStr, copyStr);
}

Example 6

Source File: CacheServiceTest.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Test
public void testCubeCRUD() throws Exception {
    final Broadcaster broadcaster = Broadcaster.getInstance(configA);
    broadcaster.getCounterAndClear();

    getStore().deleteResource("/cube/test_kylin_cube_a_new_one.json");

    //create cube

    final String cubeName = "test_kylin_cube_a_new_one";
    final CubeManager cubeManager = getCubeManager(configA);
    final CubeManager cubeManagerB = getCubeManager(configB);
    final ProjectManager projectManager = getProjectManager(configA);
    final ProjectManager projectManagerB = getProjectManager(configB);
    final CubeDescManager cubeDescManager = getCubeDescManager(configA);
    final CubeDescManager cubeDescManagerB = getCubeDescManager(configB);
    final CubeDesc cubeDesc = getCubeDescManager(configA).getCubeDesc("test_kylin_cube_with_slr_desc");

    assertTrue(cubeManager.getCube(cubeName) == null);
    assertTrue(cubeManagerB.getCube(cubeName) == null);
    assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    cubeManager.createCube(cubeName, ProjectInstance.DEFAULT_PROJECT_NAME, cubeDesc, null);
    //one for cube update, one for project update
    assertEquals(2, broadcaster.getCounterAndClear());
    waitForCounterAndClear(2);

    assertNotNull(cubeManager.getCube(cubeName));
    assertNotNull(cubeManagerB.getCube(cubeName));
    assertTrue(containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    assertTrue(containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));

    //update cube
    CubeInstance cube = cubeManager.getCube(cubeName);
    assertEquals(0, cube.getSegments().size());
    assertEquals(0, cubeManagerB.getCube(cubeName).getSegments().size());
    CubeSegment segment = cubeManager.appendSegment(cube, new TSRange(0L, 1000L));
    //one for cube update
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertEquals(1, cubeManagerB.getCube(cubeName).getSegments().size());
    assertEquals(segment.getName(), cubeManagerB.getCube(cubeName).getSegments().get(0).getName());

    //delete cube
    cubeManager.dropCube(cubeName, false);
    //one for cube update, one for project update
    assertEquals(2, broadcaster.getCounterAndClear());
    waitForCounterAndClear(2);

    assertTrue(cubeManager.getCube(cubeName) == null);
    assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    assertTrue(cubeManagerB.getCube(cubeName) == null);
    assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));

    final String cubeDescName = "test_cube_desc";
    cubeDesc.setName(cubeDescName);
    cubeDesc.setLastModified(0);
    assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
    assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);
    cubeDescManager.createCubeDesc(cubeDesc);
    //one for add cube desc
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertNotNull(cubeDescManager.getCubeDesc(cubeDescName));
    assertNotNull(cubeDescManagerB.getCubeDesc(cubeDescName));

    cubeDesc.setNotifyList(Arrays.asList("[email protected]", "[email protected]", "[email protected]"));
    cubeDescManager.updateCubeDesc(cubeDesc);
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertEquals(cubeDesc.getNotifyList(), cubeDescManagerB.getCubeDesc(cubeDescName).getNotifyList());

    cubeDescManager.removeCubeDesc(cubeDesc);
    //one for add cube desc
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
    assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);

    getStore().deleteResource("/cube/test_kylin_cube_a_new_one.json");
}

Example 7

Source File: CacheServiceTest.java From kylin with Apache License 2.0

4 votes

@Test
public void testCubeCRUD() throws Exception {
    final Broadcaster broadcaster = Broadcaster.getInstance(configA);
    broadcaster.getCounterAndClear();

    getStore().deleteResource("/cube/test_kylin_cube_a_new_one.json");

    //create cube

    final String cubeName = "test_kylin_cube_a_new_one";
    final CubeManager cubeManager = getCubeManager(configA);
    final CubeManager cubeManagerB = getCubeManager(configB);
    final ProjectManager projectManager = getProjectManager(configA);
    final ProjectManager projectManagerB = getProjectManager(configB);
    final CubeDescManager cubeDescManager = getCubeDescManager(configA);
    final CubeDescManager cubeDescManagerB = getCubeDescManager(configB);
    final CubeDesc cubeDesc = getCubeDescManager(configA).getCubeDesc("test_kylin_cube_with_slr_desc");

    assertTrue(cubeManager.getCube(cubeName) == null);
    assertTrue(cubeManagerB.getCube(cubeName) == null);
    assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    cubeManager.createCube(cubeName, ProjectInstance.DEFAULT_PROJECT_NAME, cubeDesc, null);
    //one for cube update, one for project update
    assertEquals(2, broadcaster.getCounterAndClear());
    waitForCounterAndClear(2);

    assertNotNull(cubeManager.getCube(cubeName));
    assertNotNull(cubeManagerB.getCube(cubeName));
    assertTrue(containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    assertTrue(containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));

    //update cube
    CubeInstance cube = cubeManager.getCube(cubeName);
    assertEquals(0, cube.getSegments().size());
    assertEquals(0, cubeManagerB.getCube(cubeName).getSegments().size());
    CubeSegment segment = cubeManager.appendSegment(cube, new TSRange(0L, 1000L));
    //one for cube update
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertEquals(1, cubeManagerB.getCube(cubeName).getSegments().size());
    assertEquals(segment.getName(), cubeManagerB.getCube(cubeName).getSegments().get(0).getName());

    //delete cube
    cubeManager.dropCube(cubeName, false);
    //one for cube update, one for project update
    assertEquals(2, broadcaster.getCounterAndClear());
    waitForCounterAndClear(2);

    assertTrue(cubeManager.getCube(cubeName) == null);
    assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    assertTrue(cubeManagerB.getCube(cubeName) == null);
    assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));

    final String cubeDescName = "test_cube_desc";
    cubeDesc.setName(cubeDescName);
    cubeDesc.setLastModified(0);
    assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
    assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);
    cubeDescManager.createCubeDesc(cubeDesc);
    //one for add cube desc
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertNotNull(cubeDescManager.getCubeDesc(cubeDescName));
    assertNotNull(cubeDescManagerB.getCubeDesc(cubeDescName));

    cubeDesc.setNotifyList(Arrays.asList("[email protected]", "[email protected]", "[email protected]"));
    cubeDescManager.updateCubeDesc(cubeDesc);
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertEquals(cubeDesc.getNotifyList(), cubeDescManagerB.getCubeDesc(cubeDescName).getNotifyList());

    cubeDescManager.removeCubeDesc(cubeDesc);
    //one for add cube desc
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
    assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);

    getStore().deleteResource("/cube/test_kylin_cube_a_new_one.json");
}

Example 8

Source File: CacheServiceTest.java From Kylin with Apache License 2.0

4 votes

@Test
public void testCubeCRUD() throws Exception {
    final Broadcaster broadcaster = Broadcaster.getInstance();
    broadcaster.getCounterAndClear();

    getStore().deleteResource("/cube/a_whole_new_cube.json");

    //create cube

    final String cubeName = "a_whole_new_cube";
    final CubeManager cubeManager = getCubeManager(configA);
    final CubeManager cubeManagerB = getCubeManager(configB);
    final ProjectManager projectManager = getProjectManager(configA);
    final ProjectManager projectManagerB = getProjectManager(configB);
    final CubeDescManager cubeDescManager = getCubeDescManager(configA);
    final CubeDescManager cubeDescManagerB = getCubeDescManager(configB);
    final CubeDesc cubeDesc = getCubeDescManager(configA).getCubeDesc("test_kylin_cube_with_slr_desc");

    assertTrue(cubeManager.getCube(cubeName) == null);
    assertTrue(cubeManagerB.getCube(cubeName) == null);
    assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
    assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
    cubeManager.createCube(cubeName, ProjectInstance.DEFAULT_PROJECT_NAME, cubeDesc, null);
    assertNotNull(cubeManager.getCube(cubeName));
    //one for cube update, one for project update
    assertEquals(2, broadcaster.getCounterAndClear());
    waitForCounterAndClear(2);
    assertNotNull(cubeManagerB.getCube(cubeName));
    assertTrue(containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
    assertTrue(containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));

    //update cube
    CubeInstance cube = cubeManager.getCube(cubeName);
    assertEquals(0, cube.getSegments().size());
    assertEquals(0, cubeManagerB.getCube(cubeName).getSegments().size());
    CubeSegment segment = new CubeSegment();
    segment.setName("test_segment");
    cube.getSegments().add(segment);
    cubeManager.updateCube(cube);
    //only one for update cube
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertEquals(1, cubeManagerB.getCube(cubeName).getSegments().size());
    assertEquals(segment.getName(), cubeManagerB.getCube(cubeName).getSegments().get(0).getName());

    //delete cube
    cubeManager.dropCube(cubeName, false);
    assertTrue(cubeManager.getCube(cubeName) == null);
    assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));
    //one for cube update, one for project update
    assertEquals(2, broadcaster.getCounterAndClear());
    waitForCounterAndClear(2);
    assertTrue(cubeManagerB.getCube(cubeName) == null);
    assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME), RealizationType.CUBE, cubeName));


    final String cubeDescName = "test_cube_desc";
    cubeDesc.setName(cubeDescName);
    cubeDesc.setLastModified(0);
    assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
    assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);
    cubeDescManager.createCubeDesc(cubeDesc);
    //one for add cube desc
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertNotNull(cubeDescManager.getCubeDesc(cubeDescName));
    assertNotNull(cubeDescManagerB.getCubeDesc(cubeDescName));


    cubeDesc.setNotifyList(Arrays.asList("test@email", "test@email", "test@email"));
    cubeDescManager.updateCubeDesc(cubeDesc);
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertEquals(cubeDesc.getNotifyList(), cubeDescManagerB.getCubeDesc(cubeDescName).getNotifyList());

    cubeDescManager.removeCubeDesc(cubeDesc);
    //one for add cube desc
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
    assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);



    getStore().deleteResource("/cube/a_whole_new_cube.json");
}