Java Code Examples for org.apache.kylin.cube.CubeManager#appendSegment()
The following examples show how to use
org.apache.kylin.cube.CubeManager#appendSegment() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: HybridCubeCLITest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
@Test
public void testSegmentOverlap() throws IOException {
thrown.expect(RuntimeException.class);
thrown.expectMessage("Segments has overlap");
HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
CubeInstance cube1 = cubeManager.getCube("ssb_cube1");
CubeInstance cube2 = cubeManager.getCube("ssb_cube2");
// 2012-01-01,2012-01-03
cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L));
// 2012-01-02,2012-01-04
cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325462400000L, 1325635200000L));
HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" });
}
Example 2
| Source File: HybridCubeCLITest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
@Test
public void testSegmentNotOverlap() throws IOException {
HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
CubeInstance cube1 = cubeManager.getCube("ssb_cube1");
CubeInstance cube2 = cubeManager.getCube("ssb_cube2");
// 2012-01-01,2012-01-03
cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L));
// 2012-01-03,2012-01-04
cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325548800000L, 1325635200000L));
// not throw exception
HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" });
}
Example 3
| Source File: JdbcHiveMRInputTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
@Test
public void testGenSqoopCmd_Partition() throws IOException {
ISource source = SourceManager.getSource(new JdbcSourceAware());
IMRInput input = source.adaptToBuildEngine(IMRInput.class);
Assert.assertNotNull(input);
CubeManager cubeManager = CubeManager.getInstance(getTestConfig());
CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_inner_join_cube");
CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()),
new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L));
CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg);
JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input
.getBatchCubingInputSide(flatDesc);
AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp",
cubeDesc.getName());
Assert.assertNotNull(executable);
String cmd = executable.getParam("cmd");
Assert.assertTrue(cmd.contains("org.h2.Driver"));
Assert.assertTrue(cmd.contains(
"--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\")" + System.lineSeparator()
+ "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\""));
source.close();
}
Example 4
| Source File: JdbcHiveMRInputTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
@Test
public void testGenSqoopCmd_NoPartition() throws IOException {
ISource source = SourceManager.getSource(new JdbcSourceAware());
IMRInput input = source.adaptToBuildEngine(IMRInput.class);
Assert.assertNotNull(input);
CubeManager cubeManager = CubeManager.getInstance(getTestConfig());
CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_left_join_cube");
CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()),
new SegmentRange.TSRange(0L, Long.MAX_VALUE));
CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg);
JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input
.getBatchCubingInputSide(flatDesc);
AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp",
cubeDesc.getName());
Assert.assertNotNull(executable);
String cmd = executable.getParam("cmd");
Assert.assertTrue(cmd.contains("org.h2.Driver"));
Assert.assertTrue(
cmd.contains("--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\")" + System.lineSeparator()
+ "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"\""));
source.close();
}
Example 5
| Source File: JdbcHiveMRInputTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
@Test
public void testGenSqoopCmd_WithLookupShardBy() throws IOException {
ISource source = SourceManager.getSource(new JdbcSourceAware());
IMRInput input = source.adaptToBuildEngine(IMRInput.class);
Assert.assertNotNull(input);
CubeManager cubeManager = CubeManager.getInstance(getTestConfig());
CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_jdbc_shard");
CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()),
new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L));
CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg);
JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input
.getBatchCubingInputSide(flatDesc);
AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp",
cubeDesc.getName());
Assert.assertNotNull(executable);
String cmd = executable.getParam("cmd");
Assert.assertTrue(cmd.contains("org.h2.Driver"));
Assert.assertTrue(cmd.contains(
"--boundary-query \"SELECT MIN(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\"), MAX(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\")" + System.lineSeparator()
+ "FROM \\\"DEFAULT\\\".\\\"TEST_CATEGORY_GROUPINGS\\\" AS \\\"TEST_CATEGORY_GROUPINGS\\\"\""));
source.close();
}
Example 6
| Source File: JobStepFactoryTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
@Test
public void testAddStepInCubing() throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(config);
CubeInstance cube = cubeMgr.getCube(CUBE_NAME);
cleanupSegments(CUBE_NAME);
CubeSegment oneSeg = cubeMgr.appendSegment(cube, new SegmentRange.TSRange(0L, Long.MAX_VALUE));
Set<CubeSegment> segments = Sets.newHashSet(oneSeg);
NSparkCubingJob job = NSparkCubingJob.create(segments, "ADMIN");
Assert.assertEquals(CUBE_NAME, job.getParam(MetadataConstants.P_CUBE_NAME));
NSparkExecutable resourceDetectStep = job.getResourceDetectStep();
Assert.assertEquals(ResourceDetectBeforeCubingJob.class.getName(),
resourceDetectStep.getSparkSubmitClassName());
Assert.assertEquals(ExecutableConstants.STEP_NAME_DETECT_RESOURCE, resourceDetectStep.getName());
job.getParams().forEach((key, value) -> Assert.assertEquals(value, resourceDetectStep.getParam(key)));
Assert.assertEquals(config.getJobTmpMetaStoreUrl(getProject(), resourceDetectStep.getId()).toString(),
resourceDetectStep.getDistMetaUrl());
NSparkExecutable cubeStep = job.getSparkCubingStep();
Assert.assertEquals(config.getSparkBuildClassName(), cubeStep.getSparkSubmitClassName());
Assert.assertEquals(ExecutableConstants.STEP_NAME_BUILD_SPARK_CUBE, cubeStep.getName());
job.getParams().forEach((key, value) -> Assert.assertEquals(value, cubeStep.getParam(key)));
Assert.assertEquals(config.getJobTmpMetaStoreUrl(getProject(), cubeStep.getId()).toString(),
cubeStep.getDistMetaUrl());
}
Example 7
| Source File: LocalWithSparkSessionTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
public ExecutableState buildCuboid(String cubeName, SegmentRange.TSRange tsRange) throws Exception {
KylinConfig config = KylinConfig.getInstanceFromEnv();
CubeManager cubeMgr = CubeManager.getInstance(config);
CubeInstance cube = cubeMgr.getCube(cubeName);
ExecutableManager execMgr = ExecutableManager.getInstance(config);
DataModelManager.getInstance(config).getModels();
// ready cube, segment, cuboid layout
CubeSegment oneSeg = cubeMgr.appendSegment(cube, tsRange);
NSparkCubingJob job = NSparkCubingJob.create(Sets.newHashSet(oneSeg), "ADMIN");
NSparkCubingStep sparkStep = job.getSparkCubingStep();
StorageURL distMetaUrl = StorageURL.valueOf(sparkStep.getDistMetaUrl());
Assert.assertEquals("hdfs", distMetaUrl.getScheme());
Assert.assertTrue(distMetaUrl.getParameter("path").startsWith(config.getHdfsWorkingDirectory()));
// launch the job
execMgr.addJob(job);
return wait(job);
}
Example 8
| Source File: HybridCubeCLITest.java From kylin with Apache License 2.0 | 6 votes |
|
@Test
public void testSegmentOverlap() throws IOException {
thrown.expect(RuntimeException.class);
thrown.expectMessage("Segments has overlap");
HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
CubeInstance cube1 = cubeManager.getCube("ssb_cube1");
CubeInstance cube2 = cubeManager.getCube("ssb_cube2");
// 2012-01-01,2012-01-03
cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L));
// 2012-01-02,2012-01-04
cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325462400000L, 1325635200000L));
HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" });
}
Example 9
| Source File: HybridCubeCLITest.java From kylin with Apache License 2.0 | 6 votes |
|
@Test
public void testSegmentNotOverlap() throws IOException {
HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
CubeInstance cube1 = cubeManager.getCube("ssb_cube1");
CubeInstance cube2 = cubeManager.getCube("ssb_cube2");
// 2012-01-01,2012-01-03
cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L));
// 2012-01-03,2012-01-04
cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325548800000L, 1325635200000L));
// not throw exception
HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" });
}
Example 10
| Source File: JdbcHiveMRInputTest.java From kylin with Apache License 2.0 | 6 votes |
|
@Test
public void testGenSqoopCmd_Partition() throws IOException {
ISource source = SourceManager.getSource(new JdbcSourceAware());
IMRInput input = source.adaptToBuildEngine(IMRInput.class);
Assert.assertNotNull(input);
CubeManager cubeManager = CubeManager.getInstance(getTestConfig());
CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_inner_join_cube");
CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()),
new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L));
CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg);
JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input
.getBatchCubingInputSide(flatDesc);
AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp",
cubeDesc.getName());
Assert.assertNotNull(executable);
String cmd = executable.getParam("cmd");
Assert.assertTrue(cmd.contains("org.h2.Driver"));
Assert.assertTrue(cmd.contains(
"--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\")" + System.lineSeparator()
+ "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\""));
source.close();
}
Example 11
| Source File: JdbcHiveMRInputTest.java From kylin with Apache License 2.0 | 6 votes |
|
@Test
public void testGenSqoopCmd_NoPartition() throws IOException {
ISource source = SourceManager.getSource(new JdbcSourceAware());
IMRInput input = source.adaptToBuildEngine(IMRInput.class);
Assert.assertNotNull(input);
CubeManager cubeManager = CubeManager.getInstance(getTestConfig());
CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_left_join_cube");
CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()),
new SegmentRange.TSRange(0L, Long.MAX_VALUE));
CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg);
JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input
.getBatchCubingInputSide(flatDesc);
AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp",
cubeDesc.getName());
Assert.assertNotNull(executable);
String cmd = executable.getParam("cmd");
Assert.assertTrue(cmd.contains("org.h2.Driver"));
Assert.assertTrue(
cmd.contains("--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\")" + System.lineSeparator()
+ "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"\""));
source.close();
}
Example 12
| Source File: JdbcHiveMRInputTest.java From kylin with Apache License 2.0 | 6 votes |
|
@Test
public void testGenSqoopCmd_WithLookupShardBy() throws IOException {
ISource source = SourceManager.getSource(new JdbcSourceAware());
IMRInput input = source.adaptToBuildEngine(IMRInput.class);
Assert.assertNotNull(input);
CubeManager cubeManager = CubeManager.getInstance(getTestConfig());
CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_jdbc_shard");
CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()),
new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L));
CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg);
JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input
.getBatchCubingInputSide(flatDesc);
AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp",
cubeDesc.getName());
Assert.assertNotNull(executable);
String cmd = executable.getParam("cmd");
Assert.assertTrue(cmd.contains("org.h2.Driver"));
Assert.assertTrue(cmd.contains(
"--boundary-query \"SELECT MIN(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\"), MAX(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\")" + System.lineSeparator()
+ "FROM \\\"DEFAULT\\\".\\\"TEST_CATEGORY_GROUPINGS\\\" AS \\\"TEST_CATEGORY_GROUPINGS\\\"\""));
source.close();
}
Example 13
| Source File: CsvSourceTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
@Test
public void testGetFlatTable() throws IOException {
System.out.println(getTestConfig().getMetadataUrl());
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
CubeInstance cube = cubeMgr.getCube(CUBE_NAME);
cleanupSegments(CUBE_NAME);
DataModelDesc model = cube.getModel();
CubeSegment segment = cubeMgr.appendSegment(cube, new SegmentRange.TSRange(dateToLong("2010-01-01"), dateToLong("2013-01-01")));
Dataset<Row> ds = initFlatTable(segment);
ds.show(10);
StructType schema = ds.schema();
SegmentInfo segmentInfo = MetadataConverter.getSegmentInfo(segment.getCubeInstance(), segment.getUuid(),
segment.getName(), segment.getStorageLocationIdentifier());
scala.collection.immutable.Map<String, String> map = BuildUtils.getColumnIndexMap(segmentInfo);
for (StructField field : schema.fields()) {
Assert.assertNotNull(model.findColumn(map.apply(field.name())));
}
for (LayoutEntity layoutEntity : MetadataConverter.extractEntityList2JavaList(cube)) {
Set<Integer> dims = layoutEntity.getOrderedDimensions().keySet();
Column[] modelCols = new Column[dims.size()];
int index = 0;
for (int id : dims) {
modelCols[index] = new Column(String.valueOf(id));
index++;
}
ds.select(modelCols).show(10);
}
}
Example 14
| Source File: CacheServiceTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
@Test
public void testCubeCRUD() throws Exception {
final Broadcaster broadcaster = Broadcaster.getInstance(configA);
broadcaster.getCounterAndClear();
getStore().deleteResource("/cube/test_kylin_cube_a_new_one.json");
//create cube
final String cubeName = "test_kylin_cube_a_new_one";
final CubeManager cubeManager = getCubeManager(configA);
final CubeManager cubeManagerB = getCubeManager(configB);
final ProjectManager projectManager = getProjectManager(configA);
final ProjectManager projectManagerB = getProjectManager(configB);
final CubeDescManager cubeDescManager = getCubeDescManager(configA);
final CubeDescManager cubeDescManagerB = getCubeDescManager(configB);
final CubeDesc cubeDesc = getCubeDescManager(configA).getCubeDesc("test_kylin_cube_with_slr_desc");
assertTrue(cubeManager.getCube(cubeName) == null);
assertTrue(cubeManagerB.getCube(cubeName) == null);
assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
RealizationType.CUBE, cubeName));
assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
RealizationType.CUBE, cubeName));
cubeManager.createCube(cubeName, ProjectInstance.DEFAULT_PROJECT_NAME, cubeDesc, null);
//one for cube update, one for project update
assertEquals(2, broadcaster.getCounterAndClear());
waitForCounterAndClear(2);
assertNotNull(cubeManager.getCube(cubeName));
assertNotNull(cubeManagerB.getCube(cubeName));
assertTrue(containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
RealizationType.CUBE, cubeName));
assertTrue(containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
RealizationType.CUBE, cubeName));
//update cube
CubeInstance cube = cubeManager.getCube(cubeName);
assertEquals(0, cube.getSegments().size());
assertEquals(0, cubeManagerB.getCube(cubeName).getSegments().size());
CubeSegment segment = cubeManager.appendSegment(cube, new TSRange(0L, 1000L));
//one for cube update
assertEquals(1, broadcaster.getCounterAndClear());
waitForCounterAndClear(1);
assertEquals(1, cubeManagerB.getCube(cubeName).getSegments().size());
assertEquals(segment.getName(), cubeManagerB.getCube(cubeName).getSegments().get(0).getName());
//delete cube
cubeManager.dropCube(cubeName, false);
//one for cube update, one for project update
assertEquals(2, broadcaster.getCounterAndClear());
waitForCounterAndClear(2);
assertTrue(cubeManager.getCube(cubeName) == null);
assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
RealizationType.CUBE, cubeName));
assertTrue(cubeManagerB.getCube(cubeName) == null);
assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
RealizationType.CUBE, cubeName));
final String cubeDescName = "test_cube_desc";
cubeDesc.setName(cubeDescName);
cubeDesc.setLastModified(0);
assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);
cubeDescManager.createCubeDesc(cubeDesc);
//one for add cube desc
assertEquals(1, broadcaster.getCounterAndClear());
waitForCounterAndClear(1);
assertNotNull(cubeDescManager.getCubeDesc(cubeDescName));
assertNotNull(cubeDescManagerB.getCubeDesc(cubeDescName));
cubeDesc.setNotifyList(Arrays.asList("[email protected]", "[email protected]", "[email protected]"));
cubeDescManager.updateCubeDesc(cubeDesc);
assertEquals(1, broadcaster.getCounterAndClear());
waitForCounterAndClear(1);
assertEquals(cubeDesc.getNotifyList(), cubeDescManagerB.getCubeDesc(cubeDescName).getNotifyList());
cubeDescManager.removeCubeDesc(cubeDesc);
//one for add cube desc
assertEquals(1, broadcaster.getCounterAndClear());
waitForCounterAndClear(1);
assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);
getStore().deleteResource("/cube/test_kylin_cube_a_new_one.json");
}
Example 15
| Source File: JobStepFactoryTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
@Test
public void testAddStepInMerging() throws Exception {
CubeManager cubeMgr = CubeManager.getInstance(config);
CubeInstance cube = cubeMgr.getCube(CUBE_NAME);
cleanupSegments(CUBE_NAME);
/**
* Round1. Add 2 segment
*/
CubeSegment segment1 = cubeMgr.appendSegment(cube, new SegmentRange.TSRange(dateToLong("2010-01-01"), dateToLong("2013-01-01")));
CubeSegment segment2 = cubeMgr.appendSegment(cube, new SegmentRange.TSRange(dateToLong("2013-01-01"), dateToLong("2015-01-01")));
segment1.setStatus(SegmentStatusEnum.READY);
segment2.setStatus(SegmentStatusEnum.READY);
CubeInstance reloadCube = cube.latestCopyForWrite();
Segments segments = new Segments();
segments.add(segment1);
segments.add(segment2);
reloadCube.setSegments(segments);
CubeUpdate update = new CubeUpdate(reloadCube);
cubeMgr.updateCube(update);
/**
* Round2. Merge two segments
*/
reloadCube = cubeMgr.reloadCube(CUBE_NAME);
CubeSegment mergedSegment = cubeMgr.mergeSegments(reloadCube, new SegmentRange.TSRange(dateToLong("2010-01-01"), dateToLong("2015-01-01"))
, null, true);
NSparkMergingJob job = NSparkMergingJob.merge(mergedSegment, "ADMIN");
Assert.assertEquals(CUBE_NAME, job.getParam(MetadataConstants.P_CUBE_NAME));
NSparkExecutable resourceDetectStep = job.getResourceDetectStep();
Assert.assertEquals(ResourceDetectBeforeMergingJob.class.getName(),
resourceDetectStep.getSparkSubmitClassName());
Assert.assertEquals(ExecutableConstants.STEP_NAME_DETECT_RESOURCE, resourceDetectStep.getName());
job.getParams().forEach((key, value) -> Assert.assertEquals(value, resourceDetectStep.getParam(key)));
Assert.assertEquals(config.getJobTmpMetaStoreUrl(getProject(), resourceDetectStep.getId()).toString(),
resourceDetectStep.getDistMetaUrl());
NSparkExecutable mergeStep = job.getSparkMergingStep();
Assert.assertEquals(config.getSparkMergeClassName(), mergeStep.getSparkSubmitClassName());
Assert.assertEquals(ExecutableConstants.STEP_NAME_MERGER_SPARK_SEGMENT, mergeStep.getName());
job.getParams().forEach((key, value) -> Assert.assertEquals(value, mergeStep.getParam(key)));
Assert.assertEquals(config.getJobTmpMetaStoreUrl(getProject(), mergeStep.getId()).toString(),
mergeStep.getDistMetaUrl());
CubeInstance cubeInstance = cubeMgr.reloadCube(CUBE_NAME);
NSparkUpdateMetaAndCleanupAfterMergeStep cleanStep = job.getCleanUpAfterMergeStep();
job.getParams().forEach((key, value) -> {
if (key.equalsIgnoreCase(MetadataConstants.P_SEGMENT_IDS)) {
final List<String> needDeleteSegmentNames = cubeInstance.getMergingSegments(mergedSegment).stream()
.map(CubeSegment::getName).collect(Collectors.toList());
Assert.assertEquals(needDeleteSegmentNames, Arrays.asList(cleanStep.getParam(MetadataConstants.P_SEGMENT_NAMES).split(",")));
} else {
Assert.assertEquals(value, mergeStep.getParam(key));
}
});
Assert.assertEquals(config.getJobTmpMetaStoreUrl(getProject(), cleanStep.getId()).toString(),
cleanStep.getDistMetaUrl());
}
Example 16
| Source File: CacheServiceTest.java From kylin with Apache License 2.0 | 4 votes |
|
@Test
public void testCubeCRUD() throws Exception {
final Broadcaster broadcaster = Broadcaster.getInstance(configA);
broadcaster.getCounterAndClear();
getStore().deleteResource("/cube/test_kylin_cube_a_new_one.json");
//create cube
final String cubeName = "test_kylin_cube_a_new_one";
final CubeManager cubeManager = getCubeManager(configA);
final CubeManager cubeManagerB = getCubeManager(configB);
final ProjectManager projectManager = getProjectManager(configA);
final ProjectManager projectManagerB = getProjectManager(configB);
final CubeDescManager cubeDescManager = getCubeDescManager(configA);
final CubeDescManager cubeDescManagerB = getCubeDescManager(configB);
final CubeDesc cubeDesc = getCubeDescManager(configA).getCubeDesc("test_kylin_cube_with_slr_desc");
assertTrue(cubeManager.getCube(cubeName) == null);
assertTrue(cubeManagerB.getCube(cubeName) == null);
assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
RealizationType.CUBE, cubeName));
assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
RealizationType.CUBE, cubeName));
cubeManager.createCube(cubeName, ProjectInstance.DEFAULT_PROJECT_NAME, cubeDesc, null);
//one for cube update, one for project update
assertEquals(2, broadcaster.getCounterAndClear());
waitForCounterAndClear(2);
assertNotNull(cubeManager.getCube(cubeName));
assertNotNull(cubeManagerB.getCube(cubeName));
assertTrue(containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
RealizationType.CUBE, cubeName));
assertTrue(containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
RealizationType.CUBE, cubeName));
//update cube
CubeInstance cube = cubeManager.getCube(cubeName);
assertEquals(0, cube.getSegments().size());
assertEquals(0, cubeManagerB.getCube(cubeName).getSegments().size());
CubeSegment segment = cubeManager.appendSegment(cube, new TSRange(0L, 1000L));
//one for cube update
assertEquals(1, broadcaster.getCounterAndClear());
waitForCounterAndClear(1);
assertEquals(1, cubeManagerB.getCube(cubeName).getSegments().size());
assertEquals(segment.getName(), cubeManagerB.getCube(cubeName).getSegments().get(0).getName());
//delete cube
cubeManager.dropCube(cubeName, false);
//one for cube update, one for project update
assertEquals(2, broadcaster.getCounterAndClear());
waitForCounterAndClear(2);
assertTrue(cubeManager.getCube(cubeName) == null);
assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
RealizationType.CUBE, cubeName));
assertTrue(cubeManagerB.getCube(cubeName) == null);
assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
RealizationType.CUBE, cubeName));
final String cubeDescName = "test_cube_desc";
cubeDesc.setName(cubeDescName);
cubeDesc.setLastModified(0);
assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);
cubeDescManager.createCubeDesc(cubeDesc);
//one for add cube desc
assertEquals(1, broadcaster.getCounterAndClear());
waitForCounterAndClear(1);
assertNotNull(cubeDescManager.getCubeDesc(cubeDescName));
assertNotNull(cubeDescManagerB.getCubeDesc(cubeDescName));
cubeDesc.setNotifyList(Arrays.asList("[email protected]", "[email protected]", "[email protected]"));
cubeDescManager.updateCubeDesc(cubeDesc);
assertEquals(1, broadcaster.getCounterAndClear());
waitForCounterAndClear(1);
assertEquals(cubeDesc.getNotifyList(), cubeDescManagerB.getCubeDesc(cubeDescName).getNotifyList());
cubeDescManager.removeCubeDesc(cubeDesc);
//one for add cube desc
assertEquals(1, broadcaster.getCounterAndClear());
waitForCounterAndClear(1);
assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);
getStore().deleteResource("/cube/test_kylin_cube_a_new_one.json");
}
