Java Code Examples for org.apache.kylin.cube.model.CubeDesc#getInitialCuboidScheduler()

@Test
public void testIsValid2() {
    CubeDesc cube = getTestKylinCubeWithoutSeller();
    CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();

    assertEquals(false, cuboidScheduler.isValid(toLong("111111111")));

    // base
    assertEquals(false, cuboidScheduler.isValid(0));
    assertEquals(true, cuboidScheduler.isValid(toLong("11111111")));

    // aggregation group & zero tail
    assertEquals(true, cuboidScheduler.isValid(toLong("10000111")));
    assertEquals(false, cuboidScheduler.isValid(toLong("10001111")));
    assertEquals(false, cuboidScheduler.isValid(toLong("11001111")));
    assertEquals(true, cuboidScheduler.isValid(toLong("10000001")));
    assertEquals(true, cuboidScheduler.isValid(toLong("10000101")));

    // hierarchy
    assertEquals(true, cuboidScheduler.isValid(toLong("10100000")));
    assertEquals(true, cuboidScheduler.isValid(toLong("10110000")));
    assertEquals(true, cuboidScheduler.isValid(toLong("10111000")));
    assertEquals(false, cuboidScheduler.isValid(toLong("10001000")));
    assertEquals(false, cuboidScheduler.isValid(toLong("10011000")));
}
 

@Test
public void testCuboid_onlyBaseCuboid() {
    for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc").listFiles()) {
        if (f.getName().endsWith(".bad")) {
            String path = f.getPath();
            f.renameTo(new File(path.substring(0, path.length() - 4)));
        }
    }
    getTestConfig().clearManagers();
    CubeDesc cube = getCubeDescManager().getCubeDesc("ut_large_dimension_number");
    CuboidScheduler scheduler = cube.getInitialCuboidScheduler();

    Cuboid baseCuboid = Cuboid.getBaseCuboid(cube);
    assertTrue(scheduler.isValid(baseCuboid.getId()));

    List<Long> spanningChild = scheduler.getSpanningCuboid(baseCuboid.getId());
    assertTrue(spanningChild.size() > 0);
}
 

@Test
public void testLargeCube() {
    CubeDesc cube = getFiftyDimCubeDesc();
    CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
    long start = System.currentTimeMillis();
    System.out.println(cuboidScheduler.getCuboidCount());
    System.out.println("build tree takes: " + (System.currentTimeMillis() - start) + "ms");
}
 

@Test
public void testGetSpanningCuboid2() {
    CubeDesc cube = getTestKylinCubeWithSeller();
    CuboidScheduler scheduler = cube.getInitialCuboidScheduler();

    // generate 8d
    System.out.println("Spanning for 8D Cuboids");
    testSpanningAndGetParent(scheduler, cube, new long[] { 511 }, new long[] { 504, 447, 503, 383 });
    // generate 7d
    System.out.println("Spanning for 7D Cuboids");
    testSpanningAndGetParent(scheduler, cube, new long[] { 504, 447, 503, 383 }, new long[] { 440, 496, 376, 439, 487, 319, 375 });
    // generate 6d
    System.out.println("Spanning for 6D Cuboids");
    testSpanningAndGetParent(scheduler, cube, new long[] { 440, 496, 376, 439, 487, 319, 375 }, new long[] { 432, 480, 312, 368, 423, 455, 311, 359 });
    // generate 5d
    System.out.println("Spanning for 5D Cuboids");
    testSpanningAndGetParent(scheduler, cube, new long[] { 432, 480, 312, 368, 423, 455, 311, 359 }, new long[] { 416, 448, 304, 352, 391, 295, 327 });
    // generate 4d
    System.out.println("Spanning for 4D Cuboids");
    testSpanningAndGetParent(scheduler, cube, new long[] { 416, 448, 304, 352, 391, 295, 327 }, new long[] { 384, 288, 320, 263 });
    // generate 3d
    System.out.println("Spanning for 3D Cuboids");
    testSpanningAndGetParent(scheduler, cube, new long[] { 384, 288, 320, 263 }, new long[0]);
    // generate 2d
    // generate 1d
    // generate 0d
}
 

public static int[] calculateAllLevelCount(CubeDesc cube) {
    int levels = cube.getInitialCuboidScheduler().getBuildLevel();
    int[] allLevelCounts = new int[levels + 1];

    CuboidScheduler scheduler = cube.getInitialCuboidScheduler();
    LinkedList<Long> nextQueue = new LinkedList<Long>();
    LinkedList<Long> currentQueue = new LinkedList<Long>();
    long baseCuboid = Cuboid.getBaseCuboidId(cube);
    currentQueue.push(baseCuboid);

    for (int i = 0; i <= levels; i++) {
        allLevelCounts[i] = currentQueue.size();
        while (!currentQueue.isEmpty()) {
            long cuboid = currentQueue.pop();
            Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid);

            nextQueue.addAll(spnanningCuboids);
        }
        currentQueue = nextQueue;
        nextQueue = new LinkedList<Long>();

        if (i == levels) {
            if (!currentQueue.isEmpty()) {
                throw new IllegalStateException();
            }
        }
    }

    return allLevelCounts;
}
 

@Test
public void testCuboidCounts3() {
    CubeDesc cube = getTestKylinCubeWithSeller();
    CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
    int[] counts = CuboidCLI.calculateAllLevelCount(cube);
    printCount(counts);
    int sum = 0;
    for (Integer x : counts) {
        sum += x;
    }
    assertEquals(cuboidScheduler.getCuboidCount(), sum);
}
 

@Test
public void testLargeCube() {
    CubeDesc cube = getFiftyDimCubeDesc();
    CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
    long start = System.currentTimeMillis();
    System.out.println(cuboidScheduler.getCuboidCount());
    System.out.println("build tree takes: " + (System.currentTimeMillis() - start) + "ms");
}
 

@Test(expected=RuntimeException.class)
public void testTooManyCombination() {
    File twentyFile = new File(new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc"), "twenty_dim");
    twentyFile.renameTo(new File(twentyFile.getPath().substring(0, twentyFile.getPath().length() - 4)));
    CubeDesc cube = getTwentyDimCubeDesc();
    CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
    cuboidScheduler.getCuboidCount();
    twentyFile.renameTo(new File(twentyFile.getPath() + ".bad"));
}
 

@Test
public void testTooLargeCube() {
    CubeDesc cubeDesc = getFiftyDimFiveCapCubeDesc();
    AggregationGroupRule rule = new AggregationGroupRule();
    ValidateContext context = new ValidateContext();
    rule.validate(cubeDesc, context);
    assertFalse(context.ifPass());
    assertEquals(1, context.getResults().length);

    try {
        cubeDesc.getInitialCuboidScheduler();
        Assert.fail();
    } catch (RuntimeException e) {
    }
}
 

@Test
public void testCuboidCounts6() {
    CubeDesc cube = getCIInnerJoinCube();
    CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
    int[] counts = CuboidCLI.calculateAllLevelCount(cube);
    printCount(counts);
    int sum = 0;
    for (Integer x : counts) {
        sum += x;
    }
    assertEquals(cuboidScheduler.getCuboidCount(), sum);
}
 

@Test
public void testCuboidCounts5() {
    CubeDesc cube = getStreamingCubeDesc();
    CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
    int[] counts = CuboidCLI.calculateAllLevelCount(cube);
    printCount(counts);
    int sum = 0;
    for (Integer x : counts) {
        sum += x;
    }
    assertEquals(cuboidScheduler.getCuboidCount(), sum);
}
 

@Test
public void testIsValid3() {
    CubeDesc cube = getSSBCubeDesc();
    CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();

    assertEquals(false, cuboidScheduler.isValid(toLong("10000000000")));

    // the 4th is mandatory and isMandatoryOnlyValid is true
    assertEquals(true, cuboidScheduler.isValid(toLong("10000001000")));
    assertEquals(true, cuboidScheduler.isValid(toLong("00000001000")));
}
 

@Test
public void testCuboidCounts3() {
    CubeDesc cube = getTestKylinCubeWithSeller();
    CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
    int[] counts = CuboidCLI.calculateAllLevelCount(cube);
    printCount(counts);
    int sum = 0;
    for (Integer x : counts) {
        sum += x;
    }
    assertEquals(cuboidScheduler.getCuboidCount(), sum);
}
 

@Test
public void testCuboidCounts2() {
    CubeDesc cube = getTestKylinCubeWithoutSellerLeftJoin();
    CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
    int[] counts = CuboidCLI.calculateAllLevelCount(cube);
    printCount(counts);
    int sum = 0;
    for (Integer x : counts) {
        sum += x;
    }
    assertEquals(cuboidScheduler.getCuboidCount(), sum);
}
 

@Test
public void testCuboidCounts6() {
    CubeDesc cube = getCIInnerJoinCube();
    CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
    int[] counts = CuboidCLI.calculateAllLevelCount(cube);
    printCount(counts);
    int sum = 0;
    for (Integer x : counts) {
        sum += x;
    }
    assertEquals(cuboidScheduler.getCuboidCount(), sum);
}
 

@Test
public void testCuboidCounts4() {
    CubeDesc cube = getTestKylinCubeWithSellerLeft();
    CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
    int[] counts = CuboidCLI.calculateAllLevelCount(cube);
    printCount(counts);
    int sum = 0;
    for (Integer x : counts) {
        sum += x;
    }
    assertEquals(cuboidScheduler.getCuboidCount(), sum);
}
 

@Test
public void testGetSpanningCuboid1() {
    CubeDesc cube = getTestKylinCubeWithoutSeller();
    CuboidScheduler scheduler = cube.getInitialCuboidScheduler();

    // generate 7d
    System.out.println("Spanning for 7D Cuboids");
    testSpanningAndGetParent(scheduler, cube, new long[] { 255 }, new long[] { 135, 251, 253, 254 });
    // generate 6d
    System.out.println("Spanning for 6D Cuboids");
    testSpanningAndGetParent(scheduler, cube, new long[] { 135, 251, 253, 254 }, new long[] { 131, 133, 134, 249, 250, 252 });
    // generate 5d
    System.out.println("Spanning for 5D Cuboids");
    testSpanningAndGetParent(scheduler, cube, new long[] { 131, 133, 134, 249, 250, 252 }, new long[] { 129, 130, 132, 248 });
    // generate 4d
    System.out.println("Spanning for 4D Cuboids");
    testSpanningAndGetParent(scheduler, cube, new long[] { 129, 130, 132, 248 }, new long[] { 184, 240 });
    // generate 3d
    System.out.println("Spanning for 3D Cuboids");
    testSpanningAndGetParent(scheduler, cube, new long[] { 184, 240 }, new long[] { 176, 224 });
    // generate 2d
    System.out.println("Spanning for 2D Cuboids");
    testSpanningAndGetParent(scheduler, cube, new long[] { 176, 224 }, new long[] { 160, 192 });
    // generate 1d
    System.out.println("Spanning for 1D Cuboids");
    testSpanningAndGetParent(scheduler, cube, new long[] { 160, 192 }, new long[0]);
    // generate 0d
}
 

public static int[] calculateAllLevelCount(CubeDesc cube) {
    int levels = cube.getInitialCuboidScheduler().getBuildLevel();
    int[] allLevelCounts = new int[levels + 1];

    CuboidScheduler scheduler = cube.getInitialCuboidScheduler();
    LinkedList<Long> nextQueue = new LinkedList<Long>();
    LinkedList<Long> currentQueue = new LinkedList<Long>();
    long baseCuboid = Cuboid.getBaseCuboidId(cube);
    currentQueue.push(baseCuboid);

    for (int i = 0; i <= levels; i++) {
        allLevelCounts[i] = currentQueue.size();
        while (!currentQueue.isEmpty()) {
            long cuboid = currentQueue.pop();
            Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid);

            nextQueue.addAll(spnanningCuboids);
        }
        currentQueue = nextQueue;
        nextQueue = new LinkedList<Long>();

        if (i == levels) {
            if (!currentQueue.isEmpty()) {
                throw new IllegalStateException();
            }
        }
    }

    return allLevelCounts;
}
 

public static int simulateCuboidGeneration(CubeDesc cubeDesc, boolean validate) {
    CuboidScheduler scheduler = cubeDesc.getInitialCuboidScheduler();
    long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
    Collection<Long> cuboidSet = new TreeSet<Long>();
    cuboidSet.add(baseCuboid);
    LinkedList<Long> cuboidQueue = new LinkedList<Long>();
    cuboidQueue.push(baseCuboid);
    while (!cuboidQueue.isEmpty()) {
        long cuboid = cuboidQueue.pop();
        Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid);
        for (Long sc : spnanningCuboids) {
            boolean notfound = cuboidSet.add(sc);
            if (!notfound) {
                throw new IllegalStateException("Find duplicate spanning cuboid " + sc + " from cuboid " + cuboid);
            }

            cuboidQueue.push(sc);

        }
    }

    boolean enableDimCap = false;
    for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
        if (agg.getDimCap() > 0) {
            enableDimCap = true;
            break;
        }
    }

    if (validate) {
        if (enableDimCap) {
            if (cubeDesc.getAllCuboids().size() != cuboidSet.size()) {
                throw new IllegalStateException("Expected cuboid set " + cubeDesc.getAllCuboids() + "; but actual cuboid set " + cuboidSet);
            }
        } else {
            //only run this for test purpose, performance is bad when # of dims is large
            TreeSet<Long> enumCuboids = enumCalcCuboidCount(cubeDesc);
            System.out.println(Arrays.toString(enumCuboids.toArray(new Long[enumCuboids.size()])));
            if (enumCuboids.equals(cuboidSet) == false) {
                throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet);
            }

            //check all valid and invalid
            for (long i = 0; i < baseCuboid; ++i) {
                if (cuboidSet.contains(i)) {
                    if (!scheduler.isValid(i)) {
                        throw new RuntimeException();
                    }

                    if (scheduler.findBestMatchCuboid(i) != i) {
                        throw new RuntimeException();
                    }
                } else {
                    if (scheduler.isValid(i)) {
                        throw new RuntimeException();
                    }

                    long corrected = scheduler.findBestMatchCuboid(i);
                    if (corrected == i) {
                        throw new RuntimeException();
                    }

                    if (!scheduler.isValid(corrected)) {
                        throw new RuntimeException();
                    }

                    if (scheduler.findBestMatchCuboid(corrected) != corrected) {
                        throw new RuntimeException();
                    }
                }
            }
        }
    }

    return cuboidSet.size();

}
 

public static int simulateCuboidGeneration(CubeDesc cubeDesc, boolean validate) {
    CuboidScheduler scheduler = cubeDesc.getInitialCuboidScheduler();
    long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
    Collection<Long> cuboidSet = new TreeSet<Long>();
    cuboidSet.add(baseCuboid);
    LinkedList<Long> cuboidQueue = new LinkedList<Long>();
    cuboidQueue.push(baseCuboid);
    while (!cuboidQueue.isEmpty()) {
        long cuboid = cuboidQueue.pop();
        Collection<Long> spnanningCuboids = scheduler.getSpanningCuboid(cuboid);
        for (Long sc : spnanningCuboids) {
            boolean notfound = cuboidSet.add(sc);
            if (!notfound) {
                throw new IllegalStateException("Find duplicate spanning cuboid " + sc + " from cuboid " + cuboid);
            }

            cuboidQueue.push(sc);

        }
    }

    boolean enableDimCap = false;
    for (AggregationGroup agg : cubeDesc.getAggregationGroups()) {
        if (agg.getDimCap() > 0) {
            enableDimCap = true;
            break;
        }
    }

    if (validate) {
        if (enableDimCap) {
            if (cubeDesc.getAllCuboids().size() != cuboidSet.size()) {
                throw new IllegalStateException("Expected cuboid set " + cubeDesc.getAllCuboids() + "; but actual cuboid set " + cuboidSet);
            }
        } else {
            //only run this for test purpose, performance is bad when # of dims is large
            TreeSet<Long> enumCuboids = enumCalcCuboidCount(cubeDesc);
            System.out.println(Arrays.toString(enumCuboids.toArray(new Long[enumCuboids.size()])));
            if (enumCuboids.equals(cuboidSet) == false) {
                throw new IllegalStateException("Expected cuboid set " + enumCuboids + "; but actual cuboid set " + cuboidSet);
            }

            //check all valid and invalid
            for (long i = 0; i < baseCuboid; ++i) {
                if (cuboidSet.contains(i)) {
                    if (!scheduler.isValid(i)) {
                        throw new RuntimeException();
                    }

                    if (scheduler.findBestMatchCuboid(i) != i) {
                        throw new RuntimeException();
                    }
                } else {
                    if (scheduler.isValid(i)) {
                        throw new RuntimeException();
                    }

                    long corrected = scheduler.findBestMatchCuboid(i);
                    if (corrected == i) {
                        throw new RuntimeException();
                    }

                    if (!scheduler.isValid(corrected)) {
                        throw new RuntimeException();
                    }

                    if (scheduler.findBestMatchCuboid(corrected) != corrected) {
                        throw new RuntimeException();
                    }
                }
            }
        }
    }

    return cuboidSet.size();

}