Java Code Examples for org.apache.kylin.cube.model.CubeDesc#init()

The following examples show how to use org.apache.kylin.cube.model.CubeDesc#init() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CubeDescManager.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private CubeDesc loadCubeDesc(String path) throws IOException {
    ResourceStore store = getStore();
    CubeDesc ndesc = store.getResource(path, CubeDesc.class, CUBE_DESC_SERIALIZER);
    
    if (StringUtils.isBlank(ndesc.getName())) {
        throw new IllegalStateException("CubeDesc name must not be blank");
    }

    ndesc.init(config, getMetadataManager().getAllTablesMap());

    if (ndesc.getError().isEmpty() == false) {
        throw new IllegalStateException("Cube desc at " + path + " has issues: " + ndesc.getError());
    }
    
    return ndesc;
}
Example 2
Source File: AggregationGroupRuleTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGoodBecomeBadDesc() throws IOException {
    AggregationGroupRule rule = new AggregationGroupRule() {
        @Override
        protected long getMaxCombinations(CubeDesc cubeDesc) {
            return 2;
        }
    };

    for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/").listFiles()) {
        System.out.println(f.getName());
        CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);
        try {
            desc.init(getTestConfig());
        } catch (Exception e) {
            // Ignore any exception here, validation may fail for bad json
        }
        ValidateContext vContext = new ValidateContext();
        rule.validate(desc, vContext);
        //vContext.print(System.out);
        assertTrue(vContext.getResults().length > 0);
        assertTrue(vContext.getResults()[0].getMessage().startsWith("Aggregation group 1 has too many combinations"));
    }
}
Example 3
Source File: AggregationGroupRuleTest.java    From kylin with Apache License 2.0 6 votes vote down vote up 
@Test
public void testGoodDesc() throws IOException {
    AggregationGroupRule rule = getAggregationGroupRule();

    for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/").listFiles()) {
        if (!f.getName().endsWith("json")) {
            continue;
        }
        CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);
        desc.init(getTestConfig());
        ValidateContext vContext = new ValidateContext();
        rule.validate(desc, vContext);
        //vContext.print(System.out);
        assertTrue(vContext.getResults().length == 0);
    }
}
Example 4
Source File: CubeDescTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testTooManyRowkeys() throws Exception {
    File metaFile = new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc/ut_78_rowkeys.json.bad");
    Assert.assertTrue(metaFile.exists());
    String path = metaFile.getPath();
    metaFile.renameTo(new File(path.substring(0, path.length() - 4)));

    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage(
            "Too many rowkeys (78) in CubeDesc, please try to reduce dimension number or adopt derived dimensions");
    getTestConfig().clearManagers();
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_78_rowkeys");
    cubeDesc.init(getTestConfig());
}
Example 5
Source File: CubeDescTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testBadInit9() throws Exception {
    String[] strs = new String[] { LSTG_FORMAT_NAME, META_CATEG_NAME };
    thrown.expect(IllegalStateException.class);
    thrown.expectMessage(
            "Aggregation group 1 hierarchy dimensions overlap with joint dimensions: " + sortStrs(strs));
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] {
            new String[] { META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME },
            new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID } };
    cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] {
            new String[] { META_CATEG_NAME, LSTG_FORMAT_NAME } };

    cubeDesc.init(getTestConfig());
}
Example 6
Source File: CubeDescTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testBadInit8() throws Exception {
    String[] strs = new String[] { CATEG_LVL2_NAME, META_CATEG_NAME };
    thrown.expect(IllegalStateException.class);
    thrown.expectMessage(
            "Aggregation group 1 hierarchy dimensions overlap with joint dimensions: " + sortStrs(strs));

    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] {
            new String[] { META_CATEG_NAME, CATEG_LVL2_NAME } };

    cubeDesc.init(getTestConfig());
}
Example 7
Source File: CubeDescTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testBadInit6() throws Exception {
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    cubeDesc.getAggregationGroups().get(0).getSelectRule().mandatoryDims = new String[] { SELLER_ID,
            LSTG_FORMAT_NAME };

    cubeDesc.init(getTestConfig());
}
Example 8
Source File: CubeDescTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testBadInit5() throws Exception {
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    cubeDesc.getAggregationGroups().get(0).getSelectRule().mandatoryDims = new String[] { SELLER_ID,
            META_CATEG_NAME };

    cubeDesc.init(getTestConfig());
}
Example 9
Source File: CubeDescTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testBadInit3() throws Exception {
    thrown.expect(IllegalStateException.class);
    thrown.expectMessage("Aggregation group 1 'includes' dimensions not include all the dimensions:");
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    String[] temp = Arrays.asList(cubeDesc.getAggregationGroups().get(0).getIncludes()).subList(0, 3)
            .toArray(new String[3]);
    cubeDesc.getAggregationGroups().get(0).setIncludes(temp);

    cubeDesc.init(getTestConfig());
}
Example 10
Source File: DictionaryRuleTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGoodDesc() throws IOException {
    DictionaryRule rule = new DictionaryRule();

    for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/").listFiles()) {
        if (!f.getName().endsWith("json")) {
            continue;
        }
        CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);
        desc.init(config);
        ValidateContext vContext = new ValidateContext();
        rule.validate(desc, vContext);
        assertTrue(vContext.getResults().length == 0);
    }
}
Example 11
Source File: CubeDescManager.java    From kylin with Apache License 2.0 5 votes vote down vote up 
/**
 * Create a new CubeDesc
 */
public CubeDesc createCubeDesc(CubeDesc cubeDesc) throws IOException {
    try (AutoLock lock = descMapLock.lockForWrite()) {
        if (cubeDesc.getUuid() == null || cubeDesc.getName() == null)
            throw new IllegalArgumentException();
        if (cubeDescMap.containsKey(cubeDesc.getName()))
            throw new IllegalArgumentException("CubeDesc '" + cubeDesc.getName() + "' already exists");
        if (cubeDesc.isDraft())
            throw new IllegalArgumentException(String.format(Locale.ROOT, CUBE_SHOULD_NOT_BE_DRAFT_MSG, cubeDesc.getName()));

        try {
            cubeDesc.init(config);
        } catch (Exception e) {
            logger.warn("Broken cube desc " + cubeDesc, e);
            cubeDesc.addError(e.toString());
        }
        
        postProcessCubeDesc(cubeDesc);
        // Check base validation
        if (cubeDesc.isBroken()) {
            return cubeDesc;
        }
        // Semantic validation
        CubeMetadataValidator validator = new CubeMetadataValidator(config);
        ValidateContext context = validator.validate(cubeDesc);
        if (!context.ifPass()) {
            return cubeDesc;
        }

        cubeDesc.setSignature(cubeDesc.calculateSignature());

        // save resource
        crud.save(cubeDesc);
        
        return cubeDesc;
    }
}
Example 12
Source File: CubeDescTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testBadInit10() throws Exception {
    String[] strs = new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID };
    thrown.expect(IllegalStateException.class);
    thrown.expectMessage("Aggregation group 1 a dimension exist in more than one joint: " + sortStrs(strs));

    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] {
            new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD },
            new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID, LEAF_CATEG_ID } };

    cubeDesc.init(getTestConfig());
}
Example 13
Source File: CubeDescManager.java    From kylin with Apache License 2.0 5 votes vote down vote up 
/**
 * Update CubeDesc with the input. Broadcast the event into cluster
 */
public CubeDesc updateCubeDesc(CubeDesc desc) throws IOException {
    try (AutoLock lock = descMapLock.lockForWrite()) {
        // Validate CubeDesc
        if (desc.getUuid() == null || desc.getName() == null)
            throw new IllegalArgumentException();
        String name = desc.getName();
        if (!cubeDescMap.containsKey(name))
            throw new IllegalArgumentException("CubeDesc '" + name + "' does not exist.");
        if (desc.isDraft())
            throw new IllegalArgumentException(String.format(Locale.ROOT, CUBE_SHOULD_NOT_BE_DRAFT_MSG, desc.getName()));

        try {
            desc.init(config);
        } catch (Exception e) {
            logger.warn("Broken cube desc " + desc, e);
            desc.addError(e.toString());
            return desc;
        }

        postProcessCubeDesc(desc);
        // Semantic validation
        CubeMetadataValidator validator = new CubeMetadataValidator(config);
        ValidateContext context = validator.validate(desc);
        if (!context.ifPass()) {
            return desc;
        }

        desc.setSignature(desc.calculateSignature());

        // save resource
        crud.save(desc);

        return desc;
    }
}
Example 14
Source File: CubeDescTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testBadInit9() throws Exception {
    String[] strs = new String[] { LSTG_FORMAT_NAME, META_CATEG_NAME };
    thrown.expect(IllegalStateException.class);
    thrown.expectMessage(
            "Aggregation group 1 hierarchy dimensions overlap with joint dimensions: " + sortStrs(strs));
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] {
            new String[] { META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME },
            new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID } };
    cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] {
            new String[] { META_CATEG_NAME, LSTG_FORMAT_NAME } };

    cubeDesc.init(getTestConfig());
}
Example 15
Source File: FunctionRuleTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test(expected = IllegalStateException.class)
public void testValidateMeasureNamesDuplicated() throws IOException {
    File f = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/ssb.json");
    CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);

    MeasureDesc measureDescDuplicated = desc.getMeasures().get(1);
    List<MeasureDesc> newMeasures = Lists.newArrayList(desc.getMeasures());
    newMeasures.add(measureDescDuplicated);
    desc.setMeasures(newMeasures);

    desc.init(config);
}
Example 16
Source File: CubeDescTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testValidateNotifyList() throws Exception {
    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage("Email [test] is not validation.");

    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    List<String> notify = Lists.newArrayList();
    notify.add("test");
    cubeDesc.setNotifyList(notify);
    cubeDesc.validateNotifyList();
    cubeDesc.init(getTestConfig());
}
Example 17
Source File: CubeDescManager.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
/**
 * Update CubeDesc with the input. Broadcast the event into cluster
 */
public CubeDesc updateCubeDesc(CubeDesc desc) throws IOException {
    try (AutoLock lock = descMapLock.lockForWrite()) {
        // Validate CubeDesc
        if (desc.getUuid() == null || desc.getName() == null)
            throw new IllegalArgumentException();
        String name = desc.getName();
        if (!cubeDescMap.containsKey(name))
            throw new IllegalArgumentException("CubeDesc '" + name + "' does not exist.");
        if (desc.isDraft())
            throw new IllegalArgumentException(String.format(Locale.ROOT, CUBE_SHOULD_NOT_BE_DRAFT_MSG, desc.getName()));

        try {
            desc.init(config);
        } catch (Exception e) {
            logger.warn("Broken cube desc " + desc, e);
            desc.addError(e.toString());
            return desc;
        }

        postProcessCubeDesc(desc);
        // Semantic validation
        CubeMetadataValidator validator = new CubeMetadataValidator(config);
        ValidateContext context = validator.validate(desc);
        if (!context.ifPass()) {
            return desc;
        }

        desc.setSignature(desc.calculateSignature());

        // save resource
        crud.save(desc);

        return desc;
    }
}
Example 18
Source File: CubeDescManager.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
/**
 * Create a new CubeDesc
 */
public CubeDesc createCubeDesc(CubeDesc cubeDesc) throws IOException {
    try (AutoLock lock = descMapLock.lockForWrite()) {
        if (cubeDesc.getUuid() == null || cubeDesc.getName() == null)
            throw new IllegalArgumentException();
        if (cubeDescMap.containsKey(cubeDesc.getName()))
            throw new IllegalArgumentException("CubeDesc '" + cubeDesc.getName() + "' already exists");
        if (cubeDesc.isDraft())
            throw new IllegalArgumentException(String.format(Locale.ROOT, CUBE_SHOULD_NOT_BE_DRAFT_MSG, cubeDesc.getName()));

        try {
            cubeDesc.init(config);
        } catch (Exception e) {
            logger.warn("Broken cube desc " + cubeDesc, e);
            cubeDesc.addError(e.toString());
        }
        
        postProcessCubeDesc(cubeDesc);
        // Check base validation
        if (cubeDesc.isBroken()) {
            return cubeDesc;
        }
        // Semantic validation
        CubeMetadataValidator validator = new CubeMetadataValidator(config);
        ValidateContext context = validator.validate(cubeDesc);
        if (!context.ifPass()) {
            return cubeDesc;
        }

        cubeDesc.setSignature(cubeDesc.calculateSignature());

        // save resource
        crud.save(cubeDesc);
        
        return cubeDesc;
    }
}
Example 19
Source File: CubeDescTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCombinationIntOverflow() throws Exception {
    for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc").listFiles()) {
        if (f.getName().endsWith(".bad")) {
            String path = f.getPath();
            f.renameTo(new File(path.substring(0, path.length() - 4)));
        }
    }

    thrown.expect(TooManyCuboidException.class);
    getTestConfig().clearManagers();
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig())
            .getCubeDesc("ut_cube_desc_combination_int_overflow");
    cubeDesc.init(getTestConfig());
}
Example 20
Source File: CubeDescTest.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Test
public void testGoodInit() throws Exception {
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    cubeDesc.init(getTestConfig());
}