Java Code Examples for org.apache.kylin.cube.model.CubeDesc#getModel()

The following examples show how to use org.apache.kylin.cube.model.CubeDesc#getModel() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LookupTableToHFileJob.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private String[] getLookupKeyColumns(CubeInstance cube, String tableName) {
    CubeDesc cubeDesc = cube.getDescriptor();
    DataModelDesc modelDesc = cubeDesc.getModel();
    TableRef lookupTableRef = null;
    for (TableRef tableRef : modelDesc.getLookupTables()) {
        if (tableRef.getTableIdentity().equalsIgnoreCase(tableName)) {
            lookupTableRef = tableRef;
            break;
        }
    }
    if (lookupTableRef == null) {
        throw new IllegalStateException("cannot find table in model:" + tableName);
    }
    JoinDesc joinDesc = modelDesc.getJoinByPKSide(lookupTableRef);
    TblColRef[] keyColRefs = joinDesc.getPrimaryKeyColumns();
    String[] result = new String[keyColRefs.length];
    for (int i = 0; i < keyColRefs.length; i++) {
        result[i] = keyColRefs[i].getName();
    }
    return result;
}
Example 2
Source File: LookupTableToHFileJob.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private String[] getLookupKeyColumns(CubeInstance cube, String tableName) {
    CubeDesc cubeDesc = cube.getDescriptor();
    DataModelDesc modelDesc = cubeDesc.getModel();
    TableRef lookupTableRef = null;
    for (TableRef tableRef : modelDesc.getLookupTables()) {
        if (tableRef.getTableIdentity().equalsIgnoreCase(tableName)) {
            lookupTableRef = tableRef;
            break;
        }
    }
    if (lookupTableRef == null) {
        throw new IllegalStateException("cannot find table in model:" + tableName);
    }
    JoinDesc joinDesc = modelDesc.getJoinByPKSide(lookupTableRef);
    TblColRef[] keyColRefs = joinDesc.getPrimaryKeyColumns();
    String[] result = new String[keyColRefs.length];
    for (int i = 0; i < keyColRefs.length; i++) {
        result[i] = keyColRefs[i].getName();
    }
    return result;
}
Example 3
Source File: MetadataUpgradeTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private void checkCubeDesc(String descName) {
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeDesc cubedesc1 = cubeDescMgr.getCubeDesc(descName);
    Assert.assertNotNull(cubedesc1);
    DataModelDesc model = cubedesc1.getModel();
    Assert.assertNotNull(model);
    Assert.assertTrue(model.getLookups().length > 0);

    List<DimensionDesc> dims = cubedesc1.getDimensions();

    Assert.assertTrue(dims.size() > 0);

    for (DimensionDesc dim : dims) {
        Assert.assertTrue(dim.getColumn().length > 0);
    }

    Assert.assertTrue(cubedesc1.getMeasures().size() > 0);

    CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    List<CubeInstance> cubes = cubeMgr.getCubesByDesc(descName);

    Assert.assertTrue(cubes.size() > 0);
}
Example 4
Source File: RealizationCheckTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRealizationCheck() {
    RealizationCheck realizationCheck = new RealizationCheck();
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ssb");
    DataModelDesc dataModelDesc = cubeDesc.getModel();
    IRealization iRealization = CubeInstance.create("ssb", cubeDesc);
    realizationCheck.addCubeIncapableReason(iRealization,
            RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.CUBE_NOT_CONTAIN_ALL_COLUMN));
    realizationCheck.addCubeIncapableReason(iRealization,
            RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.CUBE_NOT_CONTAIN_ALL_COLUMN));
    Assert.assertTrue(realizationCheck.getCubeIncapableReasons().size() == 1);

    realizationCheck.addModelIncapableReason(dataModelDesc,
            RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.CUBE_NOT_CONTAIN_ALL_COLUMN));
    realizationCheck.addModelIncapableReason(dataModelDesc,
            RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.CUBE_NOT_CONTAIN_ALL_MEASURE));
    realizationCheck.addModelIncapableReason(dataModelDesc,
            RealizationCheck.IncapableReason.notContainAllColumn(Lists.<TblColRef> newArrayList()));
    realizationCheck.addModelIncapableReason(dataModelDesc,
            RealizationCheck.IncapableReason.notContainAllColumn(Lists.<TblColRef> newArrayList()));
    Assert.assertTrue(realizationCheck.getModelIncapableReasons().size() == 1);
    Assert.assertTrue(realizationCheck.getModelIncapableReasons().get(dataModelDesc).size() == 3);

    realizationCheck.addModelIncapableReason(dataModelDesc, RealizationCheck.IncapableReason
            .notContainAllColumn(Lists.<TblColRef> newArrayList(dataModelDesc.findColumn("LO_DATE"))));
    Assert.assertTrue(realizationCheck.getModelIncapableReasons().size() == 1);
    Assert.assertTrue(realizationCheck.getModelIncapableReasons().get(dataModelDesc).size() == 4);
}
Example 5
Source File: CacheControllerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testClearCacheForCubeMigration() throws IOException {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    String CUBENAME = "test_kylin_cube_without_slr_desc";

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
    CubeDesc cubeDesc = cubeDescManager.getCubeDesc(CUBENAME);
    DataModelDesc modelDesc = cubeDesc.getModel();
    Map<String, String> tableToProjects = new HashMap<>();
    for (TableRef tableRef : modelDesc.getAllTables()) {
        tableToProjects.put(tableRef.getTableIdentity(), tableRef.getTableDesc().getProject());
    }

    String uuid = cubeDesc.getUuid();
    String signature = cubeDesc.getSignature();

    assertEquals(cubeDesc.getRetentionRange(), 0);

    //update cubeDesc
    cubeDesc.setRetentionRange(2018);
    cubeDesc.updateRandomUuid();

    //directly update metadata in store to simulate cube migration
    Serializer<CubeDesc> cubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class);
    getStore().checkAndPutResource(cubeDesc.getResourcePath(), cubeDesc, cubeDescSerializer);

    CubeMigrationRequest request = new CubeMigrationRequest();
    request.setCube(cubeDesc.getName());
    request.setModel(modelDesc.getName());
    request.setProject(modelDesc.getProject());
    request.setTableToProjects(tableToProjects);

    cacheController.clearCacheForCubeMigration(request);

    assertEquals(2018, cubeDescManager.getCubeDesc(CUBENAME).getRetentionRange());
    assertEquals(signature, cubeDescManager.getCubeDesc(CUBENAME).getSignature());
    assertNotEquals(uuid, cubeDescManager.getCubeDesc(CUBENAME).getUuid());
}
Example 6
Source File: StreamingCubeRule.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    DataModelDesc model = cube.getModel();
    
    if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING
            && !model.getRootFactTable().getTableDesc().isStreamingTable()) {
        return;
    }

    if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) {
        context.addResult(ResultLevel.ERROR, "Must define a partition column.");
        return;
    }

    final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef();
    boolean found = false;
    for (DimensionDesc dimensionDesc : cube.getDimensions()) {
        for (TblColRef dimCol : dimensionDesc.getColumnRefs()) {
            if (dimCol.equals(partitionCol)) {
                found = true;
                break;
            }
        }
    }

    if (found == false) {
        context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list.");
        return;
    }

}
Example 7
Source File: FunctionRule.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
/**
 * @param context
 * @param cube
 * @param value
 */
private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) {
    DataModelDesc model = cube.getModel();
    try {
        model.findColumn(value);
    } catch (IllegalArgumentException e) {
        context.addResult(ResultLevel.ERROR, e.getMessage());
    }
}
Example 8
Source File: CubeInstance.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public DataModelDesc getModel() {
    CubeDesc cubeDesc = this.getDescriptor();
    if (cubeDesc != null) {
        return cubeDesc.getModel();
    } else {
        return null;
    }
}
Example 9
Source File: RealizationCheckTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRealizationCheck() {
    RealizationCheck realizationCheck = new RealizationCheck();
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ssb");
    DataModelDesc dataModelDesc = cubeDesc.getModel();
    IRealization iRealization = CubeInstance.create("ssb", cubeDesc);
    realizationCheck.addCubeIncapableReason(iRealization,
            RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.CUBE_NOT_CONTAIN_ALL_COLUMN));
    realizationCheck.addCubeIncapableReason(iRealization,
            RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.CUBE_NOT_CONTAIN_ALL_COLUMN));
    Assert.assertTrue(realizationCheck.getCubeIncapableReasons().size() == 1);

    realizationCheck.addModelIncapableReason(dataModelDesc,
            RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.CUBE_NOT_CONTAIN_ALL_COLUMN));
    realizationCheck.addModelIncapableReason(dataModelDesc,
            RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.CUBE_NOT_CONTAIN_ALL_MEASURE));
    realizationCheck.addModelIncapableReason(dataModelDesc,
            RealizationCheck.IncapableReason.notContainAllColumn(Lists.<TblColRef> newArrayList()));
    realizationCheck.addModelIncapableReason(dataModelDesc,
            RealizationCheck.IncapableReason.notContainAllColumn(Lists.<TblColRef> newArrayList()));
    Assert.assertTrue(realizationCheck.getModelIncapableReasons().size() == 1);
    Assert.assertTrue(realizationCheck.getModelIncapableReasons().get(dataModelDesc).size() == 3);

    realizationCheck.addModelIncapableReason(dataModelDesc, RealizationCheck.IncapableReason
            .notContainAllColumn(Lists.<TblColRef> newArrayList(dataModelDesc.findColumn("LO_DATE"))));
    Assert.assertTrue(realizationCheck.getModelIncapableReasons().size() == 1);
    Assert.assertTrue(realizationCheck.getModelIncapableReasons().get(dataModelDesc).size() == 4);
}
Example 10
Source File: CacheControllerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testClearCacheForCubeMigration() throws IOException {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    String CUBENAME = "test_kylin_cube_without_slr_desc";

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
    CubeDesc cubeDesc = cubeDescManager.getCubeDesc(CUBENAME);
    DataModelDesc modelDesc = cubeDesc.getModel();
    Map<String, String> tableToProjects = new HashMap<>();
    for (TableRef tableRef : modelDesc.getAllTables()) {
        tableToProjects.put(tableRef.getTableIdentity(), tableRef.getTableDesc().getProject());
    }

    String uuid = cubeDesc.getUuid();
    String signature = cubeDesc.getSignature();

    assertEquals(cubeDesc.getRetentionRange(), 0);

    //update cubeDesc
    cubeDesc.setRetentionRange(2018);
    cubeDesc.updateRandomUuid();

    //directly update metadata in store to simulate cube migration
    Serializer<CubeDesc> cubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class);
    getStore().checkAndPutResource(cubeDesc.getResourcePath(), cubeDesc, cubeDescSerializer);

    CubeMigrationRequest request = new CubeMigrationRequest();
    request.setCube(cubeDesc.getName());
    request.setModel(modelDesc.getName());
    request.setProject(modelDesc.getProject());
    request.setTableToProjects(tableToProjects);

    cacheController.clearCacheForCubeMigration(request);

    assertEquals(2018, cubeDescManager.getCubeDesc(CUBENAME).getRetentionRange());
    assertEquals(signature, cubeDescManager.getCubeDesc(CUBENAME).getSignature());
    assertNotEquals(uuid, cubeDescManager.getCubeDesc(CUBENAME).getUuid());
}
Example 11
Source File: StreamingCubeRule.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    DataModelDesc model = cube.getModel();
    
    if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING
            && !model.getRootFactTable().getTableDesc().isStreamingTable()) {
        return;
    }

    if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) {
        context.addResult(ResultLevel.ERROR, "Must define a partition column.");
        return;
    }

    final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef();
    boolean found = false;
    for (DimensionDesc dimensionDesc : cube.getDimensions()) {
        for (TblColRef dimCol : dimensionDesc.getColumnRefs()) {
            if (dimCol.equals(partitionCol)) {
                found = true;
                break;
            }
        }
    }

    if (found == false) {
        context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list.");
        return;
    }

}
Example 12
Source File: FunctionRule.java    From kylin with Apache License 2.0 5 votes vote down vote up 
/**
 * @param context
 * @param cube
 * @param value
 */
private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) {
    DataModelDesc model = cube.getModel();
    try {
        model.findColumn(value);
    } catch (IllegalArgumentException e) {
        context.addResult(ResultLevel.ERROR, e.getMessage());
    }
}
Example 13
Source File: CubeInstance.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public DataModelDesc getModel() {
    CubeDesc cubeDesc = this.getDescriptor();
    if (cubeDesc != null) {
        return cubeDesc.getModel();
    } else {
        return null;
    }
}