org.apache.kylin.dict.lookup.SnapshotManager Java Examples

The following examples show how to use org.apache.kylin.dict.lookup.SnapshotManager. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LookupSnapshotToMetaStoreStep.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig kylinConfig = context.getConfig();
    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
    TableMetadataManager metaMgr = TableMetadataManager.getInstance(kylinConfig);
    SnapshotManager snapshotMgr = SnapshotManager.getInstance(kylinConfig);
    CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams()));
    List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams());
    String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams());
    CubeDesc cubeDesc = cube.getDescriptor();
    try {
        TableDesc tableDesc = metaMgr.getTableDesc(lookupTableName, cube.getProject());
        IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null);
        logger.info("take snapshot for table:" + lookupTableName);
        SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cube.getConfig());

        logger.info("update snapshot path to cube metadata");
        if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) {
            LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName,
                    snapshot.getResourcePath());
        } else {
            LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName,
                    snapshot.getResourcePath());
        }
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to build snapshot for:" + lookupTableName, e);
        return ExecuteResult.createError(e);
    }
}
Example #2
Source File: CubeManager.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable, String uuid)
        throws IOException {
    // work on copy instead of cached objects
    CubeInstance cubeCopy = cubeSeg.getCubeInstance().latestCopyForWrite(); // get a latest copy
    CubeSegment segCopy = cubeCopy.getSegmentById(cubeSeg.getUuid());

    TableMetadataManager metaMgr = getTableManager();
    SnapshotManager snapshotMgr = getSnapshotManager();

    TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, segCopy.getProject()));
    IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, uuid);
    SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cubeSeg.getConfig());

    CubeDesc cubeDesc = cubeSeg.getCubeDesc();
    if (!cubeDesc.isGlobalSnapshotTable(lookupTable)) {
        segCopy.putSnapshotResPath(lookupTable, snapshot.getResourcePath());
        CubeUpdate update = new CubeUpdate(cubeCopy);
        update.setToUpdateSegs(segCopy);
        updateCube(update);

        // Update the input cubeSeg after the resource store updated
        cubeSeg.putSnapshotResPath(lookupTable, segCopy.getSnapshotResPath(lookupTable));
    } else {
        CubeUpdate cubeUpdate = new CubeUpdate(cubeCopy);
        Map<String, String> map = Maps.newHashMap();
        map.put(lookupTable, snapshot.getResourcePath());
        cubeUpdate.setUpdateTableSnapshotPath(map);
        updateCube(cubeUpdate);

        cubeSeg.getCubeInstance().putSnapshotResPath(lookupTable, snapshot.getResourcePath());
    }
    return snapshot;
}
Example #3
Source File: SparkBuildDictionary.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private String buildSnapshotTable(KylinConfig config, CubeSegment cubeSeg, String lookupTable, String uuid) throws IOException{
    CubeInstance cubeCopy = cubeSeg.getCubeInstance().latestCopyForWrite(); // get a latest copy
    CubeSegment segCopy = cubeCopy.getSegmentById(cubeSeg.getUuid());

    TableMetadataManager metaMgr = TableMetadataManager.getInstance(config);
    SnapshotManager snapshotMgr = SnapshotManager.getInstance(config);

    TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, segCopy.getProject()));
    IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, uuid);
    SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cubeSeg.getConfig());
    return snapshot.getResourcePath();
}
Example #4
Source File: SrcClusterUtil.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public SrcClusterUtil(String configURI, boolean ifJobFSHAEnabled, boolean ifHBaseFSHAEnabled) throws IOException {
    super(configURI, ifJobFSHAEnabled, ifHBaseFSHAEnabled);

    this.hbaseDataDir = hbaseConf.get(hbaseRootDirConfKey) + "/data/default/";
    metadataManager = TableMetadataManager.getInstance(kylinConfig);
    modelManager = DataModelManager.getInstance(kylinConfig);
    projectManager = ProjectManager.getInstance(kylinConfig);
    hybridManager = HybridManager.getInstance(kylinConfig);
    cubeManager = CubeManager.getInstance(kylinConfig);
    cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
    dictionaryManager = DictionaryManager.getInstance(kylinConfig);
    snapshotManager = SnapshotManager.getInstance(kylinConfig);
    extSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig);
}
Example #5
Source File: LookupSnapshotToMetaStoreStep.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig kylinConfig = context.getConfig();
    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
    TableMetadataManager metaMgr = TableMetadataManager.getInstance(kylinConfig);
    SnapshotManager snapshotMgr = SnapshotManager.getInstance(kylinConfig);
    CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams()));
    List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams());
    String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams());
    CubeDesc cubeDesc = cube.getDescriptor();
    try {
        TableDesc tableDesc = metaMgr.getTableDesc(lookupTableName, cube.getProject());
        IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null);
        logger.info("take snapshot for table:" + lookupTableName);
        SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cube.getConfig());

        logger.info("update snapshot path to cube metadata");
        if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) {
            LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName,
                    snapshot.getResourcePath());
        } else {
            LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName,
                    snapshot.getResourcePath());
        }
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to build snapshot for:" + lookupTableName, e);
        return ExecuteResult.createError(e);
    }
}
Example #6
Source File: CubeManager.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable, String uuid)
        throws IOException {
    // work on copy instead of cached objects
    CubeInstance cubeCopy = cubeSeg.getCubeInstance().latestCopyForWrite(); // get a latest copy
    CubeSegment segCopy = cubeCopy.getSegmentById(cubeSeg.getUuid());

    TableMetadataManager metaMgr = getTableManager();
    SnapshotManager snapshotMgr = getSnapshotManager();

    TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, segCopy.getProject()));
    IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, uuid);
    SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cubeSeg.getConfig());

    CubeDesc cubeDesc = cubeSeg.getCubeDesc();
    if (!cubeDesc.isGlobalSnapshotTable(lookupTable)) {
        segCopy.putSnapshotResPath(lookupTable, snapshot.getResourcePath());
        CubeUpdate update = new CubeUpdate(cubeCopy);
        update.setToUpdateSegs(segCopy);
        updateCube(update);

        // Update the input cubeSeg after the resource store updated
        cubeSeg.putSnapshotResPath(lookupTable, segCopy.getSnapshotResPath(lookupTable));
    } else {
        CubeUpdate cubeUpdate = new CubeUpdate(cubeCopy);
        Map<String, String> map = Maps.newHashMap();
        map.put(lookupTable, snapshot.getResourcePath());
        cubeUpdate.setUpdateTableSnapshotPath(map);
        updateCube(cubeUpdate);

        cubeSeg.getCubeInstance().putSnapshotResPath(lookupTable, snapshot.getResourcePath());
    }
    return snapshot;
}
Example #7
Source File: SparkBuildDictionary.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private String buildSnapshotTable(KylinConfig config, CubeSegment cubeSeg, String lookupTable, String uuid) throws IOException{
    CubeInstance cubeCopy = cubeSeg.getCubeInstance().latestCopyForWrite(); // get a latest copy
    CubeSegment segCopy = cubeCopy.getSegmentById(cubeSeg.getUuid());

    TableMetadataManager metaMgr = TableMetadataManager.getInstance(config);
    SnapshotManager snapshotMgr = SnapshotManager.getInstance(config);

    TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, segCopy.getProject()));
    IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, uuid);
    SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cubeSeg.getConfig());
    return snapshot.getResourcePath();
}
Example #8
Source File: CubeManager.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable) throws IOException {
    MetadataManager metaMgr = getMetadataManager();
    SnapshotManager snapshotMgr = getSnapshotManager();

    HiveTable hiveTable = new HiveTable(metaMgr, lookupTable);
    TableDesc tableDesc = metaMgr.getTableDesc(lookupTable);
    SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc);

    cubeSeg.putSnapshotResPath(lookupTable, snapshot.getResourcePath());

    saveResource(cubeSeg.getCubeInstance());

    return snapshot;
}
Example #9
Source File: ITSnapshotManagerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Before
public void setup() throws Exception {
    createTestMetadata();
    snapshotMgr = SnapshotManager.getInstance(getTestConfig());
}
Example #10
Source File: CubeManager.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private SnapshotManager getSnapshotManager() {
    return SnapshotManager.getInstance(config);
}
Example #11
Source File: ITSnapshotManagerTest.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Before
public void setup() throws Exception {
    createTestMetadata();
    snapshotMgr = SnapshotManager.getInstance(getTestConfig());
}
Example #12
Source File: CubeManager.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private SnapshotManager getSnapshotManager() {
    return SnapshotManager.getInstance(config);
}
Example #13
Source File: SnapshotManagerTest.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
@Before
public void setup() throws Exception {
    createTestMetadata();
    snapshotMgr = SnapshotManager.getInstance(getTestConfig());
}
Example #14
Source File: CubeManager.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
private SnapshotManager getSnapshotManager() {
    return SnapshotManager.getInstance(config);
}