Java Code Examples for org.apache.kylin.dict.lookup.SnapshotManager#getInstance()

The following examples show how to use org.apache.kylin.dict.lookup.SnapshotManager#getInstance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: LookupSnapshotToMetaStoreStep.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig kylinConfig = context.getConfig();
    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
    TableMetadataManager metaMgr = TableMetadataManager.getInstance(kylinConfig);
    SnapshotManager snapshotMgr = SnapshotManager.getInstance(kylinConfig);
    CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams()));
    List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams());
    String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams());
    CubeDesc cubeDesc = cube.getDescriptor();
    try {
        TableDesc tableDesc = metaMgr.getTableDesc(lookupTableName, cube.getProject());
        IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null);
        logger.info("take snapshot for table:" + lookupTableName);
        SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cube.getConfig());

        logger.info("update snapshot path to cube metadata");
        if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) {
            LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName,
                    snapshot.getResourcePath());
        } else {
            LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName,
                    snapshot.getResourcePath());
        }
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to build snapshot for:" + lookupTableName, e);
        return ExecuteResult.createError(e);
    }
}

Example 2

Source File: SparkBuildDictionary.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private String buildSnapshotTable(KylinConfig config, CubeSegment cubeSeg, String lookupTable, String uuid) throws IOException{
    CubeInstance cubeCopy = cubeSeg.getCubeInstance().latestCopyForWrite(); // get a latest copy
    CubeSegment segCopy = cubeCopy.getSegmentById(cubeSeg.getUuid());

    TableMetadataManager metaMgr = TableMetadataManager.getInstance(config);
    SnapshotManager snapshotMgr = SnapshotManager.getInstance(config);

    TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, segCopy.getProject()));
    IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, uuid);
    SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cubeSeg.getConfig());
    return snapshot.getResourcePath();
}

Example 3

Source File: SrcClusterUtil.java From kylin with Apache License 2.0

5 votes

public SrcClusterUtil(String configURI, boolean ifJobFSHAEnabled, boolean ifHBaseFSHAEnabled) throws IOException {
    super(configURI, ifJobFSHAEnabled, ifHBaseFSHAEnabled);

    this.hbaseDataDir = hbaseConf.get(hbaseRootDirConfKey) + "/data/default/";
    metadataManager = TableMetadataManager.getInstance(kylinConfig);
    modelManager = DataModelManager.getInstance(kylinConfig);
    projectManager = ProjectManager.getInstance(kylinConfig);
    hybridManager = HybridManager.getInstance(kylinConfig);
    cubeManager = CubeManager.getInstance(kylinConfig);
    cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
    dictionaryManager = DictionaryManager.getInstance(kylinConfig);
    snapshotManager = SnapshotManager.getInstance(kylinConfig);
    extSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig);
}

Example 4

Source File: LookupSnapshotToMetaStoreStep.java From kylin with Apache License 2.0

5 votes

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig kylinConfig = context.getConfig();
    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
    TableMetadataManager metaMgr = TableMetadataManager.getInstance(kylinConfig);
    SnapshotManager snapshotMgr = SnapshotManager.getInstance(kylinConfig);
    CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams()));
    List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams());
    String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams());
    CubeDesc cubeDesc = cube.getDescriptor();
    try {
        TableDesc tableDesc = metaMgr.getTableDesc(lookupTableName, cube.getProject());
        IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null);
        logger.info("take snapshot for table:" + lookupTableName);
        SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cube.getConfig());

        logger.info("update snapshot path to cube metadata");
        if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) {
            LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName,
                    snapshot.getResourcePath());
        } else {
            LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName,
                    snapshot.getResourcePath());
        }
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to build snapshot for:" + lookupTableName, e);
        return ExecuteResult.createError(e);
    }
}

Example 5

Source File: SparkBuildDictionary.java From kylin with Apache License 2.0

5 votes

private String buildSnapshotTable(KylinConfig config, CubeSegment cubeSeg, String lookupTable, String uuid) throws IOException{
    CubeInstance cubeCopy = cubeSeg.getCubeInstance().latestCopyForWrite(); // get a latest copy
    CubeSegment segCopy = cubeCopy.getSegmentById(cubeSeg.getUuid());

    TableMetadataManager metaMgr = TableMetadataManager.getInstance(config);
    SnapshotManager snapshotMgr = SnapshotManager.getInstance(config);

    TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, segCopy.getProject()));
    IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, uuid);
    SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cubeSeg.getConfig());
    return snapshot.getResourcePath();
}

Example 6

Source File: ITSnapshotManagerTest.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Before
public void setup() throws Exception {
    createTestMetadata();
    snapshotMgr = SnapshotManager.getInstance(getTestConfig());
}

Example 7

Source File: CubeManager.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

private SnapshotManager getSnapshotManager() {
    return SnapshotManager.getInstance(config);
}

Example 8

Source File: ITSnapshotManagerTest.java From kylin with Apache License 2.0

4 votes

@Before
public void setup() throws Exception {
    createTestMetadata();
    snapshotMgr = SnapshotManager.getInstance(getTestConfig());
}

Example 9

Source File: CubeManager.java From kylin with Apache License 2.0

4 votes

private SnapshotManager getSnapshotManager() {
    return SnapshotManager.getInstance(config);
}

Example 10

Source File: SnapshotManagerTest.java From Kylin with Apache License 2.0

4 votes

@Before
public void setup() throws Exception {
    createTestMetadata();
    snapshotMgr = SnapshotManager.getInstance(getTestConfig());
}

Example 11

Source File: CubeManager.java From Kylin with Apache License 2.0

4 votes

private SnapshotManager getSnapshotManager() {
    return SnapshotManager.getInstance(config);
}