org.apache.kylin.dict.lookup.SnapshotTable Java Examples

The following examples show how to use org.apache.kylin.dict.lookup.SnapshotTable. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CubeManager.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) {

        String tableName = dim.getTable();
        String[] pkCols = dim.getJoin().getPrimaryKey();
        String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
        if (snapshotResPath == null)
            throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment);

        try {
            SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath);
            TableDesc tableDesc = getMetadataManager().getTableDesc(tableName);
            return new LookupStringTable(tableDesc, pkCols, snapshot);
        } catch (IOException e) {
            throw new IllegalStateException("Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e);
        }
    }
Example #2
Source File: LookupSnapshotToMetaStoreStep.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig kylinConfig = context.getConfig();
    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
    TableMetadataManager metaMgr = TableMetadataManager.getInstance(kylinConfig);
    SnapshotManager snapshotMgr = SnapshotManager.getInstance(kylinConfig);
    CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams()));
    List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams());
    String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams());
    CubeDesc cubeDesc = cube.getDescriptor();
    try {
        TableDesc tableDesc = metaMgr.getTableDesc(lookupTableName, cube.getProject());
        IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null);
        logger.info("take snapshot for table:" + lookupTableName);
        SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cube.getConfig());

        logger.info("update snapshot path to cube metadata");
        if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) {
            LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName,
                    snapshot.getResourcePath());
        } else {
            LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName,
                    snapshot.getResourcePath());
        }
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to build snapshot for:" + lookupTableName, e);
        return ExecuteResult.createError(e);
    }
}
Example #3
Source File: CubeManager.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable) throws IOException {
    MetadataManager metaMgr = getMetadataManager();
    SnapshotManager snapshotMgr = getSnapshotManager();

    HiveTable hiveTable = new HiveTable(metaMgr, lookupTable);
    TableDesc tableDesc = metaMgr.getTableDesc(lookupTable);
    SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc);

    cubeSeg.putSnapshotResPath(lookupTable, snapshot.getResourcePath());

    saveResource(cubeSeg.getCubeInstance());

    return snapshot;
}
Example #4
Source File: SnapshotManagerTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void basicTest() throws Exception {
    String tableName = "EDW.TEST_SITES";
    HiveTable hiveTable = new HiveTable(MetadataManager.getInstance(getTestConfig()), tableName);
    TableDesc tableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc(tableName);
    String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc).getResourcePath();

    snapshotMgr.wipeoutCache();

    SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath);

    // compare hive & snapshot
    TableReader hiveReader = hiveTable.getReader();
    TableReader snapshotReader = snapshot.getReader();

    while (true) {
        boolean hiveNext = hiveReader.next();
        boolean snapshotNext = snapshotReader.next();
        assertEquals(hiveNext, snapshotNext);

        if (hiveNext == false)
            break;

        String[] hiveRow = hiveReader.getRow();
        String[] snapshotRow = snapshotReader.getRow();
        assertArrayEquals(hiveRow, snapshotRow);
    }
}
Example #5
Source File: SparkBuildDictionary.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private String buildSnapshotTable(KylinConfig config, CubeSegment cubeSeg, String lookupTable, String uuid) throws IOException{
    CubeInstance cubeCopy = cubeSeg.getCubeInstance().latestCopyForWrite(); // get a latest copy
    CubeSegment segCopy = cubeCopy.getSegmentById(cubeSeg.getUuid());

    TableMetadataManager metaMgr = TableMetadataManager.getInstance(config);
    SnapshotManager snapshotMgr = SnapshotManager.getInstance(config);

    TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, segCopy.getProject()));
    IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, uuid);
    SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cubeSeg.getConfig());
    return snapshot.getResourcePath();
}
Example #6
Source File: CubeManager.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable, String uuid)
        throws IOException {
    // work on copy instead of cached objects
    CubeInstance cubeCopy = cubeSeg.getCubeInstance().latestCopyForWrite(); // get a latest copy
    CubeSegment segCopy = cubeCopy.getSegmentById(cubeSeg.getUuid());

    TableMetadataManager metaMgr = getTableManager();
    SnapshotManager snapshotMgr = getSnapshotManager();

    TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, segCopy.getProject()));
    IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, uuid);
    SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cubeSeg.getConfig());

    CubeDesc cubeDesc = cubeSeg.getCubeDesc();
    if (!cubeDesc.isGlobalSnapshotTable(lookupTable)) {
        segCopy.putSnapshotResPath(lookupTable, snapshot.getResourcePath());
        CubeUpdate update = new CubeUpdate(cubeCopy);
        update.setToUpdateSegs(segCopy);
        updateCube(update);

        // Update the input cubeSeg after the resource store updated
        cubeSeg.putSnapshotResPath(lookupTable, segCopy.getSnapshotResPath(lookupTable));
    } else {
        CubeUpdate cubeUpdate = new CubeUpdate(cubeCopy);
        Map<String, String> map = Maps.newHashMap();
        map.put(lookupTable, snapshot.getResourcePath());
        cubeUpdate.setUpdateTableSnapshotPath(map);
        updateCube(cubeUpdate);

        cubeSeg.getCubeInstance().putSnapshotResPath(lookupTable, snapshot.getResourcePath());
    }
    return snapshot;
}
Example #7
Source File: CubeManager.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private ILookupTable getInMemLookupTable(CubeSegment cubeSegment, JoinDesc join,
        SnapshotTableDesc snapshotTableDesc) {
    String tableName = join.getPKSide().getTableIdentity();
    String snapshotResPath = getSnapshotResPath(cubeSegment, tableName, snapshotTableDesc);
    String[] pkCols = join.getPrimaryKey();

    try {
        SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath);
        TableDesc tableDesc = getMetadataManager().getTableDesc(tableName, cubeSegment.getProject());
        return LookupProviderFactory.getInMemLookupTable(tableDesc, pkCols, snapshot);
    } catch (IOException e) {
        throw new IllegalStateException(
                "Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e);
    }
}
Example #8
Source File: LookupSnapshotToMetaStoreStep.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig kylinConfig = context.getConfig();
    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
    TableMetadataManager metaMgr = TableMetadataManager.getInstance(kylinConfig);
    SnapshotManager snapshotMgr = SnapshotManager.getInstance(kylinConfig);
    CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams()));
    List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams());
    String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams());
    CubeDesc cubeDesc = cube.getDescriptor();
    try {
        TableDesc tableDesc = metaMgr.getTableDesc(lookupTableName, cube.getProject());
        IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null);
        logger.info("take snapshot for table:" + lookupTableName);
        SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cube.getConfig());

        logger.info("update snapshot path to cube metadata");
        if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) {
            LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName,
                    snapshot.getResourcePath());
        } else {
            LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName,
                    snapshot.getResourcePath());
        }
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to build snapshot for:" + lookupTableName, e);
        return ExecuteResult.createError(e);
    }
}
Example #9
Source File: ITSnapshotManagerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void basicTest() throws Exception {
    String tableName = "EDW.TEST_SITES";
    TableDesc tableDesc = TableMetadataManager.getInstance(getTestConfig()).getTableDesc(tableName, "default");
    IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null);
    String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc, getTestConfig()).getResourcePath();

    snapshotMgr.wipeoutCache();

    SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath);

    // compare hive & snapshot
    TableReader hiveReader = hiveTable.getReader();
    TableReader snapshotReader = snapshot.getReader();

    while (true) {
        boolean hiveNext = hiveReader.next();
        boolean snapshotNext = snapshotReader.next();
        assertEquals(hiveNext, snapshotNext);

        if (hiveNext == false)
            break;

        String[] hiveRow = hiveReader.getRow();
        String[] snapshotRow = snapshotReader.getRow();
        assertArrayEquals(hiveRow, snapshotRow);
    }
}
Example #10
Source File: CubeMigrationCrossClusterCLI.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void copySnapshot(CubeInstance cube, String snapshotTablePath) throws IOException {
    if (dstCluster.exists(snapshotTablePath)) {
        logger.info("Item {} has already existed in destination cluster", snapshotTablePath);
        return;
    }
    SnapshotTable snapshotTable = srcCluster.getSnapshotTable(snapshotTablePath);
    dstCluster.saveSnapshotTable(snapshotTable);
}
Example #11
Source File: SparkBuildDictionary.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private String buildSnapshotTable(KylinConfig config, CubeSegment cubeSeg, String lookupTable, String uuid) throws IOException{
    CubeInstance cubeCopy = cubeSeg.getCubeInstance().latestCopyForWrite(); // get a latest copy
    CubeSegment segCopy = cubeCopy.getSegmentById(cubeSeg.getUuid());

    TableMetadataManager metaMgr = TableMetadataManager.getInstance(config);
    SnapshotManager snapshotMgr = SnapshotManager.getInstance(config);

    TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, segCopy.getProject()));
    IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, uuid);
    SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cubeSeg.getConfig());
    return snapshot.getResourcePath();
}
Example #12
Source File: CubeManager.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable, String uuid)
        throws IOException {
    // work on copy instead of cached objects
    CubeInstance cubeCopy = cubeSeg.getCubeInstance().latestCopyForWrite(); // get a latest copy
    CubeSegment segCopy = cubeCopy.getSegmentById(cubeSeg.getUuid());

    TableMetadataManager metaMgr = getTableManager();
    SnapshotManager snapshotMgr = getSnapshotManager();

    TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, segCopy.getProject()));
    IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, uuid);
    SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cubeSeg.getConfig());

    CubeDesc cubeDesc = cubeSeg.getCubeDesc();
    if (!cubeDesc.isGlobalSnapshotTable(lookupTable)) {
        segCopy.putSnapshotResPath(lookupTable, snapshot.getResourcePath());
        CubeUpdate update = new CubeUpdate(cubeCopy);
        update.setToUpdateSegs(segCopy);
        updateCube(update);

        // Update the input cubeSeg after the resource store updated
        cubeSeg.putSnapshotResPath(lookupTable, segCopy.getSnapshotResPath(lookupTable));
    } else {
        CubeUpdate cubeUpdate = new CubeUpdate(cubeCopy);
        Map<String, String> map = Maps.newHashMap();
        map.put(lookupTable, snapshot.getResourcePath());
        cubeUpdate.setUpdateTableSnapshotPath(map);
        updateCube(cubeUpdate);

        cubeSeg.getCubeInstance().putSnapshotResPath(lookupTable, snapshot.getResourcePath());
    }
    return snapshot;
}
Example #13
Source File: CubeManager.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private ILookupTable getInMemLookupTable(CubeSegment cubeSegment, JoinDesc join,
        SnapshotTableDesc snapshotTableDesc) {
    String tableName = join.getPKSide().getTableIdentity();
    String snapshotResPath = getSnapshotResPath(cubeSegment, tableName, snapshotTableDesc);
    String[] pkCols = join.getPrimaryKey();

    try {
        SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath);
        TableDesc tableDesc = getMetadataManager().getTableDesc(tableName, cubeSegment.getProject());
        return LookupProviderFactory.getInMemLookupTable(tableDesc, pkCols, snapshot);
    } catch (IOException e) {
        throw new IllegalStateException(
                "Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e);
    }
}
Example #14
Source File: ITSnapshotManagerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void basicTest() throws Exception {
    String tableName = "EDW.TEST_SITES";
    TableDesc tableDesc = TableMetadataManager.getInstance(getTestConfig()).getTableDesc(tableName, "default");
    IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null);
    String snapshotPath = snapshotMgr.buildSnapshot(hiveTable, tableDesc, getTestConfig()).getResourcePath();

    snapshotMgr.wipeoutCache();

    SnapshotTable snapshot = snapshotMgr.getSnapshotTable(snapshotPath);

    // compare hive & snapshot
    TableReader hiveReader = hiveTable.getReader();
    TableReader snapshotReader = snapshot.getReader();

    while (true) {
        boolean hiveNext = hiveReader.next();
        boolean snapshotNext = snapshotReader.next();
        assertEquals(hiveNext, snapshotNext);

        if (hiveNext == false)
            break;

        String[] hiveRow = hiveReader.getRow();
        String[] snapshotRow = snapshotReader.getRow();
        assertArrayEquals(hiveRow, snapshotRow);
    }
}
Example #15
Source File: SrcClusterUtil.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public SnapshotTable getSnapshotTable(String snapshotPath) throws IOException {
    return snapshotManager.getSnapshotTable(snapshotPath);
}
Example #16
Source File: DstClusterUtil.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public void saveSnapshotTable(SnapshotTable snapshotTable) throws IOException {
    putMetaResource(snapshotTable.getResourcePath(), snapshotTable, SnapshotTableSerializer.FULL_SERIALIZER);
    logger.info("saved snapshot table {}", snapshotTable.getResourcePath());
}
Example #17
Source File: DstClusterUtil.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public SnapshotTable getSnapshotTable(String snapshotPath) throws IOException {
    return resourceStore.getResource(snapshotPath, SnapshotTableSerializer.FULL_SERIALIZER);
}
Example #18
Source File: SnapshotTableDesc.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public boolean isExtSnapshotTable() {
    return !SnapshotTable.STORAGE_TYPE_METASTORE.equals(storageType);
}
Example #19
Source File: CubeManager.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable, String uuid) throws IOException {
    return dictAssist.buildSnapshotTable(cubeSeg, lookupTable, uuid);
}
Example #20
Source File: CubeManager.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable, String uuid) throws IOException {
    return dictAssist.buildSnapshotTable(cubeSeg, lookupTable, uuid);
}
Example #21
Source File: SnapshotTableDesc.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public boolean isExtSnapshotTable() {
    return !SnapshotTable.STORAGE_TYPE_METASTORE.equals(storageType);
}
Example #22
Source File: ClusterUtil.java    From kylin with Apache License 2.0 votes vote down vote up 
public abstract SnapshotTable getSnapshotTable(String snapshotPath) throws IOException;