Java Code Examples for org.apache.kylin.cube.CubeSegment#getStorageLocationIdentifier()

The following examples show how to use org.apache.kylin.cube.CubeSegment#getStorageLocationIdentifier() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: UpdateHTableHostCLI.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private static List<String> filterByCubes(List<String> allTableNames, List<String> cubeNames) {
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    List<String> result = Lists.newArrayList();
    for (String c : cubeNames) {
        c = c.trim();
        if (c.endsWith(","))
            c = c.substring(0, c.length() - 1);

        CubeInstance cubeInstance = cubeManager.getCube(c);
        for (CubeSegment segment : cubeInstance.getSegments()) {
            String tableName = segment.getStorageLocationIdentifier();
            if (allTableNames.contains(tableName)) {
                result.add(tableName);
            }
        }
    }
    return result;
}
Example 2
Source File: KylinHealthCheckJob.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private void checkHBaseTables(List<CubeInstance> cubes) throws IOException {
    reporter.log("## Checking HBase Table of segments");
    HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create());
    try {
        for (CubeInstance cube : cubes) {
            for (CubeSegment segment : cube.getSegments()) {
                if (segment.getStatus() != SegmentStatusEnum.NEW) {
                    String tableName = segment.getStorageLocationIdentifier();
                    if ((!hbaseAdmin.tableExists(tableName)) || (!hbaseAdmin.isTableEnabled(tableName))) {
                        reporter.log("HBase table: {} not exist for segment: {}, project: {}", tableName, segment,
                                cube.getProject());
                        reporter.log(
                                "The rebuild url: -d '{\"startTime\":{}, \"endTime\":{}, \"buildType\":\"REFRESH\"}' /kylin/api/cubes/{}/build",
                                segment.getTSRange().start, segment.getTSRange().end, cube.getName());
                    }
                }
            }
        }
    } finally {
        if (null != hbaseAdmin) {
            hbaseAdmin.close();
        }
    }

}
Example 3
Source File: UpdateHTableHostCLI.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private static List<String> getHTableNames(KylinConfig config) {
    CubeManager cubeMgr = CubeManager.getInstance(config);

    ArrayList<String> result = new ArrayList<>();
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
        for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) {
            String tableName = seg.getStorageLocationIdentifier();
            if (!StringUtils.isBlank(tableName)) {
                result.add(tableName);
                logger.info("added new table: {}", tableName);
            }
        }
    }

    return result;
}
Example 4
Source File: DeployCoprocessorCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private static List<String> filterByProjects(List<String> allTableNames, List<String> projectNames) {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());

    List<String> result = Lists.newArrayList();
    for (String p : projectNames) {
        p = p.trim();
        if (p.endsWith(",")) {
            p = p.substring(0, p.length() - 1);
        }

        ProjectInstance projectInstance = projectManager.getProject(p);
        List<RealizationEntry> cubeList = projectInstance.getRealizationEntries(RealizationType.CUBE);
        for (RealizationEntry cube : cubeList) {
            CubeInstance cubeInstance = cubeManager.getCube(cube.getRealization());
            for (CubeSegment segment : cubeInstance.getSegments()) {
                String tableName = segment.getStorageLocationIdentifier();
                if (allTableNames.contains(tableName)) {
                    result.add(tableName);
                }
            }
        }
    }
    return result;
}
Example 5
Source File: DeployCoprocessorCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private static List<String> filterByCubes(List<String> allTableNames, List<String> cubeNames) {
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    List<String> result = Lists.newArrayList();
    for (String c : cubeNames) {
        c = c.trim();
        if (c.endsWith(","))
            c = c.substring(0, c.length() - 1);

        CubeInstance cubeInstance = cubeManager.getCube(c);
        for (CubeSegment segment : cubeInstance.getSegments()) {
            String tableName = segment.getStorageLocationIdentifier();
            if (allTableNames.contains(tableName)) {
                result.add(tableName);
            }
        }
    }
    return result;
}
Example 6
Source File: DeployCoprocessorCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private static List<String> getHTableNames(KylinConfig config) {
    CubeManager cubeMgr = CubeManager.getInstance(config);

    ArrayList<String> result = new ArrayList<String>();
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
        if (cube.getStorageType() == IStorageAware.ID_HBASE
                || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE
                || cube.getStorageType() == IStorageAware.ID_REALTIME_AND_HBASE) {
            for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) {
                String tableName = seg.getStorageLocationIdentifier();
                if (StringUtils.isBlank(tableName) == false) {
                    result.add(tableName);
                    System.out.println("added new table: " + tableName);
                }
            }
        }
    }

    return result;
}
Example 7
Source File: UpdateHTableHostCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private static List<String> getHTableNames(KylinConfig config) {
    CubeManager cubeMgr = CubeManager.getInstance(config);

    ArrayList<String> result = new ArrayList<>();
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
        for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) {
            String tableName = seg.getStorageLocationIdentifier();
            if (!StringUtils.isBlank(tableName)) {
                result.add(tableName);
                logger.info("added new table: {}", tableName);
            }
        }
    }

    return result;
}
Example 8
Source File: CubeSegmentTupleIterator.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
public CubeSegmentTupleIterator(CubeSegment cubeSeg, Collection<HBaseKeyRange> keyRanges, HConnection conn, Collection<TblColRef> dimensions, TupleFilter filter, Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context) {
    this.cube = cubeSeg.getCubeInstance();
    this.cubeSeg = cubeSeg;
    this.dimensions = dimensions;
    this.filter = filter;
    this.groupBy = groupBy;
    this.rowValueDecoders = rowValueDecoders;
    this.context = context;
    this.tableName = cubeSeg.getStorageLocationIdentifier();
    this.rowKeyDecoder = new RowKeyDecoder(this.cubeSeg);
    this.scanCount = 0;

    try {
        this.table = conn.getTable(tableName);
    } catch (Throwable t) {
        throw new StorageException("Error when open connection to table " + tableName, t);
    }
    this.rangeIterator = keyRanges.iterator();
    scanNextRange();
}
Example 9
Source File: KylinHealthCheckJob.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private void checkHBaseTables(List<CubeInstance> cubes) throws IOException {
    reporter.log("## Checking HBase Table of segments");
    HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create());
    try {
        for (CubeInstance cube : cubes) {
            for (CubeSegment segment : cube.getSegments()) {
                if (segment.getStatus() != SegmentStatusEnum.NEW) {
                    String tableName = segment.getStorageLocationIdentifier();
                    if ((!hbaseAdmin.tableExists(tableName)) || (!hbaseAdmin.isTableEnabled(tableName))) {
                        reporter.log("HBase table: {} not exist for segment: {}, project: {}", tableName, segment,
                                cube.getProject());
                        reporter.log(
                                "The rebuild url: -d '{\"startTime\":{}, \"endTime\":{}, \"buildType\":\"REFRESH\"}' /kylin/api/cubes/{}/build",
                                segment.getTSRange().start, segment.getTSRange().end, cube.getName());
                    }
                }
            }
        }
    } finally {
        if (null != hbaseAdmin) {
            hbaseAdmin.close();
        }
    }

}
Example 10
Source File: DeployCoprocessorCLI.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private static List<String> getHTableNames(KylinConfig config) {
    CubeManager cubeMgr = CubeManager.getInstance(config);

    ArrayList<String> result = new ArrayList<String>();
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
        if (cube.getStorageType() == IStorageAware.ID_HBASE
                || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE
                || cube.getStorageType() == IStorageAware.ID_REALTIME_AND_HBASE) {
            for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) {
                String tableName = seg.getStorageLocationIdentifier();
                if (StringUtils.isBlank(tableName) == false) {
                    result.add(tableName);
                    System.out.println("added new table: " + tableName);
                }
            }
        }
    }

    return result;
}
Example 11
Source File: DeployCoprocessorCLI.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private static List<String> filterByCubes(List<String> allTableNames, List<String> cubeNames) {
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    List<String> result = Lists.newArrayList();
    for (String c : cubeNames) {
        c = c.trim();
        if (c.endsWith(","))
            c = c.substring(0, c.length() - 1);

        CubeInstance cubeInstance = cubeManager.getCube(c);
        for (CubeSegment segment : cubeInstance.getSegments()) {
            String tableName = segment.getStorageLocationIdentifier();
            if (allTableNames.contains(tableName)) {
                result.add(tableName);
            }
        }
    }
    return result;
}
Example 12
Source File: DeployCoprocessorCLI.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private static List<String> filterByProjects(List<String> allTableNames, List<String> projectNames) {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());

    List<String> result = Lists.newArrayList();
    for (String p : projectNames) {
        p = p.trim();
        if (p.endsWith(",")) {
            p = p.substring(0, p.length() - 1);
        }

        ProjectInstance projectInstance = projectManager.getProject(p);
        List<RealizationEntry> cubeList = projectInstance.getRealizationEntries(RealizationType.CUBE);
        for (RealizationEntry cube : cubeList) {
            CubeInstance cubeInstance = cubeManager.getCube(cube.getRealization());
            for (CubeSegment segment : cubeInstance.getSegments()) {
                String tableName = segment.getStorageLocationIdentifier();
                if (allTableNames.contains(tableName)) {
                    result.add(tableName);
                }
            }
        }
    }
    return result;
}
Example 13
Source File: BuildCubeWithEngine.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unused")
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
    try (Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl())) {
        String tableName = segment.getStorageLocationIdentifier();

        HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
        Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
        long totalSize = 0;
        for (Long size : sizeMap.values()) {
            totalSize += size;
        }
        if (totalSize == 0) {
            return;
        }
        Map<byte[], Pair<Integer, Integer>> countMap = cal.getRegionHFileCountMap();
        // check if there's region contains more than one hfile, which means the hfile config take effects
        boolean hasMultiHFileRegions = false;
        for (Pair<Integer, Integer> count : countMap.values()) {
            // check if hfile count is greater than store count
            if (count.getSecond() > count.getFirst()) {
                hasMultiHFileRegions = true;
                break;
            }
        }
        if (KylinConfig.getInstanceFromEnv().getHBaseHFileSizeGB() == 0 && hasMultiHFileRegions) {
            throw new IOException("hfile size set to 0, but found region contains more than one hfiles");
        } else if (KylinConfig.getInstanceFromEnv().getHBaseHFileSizeGB() > 0 && !hasMultiHFileRegions) {
            throw new IOException("hfile size set greater than 0, but all regions still has only one hfile");
        }
    }
}
Example 14
Source File: BuildCubeWithEngine.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unused")
private void checkHFilesInHBase(CubeSegment segment) throws IOException {
    try (Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl())) {
        String tableName = segment.getStorageLocationIdentifier();

        HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
        Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
        long totalSize = 0;
        for (Long size : sizeMap.values()) {
            totalSize += size;
        }
        if (totalSize == 0) {
            return;
        }
        Map<byte[], Pair<Integer, Integer>> countMap = cal.getRegionHFileCountMap();
        // check if there's region contains more than one hfile, which means the hfile config take effects
        boolean hasMultiHFileRegions = false;
        for (Pair<Integer, Integer> count : countMap.values()) {
            // check if hfile count is greater than store count
            if (count.getSecond() > count.getFirst()) {
                hasMultiHFileRegions = true;
                break;
            }
        }
        if (KylinConfig.getInstanceFromEnv().getHBaseHFileSizeGB() == 0 && hasMultiHFileRegions) {
            throw new IOException("hfile size set to 0, but found region contains more than one hfiles");
        } else if (KylinConfig.getInstanceFromEnv().getHBaseHFileSizeGB() > 0 && !hasMultiHFileRegions) {
            throw new IOException("hfile size set greater than 0, but all regions still has only one hfile");
        }
    }
}
Example 15
Source File: CubeMigrationCrossClusterCLI.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private boolean checkHTableExist(CubeSegment segment) throws IOException {
    String tableName = segment.getStorageLocationIdentifier();
    TableName htableName = TableName.valueOf(tableName);
    if (!dstCluster.checkExist(htableName, segment)) {
        return false;
    }

    if (!checkHTableEquals(tableName)) {
        logger.warn("although htable {} exists in destination, the details data are different", tableName);
        dstCluster.deleteHTable(tableName);
        return false;
    }
    return true;
}
Example 16
Source File: CubeMigrationCrossClusterCLI.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void copyHTable(CubeSegment segment) throws IOException {
    String tableName = segment.getStorageLocationIdentifier();
    if (ifExecute) {
        if (checkHTableExist(segment)) {
            logger.info("htable {} has already existed in dst, will skip the migration", tableName);
        } else {
            copyHTable(tableName, true);
            if (!checkHTableEquals(tableName)) {
                logger.error("htable {} is copied to dst with different size!!!", tableName);
            }
        }
    }
    logger.info("migrated htable {} for segment {}", tableName, segment);
}
Example 17
Source File: CubeMigrationCheckCLI.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public void addHTableNamesForCube(CubeInstance cube, List<String> segFullNameList) {
    for (CubeSegment seg : cube.getSegments()) {
        String tableName = seg.getStorageLocationIdentifier();
        segFullNameList.add(tableName + "," + cube.getName());
    }
}
Example 18
Source File: CubeMigrationCheckCLI.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public void addHTableNamesForCube(CubeInstance cube, List<String> segFullNameList) {
    for (CubeSegment seg : cube.getSegments()) {
        String tableName = seg.getStorageLocationIdentifier();
        segFullNameList.add(tableName + "," + cube.getName());
    }
}
Example 19
Source File: CubeMigrationCheckCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public void addHTableNamesForCube(CubeInstance cube, List<String> segFullNameList) {
    for (CubeSegment seg : cube.getSegments()) {
        String tableName = seg.getStorageLocationIdentifier();
        segFullNameList.add(tableName + "," + cube.getName());
    }
}
Example 20
Source File: CubeMigrationCheckCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public void addHTableNamesForCube(CubeInstance cube, List<String> segFullNameList) {
    for (CubeSegment seg : cube.getSegments()) {
        String tableName = seg.getStorageLocationIdentifier();
        segFullNameList.add(tableName + "," + cube.getName());
    }
}