Java Code Examples for org.apache.kylin.metadata.tuple.ITupleIterator#EMPTY_TUPLE_ITERATOR

The following examples show how to use org.apache.kylin.metadata.tuple.ITupleIterator#EMPTY_TUPLE_ITERATOR . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: GTCubeStorageQueryBase.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
    GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);

    List<CubeSegmentScanner> scanners = Lists.newArrayList();
    SegmentPruner segPruner = new SegmentPruner(sqlDigest.filter);
    for (CubeSegment cubeSeg : segPruner.listSegmentsForQuery(cubeInstance)) {
        CubeSegmentScanner scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), //
                request.getGroups(), request.getDynGroups(), request.getDynGroupExprs(), //
                request.getMetrics(), request.getDynFuncs(), //
                request.getFilter(), request.getHavingFilter(), request.getContext());
        if (!scanner.isSegmentSkipped())
            scanners.add(scanner);
    }

    if (scanners.isEmpty())
        return ITupleIterator.EMPTY_TUPLE_ITERATOR;

    return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(),
            request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo,
            request.getContext(), sqlDigest);
}

Example 2

Source File: GTCubeStorageQueryBase.java From kylin with Apache License 2.0

6 votes

@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
    GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);

    List<CubeSegmentScanner> scanners = Lists.newArrayList();
    SegmentPruner segPruner = new SegmentPruner(sqlDigest.filter);
    for (CubeSegment cubeSeg : segPruner.listSegmentsForQuery(cubeInstance)) {
        CubeSegmentScanner scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), //
                request.getGroups(), request.getDynGroups(), request.getDynGroupExprs(), //
                request.getMetrics(), request.getDynFuncs(), //
                request.getFilter(), request.getHavingFilter(), request.getContext());
        if (!scanner.isSegmentSkipped())
            scanners.add(scanner);
    }

    if (scanners.isEmpty())
        return ITupleIterator.EMPTY_TUPLE_ITERATOR;

    return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(),
            request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo,
            request.getContext(), sqlDigest);
}

Example 3

Source File: SerializedHBaseTupleIterator.java From Kylin with Apache License 2.0

6 votes

public SerializedHBaseTupleIterator(HConnection conn, List<HBaseKeyRange> segmentKeyRanges, CubeInstance cube, Collection<TblColRef> dimensions, TupleFilter filter, Collection<TblColRef> groupBy, Collection<RowValueDecoder> rowValueDecoders, StorageContext context) {

        this.context = context;
        int limit = context.getLimit();
        this.partialResultLimit = Math.max(limit, PARTIAL_DEFAULT_LIMIT);

        this.segmentIteratorList = new ArrayList<CubeSegmentTupleIterator>(segmentKeyRanges.size());
        Map<CubeSegment, List<HBaseKeyRange>> rangesMap = makeRangesMap(segmentKeyRanges);
        for (Map.Entry<CubeSegment, List<HBaseKeyRange>> entry : rangesMap.entrySet()) {
            CubeSegmentTupleIterator segIter = new CubeSegmentTupleIterator(entry.getKey(), entry.getValue(), conn, dimensions, filter, groupBy, rowValueDecoders, context);
            this.segmentIteratorList.add(segIter);
        }

        this.segmentIteratorIterator = this.segmentIteratorList.iterator();
        if (this.segmentIteratorIterator.hasNext()) {
            this.segmentIterator = this.segmentIteratorIterator.next();
        } else {
            this.segmentIterator = ITupleIterator.EMPTY_TUPLE_ITERATOR;
        }
    }

Example 4

Source File: StreamStorageQuery.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
    GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);

    List<CubeSegmentScanner> scanners = Lists.newArrayList();
    long maxHistorySegmentTime = -1;
    StreamingDataQueryPlanner segmentsPlanner = new StreamingDataQueryPlanner(cubeInstance.getDescriptor(),
            request.getFilter());
    for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
        TSRange segmentRange = cubeSeg.getTSRange();
        if (segmentRange.end.v > maxHistorySegmentTime) {
            maxHistorySegmentTime = cubeSeg.getTSRange().end.v;
        }
        CubeSegmentScanner scanner;

        if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) {
            logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
            continue;
        }

        if (segmentsPlanner.canSkip(segmentRange.start.v, segmentRange.end.v)) {
            logger.info("Skip cube segment {} because of not satisfy filter:{}", cubeSeg, request.getFilter());
            continue;
        }

        scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(),
                request.getGroups(), request.getDynGroups(), request.getDynGroupExprs(), request.getMetrics(),
                request.getDynFuncs(), request.getFilter(), request.getHavingFilter(), request.getContext());
        if (!scanner.isSegmentSkipped())
            scanners.add(scanner);
    }

    ITupleIterator historyResult;
    if (scanners.isEmpty()) {
        historyResult = ITupleIterator.EMPTY_TUPLE_ITERATOR;
    } else {
        historyResult = new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(),
                request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo, context, sqlDigest);
    }
    Set<TblColRef> dimensionsD = request.getDimensions();
    if (dimensionsD.isEmpty()) {
        dimensionsD = Sets.newHashSet(request.getCuboid().getColumns()); // temporary fix for query like: select count(1) from TABLE
    }

    ITupleIterator realTimeResult;
    if (segmentsPlanner.canSkip(maxHistorySegmentTime, Long.MAX_VALUE)) {
        logger.info("Skip scan realTime data, {}", maxHistorySegmentTime);
        realTimeResult = ITupleIterator.EMPTY_TUPLE_ITERATOR;
    } else {
        boolean isSelectAllQuery = isSelectAllQuery(request.getCuboid(), request.getGroups(), request.getFilter());
        int limitPushDown = isSelectAllQuery ? context.getFinalPushDownLimit() : Integer.MAX_VALUE;
        realTimeResult = realTimeSearchClient.search(maxHistorySegmentTime, cubeInstance, returnTupleInfo,
                request.getFilter(), dimensionsD, request.getGroups(), request.getMetrics(), limitPushDown,
                !isSelectAllQuery);
    }
    return new CompoundTupleIterator(Arrays.asList(historyResult, realTimeResult));
}

Example 5

Source File: StreamStorageQuery.java From kylin with Apache License 2.0

4 votes

@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
    GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);

    List<CubeSegmentScanner> scanners = Lists.newArrayList();
    long maxHistorySegmentTime = -1;
    StreamingDataQueryPlanner segmentsPlanner = new StreamingDataQueryPlanner(cubeInstance.getDescriptor(),
            request.getFilter());
    long current = System.currentTimeMillis();
    for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {
        TSRange segmentRange = cubeSeg.getTSRange();
        if (segmentRange.end.v > maxHistorySegmentTime) {
            if (cubeSeg.getTSRange().end.v < current) {
                // In normal case, the segment for future time range is not reasonable in streaming case
                maxHistorySegmentTime = cubeSeg.getTSRange().end.v;
            }
        }
        CubeSegmentScanner scanner;

        if (cubeDesc.getConfig().isSkippingEmptySegments() && cubeSeg.getInputRecords() == 0) {
            logger.info("Skip cube segment {} because its input record is 0", cubeSeg);
            continue;
        }

        if (segmentsPlanner.canSkip(segmentRange.start.v, segmentRange.end.v)) {
            logger.info("Skip cube segment {} because of not satisfy filter:{}", cubeSeg, request.getFilter());
            continue;
        }

        scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(),
                request.getGroups(), request.getDynGroups(), request.getDynGroupExprs(), request.getMetrics(),
                request.getDynFuncs(), request.getFilter(), request.getHavingFilter(), request.getContext());
        if (!scanner.isSegmentSkipped())
            scanners.add(scanner);
    }

    ITupleIterator historyResult;
    if (scanners.isEmpty()) {
        historyResult = ITupleIterator.EMPTY_TUPLE_ITERATOR;
    } else {
        historyResult = new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(),
                request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo, context, sqlDigest);
    }
    Set<TblColRef> dimensionsD = request.getDimensions();
    if (dimensionsD.isEmpty()) {
        dimensionsD = Sets.newHashSet(request.getCuboid().getColumns()); // temporary fix for query like: select count(1) from TABLE
    }

    ITupleIterator realTimeResult;
    if (segmentsPlanner.canSkip(maxHistorySegmentTime, Long.MAX_VALUE)) {
        logger.info("Skip scan realTime data, {}", maxHistorySegmentTime);
        realTimeResult = ITupleIterator.EMPTY_TUPLE_ITERATOR;
    } else {
        boolean isSelectAllQuery = isSelectAllQuery(request.getCuboid(), request.getGroups(), request.getFilter());
        int limitPushDown = isSelectAllQuery ? context.getFinalPushDownLimit() : Integer.MAX_VALUE;
        realTimeResult = realTimeSearchClient.search(maxHistorySegmentTime, cubeInstance, returnTupleInfo,
                request.getFilter(), dimensionsD, request.getGroups(), request.getMetrics(), limitPushDown,
                !isSelectAllQuery);
    }
    return new CompoundTupleIterator(Arrays.asList(historyResult, realTimeResult));
}

Example 6

Source File: EndpointTupleIterator.java From Kylin with Apache License 2.0

4 votes

public EndpointTupleIterator(IISegment segment, TupleFilter rootFilter, Collection<TblColRef> groupBy, List<FunctionDesc> measures, StorageContext context, HConnection conn) throws Throwable {

        String tableName = segment.getStorageLocationIdentifier();
        table = conn.getTable(tableName);
        factTableName = segment.getIIDesc().getFactTableName();

        if (rootFilter == null) {
            rootFilter = ConstantTupleFilter.TRUE;
        }

        if (groupBy == null) {
            groupBy = Sets.newHashSet();
        }

        if (measures == null) {
            measures = Lists.newArrayList();
        }

        //this method will change measures
        rewriteMeasureParameters(measures, segment.getColumns());

        this.seg = segment;
        this.context = context;
        this.measures = measures;

        this.columns = segment.getColumns();
        this.columnNames = getColumnNames(columns);

        this.tupleInfo = buildTupleInfo();
        this.tableRecordInfo = new TableRecordInfo(this.seg);

        this.pushedDownRowType = CoprocessorRowType.fromTableRecordInfo(tableRecordInfo, this.columns);
        this.pushedDownFilter = CoprocessorFilter.fromFilter(this.seg, rootFilter);

        for (TblColRef column : this.pushedDownFilter.getUnstrictlyFilteredColumns()) {
            groupBy.add(column);
        }

        this.pushedDownProjector = CoprocessorProjector.makeForEndpoint(tableRecordInfo, groupBy);
        this.pushedDownAggregators = EndpointAggregators.fromFunctions(tableRecordInfo, measures);

        IIProtos.IIRequest endpointRequest = prepareRequest();
        regionResponsesIterator = getResults(endpointRequest, table);

        if (this.regionResponsesIterator.hasNext()) {
            this.tupleIterator = new SingleRegionTupleIterator(this.regionResponsesIterator.next());
        } else {
            this.tupleIterator = ITupleIterator.EMPTY_TUPLE_ITERATOR;
        }
    }