Java Code Examples for org.apache.kylin.metadata.filter.TupleFilter#isEvaluableRecursively()

The following examples show how to use org.apache.kylin.metadata.filter.TupleFilter#isEvaluableRecursively() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CaseTupleExpression.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
@Override
protected boolean ifAbleToPushDown() {
    if (ifAbleToPushDown == null) {
        for (Pair<TupleFilter, TupleExpression> whenEntry : whenList) {
            ifAbleToPushDown = TupleFilter.isEvaluableRecursively(whenEntry.getFirst())
                    && whenEntry.getSecond().ifAbleToPushDown();
            if (!ifAbleToPushDown) {
                break;
            }
        }
        if (elseExpr != null && Boolean.TRUE.equals(ifAbleToPushDown)) {
            ifAbleToPushDown = elseExpr.ifAbleToPushDown();
        }
        if (ifAbleToPushDown == null) {
            ifAbleToPushDown = true;
        }
    }
    return ifAbleToPushDown;
}
Example 2
Source File: GTFilterScanner.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public GTFilterScanner(IGTScanner delegated, GTScanRequest req, IGTBypassChecker checker) {
    super(delegated);
    this.checker = checker;

    if (req != null) {
        this.filter = req.getFilterPushDown();
        this.filterCodeSystem = GTUtil.wrap(getInfo().codeSystem.getComparator());
        this.oneTuple = new IEvaluatableTuple() {
            @Override
            public Object getValue(TblColRef col) {
                return next.get(col.getColumnDesc().getZeroBasedIndex());
            }
        };

        if (!TupleFilter.isEvaluableRecursively(filter))
            throw new IllegalArgumentException();
    }
}
Example 3
Source File: CaseTupleExpression.java    From kylin with Apache License 2.0 6 votes vote down vote up 
@Override
protected boolean ifAbleToPushDown() {
    if (ifAbleToPushDown == null) {
        for (Pair<TupleFilter, TupleExpression> whenEntry : whenList) {
            ifAbleToPushDown = TupleFilter.isEvaluableRecursively(whenEntry.getFirst())
                    && whenEntry.getSecond().ifAbleToPushDown();
            if (!ifAbleToPushDown) {
                break;
            }
        }
        if (elseExpr != null && Boolean.TRUE.equals(ifAbleToPushDown)) {
            ifAbleToPushDown = elseExpr.ifAbleToPushDown();
        }
        if (ifAbleToPushDown == null) {
            ifAbleToPushDown = true;
        }
    }
    return ifAbleToPushDown;
}
Example 4
Source File: GTFilterScanner.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public GTFilterScanner(IGTScanner delegated, GTScanRequest req, IGTBypassChecker checker) {
    super(delegated);
    this.checker = checker;

    if (req != null) {
        this.filter = req.getFilterPushDown();
        this.filterCodeSystem = GTUtil.wrap(getInfo().codeSystem.getComparator());
        this.oneTuple = new IEvaluatableTuple() {
            @Override
            public Object getValue(TblColRef col) {
                return next.get(col.getColumnDesc().getZeroBasedIndex());
            }
        };

        if (!TupleFilter.isEvaluableRecursively(filter))
            throw new IllegalArgumentException();
    }
}
Example 5
Source File: GTUtil.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter onSerialize(TupleFilter filter) {
    if (filter == null)
        return null;

    // In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE,
    // Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter)
    // will always return FALSE.
    if (filter.getOperator() == FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // map to column onto grid table
    if (colMapping != null && filter instanceof ColumnTupleFilter) {
        ColumnTupleFilter colFilter = (ColumnTupleFilter) filter;
        int gtColIdx = mapCol(colFilter.getColumn());
        return new ColumnTupleFilter(info.colRef(gtColIdx));
    }

    // encode constants
    if (useEncodeConstants && filter instanceof CompareTupleFilter) {
        return encodeConstants((CompareTupleFilter) filter);
    }

    return filter;
}
Example 6
Source File: StreamingBuiltInFunctionTransformer.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter transform(TupleFilter tupleFilter) {
    TupleFilter translated = null;
    if (tupleFilter instanceof CompareTupleFilter) {
        //normal case
        translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
        if (translated != null) {
            logger.info("Translated {" + tupleFilter + "}");
        }
    } else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
        //like case
        translated = translateFunctionTupleFilter((BuiltInFunctionTupleFilter) tupleFilter);
        if (translated != null) {
            logger.info("Translated {" + tupleFilter + "}");
        }
    } else if (tupleFilter instanceof LogicalTupleFilter) {
        @SuppressWarnings("unchecked")
        ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren()
                .listIterator();
        while (childIterator.hasNext()) {
            TupleFilter transformed = transform(childIterator.next());
            if (transformed != null)
                childIterator.set(transformed);
        }
    }

    TupleFilter result = translated == null ? tupleFilter : translated;
    if (result.getOperator() == TupleFilter.FilterOperatorEnum.NOT
            && !TupleFilter.isEvaluableRecursively(result)) {
        TupleFilter.collectColumns(result, unEvaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!result.isEvaluable()) {
        TupleFilter.collectColumns(result, unEvaluableColumns);
        return ConstantTupleFilter.TRUE;
    }
    return result;
}
Example 7
Source File: FragmentFileSearcher.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter transform(TupleFilter filter) {
    if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT
            && !TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, unEvaluableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, unEvaluableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    if (filter instanceof CompareTupleFilter) {
        return translateCompareFilter((CompareTupleFilter) filter);
    } else if (filter instanceof LogicalTupleFilter) {
        @SuppressWarnings("unchecked")
        ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) filter.getChildren().listIterator();
        while (childIterator.hasNext()) {
            TupleFilter transformed = transform(childIterator.next());
            if (transformed != null) {
                childIterator.set(transformed);
            } else {
                throw new IllegalStateException("Should not be null");
            }
        }
    }
    return filter;
}
Example 8
Source File: GTUtil.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter onSerialize(TupleFilter filter) {
    if (filter == null)
        return null;

    // In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE,
    // Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter)
    // will always return FALSE.
    if (filter.getOperator() == FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // map to column onto grid table
    if (colMapping != null && filter instanceof ColumnTupleFilter) {
        ColumnTupleFilter colFilter = (ColumnTupleFilter) filter;
        int gtColIdx = mapCol(colFilter.getColumn());
        return new ColumnTupleFilter(info.colRef(gtColIdx));
    }

    // encode constants
    if (useEncodeConstants && filter instanceof CompareTupleFilter) {
        return encodeConstants((CompareTupleFilter) filter);
    }

    return filter;
}
Example 9
Source File: StreamingBuiltInFunctionTransformer.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter transform(TupleFilter tupleFilter) {
    TupleFilter translated = null;
    if (tupleFilter instanceof CompareTupleFilter) {
        //normal case
        translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
        if (translated != null) {
            logger.info("Translated {" + tupleFilter + "}");
        }
    } else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
        //like case
        translated = translateFunctionTupleFilter((BuiltInFunctionTupleFilter) tupleFilter);
        if (translated != null) {
            logger.info("Translated {" + tupleFilter + "}");
        }
    } else if (tupleFilter instanceof LogicalTupleFilter) {
        @SuppressWarnings("unchecked")
        ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren()
                .listIterator();
        while (childIterator.hasNext()) {
            TupleFilter transformed = transform(childIterator.next());
            if (transformed != null)
                childIterator.set(transformed);
        }
    }

    TupleFilter result = translated == null ? tupleFilter : translated;
    if (result.getOperator() == TupleFilter.FilterOperatorEnum.NOT
            && !TupleFilter.isEvaluableRecursively(result)) {
        TupleFilter.collectColumns(result, unEvaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!result.isEvaluable()) {
        TupleFilter.collectColumns(result, unEvaluableColumns);
        return ConstantTupleFilter.TRUE;
    }
    return result;
}
Example 10
Source File: FragmentFileSearcher.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter transform(TupleFilter filter) {
    if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT
            && !TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, unEvaluableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, unEvaluableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    if (filter instanceof CompareTupleFilter) {
        return translateCompareFilter((CompareTupleFilter) filter);
    } else if (filter instanceof LogicalTupleFilter) {
        @SuppressWarnings("unchecked")
        ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) filter.getChildren().listIterator();
        while (childIterator.hasNext()) {
            TupleFilter transformed = transform(childIterator.next());
            if (transformed != null) {
                childIterator.set(transformed);
            } else {
                throw new IllegalStateException("Should not be null");
            }
        }
    }
    return filter;
}
Example 11
Source File: CubeStorageEngine.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private void setLimit(TupleFilter filter, StorageContext context) {
    boolean goodAggr = context.isExactAggregation();
    boolean goodFilter = filter == null || (TupleFilter.isEvaluableRecursively(filter) && context.isCoprocessorEnabled());
    boolean goodSort = context.hasSort() == false;
    if (goodAggr && goodFilter && goodSort) {
        logger.info("Enable limit " + context.getLimit());
        context.enableLimit();
    }
}
Example 12
Source File: GTCubeStorageQueryBase.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private void enableStorageLimitIfPossible(Cuboid cuboid, Collection<TblColRef> groups, List<TblColRef> dynGroups,
        Set<TblColRef> derivedPostAggregation, Collection<TblColRef> groupsD, TupleFilter filter,
        Set<TblColRef> loosenedColumnD, SQLDigest sqlDigest, StorageContext context) {
    Collection<FunctionDesc> functionDescs = sqlDigest.aggregations;

    StorageLimitLevel storageLimitLevel = StorageLimitLevel.LIMIT_ON_SCAN;

    //if groupsD is clustered at "head" of the rowkey, then limit push down is possible
    int size = groupsD.size();
    if (!groupsD.containsAll(cuboid.getColumns().subList(0, size))) {
        storageLimitLevel = StorageLimitLevel.LIMIT_ON_RETURN_SIZE;
        logger.debug(
                "storageLimitLevel set to LIMIT_ON_RETURN_SIZE because groupD is not clustered at head, groupsD: {} with cuboid columns: {}",
                groupsD, cuboid.getColumns());
    }

    if (!dynGroups.isEmpty()) {
        storageLimitLevel = StorageLimitLevel.NO_LIMIT;
        logger.debug("Storage limit push down is impossible because the query has dynamic groupby {}", dynGroups);
    }

    // derived aggregation is bad, unless expanded columns are already in group by
    if (!groups.containsAll(derivedPostAggregation)) {
        storageLimitLevel = StorageLimitLevel.NO_LIMIT;
        logger.debug("storageLimitLevel set to NO_LIMIT because derived column require post aggregation: {}",
                derivedPostAggregation);
    }

    if (!TupleFilter.isEvaluableRecursively(filter)) {
        storageLimitLevel = StorageLimitLevel.NO_LIMIT;
        logger.debug("storageLimitLevel set to NO_LIMIT because the filter isn't evaluable");
    }

    if (!loosenedColumnD.isEmpty()) { // KYLIN-2173
        storageLimitLevel = StorageLimitLevel.NO_LIMIT;
        logger.debug("storageLimitLevel set to NO_LIMIT because filter is loosened: {}", loosenedColumnD);
    }

    if (context.hasSort()) {
        storageLimitLevel = StorageLimitLevel.NO_LIMIT;
        logger.debug("storageLimitLevel set to NO_LIMIT because the query has order by");
    }

    //if exists measures like max(cal_dt), then it's not a perfect cuboid match, cannot apply limit
    for (FunctionDesc functionDesc : functionDescs) {
        if (functionDesc.isDimensionAsMetric()) {
            storageLimitLevel = StorageLimitLevel.NO_LIMIT;
            logger.debug("storageLimitLevel set to NO_LIMIT because {} isDimensionAsMetric ", functionDesc);
        }
    }

    if (sqlDigest.groupByExpression) {
        storageLimitLevel = StorageLimitLevel.NO_LIMIT;
        logger.debug("storageLimitLevel set to NO_LIMIT because group by clause is an expression");
    }

    context.applyLimitPushDown(cubeInstance, storageLimitLevel);
}
Example 13
Source File: FilterDecorator.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public TupleFilter onSerialize(TupleFilter filter) {
    if (filter == null)
        return null;

    BuiltInFunctionTransformer translator = new BuiltInFunctionTransformer(dimEncMap);
    filter = translator.transform(filter);

    // un-evaluatable filter is replaced with TRUE
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (!(filter instanceof CompareTupleFilter))
        return filter;

    // double check all internal of CompareTupleFilter is evaluatable
    if (!TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (filterConstantsTreatment == FilterConstantsTreatment.AS_IT_IS) {
        return filter;
    } else {

        // extract ColumnFilter & ConstantFilter
        CompareTupleFilter compareFilter = (CompareTupleFilter) filter;
        TblColRef col = compareFilter.getColumn();

        if (col == null) {
            return filter;
        }

        Collection<String> constValues = (Collection<String>) compareFilter.getValues();
        if (constValues == null || constValues.isEmpty()) {
            return filter;
        }

        CompareTupleFilter newCompareFilter = new CompareTupleFilter(compareFilter.getOperator());
        newCompareFilter.addChild(new ColumnTupleFilter(col));

        if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_GLOBAL_DICT) {
            return replaceConstantsWithGlobalDict(compareFilter, newCompareFilter);
        } else if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_LOCAL_DICT) {
            return replaceConstantsWithLocalDict(compareFilter, newCompareFilter);
        } else {
            throw new RuntimeException("should not reach here");
        }
    }
}
Example 14
Source File: GTCubeStorageQueryBase.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private void enableStorageLimitIfPossible(Cuboid cuboid, Collection<TblColRef> groups, List<TblColRef> dynGroups,
        Set<TblColRef> derivedPostAggregation, Collection<TblColRef> groupsD, TupleFilter filter,
        Set<TblColRef> loosenedColumnD, SQLDigest sqlDigest, StorageContext context) {
    Collection<FunctionDesc> functionDescs = sqlDigest.aggregations;

    StorageLimitLevel storageLimitLevel = StorageLimitLevel.LIMIT_ON_SCAN;

    //if groupsD is clustered at "head" of the rowkey, then limit push down is possible
    int size = groupsD.size();
    if (!groupsD.containsAll(cuboid.getColumns().subList(0, size))) {
        storageLimitLevel = StorageLimitLevel.LIMIT_ON_RETURN_SIZE;
        logger.debug(
                "storageLimitLevel set to LIMIT_ON_RETURN_SIZE because groupD is not clustered at head, groupsD: {} with cuboid columns: {}",
                groupsD, cuboid.getColumns());
    }

    if (!dynGroups.isEmpty()) {
        storageLimitLevel = StorageLimitLevel.NO_LIMIT;
        logger.debug("Storage limit push down is impossible because the query has dynamic groupby {}", dynGroups);
    }

    // derived aggregation is bad, unless expanded columns are already in group by
    if (!groups.containsAll(derivedPostAggregation)) {
        storageLimitLevel = StorageLimitLevel.NO_LIMIT;
        logger.debug("storageLimitLevel set to NO_LIMIT because derived column require post aggregation: {}",
                derivedPostAggregation);
    }

    if (!TupleFilter.isEvaluableRecursively(filter)) {
        storageLimitLevel = StorageLimitLevel.NO_LIMIT;
        logger.debug("storageLimitLevel set to NO_LIMIT because the filter isn't evaluable");
    }

    if (!loosenedColumnD.isEmpty()) { // KYLIN-2173
        storageLimitLevel = StorageLimitLevel.NO_LIMIT;
        logger.debug("storageLimitLevel set to NO_LIMIT because filter is loosened: {}", loosenedColumnD);
    }

    if (context.hasSort()) {
        storageLimitLevel = StorageLimitLevel.NO_LIMIT;
        logger.debug("storageLimitLevel set to NO_LIMIT because the query has order by");
    }

    //if exists measures like max(cal_dt), then it's not a perfect cuboid match, cannot apply limit
    for (FunctionDesc functionDesc : functionDescs) {
        if (functionDesc.isDimensionAsMetric()) {
            storageLimitLevel = StorageLimitLevel.NO_LIMIT;
            logger.debug("storageLimitLevel set to NO_LIMIT because {} isDimensionAsMetric ", functionDesc);
        }
    }

    if (sqlDigest.groupByExpression) {
        storageLimitLevel = StorageLimitLevel.NO_LIMIT;
        logger.debug("storageLimitLevel set to NO_LIMIT because group by clause is an expression");
    }

    context.applyLimitPushDown(cubeInstance, storageLimitLevel);
}
Example 15
Source File: FilterDecorator.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public TupleFilter onSerialize(TupleFilter filter) {
    if (filter == null)
        return null;

    BuiltInFunctionTransformer translator = new BuiltInFunctionTransformer(dimEncMap);
    filter = translator.transform(filter);

    // un-evaluatable filter is replaced with TRUE
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (!(filter instanceof CompareTupleFilter))
        return filter;

    // double check all internal of CompareTupleFilter is evaluatable
    if (!TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (filterConstantsTreatment == FilterConstantsTreatment.AS_IT_IS) {
        return filter;
    } else {

        // extract ColumnFilter & ConstantFilter
        CompareTupleFilter compareFilter = (CompareTupleFilter) filter;
        TblColRef col = compareFilter.getColumn();

        if (col == null) {
            return filter;
        }

        Collection<String> constValues = (Collection<String>) compareFilter.getValues();
        if (constValues == null || constValues.isEmpty()) {
            return filter;
        }

        CompareTupleFilter newCompareFilter = new CompareTupleFilter(compareFilter.getOperator());
        newCompareFilter.addChild(new ColumnTupleFilter(col));

        if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_GLOBAL_DICT) {
            return replaceConstantsWithGlobalDict(compareFilter, newCompareFilter);
        } else if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_LOCAL_DICT) {
            return replaceConstantsWithLocalDict(compareFilter, newCompareFilter);
        } else {
            throw new RuntimeException("should not reach here");
        }
    }
}