Java Code Examples for org.apache.kylin.metadata.filter.TupleFilter#getOperator()

The following examples show how to use org.apache.kylin.metadata.filter.TupleFilter#getOperator() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: GTCubeStorageQueryBase.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
protected Set<CompareTupleFilter> findSingleValuesCompFilters(TupleFilter filter) {
    Collection<? extends TupleFilter> toCheck;
    if (filter instanceof CompareTupleFilter) {
        toCheck = Collections.singleton(filter);
    } else if (filter instanceof LogicalTupleFilter && filter.getOperator() == FilterOperatorEnum.AND) {
        toCheck = filter.getChildren();
    } else {
        return Collections.emptySet();
    }

    Set<CompareTupleFilter> result = Sets.newHashSet();
    for (TupleFilter f : toCheck) {
        if (f instanceof CompareTupleFilter) {
            CompareTupleFilter compFilter = (CompareTupleFilter) f;
            // is COL=const ?
            if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1
                    && compFilter.getColumn() != null) {
                result.add(compFilter);
            }
        }
    }
    return result;
}
Example 2
Source File: ScanRangePlannerBase.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
protected TupleFilter flattenToOrAndFilter(TupleFilter filter) {
    if (filter == null)
        return null;

    TupleFilter flatFilter = filter.flatFilter();

    // normalize to OR-AND filter
    if (flatFilter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
        LogicalTupleFilter f = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
        f.addChild(flatFilter);
        flatFilter = f;
    }

    if (flatFilter.getOperator() != TupleFilter.FilterOperatorEnum.OR)
        throw new IllegalStateException();

    return flatFilter;
}
Example 3
Source File: ScanRangePlannerBase.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
protected List<Collection<ColumnRange>> translateToOrAndDimRanges(TupleFilter flatFilter) {
    List<Collection<ColumnRange>> result = Lists.newArrayList();

    if (flatFilter == null) {
        result.add(Collections.<ColumnRange> emptyList());
        return result;
    }

    for (TupleFilter andFilter : flatFilter.getChildren()) {
        if (andFilter.getOperator() != TupleFilter.FilterOperatorEnum.AND)
            throw new IllegalStateException("Filter should be AND instead of " + andFilter);

        Collection<ColumnRange> andRanges = translateToAndDimRanges(andFilter.getChildren());
        if (andRanges != null) {
            result.add(andRanges);
        }
    }

    return preEvaluateConstantConditions(result);
}
Example 4
Source File: StreamingDataQueryPlanner.java    From kylin with Apache License 2.0 6 votes vote down vote up 
protected TupleFilter flattenToOrAndFilter(TupleFilter filter) {
    if (filter == null)
        return null;

    TupleFilter flatFilter = filter.flatFilter();

    // normalize to OR-AND filter
    if (flatFilter.getOperator() == FilterOperatorEnum.AND) {
        LogicalTupleFilter f = new LogicalTupleFilter(FilterOperatorEnum.OR);
        f.addChild(flatFilter);
        flatFilter = f;
    }

    if (flatFilter.getOperator() != FilterOperatorEnum.OR)
        throw new IllegalStateException();

    return flatFilter;
}
Example 5
Source File: CubeStorageEngine.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private List<Collection<ColumnValueRange>> translateToOrAndDimRanges(TupleFilter flatFilter, CubeSegment cubeSegment) {
    List<Collection<ColumnValueRange>> result = Lists.newArrayList();

    if (flatFilter == null) {
        result.add(Collections.<ColumnValueRange> emptyList());
        return result;
    }

    for (TupleFilter andFilter : flatFilter.getChildren()) {
        if (andFilter.getOperator() != FilterOperatorEnum.AND) {
            throw new IllegalStateException("Filter should be AND instead of " + andFilter);
        }

        Collection<ColumnValueRange> andRanges = translateToAndDimRanges(andFilter.getChildren(), cubeSegment);

        result.add(andRanges);
    }

    return preprocessConstantConditions(result);
}
Example 6
Source File: ScanRangePlannerBase.java    From kylin with Apache License 2.0 6 votes vote down vote up 
protected List<Collection<ColumnRange>> translateToOrAndDimRanges(TupleFilter flatFilter) {
    List<Collection<ColumnRange>> result = Lists.newArrayList();

    if (flatFilter == null) {
        result.add(Collections.<ColumnRange> emptyList());
        return result;
    }

    for (TupleFilter andFilter : flatFilter.getChildren()) {
        if (andFilter.getOperator() != TupleFilter.FilterOperatorEnum.AND)
            throw new IllegalStateException("Filter should be AND instead of " + andFilter);

        Collection<ColumnRange> andRanges = translateToAndDimRanges(andFilter.getChildren());
        if (andRanges != null) {
            result.add(andRanges);
        }
    }

    return preEvaluateConstantConditions(result);
}
Example 7
Source File: CubeStorageEngine.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private TupleFilter flattenToOrAndFilter(TupleFilter filter) {
    if (filter == null)
        return null;

    TupleFilter flatFilter = filter.flatFilter();

    // normalize to OR-AND filter
    if (flatFilter.getOperator() == FilterOperatorEnum.AND) {
        LogicalTupleFilter f = new LogicalTupleFilter(FilterOperatorEnum.OR);
        f.addChild(flatFilter);
        flatFilter = f;
    }

    if (flatFilter.getOperator() != FilterOperatorEnum.OR)
        throw new IllegalStateException();

    return flatFilter;
}
Example 8
Source File: ScanRangePlannerBase.java    From kylin with Apache License 2.0 6 votes vote down vote up 
protected TupleFilter flattenToOrAndFilter(TupleFilter filter) {
    if (filter == null)
        return null;

    TupleFilter flatFilter = filter.flatFilter();

    // normalize to OR-AND filter
    if (flatFilter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
        LogicalTupleFilter f = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
        f.addChild(flatFilter);
        flatFilter = f;
    }

    if (flatFilter.getOperator() != TupleFilter.FilterOperatorEnum.OR)
        throw new IllegalStateException();

    return flatFilter;
}
Example 9
Source File: FragmentFileSearcher.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter transform(TupleFilter filter) {
    if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT
            && !TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, unEvaluableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, unEvaluableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    if (filter instanceof CompareTupleFilter) {
        return translateCompareFilter((CompareTupleFilter) filter);
    } else if (filter instanceof LogicalTupleFilter) {
        @SuppressWarnings("unchecked")
        ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) filter.getChildren().listIterator();
        while (childIterator.hasNext()) {
            TupleFilter transformed = transform(childIterator.next());
            if (transformed != null) {
                childIterator.set(transformed);
            } else {
                throw new IllegalStateException("Should not be null");
            }
        }
    }
    return filter;
}
Example 10
Source File: CubeStorageEngine.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private TupleFilter replaceChildren(TupleFilter filter, List<TupleFilter> newChildren) {
    if (filter instanceof LogicalTupleFilter) {
        LogicalTupleFilter r = new LogicalTupleFilter(filter.getOperator());
        r.addChildren(newChildren);
        return r;
    } else
        throw new IllegalStateException("Cannot replaceChildren on " + filter);
}
Example 11
Source File: StreamingDataQueryPlanner.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public boolean canSkip(long timeStart, long timeEnd, boolean endInclude) {
    if (flatFilter == null) {
        return false;
    }
    CompareFilterTimeRangeChecker timeRangeChecker = new CompareFilterTimeRangeChecker(timeStart, timeEnd, endInclude);
    for (TupleFilter andFilter : flatFilter.getChildren()) {
        if (andFilter.getOperator() != FilterOperatorEnum.AND) {
            throw new IllegalStateException("Filter should be AND instead of " + andFilter);
        }
        if (!canSkipForAndFilter(andFilter, timeRangeChecker)) {
            return false;
        }
    }
    return true;
}
Example 12
Source File: GTUtil.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter onSerialize(TupleFilter filter) {
    if (filter == null)
        return null;

    // In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE,
    // Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter)
    // will always return FALSE.
    if (filter.getOperator() == FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // map to column onto grid table
    if (colMapping != null && filter instanceof ColumnTupleFilter) {
        ColumnTupleFilter colFilter = (ColumnTupleFilter) filter;
        int gtColIdx = mapCol(colFilter.getColumn());
        return new ColumnTupleFilter(info.colRef(gtColIdx));
    }

    // encode constants
    if (useEncodeConstants && filter instanceof CompareTupleFilter) {
        return encodeConstants((CompareTupleFilter) filter);
    }

    return filter;
}
Example 13
Source File: OLAPFilterRel.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private CompareTupleFilter mergeToInClause(TupleFilter filter) {
    List<? extends TupleFilter> children = filter.getChildren();
    TblColRef inColumn = null;
    List<String> inValues = new LinkedList<String>();
    for (TupleFilter child : children) {
        if (child.getOperator() == FilterOperatorEnum.EQ) {
            CompareTupleFilter compFilter = (CompareTupleFilter) child;
            TblColRef column = compFilter.getColumn();
            if (inColumn == null) {
                inColumn = column;
            }

            if (column == null || !column.equals(inColumn)) {
                return null;
            }
            inValues.addAll(compFilter.getValues());
        } else {
            return null;
        }
    }

    children.clear();

    CompareTupleFilter inFilter = new CompareTupleFilter(FilterOperatorEnum.IN);
    inFilter.addChild(new ColumnTupleFilter(inColumn));
    inFilter.addChild(new ConstantTupleFilter(inValues));
    return inFilter;
}
Example 14
Source File: FragmentFileSearcher.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter transform(TupleFilter filter) {
    if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT
            && !TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, unEvaluableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, unEvaluableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    if (filter instanceof CompareTupleFilter) {
        return translateCompareFilter((CompareTupleFilter) filter);
    } else if (filter instanceof LogicalTupleFilter) {
        @SuppressWarnings("unchecked")
        ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) filter.getChildren().listIterator();
        while (childIterator.hasNext()) {
            TupleFilter transformed = transform(childIterator.next());
            if (transformed != null) {
                childIterator.set(transformed);
            } else {
                throw new IllegalStateException("Should not be null");
            }
        }
    }
    return filter;
}
Example 15
Source File: StreamingBuiltInFunctionTransformer.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter transform(TupleFilter tupleFilter) {
    TupleFilter translated = null;
    if (tupleFilter instanceof CompareTupleFilter) {
        //normal case
        translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
        if (translated != null) {
            logger.info("Translated {" + tupleFilter + "}");
        }
    } else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
        //like case
        translated = translateFunctionTupleFilter((BuiltInFunctionTupleFilter) tupleFilter);
        if (translated != null) {
            logger.info("Translated {" + tupleFilter + "}");
        }
    } else if (tupleFilter instanceof LogicalTupleFilter) {
        @SuppressWarnings("unchecked")
        ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren()
                .listIterator();
        while (childIterator.hasNext()) {
            TupleFilter transformed = transform(childIterator.next());
            if (transformed != null)
                childIterator.set(transformed);
        }
    }

    TupleFilter result = translated == null ? tupleFilter : translated;
    if (result.getOperator() == TupleFilter.FilterOperatorEnum.NOT
            && !TupleFilter.isEvaluableRecursively(result)) {
        TupleFilter.collectColumns(result, unEvaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!result.isEvaluable()) {
        TupleFilter.collectColumns(result, unEvaluableColumns);
        return ConstantTupleFilter.TRUE;
    }
    return result;
}
Example 16
Source File: CubeStorageEngine.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
private Set<TblColRef> findSingleValueColumns(TupleFilter filter) {
    Collection<? extends TupleFilter> toCheck;
    if (filter instanceof CompareTupleFilter) {
        toCheck = Collections.singleton(filter);
    } else if (filter instanceof LogicalTupleFilter && filter.getOperator() == FilterOperatorEnum.AND) {
        toCheck = filter.getChildren();
    } else {
        return (Set<TblColRef>) Collections.EMPTY_SET;
    }

    Set<TblColRef> result = Sets.newHashSet();
    for (TupleFilter f : toCheck) {
        if (f instanceof CompareTupleFilter) {
            CompareTupleFilter compFilter = (CompareTupleFilter) f;
            // is COL=const ?
            if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1 && compFilter.getColumn() != null) {
                result.add(compFilter.getColumn());
            }
        }
    }

    // expand derived
    Set<TblColRef> resultD = Sets.newHashSet();
    for (TblColRef col : result) {
        if (cubeDesc.isDerived(col)) {
            DeriveInfo hostInfo = cubeDesc.getHostInfo(col);
            if (hostInfo.isOneToOne) {
                for (TblColRef hostCol : hostInfo.columns) {
                    resultD.add(hostCol);
                }
            }
            //if not one2one, it will be pruned
        } else {
            resultD.add(col);
        }
    }
    return resultD;
}
Example 17
Source File: GTUtil.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter onSerialize(TupleFilter filter) {
    if (filter == null)
        return null;

    // In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE,
    // Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter)
    // will always return FALSE.
    if (filter.getOperator() == FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // map to column onto grid table
    if (colMapping != null && filter instanceof ColumnTupleFilter) {
        ColumnTupleFilter colFilter = (ColumnTupleFilter) filter;
        int gtColIdx = mapCol(colFilter.getColumn());
        return new ColumnTupleFilter(info.colRef(gtColIdx));
    }

    // encode constants
    if (useEncodeConstants && filter instanceof CompareTupleFilter) {
        return encodeConstants((CompareTupleFilter) filter);
    }

    return filter;
}
Example 18
Source File: TsConditionExtractor.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private static Range<Long> extractTsConditionInternal(TupleFilter filter, TblColRef colRef) {
    if (filter == null) {
        return Range.all();
    }

    if (filter instanceof LogicalTupleFilter) {
        if (filter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
            Range<Long> ret = Range.all();
            for (TupleFilter child : filter.getChildren()) {
                Range childRange = extractTsConditionInternal(child, colRef);
                if (childRange != null) {
                    if (ret.isConnected(childRange) && !ret.intersection(childRange).isEmpty()) {
                        ret = ret.intersection(childRange);
                    } else {
                        return null;
                    }
                } else {
                    return null;
                }
            }
            return ret.isEmpty() ? null : ret;
        } else {
            //for conditions like date > DATE'2000-11-11' OR date < DATE '1999-01-01'
            //we will use Ranges.all() rather than two ranges to represent them
            return Range.all();
        }
    }

    if (filter instanceof CompareTupleFilter) {
        CompareTupleFilter compareTupleFilter = (CompareTupleFilter) filter;
        if (compareTupleFilter.getColumn() == null)// column will be null at filters like " 1<>1"
            return Range.all();

        if (compareTupleFilter.getColumn().equals(colRef)) {
            Object firstValue = compareTupleFilter.getFirstValue();
            long t;
            switch (compareTupleFilter.getOperator()) {
            case EQ:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.closed(t, t);
            case LT:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.lessThan(t);
            case LTE:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.atMost(t);
            case GT:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.greaterThan(t);
            case GTE:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.atLeast(t);
            case NEQ:
            case IN://not handled for now
                break;
            default:
            }
        }
    }
    return Range.all();
}
Example 19
Source File: TsConditionExtractor.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private static Range<Long> extractTsConditionInternal(TupleFilter filter, TblColRef colRef) {
    if (filter == null) {
        return Range.all();
    }

    if (filter instanceof LogicalTupleFilter) {
        if (filter.getOperator() == TupleFilter.FilterOperatorEnum.AND) {
            Range<Long> ret = Range.all();
            for (TupleFilter child : filter.getChildren()) {
                Range childRange = extractTsConditionInternal(child, colRef);
                if (childRange != null) {
                    if (ret.isConnected(childRange) && !ret.intersection(childRange).isEmpty()) {
                        ret = ret.intersection(childRange);
                    } else {
                        return null;
                    }
                } else {
                    return null;
                }
            }
            return ret.isEmpty() ? null : ret;
        } else {
            //for conditions like date > DATE'2000-11-11' OR date < DATE '1999-01-01'
            //we will use Ranges.all() rather than two ranges to represent them
            return Range.all();
        }
    }

    if (filter instanceof CompareTupleFilter) {
        CompareTupleFilter compareTupleFilter = (CompareTupleFilter) filter;
        if (compareTupleFilter.getColumn() == null)// column will be null at filters like " 1<>1"
            return Range.all();

        if (compareTupleFilter.getColumn().equals(colRef)) {
            Object firstValue = compareTupleFilter.getFirstValue();
            long t;
            switch (compareTupleFilter.getOperator()) {
            case EQ:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.closed(t, t);
            case LT:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.lessThan(t);
            case LTE:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.atMost(t);
            case GT:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.greaterThan(t);
            case GTE:
                t = DateFormat.stringToMillis((String) firstValue);
                return Range.atLeast(t);
            case NEQ:
            case IN://not handled for now
                break;
            default:
            }
        }
    }
    return Range.all();
}
Example 20
Source File: TupleFilterVisitor.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@VisibleForTesting
static TupleFilter mergeToInClause(TupleFilter filter) {
    List<? extends TupleFilter> children = filter.getChildren();
    if (children.isEmpty()) {
        return filter;
    }
    // key: inColumn
    // Value: first: inValues
    // Value: second: dynamicVariables
    Map<TblColRef, Pair<Set<Object>, Map<String, Object>>> inColumnMap = Maps.newHashMap();
    List<TupleFilter> extraFilters = Lists.newLinkedList();
    for (TupleFilter child : children) {
        if (child.getOperator() == TupleFilter.FilterOperatorEnum.EQ) {
            CompareTupleFilter compFilter = (CompareTupleFilter) child;
            TblColRef column = compFilter.getColumn();
            if (column != null) {
                Pair<Set<Object>, Map<String, Object>> tmpValue = inColumnMap.get(column);
                if (tmpValue == null) {
                    Set<Object> inValues = Sets.newHashSet();
                    Map<String, Object> dynamicVariables = Maps.newHashMap();
                    tmpValue = new Pair<>(inValues, dynamicVariables);
                    inColumnMap.put(column, tmpValue);
                }

                tmpValue.getFirst().addAll(compFilter.getValues());
                tmpValue.getSecond().putAll(compFilter.getVariables());
                continue;
            }
        }
        extraFilters.add(child);
    }

    children.clear();

    TupleFilter ret = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
    ret.addChildren(extraFilters);
    for (Map.Entry<TblColRef, Pair<Set<Object>, Map<String, Object>>> entry : inColumnMap.entrySet()) {
        CompareTupleFilter inFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.IN);
        inFilter.addChild(new ColumnTupleFilter(entry.getKey()));
        inFilter.addChild(new ConstantTupleFilter(entry.getValue().getFirst()));
        inFilter.getVariables().putAll(entry.getValue().getSecond());
        ret.addChild(inFilter);
    }

    return ret.getChildren().size() == 1 ? ret.getChildren().get(0) : ret;
}