Java Code Examples for org.apache.kylin.metadata.filter.CompareTupleFilter#getValues()

The following examples show how to use org.apache.kylin.metadata.filter.CompareTupleFilter#getValues() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CubeStorageEngine.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private Collection<ColumnValueRange> translateToAndDimRanges(List<? extends TupleFilter> andFilters, CubeSegment cubeSegment) {
    Map<TblColRef, ColumnValueRange> rangeMap = new HashMap<TblColRef, ColumnValueRange>();
    for (TupleFilter filter : andFilters) {
        if ((filter instanceof CompareTupleFilter) == false) {
            continue;
        }

        CompareTupleFilter comp = (CompareTupleFilter) filter;
        if (comp.getColumn() == null) {
            continue;
        }

        ColumnValueRange range = new ColumnValueRange(comp.getColumn(), comp.getValues(), comp.getOperator());
        andMerge(range, rangeMap);

    }
    return rangeMap.values();
}
Example 2
Source File: ScanRangePlannerBase.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private Collection<ColumnRange> translateToAndDimRanges(List<? extends TupleFilter> andFilters) {
    Map<TblColRef, ColumnRange> rangeMap = new HashMap<TblColRef, ColumnRange>();
    for (TupleFilter filter : andFilters) {
        if ((filter instanceof CompareTupleFilter) == false) {
            if (filter instanceof ConstantTupleFilter && !filter.evaluate(null, null)) {
                return null;
            } else {
                continue;
            }
        }

        CompareTupleFilter comp = (CompareTupleFilter) filter;
        if (comp.getColumn() == null) {
            continue;
        }

        @SuppressWarnings("unchecked")
        ColumnRange newRange = new ColumnRange(comp.getColumn(), (Set<ByteArray>) comp.getValues(), comp.getOperator());
        ColumnRange existing = rangeMap.get(newRange.column);
        if (existing == null) {
            rangeMap.put(newRange.column, newRange);
        } else {
            existing.andMerge(newRange);
        }
    }
    return rangeMap.values();
}
Example 3
Source File: SegmentPruner.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public static boolean satisfy(CompareTupleFilter comp, String minVal, String maxVal) {

        // When both min and max are null, it means all cells of the column are null.
        // In such case, return true to let query engine scan the segment, since the
        // result of null comparison is query engine specific.
        if (minVal == null && maxVal == null)
            return true;
        
        // pass on non-constant filter
        if (comp.getChildren().size() > 1 && !(comp.getChildren().get(1) instanceof ConstantTupleFilter))
            return true;

        TblColRef col = comp.getColumn();
        DataTypeOrder order = col.getType().getOrder();
        String filterVal = toString(comp.getFirstValue());
        
        switch (comp.getOperator()) {
        case EQ:
        case IN:
            for (String filterValue : (Set<String>) comp.getValues()) {
                if (order.compare(filterValue, maxVal) <= 0 && order.compare(minVal, filterValue) <= 0)
                    return true;
            }
            return false;
        case LT:
            return order.compare(minVal, filterVal) < 0;
        case LTE:
            return order.compare(minVal, filterVal) <= 0;
        case GT:
            return order.compare(maxVal, filterVal) > 0;
        case GTE:
            return order.compare(maxVal, filterVal) >= 0;
        case NEQ:
        case NOTIN:
        case ISNULL:
        case ISNOTNULL:
        default:
            return true;
        }
    }
Example 4
Source File: InvertIndexSearcher.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private EvalResult doEvalCompareIn(CompareTupleFilter filter) {
    EvalResult result = new EvalResult();
    String column = filter.getColumn().getName();
    ColInvertIndexSearcher colSearcher = colIndexSearchers.get(column);
    if (colSearcher == null) {
        return EvalResult.ALL_MATCH;
    }
    List<ImmutableRoaringBitmap> bitmaps = Lists.newArrayList();
    for (Object value : filter.getValues()) {
        byte[] bytes = null;
        if (value instanceof ByteArray) {
            bytes = ((ByteArray)value).array();
        } else if (value instanceof byte[]) {
            bytes = (byte[])value;
        } else if (value instanceof String) {
            bytes = Bytes.toBytes((String)value);
        }
        ImmutableRoaringBitmap bitmap = colSearcher.searchValue(bytes);
        if (bitmap != null) {
            bitmaps.add(bitmap);
        }
    }
    if (bitmaps.isEmpty()) {
        return result;
    }

    result.bitmap = ImmutableRoaringBitmap.or(bitmaps.toArray(new ImmutableRoaringBitmap[bitmaps.size()]));
    return result;
}
Example 5
Source File: ScanRangePlannerBase.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private Collection<ColumnRange> translateToAndDimRanges(List<? extends TupleFilter> andFilters) {
    Map<TblColRef, ColumnRange> rangeMap = new HashMap<TblColRef, ColumnRange>();
    for (TupleFilter filter : andFilters) {
        if ((filter instanceof CompareTupleFilter) == false) {
            if (filter instanceof ConstantTupleFilter && !filter.evaluate(null, null)) {
                return null;
            } else {
                continue;
            }
        }

        CompareTupleFilter comp = (CompareTupleFilter) filter;
        if (comp.getColumn() == null) {
            continue;
        }

        @SuppressWarnings("unchecked")
        ColumnRange newRange = new ColumnRange(comp.getColumn(), (Set<ByteArray>) comp.getValues(),
                comp.getOperator());
        ColumnRange existing = rangeMap.get(newRange.column);
        if (existing == null) {
            rangeMap.put(newRange.column, newRange);
        } else {
            existing.andMerge(newRange);
        }
    }
    return rangeMap.values();
}
Example 6
Source File: SegmentPruner.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public static boolean satisfy(CompareTupleFilter comp, String minVal, String maxVal) {

        // When both min and max are null, it means all cells of the column are null.
        // In such case, return true to let query engine scan the segment, since the
        // result of null comparison is query engine specific.
        if (minVal == null && maxVal == null)
            return true;
        
        // pass on non-constant filter
        if (comp.getChildren().size() <= 1 || !isConstantValue(comp.getChildren().get(1)))
            return true;

        TblColRef col = comp.getColumn();
        DataTypeOrder order = col.getType().getOrder();
        String filterVal = toString(comp.getFirstValue());
        
        switch (comp.getOperator()) {
        case EQ:
        case IN:
            for (String filterValue : (Set<String>) comp.getValues()) {
                if (order.compare(filterValue, maxVal) <= 0 && order.compare(minVal, filterValue) <= 0)
                    return true;
            }
            return false;
        case LT:
            return order.compare(minVal, filterVal) < 0;
        case LTE:
            return order.compare(minVal, filterVal) <= 0;
        case GT:
            return order.compare(maxVal, filterVal) > 0;
        case GTE:
            return order.compare(maxVal, filterVal) >= 0;
        case NEQ:
        case NOTIN:
        case ISNULL:
        case ISNOTNULL:
        default:
            return true;
        }
    }
Example 7
Source File: InvertIndexSearcher.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private EvalResult doEvalCompareIn(CompareTupleFilter filter) {
    EvalResult result = new EvalResult();
    String column = filter.getColumn().getName();
    ColInvertIndexSearcher colSearcher = colIndexSearchers.get(column);
    if (colSearcher == null) {
        return EvalResult.ALL_MATCH;
    }
    List<ImmutableRoaringBitmap> bitmaps = Lists.newArrayList();
    for (Object value : filter.getValues()) {
        byte[] bytes = null;
        if (value instanceof ByteArray) {
            bytes = ((ByteArray)value).array();
        } else if (value instanceof byte[]) {
            bytes = (byte[])value;
        } else if (value instanceof String) {
            bytes = Bytes.toBytes((String)value);
        }
        ImmutableRoaringBitmap bitmap = colSearcher.searchValue(bytes);
        if (bitmap != null) {
            bitmaps.add(bitmap);
        }
    }
    if (bitmaps.isEmpty()) {
        return result;
    }

    result.bitmap = ImmutableRoaringBitmap.or(bitmaps.toArray(new ImmutableRoaringBitmap[bitmaps.size()]));
    return result;
}
Example 8
Source File: BitMapFilterEvaluator.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private ConciseSet evalCompareIn(CompareTupleFilter filter) {
    ConciseSet set = new ConciseSet();
    for (String value : filter.getValues()) {
        int id = Dictionary.stringToDictId(value);
        ConciseSet bitMap = provider.getBitMap(filter.getColumn(), id, id);
        if (bitMap == null)
            return null;
        set.addAll(bitMap);
    }
    return set;
}
Example 9
Source File: FilterDecorator.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public TupleFilter onSerialize(TupleFilter filter) {
    if (filter == null)
        return null;

    BuiltInFunctionTransformer translator = new BuiltInFunctionTransformer(dimEncMap);
    filter = translator.transform(filter);

    // un-evaluatable filter is replaced with TRUE
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (!(filter instanceof CompareTupleFilter))
        return filter;

    // double check all internal of CompareTupleFilter is evaluatable
    if (!TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (filterConstantsTreatment == FilterConstantsTreatment.AS_IT_IS) {
        return filter;
    } else {

        // extract ColumnFilter & ConstantFilter
        CompareTupleFilter compareFilter = (CompareTupleFilter) filter;
        TblColRef col = compareFilter.getColumn();

        if (col == null) {
            return filter;
        }

        Collection<String> constValues = (Collection<String>) compareFilter.getValues();
        if (constValues == null || constValues.isEmpty()) {
            return filter;
        }

        CompareTupleFilter newCompareFilter = new CompareTupleFilter(compareFilter.getOperator());
        newCompareFilter.addChild(new ColumnTupleFilter(col));

        if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_GLOBAL_DICT) {
            return replaceConstantsWithGlobalDict(compareFilter, newCompareFilter);
        } else if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_LOCAL_DICT) {
            return replaceConstantsWithLocalDict(compareFilter, newCompareFilter);
        } else {
            throw new RuntimeException("should not reach here");
        }
    }
}
Example 10
Source File: CompareFilterTimeRangeChecker.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public CheckResult check(CompareTupleFilter compFilter, TimeDerivedColumnType timeDerivedColumnType, long timezoneOffset) {
    Object timestampValue = compFilter.getFirstValue();
    Set conditionValues = compFilter.getValues();
    Pair<Long, Long> timeUnitRange;
    if (timeDerivedColumnType != TimeDerivedColumnType.MINUTE_START
            && timeDerivedColumnType != TimeDerivedColumnType.HOUR_START) {
        timeUnitRange = timezoneOffset == 0 ? timeDerivedColumnType.getTimeUnitRange(timestampValue)
                : timeDerivedColumnType.getTimeUnitRangeTimezoneAware(timestampValue, timezoneOffset);
    } else {
        timeUnitRange = timeDerivedColumnType.getTimeUnitRange(timestampValue);
    }
    switch (compFilter.getOperator()) {
    case EQ:
        return checkForEqValue(timeUnitRange);
    case NEQ:
        if (timeUnitRange.getFirst() <= timeStart && timeUnitRange.getSecond() >= timeEnd) {
            return CheckResult.EXCLUDED;
        }
        if (timeUnitRange.getSecond() <= timeStart || timeUnitRange.getFirst() >= timeEnd) {
            return CheckResult.INCLUDED;
        }
        return CheckResult.OVERLAP;
    case LT:
        if ((!endClose && timeUnitRange.getFirst() >= timeEnd) || (endClose && timeUnitRange.getFirst() > timeEnd)) {
            return CheckResult.INCLUDED;
        }
        if (timeUnitRange.getFirst() <= timeStart) {
            return CheckResult.EXCLUDED;
        }
        return CheckResult.OVERLAP;
    case LTE:
        if (timeUnitRange.getFirst() >= timeEnd) {
            return CheckResult.INCLUDED;
        }
        if (timeUnitRange.getSecond() < timeStart) {
            return CheckResult.EXCLUDED;
        }
        return CheckResult.OVERLAP;
    case GT:
        if (timeUnitRange.getSecond() < timeStart) {
            return CheckResult.INCLUDED;
        }
        if (timeUnitRange.getFirst() >= timeEnd) {
            return CheckResult.EXCLUDED;
        }
        return CheckResult.OVERLAP;
    case GTE:
        if (timeUnitRange.getFirst() <= timeStart) {
            return CheckResult.INCLUDED;
        }
        if ((!endClose && timeUnitRange.getFirst() >= timeEnd) || (endClose && timeUnitRange.getFirst() > timeEnd)) {
            return CheckResult.EXCLUDED;
        }
        return CheckResult.OVERLAP;
    case IN:
        return checkForInValues(timeDerivedColumnType, conditionValues, timezoneOffset);
    default:
        return CheckResult.OVERLAP;
    }
}
Example 11
Source File: FilterDecorator.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public TupleFilter onSerialize(TupleFilter filter) {
    if (filter == null)
        return null;

    BuiltInFunctionTransformer translator = new BuiltInFunctionTransformer(dimEncMap);
    filter = translator.transform(filter);

    // un-evaluatable filter is replaced with TRUE
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (!(filter instanceof CompareTupleFilter))
        return filter;

    // double check all internal of CompareTupleFilter is evaluatable
    if (!TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (filterConstantsTreatment == FilterConstantsTreatment.AS_IT_IS) {
        return filter;
    } else {

        // extract ColumnFilter & ConstantFilter
        CompareTupleFilter compareFilter = (CompareTupleFilter) filter;
        TblColRef col = compareFilter.getColumn();

        if (col == null) {
            return filter;
        }

        Collection<String> constValues = (Collection<String>) compareFilter.getValues();
        if (constValues == null || constValues.isEmpty()) {
            return filter;
        }

        CompareTupleFilter newCompareFilter = new CompareTupleFilter(compareFilter.getOperator());
        newCompareFilter.addChild(new ColumnTupleFilter(col));

        if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_GLOBAL_DICT) {
            return replaceConstantsWithGlobalDict(compareFilter, newCompareFilter);
        } else if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_LOCAL_DICT) {
            return replaceConstantsWithLocalDict(compareFilter, newCompareFilter);
        } else {
            throw new RuntimeException("should not reach here");
        }
    }
}
Example 12
Source File: CompareFilterTimeRangeChecker.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public CheckResult check(CompareTupleFilter compFilter, TimeDerivedColumnType timeDerivedColumnType, long timezoneOffset) {
    Object timestampValue = compFilter.getFirstValue();
    Set conditionValues = compFilter.getValues();
    Pair<Long, Long> timeUnitRange;
    if (timeDerivedColumnType != TimeDerivedColumnType.MINUTE_START
            && timeDerivedColumnType != TimeDerivedColumnType.HOUR_START) {
        timeUnitRange = timezoneOffset == 0 ? timeDerivedColumnType.getTimeUnitRange(timestampValue)
                : timeDerivedColumnType.getTimeUnitRangeTimezoneAware(timestampValue, timezoneOffset);
    } else {
        timeUnitRange = timeDerivedColumnType.getTimeUnitRange(timestampValue);
    }
    switch (compFilter.getOperator()) {
    case EQ:
        return checkForEqValue(timeUnitRange);
    case NEQ:
        if (timeUnitRange.getFirst() <= timeStart && timeUnitRange.getSecond() >= timeEnd) {
            return CheckResult.EXCLUDED;
        }
        if (timeUnitRange.getSecond() <= timeStart || timeUnitRange.getFirst() >= timeEnd) {
            return CheckResult.INCLUDED;
        }
        return CheckResult.OVERLAP;
    case LT:
        if ((!endClose && timeUnitRange.getFirst() >= timeEnd) || (endClose && timeUnitRange.getFirst() > timeEnd)) {
            return CheckResult.INCLUDED;
        }
        if (timeUnitRange.getFirst() <= timeStart) {
            return CheckResult.EXCLUDED;
        }
        return CheckResult.OVERLAP;
    case LTE:
        if (timeUnitRange.getFirst() >= timeEnd) {
            return CheckResult.INCLUDED;
        }
        if (timeUnitRange.getSecond() < timeStart) {
            return CheckResult.EXCLUDED;
        }
        return CheckResult.OVERLAP;
    case GT:
        if (timeUnitRange.getSecond() < timeStart) {
            return CheckResult.INCLUDED;
        }
        if (timeUnitRange.getFirst() >= timeEnd) {
            return CheckResult.EXCLUDED;
        }
        return CheckResult.OVERLAP;
    case GTE:
        if (timeUnitRange.getFirst() <= timeStart) {
            return CheckResult.INCLUDED;
        }
        if ((!endClose && timeUnitRange.getFirst() >= timeEnd) || (endClose && timeUnitRange.getFirst() > timeEnd)) {
            return CheckResult.EXCLUDED;
        }
        return CheckResult.OVERLAP;
    case IN:
        return checkForInValues(timeDerivedColumnType, conditionValues, timezoneOffset);
    default:
        return CheckResult.OVERLAP;
    }
}