Java Code Examples for org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum#IN

The following examples show how to use org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum#IN . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DerivedFilterTranslator.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private static TupleFilter buildInFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    if (satisfyingHostRecords.size() == 0) {
        return ConstantTupleFilter.FALSE;
    }

    int hn = hostCols.length;
    if (hn == 1) {
        CompareTupleFilter in = new CompareTupleFilter(FilterOperatorEnum.IN);
        in.addChild(new ColumnTupleFilter(hostCols[0]));
        in.addChild(new ConstantTupleFilter(asValues(satisfyingHostRecords)));
        return in;
    } else {
        LogicalTupleFilter or = new LogicalTupleFilter(FilterOperatorEnum.OR);
        for (Array<String> rec : satisfyingHostRecords) {
            LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
            for (int i = 0; i < hn; i++) {
                CompareTupleFilter eq = new CompareTupleFilter(FilterOperatorEnum.EQ);
                eq.addChild(new ColumnTupleFilter(hostCols[i]));
                eq.addChild(new ConstantTupleFilter(rec.data[i]));
                and.addChild(eq);
            }
            or.addChild(and);
        }
        return or;
    }
}
Example 2
Source File: FilterBaseTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
protected CompareTupleFilter buildINCompareFilter(TblColRef dateColumn) throws ParseException {
    CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.IN);
    ColumnTupleFilter columnFilter = new ColumnTupleFilter(dateColumn);
    compareFilter.addChild(columnFilter);

    List<String> inValues = Lists.newArrayList();
    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.ROOT);
    Date startDate = simpleDateFormat.parse("1970-01-01");
    Date endDate = simpleDateFormat.parse("2100-01-01");
    Calendar start = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
    start.setTime(startDate);
    Calendar end = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
    end.setTime(endDate);
    for (Date date = start.getTime(); start.before(end); start.add(Calendar.DATE, 1), date = start.getTime()) {
        inValues.add(simpleDateFormat.format(date));
    }

    ConstantTupleFilter constantFilter = new ConstantTupleFilter(inValues);
    compareFilter.addChild(constantFilter);
    return compareFilter;
}
Example 3
Source File: DerivedFilterTranslator.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private static TupleFilter buildInFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    if (satisfyingHostRecords.size() == 0) {
        return ConstantTupleFilter.FALSE;
    }

    int hn = hostCols.length;
    if (hn == 1) {
        CompareTupleFilter in = new CompareTupleFilter(FilterOperatorEnum.IN);
        in.addChild(new ColumnTupleFilter(hostCols[0]));
        in.addChild(new ConstantTupleFilter(asValues(satisfyingHostRecords)));
        return in;
    } else {
        LogicalTupleFilter or = new LogicalTupleFilter(FilterOperatorEnum.OR);
        for (Array<String> rec : satisfyingHostRecords) {
            LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
            for (int i = 0; i < hn; i++) {
                CompareTupleFilter eq = new CompareTupleFilter(FilterOperatorEnum.EQ);
                eq.addChild(new ColumnTupleFilter(hostCols[i]));
                eq.addChild(new ConstantTupleFilter(rec.data[i]));
                and.addChild(eq);
            }
            or.addChild(and);
        }
        return or;
    }
}
Example 4
Source File: FilterBaseTest.java    From kylin with Apache License 2.0 6 votes vote down vote up 
protected CompareTupleFilter buildINCompareFilter(TblColRef dateColumn) throws ParseException {
    CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.IN);
    ColumnTupleFilter columnFilter = new ColumnTupleFilter(dateColumn);
    compareFilter.addChild(columnFilter);

    List<String> inValues = Lists.newArrayList();
    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.ROOT);
    Date startDate = simpleDateFormat.parse("1970-01-01");
    Date endDate = simpleDateFormat.parse("2100-01-01");
    Calendar start = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
    start.setTime(startDate);
    Calendar end = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
    end.setTime(endDate);
    for (Date date = start.getTime(); start.before(end); start.add(Calendar.DATE, 1), date = start.getTime()) {
        inValues.add(simpleDateFormat.format(date));
    }

    ConstantTupleFilter constantFilter = new ConstantTupleFilter(inValues);
    compareFilter.addChild(constantFilter);
    return compareFilter;
}
Example 5
Source File: DerivedFilterTranslator.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private static TupleFilter buildInFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    if (satisfyingHostRecords.size() == 0) {
        return ConstantTupleFilter.FALSE;
    }

    int hn = hostCols.length;
    if (hn == 1) {
        CompareTupleFilter in = new CompareTupleFilter(FilterOperatorEnum.IN);
        in.addChild(new ColumnTupleFilter(hostCols[0]));
        in.addChild(new ConstantTupleFilter(asValues(satisfyingHostRecords)));
        return in;
    } else {
        LogicalTupleFilter or = new LogicalTupleFilter(FilterOperatorEnum.OR);
        for (Array<String> rec : satisfyingHostRecords) {
            LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
            for (int i = 0; i < hn; i++) {
                CompareTupleFilter eq = new CompareTupleFilter(FilterOperatorEnum.EQ);
                eq.addChild(new ColumnTupleFilter(hostCols[i]));
                eq.addChild(new ConstantTupleFilter(rec.data[i]));
                and.addChild(eq);
            }
            or.addChild(and);
        }
        return or;
    }
}
Example 6
Source File: BuiltInFunctionTransformer.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private TupleFilter translateFunctionTupleFilter(BuiltInFunctionTupleFilter builtInFunctionTupleFilter) {
    if (!builtInFunctionTupleFilter.isValid())
        return null;

    TblColRef columnRef = builtInFunctionTupleFilter.getColumn();
    Dictionary<?> dict = dimEncMap.getDictionary(columnRef);
    if (dict == null)
        return null;

    CompareTupleFilter translated = new CompareTupleFilter(
            builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN);
    translated.addChild(new ColumnTupleFilter(columnRef));

    try {
        int translatedInClauseMaxSize = KylinConfig.getInstanceFromEnv().getTranslatedInClauseMaxSize();

        for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
            Object dictVal = dict.getValueFromId(i);
            if ((Boolean) builtInFunctionTupleFilter.invokeFunction(dictVal)) {
                translated.addChild(new ConstantTupleFilter(dictVal));

                if (translated.getChildren().size() > translatedInClauseMaxSize) {
                    return null;
                }
            }
        }
        logger.debug("getting a in clause with {} children", translated.getChildren().size());
    } catch (Exception e) {
        logger.debug(e.getMessage());
        return null;
    }
    return translated;
}
Example 7
Source File: GTScannerBenchmark.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private CompareTupleFilter eq(ColumnTupleFilter col, int... values) {
    CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.IN);
    r.addChild(col);

    List<ByteArray> list = Lists.newArrayList();
    for (int v : values) {
        int c = col.getColumn().getColumnDesc().getZeroBasedIndex();
        int len = info.getCodeSystem().maxCodeLength(c);
        ByteArray bytes = new ByteArray(len);
        BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len);
        list.add(bytes);
    }
    r.addChild(new ConstantTupleFilter(list));
    return r;
}
Example 8
Source File: GTScannerBenchmark2.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private CompareTupleFilter eq(ColumnTupleFilter col, int... values) {
    CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.IN);
    r.addChild(col);

    List<ByteArray> list = Lists.newArrayList();
    for (int v : values) {
        int c = col.getColumn().getColumnDesc().getZeroBasedIndex();
        int len = info.getCodeSystem().maxCodeLength(c);
        ByteArray bytes = new ByteArray(len);
        BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len);
        list.add(bytes);
    }
    r.addChild(new ConstantTupleFilter(list));
    return r;
}
Example 9
Source File: BuiltInFunctionTransformer.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private TupleFilter translateFunctionTupleFilter(BuiltInFunctionTupleFilter builtInFunctionTupleFilter) {
    if (!builtInFunctionTupleFilter.isValid())
        return null;

    TblColRef columnRef = builtInFunctionTupleFilter.getColumn();
    Dictionary<?> dict = dimEncMap.getDictionary(columnRef);
    if (dict == null)
        return null;

    CompareTupleFilter translated = new CompareTupleFilter(
            builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN);
    translated.addChild(new ColumnTupleFilter(columnRef));

    try {
        int translatedInClauseMaxSize = KylinConfig.getInstanceFromEnv().getTranslatedInClauseMaxSize();

        for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
            Object dictVal = dict.getValueFromId(i);
            if ((Boolean) builtInFunctionTupleFilter.invokeFunction(dictVal)) {
                translated.addChild(new ConstantTupleFilter(dictVal));

                if (translated.getChildren().size() > translatedInClauseMaxSize) {
                    return null;
                }
            }
        }
        logger.debug("getting a in clause with {} children", translated.getChildren().size());
    } catch (Exception e) {
        logger.debug(e.getMessage());
        return null;
    }
    return translated;
}
Example 10
Source File: GTScannerBenchmark.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private CompareTupleFilter eq(ColumnTupleFilter col, int... values) {
    CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.IN);
    r.addChild(col);

    List<ByteArray> list = Lists.newArrayList();
    for (int v : values) {
        int c = col.getColumn().getColumnDesc().getZeroBasedIndex();
        int len = info.getCodeSystem().maxCodeLength(c);
        ByteArray bytes = new ByteArray(len);
        BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len);
        list.add(bytes);
    }
    r.addChild(new ConstantTupleFilter(list));
    return r;
}
Example 11
Source File: GTScannerBenchmark2.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private CompareTupleFilter eq(ColumnTupleFilter col, int... values) {
    CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.IN);
    r.addChild(col);

    List<ByteArray> list = Lists.newArrayList();
    for (int v : values) {
        int c = col.getColumn().getColumnDesc().getZeroBasedIndex();
        int len = info.getCodeSystem().maxCodeLength(c);
        ByteArray bytes = new ByteArray(len);
        BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len);
        list.add(bytes);
    }
    r.addChild(new ConstantTupleFilter(list));
    return r;
}
Example 12
Source File: OLAPFilterRel.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private CompareTupleFilter mergeToInClause(TupleFilter filter) {
    List<? extends TupleFilter> children = filter.getChildren();
    TblColRef inColumn = null;
    List<String> inValues = new LinkedList<String>();
    for (TupleFilter child : children) {
        if (child.getOperator() == FilterOperatorEnum.EQ) {
            CompareTupleFilter compFilter = (CompareTupleFilter) child;
            TblColRef column = compFilter.getColumn();
            if (inColumn == null) {
                inColumn = column;
            }

            if (column == null || !column.equals(inColumn)) {
                return null;
            }
            inValues.addAll(compFilter.getValues());
        } else {
            return null;
        }
    }

    children.clear();

    CompareTupleFilter inFilter = new CompareTupleFilter(FilterOperatorEnum.IN);
    inFilter.addChild(new ColumnTupleFilter(inColumn));
    inFilter.addChild(new ConstantTupleFilter(inValues));
    return inFilter;
}
Example 13
Source File: BuiltInFunctionTransformer.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings({ "unchecked", "rawtypes" })
private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) {
    if (compTupleFilter.getFunction() == null
            || (!(compTupleFilter.getFunction() instanceof BuiltInFunctionTupleFilter)))
        return null;

    BuiltInFunctionTupleFilter builtInFunctionTupleFilter = (BuiltInFunctionTupleFilter) compTupleFilter
            .getFunction();

    if (!builtInFunctionTupleFilter.isValid())
        return null;

    TblColRef columnRef = builtInFunctionTupleFilter.getColumn();
    if (columnRef == null) {
        return null;
    }
    Dictionary<?> dict = dimEncMap.getDictionary(columnRef);
    if (dict == null)
        return null;

    CompareTupleFilter translated = new CompareTupleFilter(
            builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN);
    translated.addChild(new ColumnTupleFilter(columnRef));

    try {
        Collection<Object> inValues = Lists.newArrayList();
        for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
            Object dictVal = dict.getValueFromId(i);
            Object computedVal = builtInFunctionTupleFilter.invokeFunction(dictVal);
            Class clazz = Primitives.wrap(computedVal.getClass());
            Object targetVal = compTupleFilter.getFirstValue();
            if (Primitives.isWrapperType(clazz))
                targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null,
                        compTupleFilter.getFirstValue()));

            int comp = ((Comparable) computedVal).compareTo(targetVal);
            boolean compResult = false;
            switch (compTupleFilter.getOperator()) {
            case EQ:
                compResult = comp == 0;
                break;
            case NEQ:
                compResult = comp != 0;
                break;
            case LT:
                compResult = comp < 0;
                break;
            case LTE:
                compResult = comp <= 0;
                break;
            case GT:
                compResult = comp > 0;
                break;
            case GTE:
                compResult = comp >= 0;
                break;
            case IN:
                compResult = compTupleFilter.getValues().contains(computedVal.toString());
                break;
            case NOTIN:
                compResult = !compTupleFilter.getValues().contains(computedVal.toString());
                break;
            default:
                break;
            }
            if (compResult) {
                inValues.add(dictVal);
            }
        }
        translated.addChild(new ConstantTupleFilter(inValues));
    } catch (Exception e) {
        logger.debug(e.getMessage());
        return null;
    }
    return translated;
}
Example 14
Source File: BuiltInFunctionTransformer.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings({ "unchecked", "rawtypes" })
private TupleFilter translateCompareTupleFilter(CompareTupleFilter compTupleFilter) {
    if (compTupleFilter.getFunction() == null
            || (!(compTupleFilter.getFunction() instanceof BuiltInFunctionTupleFilter)))
        return null;

    BuiltInFunctionTupleFilter builtInFunctionTupleFilter = (BuiltInFunctionTupleFilter) compTupleFilter
            .getFunction();

    if (!builtInFunctionTupleFilter.isValid())
        return null;

    TblColRef columnRef = builtInFunctionTupleFilter.getColumn();
    if (columnRef == null) {
        return null;
    }
    Dictionary<?> dict = dimEncMap.getDictionary(columnRef);
    if (dict == null)
        return null;

    CompareTupleFilter translated = new CompareTupleFilter(
            builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN);
    translated.addChild(new ColumnTupleFilter(columnRef));

    try {
        Collection<Object> inValues = Lists.newArrayList();
        for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
            Object dictVal = dict.getValueFromId(i);
            Object computedVal = builtInFunctionTupleFilter.invokeFunction(dictVal);
            Class clazz = Primitives.wrap(computedVal.getClass());
            Object targetVal = compTupleFilter.getFirstValue();
            if (Primitives.isWrapperType(clazz))
                targetVal = clazz.cast(clazz.getDeclaredMethod("valueOf", String.class).invoke(null,
                        compTupleFilter.getFirstValue()));

            int comp = ((Comparable) computedVal).compareTo(targetVal);
            boolean compResult = false;
            switch (compTupleFilter.getOperator()) {
            case EQ:
                compResult = comp == 0;
                break;
            case NEQ:
                compResult = comp != 0;
                break;
            case LT:
                compResult = comp < 0;
                break;
            case LTE:
                compResult = comp <= 0;
                break;
            case GT:
                compResult = comp > 0;
                break;
            case GTE:
                compResult = comp >= 0;
                break;
            case IN:
                compResult = compTupleFilter.getValues().contains(computedVal.toString());
                break;
            case NOTIN:
                compResult = !compTupleFilter.getValues().contains(computedVal.toString());
                break;
            default:
                break;
            }
            if (compResult) {
                inValues.add(dictVal);
            }
        }
        translated.addChild(new ConstantTupleFilter(inValues));
    } catch (Exception e) {
        logger.debug(e.getMessage());
        return null;
    }
    return translated;
}