Java Code Examples for org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum#AND

The following examples show how to use org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum#AND . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DerivedFilterTranslator.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private static TupleFilter buildInFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    if (satisfyingHostRecords.size() == 0) {
        return ConstantTupleFilter.FALSE;
    }

    int hn = hostCols.length;
    if (hn == 1) {
        CompareTupleFilter in = new CompareTupleFilter(FilterOperatorEnum.IN);
        in.addChild(new ColumnTupleFilter(hostCols[0]));
        in.addChild(new ConstantTupleFilter(asValues(satisfyingHostRecords)));
        return in;
    } else {
        LogicalTupleFilter or = new LogicalTupleFilter(FilterOperatorEnum.OR);
        for (Array<String> rec : satisfyingHostRecords) {
            LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
            for (int i = 0; i < hn; i++) {
                CompareTupleFilter eq = new CompareTupleFilter(FilterOperatorEnum.EQ);
                eq.addChild(new ColumnTupleFilter(hostCols[i]));
                eq.addChild(new ConstantTupleFilter(rec.data[i]));
                and.addChild(eq);
            }
            or.addChild(and);
        }
        return or;
    }
}
Example 2
Source File: DerivedFilterTranslator.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private static LogicalTupleFilter buildRangeFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    int hn = hostCols.length;
    String[] min = new String[hn];
    String[] max = new String[hn];
    findMinMax(satisfyingHostRecords, hostCols, min, max);
    LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
    for (int i = 0; i < hn; i++) {
        CompareTupleFilter compMin = new CompareTupleFilter(FilterOperatorEnum.GTE);
        compMin.addChild(new ColumnTupleFilter(hostCols[i]));
        compMin.addChild(new ConstantTupleFilter(min[i]));
        and.addChild(compMin);
        CompareTupleFilter compMax = new CompareTupleFilter(FilterOperatorEnum.LTE);
        compMax.addChild(new ColumnTupleFilter(hostCols[i]));
        compMax.addChild(new ConstantTupleFilter(max[i]));
        and.addChild(compMax);
    }
    return and;
}
Example 3
Source File: StreamingDataQueryPlanner.java    From kylin with Apache License 2.0 6 votes vote down vote up 
protected TupleFilter flattenToOrAndFilter(TupleFilter filter) {
    if (filter == null)
        return null;

    TupleFilter flatFilter = filter.flatFilter();

    // normalize to OR-AND filter
    if (flatFilter.getOperator() == FilterOperatorEnum.AND) {
        LogicalTupleFilter f = new LogicalTupleFilter(FilterOperatorEnum.OR);
        f.addChild(flatFilter);
        flatFilter = f;
    }

    if (flatFilter.getOperator() != FilterOperatorEnum.OR)
        throw new IllegalStateException();

    return flatFilter;
}
Example 4
Source File: GTCubeStorageQueryBase.java    From kylin with Apache License 2.0 6 votes vote down vote up 
protected Set<CompareTupleFilter> findSingleValuesCompFilters(TupleFilter filter) {
    Collection<? extends TupleFilter> toCheck;
    if (filter instanceof CompareTupleFilter) {
        toCheck = Collections.singleton(filter);
    } else if (filter instanceof LogicalTupleFilter && filter.getOperator() == FilterOperatorEnum.AND) {
        toCheck = filter.getChildren();
    } else {
        return Collections.emptySet();
    }

    Set<CompareTupleFilter> result = Sets.newHashSet();
    for (TupleFilter f : toCheck) {
        if (f instanceof CompareTupleFilter) {
            CompareTupleFilter compFilter = (CompareTupleFilter) f;
            // is COL=const ?
            if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1
                    && compFilter.getColumn() != null) {
                result.add(compFilter);
            }
        }
    }
    return result;
}
Example 5
Source File: DerivedFilterTranslator.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private static LogicalTupleFilter buildRangeFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    int hn = hostCols.length;
    String[] min = new String[hn];
    String[] max = new String[hn];
    findMinMax(satisfyingHostRecords, hostCols, min, max);
    LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
    for (int i = 0; i < hn; i++) {
        CompareTupleFilter compMin = new CompareTupleFilter(FilterOperatorEnum.GTE);
        compMin.addChild(new ColumnTupleFilter(hostCols[i]));
        compMin.addChild(new ConstantTupleFilter(min[i]));
        and.addChild(compMin);
        CompareTupleFilter compMax = new CompareTupleFilter(FilterOperatorEnum.LTE);
        compMax.addChild(new ColumnTupleFilter(hostCols[i]));
        compMax.addChild(new ConstantTupleFilter(max[i]));
        and.addChild(compMax);
    }
    return and;
}
Example 6
Source File: DerivedFilterTranslator.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private static TupleFilter buildInFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    if (satisfyingHostRecords.size() == 0) {
        return ConstantTupleFilter.FALSE;
    }

    int hn = hostCols.length;
    if (hn == 1) {
        CompareTupleFilter in = new CompareTupleFilter(FilterOperatorEnum.IN);
        in.addChild(new ColumnTupleFilter(hostCols[0]));
        in.addChild(new ConstantTupleFilter(asValues(satisfyingHostRecords)));
        return in;
    } else {
        LogicalTupleFilter or = new LogicalTupleFilter(FilterOperatorEnum.OR);
        for (Array<String> rec : satisfyingHostRecords) {
            LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
            for (int i = 0; i < hn; i++) {
                CompareTupleFilter eq = new CompareTupleFilter(FilterOperatorEnum.EQ);
                eq.addChild(new ColumnTupleFilter(hostCols[i]));
                eq.addChild(new ConstantTupleFilter(rec.data[i]));
                and.addChild(eq);
            }
            or.addChild(and);
        }
        return or;
    }
}
Example 7
Source File: StreamingDataQueryPlanner.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
protected TupleFilter flattenToOrAndFilter(TupleFilter filter) {
    if (filter == null)
        return null;

    TupleFilter flatFilter = filter.flatFilter();

    // normalize to OR-AND filter
    if (flatFilter.getOperator() == FilterOperatorEnum.AND) {
        LogicalTupleFilter f = new LogicalTupleFilter(FilterOperatorEnum.OR);
        f.addChild(flatFilter);
        flatFilter = f;
    }

    if (flatFilter.getOperator() != FilterOperatorEnum.OR)
        throw new IllegalStateException();

    return flatFilter;
}
Example 8
Source File: GTCubeStorageQueryBase.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
protected Set<CompareTupleFilter> findSingleValuesCompFilters(TupleFilter filter) {
    Collection<? extends TupleFilter> toCheck;
    if (filter instanceof CompareTupleFilter) {
        toCheck = Collections.singleton(filter);
    } else if (filter instanceof LogicalTupleFilter && filter.getOperator() == FilterOperatorEnum.AND) {
        toCheck = filter.getChildren();
    } else {
        return Collections.emptySet();
    }

    Set<CompareTupleFilter> result = Sets.newHashSet();
    for (TupleFilter f : toCheck) {
        if (f instanceof CompareTupleFilter) {
            CompareTupleFilter compFilter = (CompareTupleFilter) f;
            // is COL=const ?
            if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1
                    && compFilter.getColumn() != null) {
                result.add(compFilter);
            }
        }
    }
    return result;
}
Example 9
Source File: CubeStorageEngine.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private List<Collection<ColumnValueRange>> translateToOrAndDimRanges(TupleFilter flatFilter, CubeSegment cubeSegment) {
    List<Collection<ColumnValueRange>> result = Lists.newArrayList();

    if (flatFilter == null) {
        result.add(Collections.<ColumnValueRange> emptyList());
        return result;
    }

    for (TupleFilter andFilter : flatFilter.getChildren()) {
        if (andFilter.getOperator() != FilterOperatorEnum.AND) {
            throw new IllegalStateException("Filter should be AND instead of " + andFilter);
        }

        Collection<ColumnValueRange> andRanges = translateToAndDimRanges(andFilter.getChildren(), cubeSegment);

        result.add(andRanges);
    }

    return preprocessConstantConditions(result);
}
Example 10
Source File: StreamingDataQueryPlanner.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public boolean canSkip(long timeStart, long timeEnd, boolean endInclude) {
    if (flatFilter == null) {
        return false;
    }
    CompareFilterTimeRangeChecker timeRangeChecker = new CompareFilterTimeRangeChecker(timeStart, timeEnd, endInclude);
    for (TupleFilter andFilter : flatFilter.getChildren()) {
        if (andFilter.getOperator() != FilterOperatorEnum.AND) {
            throw new IllegalStateException("Filter should be AND instead of " + andFilter);
        }
        if (!canSkipForAndFilter(andFilter, timeRangeChecker)) {
            return false;
        }
    }
    return true;
}
Example 11
Source File: StorageTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unused")
private TupleFilter buildAndFilter(List<TblColRef> columns) {
    CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
    CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
    LogicalTupleFilter andFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
    andFilter.addChild(compareFilter1);
    andFilter.addChild(compareFilter2);
    return andFilter;
}
Example 12
Source File: FilterSerializeTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSerialize11() {
    List<TblColRef> groups = buildGroups();
    TupleFilter orFilter = buildOrFilter(groups);
    TupleFilter andFilter = buildAndFilter(groups);

    LogicalTupleFilter logicFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
    logicFilter.addChild(orFilter);
    logicFilter.addChild(andFilter);

    assertFilterSerDe(logicFilter);
}
Example 13
Source File: FilterBaseTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
protected TupleFilter buildAndFilter(List<TblColRef> columns) {
    CompareTupleFilter compareFilter1 = buildEQCompareFilter(columns, 0);
    CompareTupleFilter compareFilter2 = buildEQCompareFilter(columns, 1);
    LogicalTupleFilter andFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
    andFilter.addChild(compareFilter1);
    andFilter.addChild(compareFilter2);
    return andFilter;
}
Example 14
Source File: FilterBaseTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
protected TupleFilter buildAndFilter(List<TblColRef> columns) {
    CompareTupleFilter compareFilter1 = buildEQCompareFilter(columns, 0);
    CompareTupleFilter compareFilter2 = buildEQCompareFilter(columns, 1);
    LogicalTupleFilter andFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
    andFilter.addChild(compareFilter1);
    andFilter.addChild(compareFilter2);
    return andFilter;
}
Example 15
Source File: FilterSerializeTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSerialize11() {
    List<TblColRef> groups = buildGroups();
    TupleFilter orFilter = buildOrFilter(groups);
    TupleFilter andFilter = buildAndFilter(groups);

    LogicalTupleFilter logicFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
    logicFilter.addChild(orFilter);
    logicFilter.addChild(andFilter);

    assertFilterSerDe(logicFilter);
}
Example 16
Source File: StreamingDataQueryPlanner.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public boolean canSkip(long timeStart, long timeEnd, boolean endInclude) {
    if (flatFilter == null) {
        return false;
    }
    CompareFilterTimeRangeChecker timeRangeChecker = new CompareFilterTimeRangeChecker(timeStart, timeEnd, endInclude);
    for (TupleFilter andFilter : flatFilter.getChildren()) {
        if (andFilter.getOperator() != FilterOperatorEnum.AND) {
            throw new IllegalStateException("Filter should be AND instead of " + andFilter);
        }
        if (!canSkipForAndFilter(andFilter, timeRangeChecker)) {
            return false;
        }
    }
    return true;
}
Example 17
Source File: FilterSerializeTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSerialize11() {
    List<TblColRef> groups = buildGroups();
    TupleFilter orFilter = buildOrFilter(groups);
    TupleFilter andFilter = buildAndFilter(groups);

    LogicalTupleFilter logicFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
    logicFilter.addChild(orFilter);
    logicFilter.addChild(andFilter);

    byte[] bytes = TupleFilterSerializer.serialize(logicFilter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(logicFilter, newFilter);
}
Example 18
Source File: TupleFilterTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void flatFilterTest() {
    TupleFilter topAndFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
    topAndFilter.addChild(createEQFilter("c1", "v1"));
    TupleFilter orFilter1 = new LogicalTupleFilter(FilterOperatorEnum.OR);
    TupleFilter andFilter11 = new LogicalTupleFilter(FilterOperatorEnum.AND);
    andFilter11.addChild(createEQFilter("c2", "v2"));
    andFilter11.addChild(createEQFilter("c3", "v3"));

    TupleFilter andFilter12 = new LogicalTupleFilter(FilterOperatorEnum.AND);
    andFilter12.addChild(createEQFilter("c2", "v21"));
    andFilter12.addChild(createEQFilter("c3", "v31"));

    orFilter1.addChild(andFilter11);
    orFilter1.addChild(andFilter12);

    TupleFilter orFilter2 = new LogicalTupleFilter(FilterOperatorEnum.OR);
    TupleFilter andFilter21 = new LogicalTupleFilter(FilterOperatorEnum.AND);
    andFilter21.addChild(createEQFilter("c4", "v4"));
    andFilter21.addChild(createEQFilter("c5", "v5"));

    TupleFilter andFilter22 = new LogicalTupleFilter(FilterOperatorEnum.AND);
    andFilter22.addChild(createEQFilter("c4", "v41"));
    andFilter22.addChild(createEQFilter("c5", "v51"));

    TupleFilter andFilter23 = new LogicalTupleFilter(FilterOperatorEnum.AND);
    andFilter23.addChild(createEQFilter("c4", "v42"));
    andFilter23.addChild(createEQFilter("c5", "v52"));

    orFilter2.addChild(andFilter21);
    orFilter2.addChild(andFilter22);
    orFilter2.addChild(andFilter23);

    topAndFilter.addChild(orFilter1);
    topAndFilter.addChild(orFilter2);

    TupleFilter flatFilter = topAndFilter.flatFilter(500000);
    Assert.assertEquals(6, flatFilter.children.size());
}
Example 19
Source File: CubeStorageEngine.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
private Set<TblColRef> findSingleValueColumns(TupleFilter filter) {
    Collection<? extends TupleFilter> toCheck;
    if (filter instanceof CompareTupleFilter) {
        toCheck = Collections.singleton(filter);
    } else if (filter instanceof LogicalTupleFilter && filter.getOperator() == FilterOperatorEnum.AND) {
        toCheck = filter.getChildren();
    } else {
        return (Set<TblColRef>) Collections.EMPTY_SET;
    }

    Set<TblColRef> result = Sets.newHashSet();
    for (TupleFilter f : toCheck) {
        if (f instanceof CompareTupleFilter) {
            CompareTupleFilter compFilter = (CompareTupleFilter) f;
            // is COL=const ?
            if (compFilter.getOperator() == FilterOperatorEnum.EQ && compFilter.getValues().size() == 1 && compFilter.getColumn() != null) {
                result.add(compFilter.getColumn());
            }
        }
    }

    // expand derived
    Set<TblColRef> resultD = Sets.newHashSet();
    for (TblColRef col : result) {
        if (cubeDesc.isDerived(col)) {
            DeriveInfo hostInfo = cubeDesc.getHostInfo(col);
            if (hostInfo.isOneToOne) {
                for (TblColRef hostCol : hostInfo.columns) {
                    resultD.add(hostCol);
                }
            }
            //if not one2one, it will be pruned
        } else {
            resultD.add(col);
        }
    }
    return resultD;
}
Example 20
Source File: FilterBaseTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
protected TupleFilter buildAndFilter(List<TblColRef> columns) {
    CompareTupleFilter compareFilter1 = buildCompareFilter(columns, 0);
    CompareTupleFilter compareFilter2 = buildCompareFilter(columns, 1);
    LogicalTupleFilter andFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
    andFilter.addChild(compareFilter1);
    andFilter.addChild(compareFilter2);
    return andFilter;
}