Java Code Examples for org.apache.kylin.metadata.filter.ConstantTupleFilter#TRUE

The following examples show how to use org.apache.kylin.metadata.filter.ConstantTupleFilter#TRUE . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: GTUtil.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter onSerialize(TupleFilter filter) {
    if (filter == null)
        return null;

    // In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE,
    // Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter)
    // will always return FALSE.
    if (filter.getOperator() == FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // map to column onto grid table
    if (colMapping != null && filter instanceof ColumnTupleFilter) {
        ColumnTupleFilter colFilter = (ColumnTupleFilter) filter;
        int gtColIdx = mapCol(colFilter.getColumn());
        return new ColumnTupleFilter(info.colRef(gtColIdx));
    }

    // encode constants
    if (useEncodeConstants && filter instanceof CompareTupleFilter) {
        return encodeConstants((CompareTupleFilter) filter);
    }

    return filter;
}
Example 2
Source File: StreamingBuiltInFunctionTransformer.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter transform(TupleFilter tupleFilter) {
    TupleFilter translated = null;
    if (tupleFilter instanceof CompareTupleFilter) {
        //normal case
        translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
        if (translated != null) {
            logger.info("Translated {" + tupleFilter + "}");
        }
    } else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
        //like case
        translated = translateFunctionTupleFilter((BuiltInFunctionTupleFilter) tupleFilter);
        if (translated != null) {
            logger.info("Translated {" + tupleFilter + "}");
        }
    } else if (tupleFilter instanceof LogicalTupleFilter) {
        @SuppressWarnings("unchecked")
        ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren()
                .listIterator();
        while (childIterator.hasNext()) {
            TupleFilter transformed = transform(childIterator.next());
            if (transformed != null)
                childIterator.set(transformed);
        }
    }

    TupleFilter result = translated == null ? tupleFilter : translated;
    if (result.getOperator() == TupleFilter.FilterOperatorEnum.NOT
            && !TupleFilter.isEvaluableRecursively(result)) {
        TupleFilter.collectColumns(result, unEvaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!result.isEvaluable()) {
        TupleFilter.collectColumns(result, unEvaluableColumns);
        return ConstantTupleFilter.TRUE;
    }
    return result;
}
Example 3
Source File: FragmentFileSearcher.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter transform(TupleFilter filter) {
    if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT
            && !TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, unEvaluableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, unEvaluableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    if (filter instanceof CompareTupleFilter) {
        return translateCompareFilter((CompareTupleFilter) filter);
    } else if (filter instanceof LogicalTupleFilter) {
        @SuppressWarnings("unchecked")
        ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) filter.getChildren().listIterator();
        while (childIterator.hasNext()) {
            TupleFilter transformed = transform(childIterator.next());
            if (transformed != null) {
                childIterator.set(transformed);
            } else {
                throw new IllegalStateException("Should not be null");
            }
        }
    }
    return filter;
}
Example 4
Source File: GTUtil.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter onSerialize(TupleFilter filter) {
    if (filter == null)
        return null;

    // In case of NOT(unEvaluatableFilter), we should immediately replace it as TRUE,
    // Otherwise, unEvaluatableFilter will later be replace with TRUE and NOT(unEvaluatableFilter)
    // will always return FALSE.
    if (filter.getOperator() == FilterOperatorEnum.NOT && !TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, unevaluatableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // map to column onto grid table
    if (colMapping != null && filter instanceof ColumnTupleFilter) {
        ColumnTupleFilter colFilter = (ColumnTupleFilter) filter;
        int gtColIdx = mapCol(colFilter.getColumn());
        return new ColumnTupleFilter(info.colRef(gtColIdx));
    }

    // encode constants
    if (useEncodeConstants && filter instanceof CompareTupleFilter) {
        return encodeConstants((CompareTupleFilter) filter);
    }

    return filter;
}
Example 5
Source File: StreamingBuiltInFunctionTransformer.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter transform(TupleFilter tupleFilter) {
    TupleFilter translated = null;
    if (tupleFilter instanceof CompareTupleFilter) {
        //normal case
        translated = translateCompareTupleFilter((CompareTupleFilter) tupleFilter);
        if (translated != null) {
            logger.info("Translated {" + tupleFilter + "}");
        }
    } else if (tupleFilter instanceof BuiltInFunctionTupleFilter) {
        //like case
        translated = translateFunctionTupleFilter((BuiltInFunctionTupleFilter) tupleFilter);
        if (translated != null) {
            logger.info("Translated {" + tupleFilter + "}");
        }
    } else if (tupleFilter instanceof LogicalTupleFilter) {
        @SuppressWarnings("unchecked")
        ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) tupleFilter.getChildren()
                .listIterator();
        while (childIterator.hasNext()) {
            TupleFilter transformed = transform(childIterator.next());
            if (transformed != null)
                childIterator.set(transformed);
        }
    }

    TupleFilter result = translated == null ? tupleFilter : translated;
    if (result.getOperator() == TupleFilter.FilterOperatorEnum.NOT
            && !TupleFilter.isEvaluableRecursively(result)) {
        TupleFilter.collectColumns(result, unEvaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!result.isEvaluable()) {
        TupleFilter.collectColumns(result, unEvaluableColumns);
        return ConstantTupleFilter.TRUE;
    }
    return result;
}
Example 6
Source File: FragmentFileSearcher.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public TupleFilter transform(TupleFilter filter) {
    if (filter.getOperator() == TupleFilter.FilterOperatorEnum.NOT
            && !TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, unEvaluableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    // shortcut for unEvaluatable filter
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, unEvaluableColumnCollector);
        return ConstantTupleFilter.TRUE;
    }

    if (filter instanceof CompareTupleFilter) {
        return translateCompareFilter((CompareTupleFilter) filter);
    } else if (filter instanceof LogicalTupleFilter) {
        @SuppressWarnings("unchecked")
        ListIterator<TupleFilter> childIterator = (ListIterator<TupleFilter>) filter.getChildren().listIterator();
        while (childIterator.hasNext()) {
            TupleFilter transformed = transform(childIterator.next());
            if (transformed != null) {
                childIterator.set(transformed);
            } else {
                throw new IllegalStateException("Should not be null");
            }
        }
    }
    return filter;
}
Example 7
Source File: FilterDecorator.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public TupleFilter onSerialize(TupleFilter filter) {
    if (filter == null)
        return null;

    BuiltInFunctionTransformer translator = new BuiltInFunctionTransformer(dimEncMap);
    filter = translator.transform(filter);

    // un-evaluatable filter is replaced with TRUE
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (!(filter instanceof CompareTupleFilter))
        return filter;

    // double check all internal of CompareTupleFilter is evaluatable
    if (!TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (filterConstantsTreatment == FilterConstantsTreatment.AS_IT_IS) {
        return filter;
    } else {

        // extract ColumnFilter & ConstantFilter
        CompareTupleFilter compareFilter = (CompareTupleFilter) filter;
        TblColRef col = compareFilter.getColumn();

        if (col == null) {
            return filter;
        }

        Collection<String> constValues = (Collection<String>) compareFilter.getValues();
        if (constValues == null || constValues.isEmpty()) {
            return filter;
        }

        CompareTupleFilter newCompareFilter = new CompareTupleFilter(compareFilter.getOperator());
        newCompareFilter.addChild(new ColumnTupleFilter(col));

        if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_GLOBAL_DICT) {
            return replaceConstantsWithGlobalDict(compareFilter, newCompareFilter);
        } else if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_LOCAL_DICT) {
            return replaceConstantsWithLocalDict(compareFilter, newCompareFilter);
        } else {
            throw new RuntimeException("should not reach here");
        }
    }
}
Example 8
Source File: FragmentFileSearcher.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
public void search(StreamingSearchContext searchContext, ResultCollector collector) throws IOException {
    String timezone = searchContext.getCubeDesc().getConfig().getStreamingDerivedTimeTimezone();
    long timezoneOffset = 0;
    if (timezone != null && timezone.length() > 0) {
        timezoneOffset = TimeZone.getTimeZone(timezone).getRawOffset();
    }
    FragmentMetaInfo fragmentMetaInfo = fragmentData.getFragmentMetaInfo();
    CuboidMetaInfo cuboidMetaInfo;
    if (searchContext.hitBasicCuboid()) {
        cuboidMetaInfo = fragmentMetaInfo.getBasicCuboidMetaInfo();
    } else {
        cuboidMetaInfo = fragmentMetaInfo.getCuboidMetaInfo(searchContext.getHitCuboid());
        if (cuboidMetaInfo == null) {
            logger.warn("the cuboid:{} is not exist in the fragment:{}, use basic cuboid instead",
                    searchContext.getHitCuboid(), fragment.getFragmentId());
            cuboidMetaInfo = fragmentMetaInfo.getBasicCuboidMetaInfo();
        }
    }

    ResponseResultSchema responseSchema = searchContext.getRespResultSchema();
    TblColRef[] dimensions = responseSchema.getDimensions();
    FunctionDesc[] metrics = responseSchema.getMetrics();
    Map<TblColRef, Dictionary<String>> dictMap = fragmentData.getDimensionDictionaries(dimensions);

    CubeDesc cubeDesc = responseSchema.getCubeDesc();
    List<MeasureDesc> allMeasures = cubeDesc.getMeasures();
    Map<FunctionDesc, MeasureDesc> funcMeasureMap = Maps.newHashMap();
    for (MeasureDesc measure : allMeasures) {
        funcMeasureMap.put(measure.getFunction(), measure);
    }
    MeasureDesc[] measures = new MeasureDesc[metrics.length];
    for (int i = 0; i < measures.length; i++) {
        measures[i] = funcMeasureMap.get(metrics[i]);
    }
    DimensionEncoding[] dimensionEncodings = ParsedStreamingCubeInfo.getDimensionEncodings(cubeDesc, dimensions,
            dictMap);
    ColumnarMetricsEncoding[] metricsEncodings = ParsedStreamingCubeInfo.getMetricsEncodings(measures);
    ColumnarRecordCodec recordCodec = new ColumnarRecordCodec(dimensionEncodings, metricsEncodings);

    // change the unEvaluable dimensions to groupBy
    Set<TblColRef> unEvaluateDims = Sets.newHashSet();
    TupleFilter fragmentFilter = null;
    if (searchContext.getFilter() != null) {
        fragmentFilter = convertFilter(fragmentMetaInfo, searchContext.getFilter(), recordCodec, dimensions,
                new CubeDimEncMap(cubeDesc, dictMap), unEvaluateDims, timezoneOffset);
    }
    if (ConstantTupleFilter.TRUE == fragmentFilter) {
        fragmentFilter = null;
    } else if (ConstantTupleFilter.FALSE == fragmentFilter) {
        collector.collectSearchResult(IStreamingSearchResult.EMPTY_RESULT);
    }
    Set<TblColRef> groups = searchContext.getGroups();
    if (!unEvaluateDims.isEmpty()) {
        searchContext.addNewGroups(unEvaluateDims);
        groups = Sets.union(groups, unEvaluateDims);
    }
    collector.collectSearchResult(new FragmentSearchResult(fragment, fragmentData, cuboidMetaInfo, responseSchema, fragmentFilter, groups, searchContext.getHavingFilter(),
            recordCodec));
}
Example 9
Source File: FilterDecorator.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public TupleFilter onSerialize(TupleFilter filter) {
    if (filter == null)
        return null;

    BuiltInFunctionTransformer translator = new BuiltInFunctionTransformer(dimEncMap);
    filter = translator.transform(filter);

    // un-evaluatable filter is replaced with TRUE
    if (!filter.isEvaluable()) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (!(filter instanceof CompareTupleFilter))
        return filter;

    // double check all internal of CompareTupleFilter is evaluatable
    if (!TupleFilter.isEvaluableRecursively(filter)) {
        TupleFilter.collectColumns(filter, inevaluableColumns);
        return ConstantTupleFilter.TRUE;
    }

    if (filterConstantsTreatment == FilterConstantsTreatment.AS_IT_IS) {
        return filter;
    } else {

        // extract ColumnFilter & ConstantFilter
        CompareTupleFilter compareFilter = (CompareTupleFilter) filter;
        TblColRef col = compareFilter.getColumn();

        if (col == null) {
            return filter;
        }

        Collection<String> constValues = (Collection<String>) compareFilter.getValues();
        if (constValues == null || constValues.isEmpty()) {
            return filter;
        }

        CompareTupleFilter newCompareFilter = new CompareTupleFilter(compareFilter.getOperator());
        newCompareFilter.addChild(new ColumnTupleFilter(col));

        if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_GLOBAL_DICT) {
            return replaceConstantsWithGlobalDict(compareFilter, newCompareFilter);
        } else if (filterConstantsTreatment == FilterConstantsTreatment.REPLACE_WITH_LOCAL_DICT) {
            return replaceConstantsWithLocalDict(compareFilter, newCompareFilter);
        } else {
            throw new RuntimeException("should not reach here");
        }
    }
}
Example 10
Source File: FragmentFileSearcher.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
public void search(StreamingSearchContext searchContext, ResultCollector collector) throws IOException {
    String timezone = searchContext.getCubeDesc().getConfig().getStreamingDerivedTimeTimezone();
    long timezoneOffset = 0;
    if (timezone != null && timezone.length() > 0) {
        timezoneOffset = TimeZone.getTimeZone(timezone).getRawOffset();
    }
    FragmentMetaInfo fragmentMetaInfo = fragmentData.getFragmentMetaInfo();
    CuboidMetaInfo cuboidMetaInfo;
    if (searchContext.hitBasicCuboid()) {
        cuboidMetaInfo = fragmentMetaInfo.getBasicCuboidMetaInfo();
    } else {
        cuboidMetaInfo = fragmentMetaInfo.getCuboidMetaInfo(searchContext.getHitCuboid());
        if (cuboidMetaInfo == null) {
            logger.warn("the cuboid:{} is not exist in the fragment:{}, use basic cuboid instead",
                    searchContext.getHitCuboid(), fragment.getFragmentId());
            cuboidMetaInfo = fragmentMetaInfo.getBasicCuboidMetaInfo();
        }
    }

    ResponseResultSchema responseSchema = searchContext.getRespResultSchema();
    TblColRef[] dimensions = responseSchema.getDimensions();
    FunctionDesc[] metrics = responseSchema.getMetrics();
    Map<TblColRef, Dictionary<String>> dictMap = fragmentData.getDimensionDictionaries(dimensions);

    CubeDesc cubeDesc = responseSchema.getCubeDesc();
    List<MeasureDesc> allMeasures = cubeDesc.getMeasures();
    Map<FunctionDesc, MeasureDesc> funcMeasureMap = Maps.newHashMap();
    for (MeasureDesc measure : allMeasures) {
        funcMeasureMap.put(measure.getFunction(), measure);
    }
    MeasureDesc[] measures = new MeasureDesc[metrics.length];
    for (int i = 0; i < measures.length; i++) {
        measures[i] = funcMeasureMap.get(metrics[i]);
    }
    DimensionEncoding[] dimensionEncodings = ParsedStreamingCubeInfo.getDimensionEncodings(cubeDesc, dimensions,
            dictMap);
    ColumnarMetricsEncoding[] metricsEncodings = ParsedStreamingCubeInfo.getMetricsEncodings(measures);
    ColumnarRecordCodec recordCodec = new ColumnarRecordCodec(dimensionEncodings, metricsEncodings);

    // change the unEvaluable dimensions to groupBy
    Set<TblColRef> unEvaluateDims = Sets.newHashSet();
    TupleFilter fragmentFilter = null;
    if (searchContext.getFilter() != null) {
        fragmentFilter = convertFilter(fragmentMetaInfo, searchContext.getFilter(), recordCodec, dimensions,
                new CubeDimEncMap(cubeDesc, dictMap), unEvaluateDims, timezoneOffset);
    }
    if (ConstantTupleFilter.TRUE == fragmentFilter) {
        fragmentFilter = null;
    } else if (ConstantTupleFilter.FALSE == fragmentFilter) {
        collector.collectSearchResult(IStreamingSearchResult.EMPTY_RESULT);
    }
    Set<TblColRef> groups = searchContext.getGroups();
    if (!unEvaluateDims.isEmpty()) {
        searchContext.addNewGroups(unEvaluateDims);
        groups = Sets.union(groups, unEvaluateDims);
    }
    collector.collectSearchResult(new FragmentSearchResult(fragment, fragmentData, cuboidMetaInfo, responseSchema, fragmentFilter, groups, searchContext.getHavingFilter(),
            recordCodec));
}
Example 11
Source File: EndpointTupleIterator.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
public EndpointTupleIterator(IISegment segment, TupleFilter rootFilter, Collection<TblColRef> groupBy, List<FunctionDesc> measures, StorageContext context, HConnection conn) throws Throwable {

        String tableName = segment.getStorageLocationIdentifier();
        table = conn.getTable(tableName);
        factTableName = segment.getIIDesc().getFactTableName();

        if (rootFilter == null) {
            rootFilter = ConstantTupleFilter.TRUE;
        }

        if (groupBy == null) {
            groupBy = Sets.newHashSet();
        }

        if (measures == null) {
            measures = Lists.newArrayList();
        }

        //this method will change measures
        rewriteMeasureParameters(measures, segment.getColumns());

        this.seg = segment;
        this.context = context;
        this.measures = measures;

        this.columns = segment.getColumns();
        this.columnNames = getColumnNames(columns);

        this.tupleInfo = buildTupleInfo();
        this.tableRecordInfo = new TableRecordInfo(this.seg);

        this.pushedDownRowType = CoprocessorRowType.fromTableRecordInfo(tableRecordInfo, this.columns);
        this.pushedDownFilter = CoprocessorFilter.fromFilter(this.seg, rootFilter);

        for (TblColRef column : this.pushedDownFilter.getUnstrictlyFilteredColumns()) {
            groupBy.add(column);
        }

        this.pushedDownProjector = CoprocessorProjector.makeForEndpoint(tableRecordInfo, groupBy);
        this.pushedDownAggregators = EndpointAggregators.fromFunctions(tableRecordInfo, measures);

        IIProtos.IIRequest endpointRequest = prepareRequest();
        regionResponsesIterator = getResults(endpointRequest, table);

        if (this.regionResponsesIterator.hasNext()) {
            this.tupleIterator = new SingleRegionTupleIterator(this.regionResponsesIterator.next());
        } else {
            this.tupleIterator = ITupleIterator.EMPTY_TUPLE_ITERATOR;
        }
    }