org.apache.kylin.metadata.filter.ColumnTupleFilter Java Examples

private static TupleFilter buildInFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    if (satisfyingHostRecords.size() == 0) {
        return ConstantTupleFilter.FALSE;
    }

    int hn = hostCols.length;
    if (hn == 1) {
        CompareTupleFilter in = new CompareTupleFilter(FilterOperatorEnum.IN);
        in.addChild(new ColumnTupleFilter(hostCols[0]));
        in.addChild(new ConstantTupleFilter(asValues(satisfyingHostRecords)));
        return in;
    } else {
        LogicalTupleFilter or = new LogicalTupleFilter(FilterOperatorEnum.OR);
        for (Array<String> rec : satisfyingHostRecords) {
            LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
            for (int i = 0; i < hn; i++) {
                CompareTupleFilter eq = new CompareTupleFilter(FilterOperatorEnum.EQ);
                eq.addChild(new ColumnTupleFilter(hostCols[i]));
                eq.addChild(new ConstantTupleFilter(rec.data[i]));
                and.addChild(eq);
            }
            or.addChild(and);
        }
        return or;
    }
}
 

protected CompareTupleFilter buildINCompareFilter(TblColRef dateColumn) throws ParseException {
    CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.IN);
    ColumnTupleFilter columnFilter = new ColumnTupleFilter(dateColumn);
    compareFilter.addChild(columnFilter);

    List<String> inValues = Lists.newArrayList();
    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.ROOT);
    Date startDate = simpleDateFormat.parse("1970-01-01");
    Date endDate = simpleDateFormat.parse("2100-01-01");
    Calendar start = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
    start.setTime(startDate);
    Calendar end = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
    end.setTime(endDate);
    for (Date date = start.getTime(); start.before(end); start.add(Calendar.DATE, 1), date = start.getTime()) {
        inValues.add(simpleDateFormat.format(date));
    }

    ConstantTupleFilter constantFilter = new ConstantTupleFilter(inValues);
    compareFilter.addChild(constantFilter);
    return compareFilter;
}
 

protected CompareTupleFilter buildINCompareFilter(TblColRef dateColumn) throws ParseException {
    CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.IN);
    ColumnTupleFilter columnFilter = new ColumnTupleFilter(dateColumn);
    compareFilter.addChild(columnFilter);

    List<String> inValues = Lists.newArrayList();
    SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.ROOT);
    Date startDate = simpleDateFormat.parse("1970-01-01");
    Date endDate = simpleDateFormat.parse("2100-01-01");
    Calendar start = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
    start.setTime(startDate);
    Calendar end = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
    end.setTime(endDate);
    for (Date date = start.getTime(); start.before(end); start.add(Calendar.DATE, 1), date = start.getTime()) {
        inValues.add(simpleDateFormat.format(date));
    }

    ConstantTupleFilter constantFilter = new ConstantTupleFilter(inValues);
    compareFilter.addChild(constantFilter);
    return compareFilter;
}
 

private static LogicalTupleFilter buildRangeFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    int hn = hostCols.length;
    String[] min = new String[hn];
    String[] max = new String[hn];
    findMinMax(satisfyingHostRecords, hostCols, min, max);
    LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
    for (int i = 0; i < hn; i++) {
        CompareTupleFilter compMin = new CompareTupleFilter(FilterOperatorEnum.GTE);
        compMin.addChild(new ColumnTupleFilter(hostCols[i]));
        compMin.addChild(new ConstantTupleFilter(min[i]));
        and.addChild(compMin);
        CompareTupleFilter compMax = new CompareTupleFilter(FilterOperatorEnum.LTE);
        compMax.addChild(new ColumnTupleFilter(hostCols[i]));
        compMax.addChild(new ConstantTupleFilter(max[i]));
        and.addChild(compMax);
    }
    return and;
}
 

private static LogicalTupleFilter buildRangeFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    int hn = hostCols.length;
    String[] min = new String[hn];
    String[] max = new String[hn];
    findMinMax(satisfyingHostRecords, hostCols, min, max);
    LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
    for (int i = 0; i < hn; i++) {
        CompareTupleFilter compMin = new CompareTupleFilter(FilterOperatorEnum.GTE);
        compMin.addChild(new ColumnTupleFilter(hostCols[i]));
        compMin.addChild(new ConstantTupleFilter(min[i]));
        and.addChild(compMin);
        CompareTupleFilter compMax = new CompareTupleFilter(FilterOperatorEnum.LTE);
        compMax.addChild(new ColumnTupleFilter(hostCols[i]));
        compMax.addChild(new ConstantTupleFilter(max[i]));
        and.addChild(compMax);
    }
    return and;
}
 

private static TupleFilter buildInFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    if (satisfyingHostRecords.size() == 0) {
        return ConstantTupleFilter.FALSE;
    }

    int hn = hostCols.length;
    if (hn == 1) {
        CompareTupleFilter in = new CompareTupleFilter(FilterOperatorEnum.IN);
        in.addChild(new ColumnTupleFilter(hostCols[0]));
        in.addChild(new ConstantTupleFilter(asValues(satisfyingHostRecords)));
        return in;
    } else {
        LogicalTupleFilter or = new LogicalTupleFilter(FilterOperatorEnum.OR);
        for (Array<String> rec : satisfyingHostRecords) {
            LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
            for (int i = 0; i < hn; i++) {
                CompareTupleFilter eq = new CompareTupleFilter(FilterOperatorEnum.EQ);
                eq.addChild(new ColumnTupleFilter(hostCols[i]));
                eq.addChild(new ConstantTupleFilter(rec.data[i]));
                and.addChild(eq);
            }
            or.addChild(and);
        }
        return or;
    }
}
 

private static TupleFilter buildInFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    if (satisfyingHostRecords.size() == 0) {
        return ConstantTupleFilter.FALSE;
    }

    int hn = hostCols.length;
    if (hn == 1) {
        CompareTupleFilter in = new CompareTupleFilter(FilterOperatorEnum.IN);
        in.addChild(new ColumnTupleFilter(hostCols[0]));
        in.addChild(new ConstantTupleFilter(asValues(satisfyingHostRecords)));
        return in;
    } else {
        LogicalTupleFilter or = new LogicalTupleFilter(FilterOperatorEnum.OR);
        for (Array<String> rec : satisfyingHostRecords) {
            LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
            for (int i = 0; i < hn; i++) {
                CompareTupleFilter eq = new CompareTupleFilter(FilterOperatorEnum.EQ);
                eq.addChild(new ColumnTupleFilter(hostCols[i]));
                eq.addChild(new ConstantTupleFilter(rec.data[i]));
                and.addChild(eq);
            }
            or.addChild(and);
        }
        return or;
    }
}
 

private static LogicalTupleFilter buildRangeFilter(TblColRef[] hostCols, Set<Array<String>> satisfyingHostRecords) {
    int hn = hostCols.length;
    String[] min = new String[hn];
    String[] max = new String[hn];
    findMinMax(satisfyingHostRecords, hostCols, min, max);
    LogicalTupleFilter and = new LogicalTupleFilter(FilterOperatorEnum.AND);
    for (int i = 0; i < hn; i++) {
        CompareTupleFilter compMin = new CompareTupleFilter(FilterOperatorEnum.GTE);
        compMin.addChild(new ColumnTupleFilter(hostCols[i]));
        compMin.addChild(new ConstantTupleFilter(min[i]));
        and.addChild(compMin);
        CompareTupleFilter compMax = new CompareTupleFilter(FilterOperatorEnum.LTE);
        compMax.addChild(new ColumnTupleFilter(hostCols[i]));
        compMax.addChild(new ConstantTupleFilter(max[i]));
        and.addChild(compMax);
    }
    return and;
}
 

@Test
public void testSerialization() {
    TblColRef colD = TblColRef.mockup(t, 1, "C1", "decimal");
    TblColRef colM = TblColRef.mockup(t, 2, "C2", "string");
    BigDecimal value = BigDecimal.valueOf(10L);

    ColumnTupleFilter colFilter = new ColumnTupleFilter(colD);
    ConstantTupleFilter constFilter = new ConstantTupleFilter("col");
    CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
    compareFilter.addChild(colFilter);
    compareFilter.addChild(constFilter);

    ColumnTupleExpression colTuple = new ColumnTupleExpression(colM);
    NumberTupleExpression constTuple = new NumberTupleExpression(value);

    Pair<TupleFilter, TupleExpression> whenEntry = new Pair<TupleFilter, TupleExpression>(compareFilter, colTuple);
    CaseTupleExpression caseTuple = new CaseTupleExpression(Lists.newArrayList(whenEntry), constTuple);

    byte[] result = TupleExpressionSerializer.serialize(caseTuple, StringCodeSystem.INSTANCE);

    TupleExpression desTuple = TupleExpressionSerializer.deserialize(result, StringCodeSystem.INSTANCE);
    assertEquals(caseTuple, desTuple);
}
 

@Before
public void setup() throws IOException {
    this.createTestMetadata();
    this.ii = IIManager.getInstance(getTestConfig()).getII("test_kylin_ii");
    this.tableRecordInfo = new TableRecordInfo(ii.getFirstSegment());
    factTableDesc = MetadataManager.getInstance(getTestConfig()).getTableDesc("DEFAULT.TEST_KYLIN_FACT");
    TblColRef formatName = this.ii.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_FORMAT_NAME");
    TblColRef siteId = this.ii.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_SITE_ID");

    Collection<TblColRef> dims = new HashSet<>();
    dims.add(formatName);
    projector = CoprocessorProjector.makeForEndpoint(tableRecordInfo, dims);
    aggregators = EndpointAggregators.fromFunctions(tableRecordInfo, buildAggregations());

    CompareTupleFilter rawFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
    rawFilter.addChild(new ColumnTupleFilter(siteId));
    rawFilter.addChild(new ConstantTupleFilter("0"));
    filter = CoprocessorFilter.fromFilter(this.ii.getFirstSegment(), rawFilter);

    aggCache = new EndpointAggregationCache(aggregators);
    tableData = mockTable();
}
 

@Test
public void testSerialization() {
    TblColRef colD = TblColRef.mockup(t, 1, "C1", "string");
    TblColRef colM = TblColRef.mockup(t, 2, "C2", "decimal");
    BigDecimal value = BigDecimal.valueOf(10L);

    ColumnTupleFilter colFilter = new ColumnTupleFilter(colD);
    ConstantTupleFilter constFilter = new ConstantTupleFilter("col");
    CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
    compareFilter.addChild(colFilter);
    compareFilter.addChild(constFilter);

    ColumnTupleExpression colTuple = new ColumnTupleExpression(colM);
    ConstantTupleExpression constTuple = new ConstantTupleExpression(value);

    Pair<TupleFilter, TupleExpression> whenEntry = new Pair<TupleFilter, TupleExpression>(compareFilter, colTuple);
    CaseTupleExpression caseTuple = new CaseTupleExpression(Lists.newArrayList(whenEntry), constTuple);

    byte[] result = TupleExpressionSerializer.serialize(caseTuple, StringCodeSystem.INSTANCE);

    TupleExpression desTuple = TupleExpressionSerializer.deserialize(result, StringCodeSystem.INSTANCE);
    assertEquals(caseTuple, desTuple);
}
 

private CompareTupleFilter eq(ColumnTupleFilter col, int... values) {
    CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.IN);
    r.addChild(col);

    List<ByteArray> list = Lists.newArrayList();
    for (int v : values) {
        int c = col.getColumn().getColumnDesc().getZeroBasedIndex();
        int len = info.getCodeSystem().maxCodeLength(c);
        ByteArray bytes = new ByteArray(len);
        BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len);
        list.add(bytes);
    }
    r.addChild(new ConstantTupleFilter(list));
    return r;
}
 

public CompareTupleFilter buildFilter1(TblColRef column) {
    CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.LTE);
    ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
    compareFilter.addChild(columnFilter1);
    ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2012-05-23");
    compareFilter.addChild(constantFilter1);
    return compareFilter;
}
 

private CompareTupleFilter mergeToInClause(TupleFilter filter) {
    List<? extends TupleFilter> children = filter.getChildren();
    TblColRef inColumn = null;
    List<String> inValues = new LinkedList<String>();
    for (TupleFilter child : children) {
        if (child.getOperator() == FilterOperatorEnum.EQ) {
            CompareTupleFilter compFilter = (CompareTupleFilter) child;
            TblColRef column = compFilter.getColumn();
            if (inColumn == null) {
                inColumn = column;
            }

            if (column == null || !column.equals(inColumn)) {
                return null;
            }
            inValues.addAll(compFilter.getValues());
        } else {
            return null;
        }
    }

    children.clear();

    CompareTupleFilter inFilter = new CompareTupleFilter(FilterOperatorEnum.IN);
    inFilter.addChild(new ColumnTupleFilter(inColumn));
    inFilter.addChild(new ConstantTupleFilter(inValues));
    return inFilter;
}
 

@Override
public TupleFilter visitInputRef(RexInputRef inputRef) {
    TblColRef column = inputRowType.getColumnByIndex(inputRef.getIndex());
    context.allColumns.add(column);
    ColumnTupleFilter filter = new ColumnTupleFilter(column);
    return filter;
}
 

private void collectColumnsRecursively(TupleFilter filter, Set<TblColRef> collector) {
    if (filter instanceof ColumnTupleFilter) {
        collector.add(((ColumnTupleFilter) filter).getColumn());
    }
    for (TupleFilter child : filter.getChildren()) {
        collectColumnsRecursively(child, collector);
    }
}
 

private void collectColumnsRecursively(TupleFilter filter, Set<TblColRef> collector) {
    if (filter instanceof ColumnTupleFilter) {
        collectColumns(((ColumnTupleFilter) filter).getColumn(), collector);
    }
    for (TupleFilter child : filter.getChildren()) {
        collectColumnsRecursively(child, collector);
    }
}
 

private CompareTupleFilter buildFilter1(TblColRef column) {
    CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.EQ);
    ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
    compareFilter.addChild(columnFilter1);
    ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2013-03-10");
    compareFilter.addChild(constantFilter1);
    return compareFilter;
}
 

private CompareTupleFilter buildFilter2(TblColRef column) {
    CompareTupleFilter compareFilter = new CompareTupleFilter(FilterOperatorEnum.EQ);
    ColumnTupleFilter columnFilter2 = new ColumnTupleFilter(column);
    compareFilter.addChild(columnFilter2);
    ConstantTupleFilter constantFilter2 = new ConstantTupleFilter("ClothinShoes & Accessories");
    compareFilter.addChild(constantFilter2);
    return compareFilter;
}
 

public CompareTupleFilter buildCompareFilter(String columnName, FilterOperatorEnum op, Object value) {
    CompareTupleFilter compareFilter = new CompareTupleFilter(op);
    ColumnTupleFilter columnFilter1 = buildColumnFilter(columnName);
    compareFilter.addChild(columnFilter1);
    ConstantTupleFilter constantFilter1 = new ConstantTupleFilter(value);
    compareFilter.addChild(constantFilter1);
    return compareFilter;
}
 

@Override
public void addChild(TupleFilter child) {
    if (child instanceof ColumnTupleFilter) {
        super.addChild(child);
        ColumnTupleFilter columnFilter = (ColumnTupleFilter) child;
        if (this.column != null) {
            throw new IllegalStateException("Duplicate columns! old is " + column.getName() + " and new is " + columnFilter.getColumn().getName());
        }
        this.column = columnFilter.getColumn();

    } else if (child instanceof ConstantTupleFilter) {
        // super.addChild(child) is omitted because the filter table name is useless at storage side, 
        // we'll extract the useful filterTableResourceIdentifier,filterTableType etc and save it at the MassInTupleFilter itself

        if (filterTableName == null) {
            filterTableName = (String) child.getValues().iterator().next();
            ExternalFilterDesc externalFilterDesc = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getExtFilterDesc(filterTableName);
            if (externalFilterDesc == null) {
                throw new IllegalArgumentException("External filter named " + filterTableName + " is not found");
            }
            filterTableType = externalFilterDesc.getFilterTableType();
            filterTableResourceIdentifier = externalFilterDesc.getFilterResourceIdentifier();
        }
    } else {
        throw new IllegalStateException("MassInTupleFilter only has two children: one ColumnTupleFilter and one ConstantTupleFilter");
    }
}
 

public static CompareTupleFilter getCompareTupleFilter(int col, Object value) {
    TblColRef colRef = gtInfo.colRef(col);
    ColumnTupleFilter colFilter = new ColumnTupleFilter(colRef);

    ByteArray space = new ByteArray(gtInfo.getCodeSystem().maxCodeLength(col));
    gtInfo.getCodeSystem().encodeColumnValue(col, value, space.asBuffer());
    ConstantTupleFilter constFilter = new ConstantTupleFilter(space);

    CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
    compareFilter.addChild(colFilter);
    compareFilter.addChild(constFilter);

    return compareFilter;
}
 

public CompareTupleFilter buildFilter3(TblColRef column) {
    CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
    ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
    compareFilter.addChild(columnFilter1);
    ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2012-05-23");
    compareFilter.addChild(constantFilter1);
    return compareFilter;
}
 

public CompareTupleFilter buildFilter2(TblColRef column) {
    CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
    ColumnTupleFilter columnFilter2 = new ColumnTupleFilter(column);
    compareFilter.addChild(columnFilter2);
    ConstantTupleFilter constantFilter2 = new ConstantTupleFilter("ClothinShoes & Accessories");
    compareFilter.addChild(constantFilter2);
    return compareFilter;
}
 

private CompareTupleFilter gt(ColumnTupleFilter col, int v) {
    CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.GT);
    r.addChild(col);

    int c = col.getColumn().getColumnDesc().getZeroBasedIndex();
    int len = info.getCodeSystem().maxCodeLength(c);
    ByteArray bytes = new ByteArray(len);
    BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len);
    r.addChild(new ConstantTupleFilter(bytes));

    return r;
}
 

public CompareTupleFilter buildTs2011Filter(TblColRef column) {
    CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
    ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
    compareFilter.addChild(columnFilter1);
    ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2011-01-01");
    compareFilter.addChild(constantFilter1);
    return compareFilter;
}
 

public CompareTupleFilter buildTs2010Filter(TblColRef column) {
    CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
    ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
    compareFilter.addChild(columnFilter1);
    ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2010-01-01");
    compareFilter.addChild(constantFilter1);
    return compareFilter;
}
 

private void collectColumnsRecursively(TupleFilter filter, Set<TblColRef> collector) {
    if (filter == null)
        return;

    if (filter instanceof ColumnTupleFilter) {
        collector.add(((ColumnTupleFilter) filter).getColumn());
    }
    for (TupleFilter child : filter.getChildren()) {
        collectColumnsRecursively(child, collector);
    }
}
 

private TupleFilter translateFunctionTupleFilter(BuiltInFunctionTupleFilter builtInFunctionTupleFilter) {
    if (!builtInFunctionTupleFilter.isValid())
        return null;

    TblColRef columnRef = builtInFunctionTupleFilter.getColumn();
    Dictionary<?> dict = dimEncMap.getDictionary(columnRef);
    if (dict == null)
        return null;

    CompareTupleFilter translated = new CompareTupleFilter(
            builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN);
    translated.addChild(new ColumnTupleFilter(columnRef));

    try {
        int translatedInClauseMaxSize = KylinConfig.getInstanceFromEnv().getTranslatedInClauseMaxSize();

        for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) {
            Object dictVal = dict.getValueFromId(i);
            if ((Boolean) builtInFunctionTupleFilter.invokeFunction(dictVal)) {
                translated.addChild(new ConstantTupleFilter(dictVal));

                if (translated.getChildren().size() > translatedInClauseMaxSize) {
                    return null;
                }
            }
        }
        logger.debug("getting a in clause with {} children", translated.getChildren().size());
    } catch (Exception e) {
        logger.debug(e.getMessage());
        return null;
    }
    return translated;
}
 

private TupleExpression getCountColumnExpression(TblColRef colRef) {
    List<Pair<TupleFilter, TupleExpression>> whenList = Lists.newArrayListWithExpectedSize(1);
    TupleFilter whenFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.ISNULL);
    whenFilter.addChild(new ColumnTupleFilter(colRef));
    whenList.add(new Pair<TupleFilter, TupleExpression>(whenFilter, new NumberTupleExpression(0)));

    TupleExpression elseExpr = new ColumnTupleExpression(SumDynamicFunctionDesc.mockCntCol);
    TupleExpression ret = new CaseTupleExpression(whenList, elseExpr);
    ret.setDigest("_KY_COUNT(" + colRef.getName() + ")");
    return ret;
}