org.apache.kylin.metadata.filter.TupleFilterSerializer Java Examples

private TupleFilter convertFilter(FragmentMetaInfo fragmentMetaInfo, TupleFilter rootFilter,
        ColumnarRecordCodec recordCodec, final TblColRef[] dimensions, final IDimensionEncodingMap dimEncodingMap, //
        final Set<TblColRef> unEvaluableColumnCollector, long timezoneOffset) {
    Map<TblColRef, Integer> colMapping = Maps.newHashMap();
    for (int i = 0; i < dimensions.length; i++) {
        colMapping.put(dimensions[i], i);
    }
    byte[] bytes = TupleFilterSerializer.serialize(rootFilter, null, StringCodeSystem.INSTANCE);
    TupleFilter filter = TupleFilterSerializer.deserialize(bytes, StringCodeSystem.INSTANCE);

    BuiltInFunctionTransformer builtInFunctionTransformer = new BuiltInFunctionTransformer(dimEncodingMap);
    filter = builtInFunctionTransformer.transform(filter);
    FragmentFilterConverter fragmentFilterConverter = new FragmentFilterConverter(fragmentMetaInfo, unEvaluableColumnCollector,
            colMapping, recordCodec);
    fragmentFilterConverter.setTimezoneOffset(timezoneOffset);
    filter = fragmentFilterConverter.transform(filter);

    filter = new FilterOptimizeTransformer().transform(filter);
    return filter;
}
 

@Override
public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
    int nWhenEntries = BytesUtil.readVInt(buffer);
    List<Pair<TupleFilter, TupleExpression>> whenList = Lists.newArrayListWithExpectedSize(nWhenEntries);
    for (int i = 0; i < nWhenEntries; i++) {
        TupleFilter tupleFilter = TupleFilterSerializer.deserialize(BytesUtil.readByteArray(buffer), cs);
        TupleExpression tupleExpression = TupleExpressionSerializer.deserialize(BytesUtil.readByteArray(buffer),
                cs);
        whenList.add(new Pair<>(tupleFilter, tupleExpression));
    }
    this.whenList = whenList;
    int flag = BytesUtil.readVInt(buffer);
    if (flag == 1) {
        this.elseExpr = TupleExpressionSerializer.deserialize(BytesUtil.readByteArray(buffer), cs);
    }
}
 

@Override
public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
    BytesUtil.writeVInt(whenList.size(), buffer);
    for (Pair<TupleFilter, TupleExpression> whenEntry : whenList) {
        byte[] whenBytes = TupleFilterSerializer.serialize(whenEntry.getFirst(), cs);
        BytesUtil.writeByteArray(whenBytes, buffer);

        byte[] thenBytes = TupleExpressionSerializer.serialize(whenEntry.getSecond(), cs);
        BytesUtil.writeByteArray(thenBytes, buffer);
    }
    if (elseExpr != null) {
        BytesUtil.writeVInt(1, buffer);
        byte[] elseBytes = TupleExpressionSerializer.serialize(elseExpr, cs);
        BytesUtil.writeByteArray(elseBytes, buffer);
    } else {
        BytesUtil.writeVInt(-1, buffer);
    }
}
 

private TupleFilter convertFilter(FragmentMetaInfo fragmentMetaInfo, TupleFilter rootFilter,
        ColumnarRecordCodec recordCodec, final TblColRef[] dimensions, final IDimensionEncodingMap dimEncodingMap, //
        final Set<TblColRef> unEvaluableColumnCollector, long timezoneOffset) {
    Map<TblColRef, Integer> colMapping = Maps.newHashMap();
    for (int i = 0; i < dimensions.length; i++) {
        colMapping.put(dimensions[i], i);
    }
    byte[] bytes = TupleFilterSerializer.serialize(rootFilter, null, StringCodeSystem.INSTANCE);
    TupleFilter filter = TupleFilterSerializer.deserialize(bytes, StringCodeSystem.INSTANCE);

    BuiltInFunctionTransformer builtInFunctionTransformer = new BuiltInFunctionTransformer(dimEncodingMap);
    filter = builtInFunctionTransformer.transform(filter);
    FragmentFilterConverter fragmentFilterConverter = new FragmentFilterConverter(fragmentMetaInfo, unEvaluableColumnCollector,
            colMapping, recordCodec);
    fragmentFilterConverter.setTimezoneOffset(timezoneOffset);
    filter = fragmentFilterConverter.transform(filter);

    filter = new FilterOptimizeTransformer().transform(filter);
    return filter;
}
 

@Test
public void testSerialize07() {
    TableDesc table = new TableDesc();
    table.setName("TEST_KYLIN_FACT");
    table.setDatabase("DEFAULT");

    ColumnDesc column = new ColumnDesc();
    column.setTable(table);
    TblColRef colRef = new TblColRef(column);
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);
    TupleFilter filter = buildCompareFilter(groups, 0);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}
 

@Test
public void testSerialize08() {
    TableDesc table = new TableDesc();
    table.setDatabase("DEFAULT");

    ColumnDesc column = new ColumnDesc();
    column.setTable(table);
    TblColRef colRef = new TblColRef(column);
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);
    TupleFilter filter = buildCompareFilter(groups, 0);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}
 

@Override
public void serialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
    BytesUtil.writeVInt(whenList.size(), buffer);
    for (Pair<TupleFilter, TupleExpression> whenEntry : whenList) {
        byte[] whenBytes = TupleFilterSerializer.serialize(whenEntry.getFirst(), cs);
        BytesUtil.writeByteArray(whenBytes, buffer);

        byte[] thenBytes = TupleExpressionSerializer.serialize(whenEntry.getSecond(), cs);
        BytesUtil.writeByteArray(thenBytes, buffer);
    }
    if (elseExpr != null) {
        BytesUtil.writeVInt(1, buffer);
        byte[] elseBytes = TupleExpressionSerializer.serialize(elseExpr, cs);
        BytesUtil.writeByteArray(elseBytes, buffer);
    } else {
        BytesUtil.writeVInt(-1, buffer);
    }
}
 

@Override
public void deserialize(IFilterCodeSystem<?> cs, ByteBuffer buffer) {
    int nWhenEntries = BytesUtil.readVInt(buffer);
    List<Pair<TupleFilter, TupleExpression>> whenList = Lists.newArrayListWithExpectedSize(nWhenEntries);
    for (int i = 0; i < nWhenEntries; i++) {
        TupleFilter tupleFilter = TupleFilterSerializer.deserialize(BytesUtil.readByteArray(buffer), cs);
        TupleExpression tupleExpression = TupleExpressionSerializer.deserialize(BytesUtil.readByteArray(buffer),
                cs);
        whenList.add(new Pair<>(tupleFilter, tupleExpression));
    }
    this.whenList = whenList;
    int flag = BytesUtil.readVInt(buffer);
    if (flag == 1) {
        this.elseExpr = TupleExpressionSerializer.deserialize(BytesUtil.readByteArray(buffer), cs);
    }
}
 

@Test
public void testSerialize02() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareFilter(groups, 1);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}
 

@Test
public void testSerialize06() {
    ColumnDesc column = new ColumnDesc();
    column.setName("META_CATEG_NAME");
    TblColRef colRef = new TblColRef(column);
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);
    TupleFilter filter = buildCompareFilter(groups, 0);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}
 

@Test
public void testEvaluate07() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "3");

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, 0);
}
 

@Test
public void testSerialize11() {
    List<TblColRef> groups = buildGroups();
    TupleFilter orFilter = buildOrFilter(groups);
    TupleFilter andFilter = buildAndFilter(groups);

    LogicalTupleFilter logicFilter = new LogicalTupleFilter(FilterOperatorEnum.AND);
    logicFilter.addChild(orFilter);
    logicFilter.addChild(andFilter);

    byte[] bytes = TupleFilterSerializer.serialize(logicFilter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(logicFilter, newFilter);
}
 

@Test
public void testEvaluate03() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildAndFilter(groups);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[2]);
}
 

@Test
public void testEvaluate02() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildOrFilter(groups);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[0] + matcheCounts[1] - matcheCounts[2]);
}
 

@Test
public void testEvaluate01() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareFilter(groups, 1);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[1]);
}
 

@Test
public void testEvaluate00() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareFilter(groups, 0);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[0]);
}
 

@Test
public void testSerialize13() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "0");

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}
 

@Test
public void testSerialize12() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCaseFilter(groups);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}
 

@Test
public void testSerialize05() {
    ColumnDesc column = new ColumnDesc();

    TblColRef colRef = new TblColRef(column);
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);
    TupleFilter filter = buildCompareFilter(groups, 0);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}
 

@Test
public void testSerialize01() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareFilter(groups, 0);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}
 

public StreamSQLDigest(SQLDigest sqlDigest, TblColRef tsCol) {
    this.sqlDigest = sqlDigest;

    //must use new instance of IgnoreTsCondition
    TsConditionEraser decorator = new TsConditionEraser(tsCol, sqlDigest.filter);
    filterSerialized = TupleFilterSerializer.serialize(sqlDigest.filter, decorator, StringCodeSystem.INSTANCE);

    int nonFilterHashCode = calculateNonFilterHashCode();
    this.hashCode = 31 * nonFilterHashCode + Arrays.hashCode(filterSerialized);
}
 

@SuppressWarnings("rawtypes")
public AggregationBufferSearchResult(StreamingSearchContext searchRequest,
                                     TblColRef[] selectedDimensions, FunctionDesc[] selectedMetrics) {
    long hitCuboid = searchRequest.getHitCuboid();
    this.filter = searchRequest.getFilter();
    this.aggBufMap = getCuboidData(hitCuboid);
    this.dimIndexes = new int[selectedDimensions.length];
    this.metricsIndexes = new int[selectedMetrics.length];
    this.dimColIdxMap = Maps.newHashMap();

    CuboidInfo cuboidInfo = parsedStreamingCubeInfo.getCuboidInfo(hitCuboid);
    int idx = 0;
    for (TblColRef dimension : selectedDimensions) {
        int dimIdx = cuboidInfo.getIndexOf(dimension);
        dimIndexes[idx] = dimIdx;
        dimColIdxMap.put(dimension, dimIdx);
        idx++;
    }
    idx = 0;
    for (FunctionDesc metric : selectedMetrics) {
        metricsIndexes[idx] = parsedStreamingCubeInfo.getMetricIndexInAllMetrics(metric);
        idx++;
    }
    this.queryProfile = StreamingQueryProfile.get();
    if (filter != null && aggBufMap != null && !aggBufMap.isEmpty()) {
        byte[] bytes = TupleFilterSerializer.serialize(filter, null, StringCodeSystem.INSTANCE);
        filter = TupleFilterSerializer.deserialize(bytes, StringCodeSystem.INSTANCE);
        Set<TblColRef> unEvaluableColumns = Sets.newHashSet();
        filter = new StreamingBuiltInFunctionTransformer(unEvaluableColumns).transform(filter);
        if (!unEvaluableColumns.isEmpty()) {
            searchRequest.addNewGroups(unEvaluableColumns);
        }
    }
}
 

@Test
public void testEvaluate08() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "4");

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, number - matcheCounts[0] - matcheCounts[1] + matcheCounts[2]);
}
 

private DataRequest createDataRequest(String queryId, String cubeName, long minSegmentTime, TupleInfo tupleInfo,
        TupleFilter tupleFilter, Set<TblColRef> dimensions, Set<TblColRef> groups, Set<FunctionDesc> metrics,
        int storagePushDownLimit, boolean allowStorageAggregation) {
    DataRequest request = new DataRequest();
    request.setCubeName(cubeName);
    request.setQueryId(queryId);
    request.setMinSegmentTime(minSegmentTime);
    request.setTupleFilter(
            Base64.encodeBase64String(TupleFilterSerializer.serialize(tupleFilter, StringCodeSystem.INSTANCE)));
    request.setStoragePushDownLimit(storagePushDownLimit);
    request.setAllowStorageAggregation(allowStorageAggregation);
    request.setRequestSendTime(System.currentTimeMillis());
    request.setEnableDetailProfile(BackdoorToggles.isStreamingProfileEnable());
    request.setStorageBehavior(BackdoorToggles.getCoprocessorBehavior());

    Set<String> dimensionSet = Sets.newHashSet();
    for (TblColRef dimension : dimensions) {
        dimensionSet.add(dimension.getCanonicalName());
    }
    request.setDimensions(dimensionSet);

    Set<String> groupSet = Sets.newHashSet();
    for (TblColRef group : groups) {
        groupSet.add(group.getCanonicalName());
    }
    request.setGroups(groupSet);

    request.setMetrics(Lists.newArrayList(metrics));

    return request;
}
 

private static TupleFilter convertFilter(TupleFilter rootFilter, final GTInfo info, //
        final Map<TblColRef, Integer> colMapping, final boolean encodeConstants, //
        final Set<TblColRef> unevaluatableColumnCollector) {

    IFilterCodeSystem<ByteArray> filterCodeSystem = wrap(info.codeSystem.getComparator());

    GTConvertDecorator decorator = new GTConvertDecorator(unevaluatableColumnCollector, colMapping, info,
            encodeConstants);

    byte[] bytes = TupleFilterSerializer.serialize(rootFilter, decorator, filterCodeSystem);
    return TupleFilterSerializer.deserialize(bytes, filterCodeSystem);
}
 

@Test
public void testEvaluate08() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "4");

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, number - matcheCounts[0] - matcheCounts[1] + matcheCounts[2]);
}
 

@Test
public void testEvaluate07() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "3");

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, 0);
}
 

@Test
public void testEvaluate06() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "2");

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 1;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[1] - matcheCounts[2]);
}
 

@Test
public void testEvaluate05() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildCompareCaseFilter(groups, "1");

    byte[] bytes = TupleFilterSerializer.serialize(filter, CS);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes, CS);

    int number = 10000;
    int[] matcheCounts = new int[] { 0, 0, 0 };
    Collection<Tuple> tuples = generateTuple(number, groups, matcheCounts);
    int match = evaluateTuples(tuples, newFilter);

    assertEquals(match, matcheCounts[0] - matcheCounts[2]);
}
 

@Test
public void testSerialize03() {
    List<TblColRef> groups = buildGroups();
    TupleFilter filter = buildAndFilter(groups);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}