it.uniroma3.mat.extendedset.intset.ConciseSet Java Examples

@Override
public ConciseSet getBitMap(Integer startId, Integer endId) {
    if (startId == null && endId == null) {
        return sets[this.nValues];
    }

    int start = 0;
    int end = this.nValues - 1;
    if (startId != null) {
        start = startId;
    }
    if (endId != null) {
        end = endId;
    }

    ConciseSet ret = new ConciseSet();
    for (int i = start; i <= end; ++i) {
        ConciseSet temp = getBitMap(i);
        ret.addAll(temp);
    }
    return ret;
}
 

@Test
public void testGetOutOfBounds()
{
  final ConciseSet conciseSet = new ConciseSet();
  final Set<Integer> ints = ImmutableSet.of(0, 4, 9);
  for (int i : ints) {
    conciseSet.add(i);
  }
  final ImmutableBitmap immutableBitmap = new WrappedImmutableConciseBitmap(
      ImmutableConciseSet.newImmutableFromMutable(conciseSet));
  final MutableBitmap mutableBitmap = new WrappedConciseBitmap(conciseSet);
  for (int i = 0; i < 10; ++i) {
    Assert.assertEquals(Integer.toString(i), ints.contains(i), mutableBitmap.get(i));
    Assert.assertEquals(Integer.toString(i), ints.contains(i), immutableBitmap.get(i));
  }
}
 

private ConciseSet evalCompare(CompareTupleFilter filter) {
    switch (filter.getOperator()) {
    case ISNULL:
        return evalCompareIsNull(filter);
    case ISNOTNULL:
        return evalCompareIsNotNull(filter);
    case EQ:
        return evalCompareEqual(filter);
    case NEQ:
        return evalCompareNotEqual(filter);
    case IN:
        return evalCompareIn(filter);
    case NOTIN:
        return evalCompareNotIn(filter);
    case LT:
        return evalCompareLT(filter);
    case LTE:
        return evalCompareLTE(filter);
    case GT:
        return evalCompareGT(filter);
    case GTE:
        return evalCompareGTE(filter);
    default:
        throw new IllegalStateException("Unsupported operator " + filter.getOperator());
    }
}
 

private IIProtos.IIResponse getNonAggregatedResponse(Iterable<Slice> slices, CoprocessorFilter filter, CoprocessorRowType type) {
    IIProtos.IIResponse.Builder responseBuilder = IIProtos.IIResponse.newBuilder();
    for (Slice slice : slices) {
        ConciseSet result = null;
        if (filter != null) {
            result = new BitMapFilterEvaluator(new SliceBitMapProvider(slice, type)).evaluate(filter.getFilter());
        }

        Iterator<RawTableRecord> iterator = slice.iterateWithBitmap(result);
        while (iterator.hasNext()) {
            byte[] data = iterator.next().getBytes();
            IIProtos.IIResponse.IIRow.Builder rowBuilder = IIProtos.IIResponse.IIRow.newBuilder().setColumns(ByteString.copyFrom(data));
            responseBuilder.addRows(rowBuilder.build());
        }
    }

    return responseBuilder.build();
}
 

static ConciseSet toConcise(int[] dat) {
  ConciseSet ans = new ConciseSet();
  for (int i : dat) {
    ans.add(i);
  }
  return ans;
}
 

@Override
public ConciseSet getBitMap(TblColRef col, Integer startId, Integer endId) {
    if (!col.equals(colA))
        return null;

    // i-th record has value ID i, and last record has value null
    if (startId == null && endId == null) {
        //entry for getting null value
        ConciseSet s = new ConciseSet();
        s.add(getRecordCount() - 1);
        return s;
    }

    int start = 0;
    int end = MAX_ID;
    if (startId != null) {
        start = startId;
    }
    if (endId != null) {
        end = endId;
    }

    ConciseSet ret = new ConciseSet();
    for (int i = start; i <= end; ++i) {
        ConciseSet temp = getBitMap(col, i);
        ret.addAll(temp);
    }
    return ret;
}
 

public ConciseSet getBitMap(TblColRef col, int valueId) {
    if (!col.equals(colA))
        return null;

    // i-th record has value ID i, and last record has value null
    ConciseSet bitMap = new ConciseSet();
    if (valueId < 0 || valueId > getMaxValueId(col)) // null
        bitMap.add(getRecordCount() - 1);
    else
        bitMap.add(valueId);

    return bitMap;
}
 

@Test
public void testLogicalAnd() {
    for (int i = 0; i < basicFilters.size(); i++) {
        for (int j = 0; j < basicFilters.size(); j++) {
            LogicalTupleFilter f = logical(FilterOperatorEnum.AND, basicFilters.get(i), basicFilters.get(j));
            ConciseSet r = basicResults.get(i).clone();
            r.retainAll(basicResults.get(j));
            assertEquals(r, eval.evaluate(f));
        }
    }
}
 

@Test
public void testLogicalOr() {
    for (int i = 0; i < basicFilters.size(); i++) {
        for (int j = 0; j < basicFilters.size(); j++) {
            LogicalTupleFilter f = logical(FilterOperatorEnum.OR, basicFilters.get(i), basicFilters.get(j));
            ConciseSet r = basicResults.get(i).clone();
            r.addAll(basicResults.get(j));
            assertEquals(r, eval.evaluate(f));
        }
    }
}
 

static ConciseSet toConcise(int[] dat) {
  ConciseSet ans = new ConciseSet();
  for (int i : dat) {
    ans.add(i);
  }
  return ans;
}
 

static ConciseSet toWAH(int[] dat) {
  ConciseSet ans = new ConciseSet(true);
  for (int i : dat) {
    ans.add(i);
  }
  return ans;
}
 

@Benchmark
public int OrConcise(BenchmarkState benchmarkState) {
  ConciseSet base = benchmarkState.cc.get(0);
  for (int k = 1; k < benchmarkState.cc.size(); ++k)
    base.union(benchmarkState.cc.get(k));
  return base.size();
}
 

@Benchmark
public int OrWAH(BenchmarkState benchmarkState) {
  ConciseSet base = benchmarkState.wah.get(0);
  for (int k = 1; k < benchmarkState.wah.size(); ++k)
    base.union(benchmarkState.wah.get(k));
  return base.size();
}
 

public BenchmarkState() {
  int N = 30;
  Random rand = new Random(1234);
  for (int k = 0; k < N; ++k) {
    RoaringBitmap rb = new RoaringBitmap();
    int start = rand.nextInt(10000);

    for (int z = 0; z < 50; ++z) {
      int end = start + rand.nextInt(10000);
      rb.add(start, end);
      start = end + rand.nextInt(1000);
    }
    ConciseSet ccs = toConcise(rb.toArray());
    cc.add(ccs);
    wah.add(toWAH(rb.toArray()));
    icc.add(ImmutableConciseSet.newImmutableFromMutable(ccs));

    ac.add(rb);

    rb = rb.clone();
    rb.runOptimize();
    rc.add(rb);
    ewah.add(EWAHCompressedBitmap.bitmapOf(rb.toArray()));
    ewah32.add(EWAHCompressedBitmap32.bitmapOf(rb.toArray()));

  }
}
 

static ConciseSet toConcise(int[] dat) {
  ConciseSet ans = new ConciseSet();
  for (int i : dat) {
    ans.add(i);
  }
  return ans;
}
 

static ConciseSet toConcise(int[] dat) {
  ConciseSet ans = new ConciseSet();
  for (int i : dat) {
    ans.add(i);
  }
  return ans;
}
 

private ConciseSet bytesToSet(ImmutableBytesWritable bytes) {
    if (bytes.get() == null || bytes.getLength() == 0) {
        return new ConciseSet();
    } else {
        IntBuffer intBuffer = ByteBuffer.wrap(bytes.get(), bytes.getOffset(), bytes.getLength()).asIntBuffer();
        int[] words = new int[intBuffer.capacity()];
        intBuffer.get(words);
        return new ConciseSet(words, false);
    }
}
 

static ConciseSet toConcise(int[] dat) {
  ConciseSet ans = new ConciseSet();
  for (int i : dat) {
    ans.add(i);
  }
  return ans;
}
 

private static ConciseSet newConciseSet(int[] data, boolean simulateWAH) {
  ConciseSet concise = new ConciseSet(simulateWAH);
  for (int i : data) {
    concise.add(i);
  }
  return concise;
}
 

@Override
public BitmapAggregator naiveAndAggregator() {
  return new BitmapAggregator() {
    @Override
    public Bitmap aggregate(Iterable<Bitmap> bitmaps) {
      final Iterator<Bitmap> i = bitmaps.iterator();
      ConciseSet bitmap = ((ConciseSetWrapper) i.next()).bitmap;
      while (i.hasNext()) {
        bitmap = bitmap.intersection(((ConciseSetWrapper) i.next()).bitmap);
      }
      return new ConciseSetWrapper(bitmap);
    }
  };
}
 

@Override
public BitmapAggregator naiveOrAggregator() {
  return new BitmapAggregator() {
    @Override
    public Bitmap aggregate(Iterable<Bitmap> bitmaps) {
      final Iterator<Bitmap> i = bitmaps.iterator();
      ConciseSet bitmap = ((ConciseSetWrapper) i.next()).bitmap;
      while (i.hasNext()) {
        bitmap = bitmap.union(((ConciseSetWrapper) i.next()).bitmap);
      }
      return new ConciseSetWrapper(bitmap);
    }
  };
}
 

@Override
public void or(MutableBitmap mutableBitmap)
{
  WrappedConciseBitmap other = (WrappedConciseBitmap) mutableBitmap;
  ConciseSet unwrappedOtherBitmap = other.bitmap;
  bitmap.addAll(unwrappedOtherBitmap);
}
 

@Override
public void and(MutableBitmap mutableBitmap)
{
  WrappedConciseBitmap other = (WrappedConciseBitmap) mutableBitmap;
  ConciseSet unwrappedOtherBitmap = other.bitmap;
  bitmap = bitmap.intersection(unwrappedOtherBitmap);
}
 

@Override
public void xor(MutableBitmap mutableBitmap)
{
  WrappedConciseBitmap other = (WrappedConciseBitmap) mutableBitmap;
  ConciseSet unwrappedOtherBitmap = other.bitmap;
  bitmap = bitmap.symmetricDifference(unwrappedOtherBitmap);
}
 

@Override
public void andNot(MutableBitmap mutableBitmap)
{
  WrappedConciseBitmap other = (WrappedConciseBitmap) mutableBitmap;
  ConciseSet unwrappedOtherBitmap = other.bitmap;
  bitmap = bitmap.difference(unwrappedOtherBitmap);
}
 

@Override
public ImmutableBitmap union(ImmutableBitmap otherBitmap)
{
  WrappedConciseBitmap other = (WrappedConciseBitmap) otherBitmap;
  ConciseSet unwrappedOtherBitmap = other.bitmap;
  return new WrappedConciseBitmap(bitmap.clone().union(unwrappedOtherBitmap));
}
 

@Override
public ImmutableBitmap intersection(ImmutableBitmap otherBitmap)
{
  WrappedConciseBitmap other = (WrappedConciseBitmap) otherBitmap;
  ConciseSet unwrappedOtherBitmap = other.bitmap;
  return new WrappedConciseBitmap(bitmap.clone().intersection(unwrappedOtherBitmap));
}
 

@Override
public ImmutableBitmap difference(ImmutableBitmap otherBitmap)
{
  WrappedConciseBitmap other = (WrappedConciseBitmap) otherBitmap;
  ConciseSet unwrappedOtherBitmap = other.bitmap;
  return new WrappedConciseBitmap(bitmap.clone().difference(unwrappedOtherBitmap));
}
 

private void not(ConciseSet set) {
    if (set == null)
        return;

    set.add(provider.getRecordCount());
    set.complement();
}
 

/**
 * @param filter
 * @return a set of records that match the filter; or null if filter is null or unable to evaluate
 */
public ConciseSet evaluate(TupleFilter filter) {
    if (filter == null)
        return null;

    if (filter instanceof LogicalTupleFilter)
        return evalLogical((LogicalTupleFilter) filter);

    if (filter instanceof CompareTupleFilter)
        return evalCompare((CompareTupleFilter) filter);

    return null; // unable to evaluate
}