Java Code Examples for org.roaringbitmap.RoaringBitmap#runOptimize()

public static void main(String[] args) {
  RoaringBitmap mrb = RoaringBitmap.bitmapOf(1, 2, 3, 1000);
  System.out.println("starting with  bitmap " + mrb);
  mrb.runOptimize(); // to improve compression
  byte[] array = new byte[mrb.serializedSizeInBytes()];
  mrb.serialize(ByteBuffer.wrap(array));
  RoaringBitmap ret = new RoaringBitmap();
  try {
    ret.deserialize(ByteBuffer.wrap(array));
  } catch(IOException ioe) {
    ioe.printStackTrace(); // should not happen
  }
  if (!ret.equals(mrb))
    throw new RuntimeException("bug");
  System.out.println("decoded from byte array : " + ret);

}
 

public static void testSuperSparse() {
  System.out.println("Sparse case... universe = [0,"+universe_size+")");
  RoaringBitmap r = new RoaringBitmap();
  int howmany = 100;
  int gap = universe_size / howmany;
  System.out.println("Adding "+howmany+" values separated by gaps of "+gap+ "...");
  System.out.println("As a bitmap it would look like 1000...001000... ");
  for (int i = 1; i < howmany; i++) {
    r.add(i * gap);
  }
  System.out.println("Bits used per value = "+F.format(r.getSizeInBytes()*8.0/howmany));
  r.runOptimize();
  System.out.println("Bits used per value after run optimize = "+F.format(r.getSizeInBytes()*8.0/howmany));
  System.out.println("An uncompressed bitset might use "+F.format(universe_size*1.0/howmany)+" bits per value set");
  System.out.println();

}
 

public static void testSuperDense() {
  System.out.println("Sparse case... universe = [0,"+universe_size+")");
  RoaringBitmap r = new RoaringBitmap();
  int howmany = 100;
  int gap = universe_size / howmany;
  for (int i = 1; i < howmany; i++) {
    r.add(i * gap + 1,((i + 1) * gap));
  }
  System.out.println("Adding "+r.getCardinality()+" values partionned by "+howmany+" gaps of 1 ...");
  System.out.println("As a bitmap it would look like 01111...11011111... ");

  System.out.println("Bits used per value = "+F.format(r.getSizeInBytes()*8.0/r.getCardinality()));
  r.runOptimize();
  System.out.println("Bits used per value after run optimize = "+F.format(r.getSizeInBytes()*8.0/r.getCardinality()));
  System.out.println("Bits used per gap after run optimize = "+F.format(r.getSizeInBytes()*8.0/howmany));

  System.out.println("An uncompressed bitset might use "+F.format(universe_size*1.0/r.getCardinality())+" bits per value set");
  System.out.println();
}
 

@Setup(Level.Trial)
public void setup() throws Exception {
  ZipRealDataRetriever dataRetriever = new ZipRealDataRetriever(dataset);
  RoaringBitmap[] bitmaps = StreamSupport.stream(dataRetriever.fetchBitPositions().spliterator(), false)
          .map(RoaringBitmap::bitmapOf)
          .toArray(RoaringBitmap[]::new);
  buffers = new byte[bitmaps.length][];
  int i = 0;
  for (RoaringBitmap bitmap : bitmaps) {
    if (runOptimise) {
      bitmap.runOptimize();
    }
    ByteArrayOutputStream bos = new ByteArrayOutputStream(bitmap.serializedSizeInBytes());
    bitmap.serialize(new DataOutputStream(bos));
    buffers[i++] = bos.toByteArray();
  }
}
 

@Nonnull
public static byte[] serializeRoaring(@Nonnull final RoaringBitmap r) {
    r.runOptimize(); // might improve compression
    // next we create the ByteBuffer where the data will be stored
    final byte[] buf = new byte[r.serializedSizeInBytes()];
    // then we can serialize on a custom OutputStream
    try {
        r.serialize(new DataOutputStream(new FastByteArrayOutputStream(buf)));
    } catch (IOException e) {
        throw new IllegalStateException("Failed to serialize RoaringBitmap", e);
    }
    return buf;
}
 

public static void testAlternating() {
  System.out.println("Alternating case... universe = [0,"+universe_size+")");
  RoaringBitmap r = new RoaringBitmap();
  for (int i = 1; i < universe_size; i++) {
    if(i%2 == 0)
      r.add(i);
  }
  System.out.println("Adding all even values in the universe");
  System.out.println("As a bitmap it would look like 01010101... ");
  System.out.println("Bits used per value = "+F.format(r.getSizeInBytes()*8.0/r.getCardinality()));
  r.runOptimize();
  System.out.println("Bits used per value after run optimize = "+F.format(r.getSizeInBytes()*8.0/r.getCardinality()));
  System.out.println("An uncompressed bitset might use "+F.format(universe_size*1.0/r.getCardinality())+" bits per value set");
  System.out.println();
}
 

@Setup
public void setup() throws Exception {
  ZipRealDataRetriever dataRetriever = new ZipRealDataRetriever(dataset);
  System.out.println();
  System.out.println("Loading files from " + dataRetriever.getName());

  for (int[] data : dataRetriever.fetchBitPositions()) {
    RoaringBitmap basic = RoaringBitmap.bitmapOf(data);
    basic.runOptimize();
    rc.add(basic);
  }
  System.out.println("loaded " + rc.size() + " bitmaps");
}
 

@Setup
public void setup() throws Exception {
  ZipRealDataRetriever dataRetriever = new ZipRealDataRetriever(dataset);
  System.out.println();
  System.out.println("Loading files from " + dataRetriever.getName());

  for (int[] data : dataRetriever.fetchBitPositions()) {
    RoaringBitmap basic = RoaringBitmap.bitmapOf(data);
    basic.runOptimize();
    rc.add(basic);
  }
  System.out.println("loaded " + rc.size() + " bitmaps");
}
 

@Setup
public void setup() throws Exception {
  ZipRealDataRetriever dataRetriever = new ZipRealDataRetriever(dataset);
  System.out.println();
  System.out.println("Loading files from " + dataRetriever.getName());

  for (int[] data : dataRetriever.fetchBitPositions()) {
    RoaringBitmap basic = RoaringBitmap.bitmapOf(data);
    basic.runOptimize();
    rc.add(basic);
  }
  System.out.println("loaded " + rc.size() + " bitmaps");
}
 

public BenchmarkState() {
  int N = 30;
  Random rand = new Random(1234);
  for (int k = 0; k < N; ++k) {
    RoaringBitmap rb = new RoaringBitmap();
    int start = rand.nextInt(10000);

    for (int z = 0; z < 50; ++z) {
      int end = start + rand.nextInt(10000);
      rb.add(start, end);
      start = end + rand.nextInt(1000);
    }
    ConciseSet ccs = toConcise(rb.toArray());
    cc.add(ccs);
    wah.add(toWAH(rb.toArray()));
    icc.add(ImmutableConciseSet.newImmutableFromMutable(ccs));

    ac.add(rb);

    rb = rb.clone();
    rb.runOptimize();
    rc.add(rb);
    ewah.add(EWAHCompressedBitmap.bitmapOf(rb.toArray()));
    ewah32.add(EWAHCompressedBitmap32.bitmapOf(rb.toArray()));

  }
}
 

@Benchmark
public int cloneAndrunOptimize(BenchmarkState benchmarkState) {
  int total = 0;
  for (int i = 0; i < benchmarkState.ac.size(); i++) {
    RoaringBitmap bitmap = benchmarkState.ac.get(i).clone();
    bitmap.runOptimize();
    total += bitmap.getCardinality();
  }
  return total;
}
 

@Benchmark
public int runOptimizeAndserializeToBAOSFromClone(BenchmarkState benchmarkState)
    throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.ac.size(); i++) {
    RoaringBitmap bitmap = benchmarkState.ac.get(i).clone();
    bitmap.runOptimize();
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}
 

@Setup
public void setup() throws Exception {
  ZipRealDataRetriever dataRetriever = new ZipRealDataRetriever(dataset);
  System.out.println();
  System.out.println("Loading files from " + dataRetriever.getName());

  for (int[] data : dataRetriever.fetchBitPositions()) {
    RoaringBitmap basic = RoaringBitmap.bitmapOf(data);
    ac.add(basic);
    RoaringBitmap opti = basic.clone();
    opti.runOptimize();
    rc.add(opti);
  }
}
 

private static Bitmap newRoaringBitmap(int[] data, boolean optimize) {
  RoaringBitmap roaring = RoaringBitmap.bitmapOf(data);
  if (optimize) {
    roaring.runOptimize();
  }
  return new RoaringBitmapWrapper(roaring);
}