Java Code Examples for org.roaringbitmap.buffer.MutableRoaringBitmap#serialize()

The following examples show how to use org.roaringbitmap.buffer.MutableRoaringBitmap#serialize() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FixLenColInvertIndexWriter.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public void write(OutputStream out) throws IOException {
    int cardinality = valueBitmaps.size();
    int footLen = II_DICT_TYPE_SORT_VAL_FOOT_LEN;
    int dictAndFootLen = cardinality * (valueLenInBytes + 4) + footLen;

    DataOutputStream bitmapOut = new DataOutputStream(out);
    ByteBuffer dictBuffer = ByteBuffer.allocate(dictAndFootLen);
    int offset = 0;
    for (Map.Entry<byte[], MutableRoaringBitmap> bitmapEntry : valueBitmaps.entrySet()) {
        byte[] colValue = bitmapEntry.getKey();
        MutableRoaringBitmap bitmap = bitmapEntry.getValue();
        bitmap.runOptimize();
        int bitmapSize = bitmap.serializedSizeInBytes();
        bitmap.serialize(bitmapOut);

        dictBuffer.put(colValue);
        dictBuffer.putInt(offset);
        offset += bitmapSize;
    }

    dictBuffer.putInt(cardinality);
    dictBuffer.putInt(valueLenInBytes);
    dictBuffer.putInt(II_DICT_TYPE_SORT_VAL); // type
    out.write(dictBuffer.array(), 0, dictBuffer.position());

    out.flush();
}
Example 2
Source File: SeqColInvertIndexWriter.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public void write(OutputStream out) throws IOException {
    int cardinality = valueBitmaps.length;
    if (nullValueBitmap != null) {
        cardinality++;
    }
    int dictionaryPartLen = cardinality * 4;

    DataOutputStream bitmapOut = new DataOutputStream(out);
    int footAndDictLen = II_DICT_TYPE_SEQ_FOOT_LEN + dictionaryPartLen;
    ByteBuffer footBuffer = ByteBuffer.allocate(footAndDictLen);

    int offset = 0;
    for (MutableRoaringBitmap bitmap : valueBitmaps) {
        bitmap.runOptimize();
        int bitmapSize = bitmap.serializedSizeInBytes();
        bitmap.serialize(bitmapOut);

        footBuffer.putInt(offset);
        offset += bitmapSize;
    }
    if (nullValueBitmap != null) {
        nullValueBitmap.serialize(bitmapOut);
        footBuffer.putInt(offset);
    }

    footBuffer.putInt(cardinality); // cardinality
    footBuffer.putInt(minColVal);
    footBuffer.putInt(maxColVal);
    footBuffer.putInt(II_DICT_TYPE_SEQ);// type
    if (nullValueBitmap != null) {
        footBuffer.put((byte) 1);
    } else {
        footBuffer.put((byte) 0);
    }
    out.write(footBuffer.array(), 0, footBuffer.position());

    out.flush();
}
Example 3
Source File: RoaringBitMapUtils.java    From yuvi with Apache License 2.0 5 votes vote down vote up 
public static ByteBuffer toByteBuffer(MutableRoaringBitmap rb) {
  // we add tests
  ByteBuffer outbb = ByteBuffer.allocate(rb.serializedSizeInBytes());
  try {
    rb.serialize(new DataOutputStream(new ByteBufferBackedOutputStream(outbb)));
  } catch (IOException e) {
    e.printStackTrace();
  }
  //
  outbb.flip();
  outbb.order(ByteOrder.LITTLE_ENDIAN);
  return outbb;
}
Example 4
Source File: FixLenColInvertIndexWriter.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void write(OutputStream out) throws IOException {
    int cardinality = valueBitmaps.size();
    int footLen = II_DICT_TYPE_SORT_VAL_FOOT_LEN;
    int dictAndFootLen = cardinality * (valueLenInBytes + 4) + footLen;

    DataOutputStream bitmapOut = new DataOutputStream(out);
    ByteBuffer dictBuffer = ByteBuffer.allocate(dictAndFootLen);
    int offset = 0;
    for (Map.Entry<byte[], MutableRoaringBitmap> bitmapEntry : valueBitmaps.entrySet()) {
        byte[] colValue = bitmapEntry.getKey();
        MutableRoaringBitmap bitmap = bitmapEntry.getValue();
        bitmap.runOptimize();
        int bitmapSize = bitmap.serializedSizeInBytes();
        bitmap.serialize(bitmapOut);

        dictBuffer.put(colValue);
        dictBuffer.putInt(offset);
        offset += bitmapSize;
    }

    dictBuffer.putInt(cardinality);
    dictBuffer.putInt(valueLenInBytes);
    dictBuffer.putInt(II_DICT_TYPE_SORT_VAL); // type
    out.write(dictBuffer.array(), 0, dictBuffer.position());

    out.flush();
}
Example 5
Source File: SeqColInvertIndexWriter.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void write(OutputStream out) throws IOException {
    int cardinality = valueBitmaps.length;
    if (nullValueBitmap != null) {
        cardinality++;
    }
    int dictionaryPartLen = cardinality * 4;

    DataOutputStream bitmapOut = new DataOutputStream(out);
    int footAndDictLen = II_DICT_TYPE_SEQ_FOOT_LEN + dictionaryPartLen;
    ByteBuffer footBuffer = ByteBuffer.allocate(footAndDictLen);

    int offset = 0;
    for (MutableRoaringBitmap bitmap : valueBitmaps) {
        bitmap.runOptimize();
        int bitmapSize = bitmap.serializedSizeInBytes();
        bitmap.serialize(bitmapOut);

        footBuffer.putInt(offset);
        offset += bitmapSize;
    }
    if (nullValueBitmap != null) {
        nullValueBitmap.serialize(bitmapOut);
        footBuffer.putInt(offset);
    }

    footBuffer.putInt(cardinality); // cardinality
    footBuffer.putInt(minColVal);
    footBuffer.putInt(maxColVal);
    footBuffer.putInt(II_DICT_TYPE_SEQ);// type
    if (nullValueBitmap != null) {
        footBuffer.put((byte) 1);
    } else {
        footBuffer.put((byte) 0);
    }
    out.write(footBuffer.array(), 0, footBuffer.position());

    out.flush();
}
Example 6
Source File: TestConcatenation.java    From RoaringBitmap with Apache License 2.0 5 votes vote down vote up 
@ParameterizedTest(name = "{1}")
@MethodSource("params")
public void canSerializeAndDeserializeBuffer(RoaringBitmap bitmap, int offset) throws IOException {
  MutableRoaringBitmap shifted = MutableRoaringBitmap.addOffset(bitmap.toMutableRoaringBitmap(), offset);
  ByteArrayDataOutput out = ByteStreams.newDataOutput();
  shifted.serialize(out);
  MutableRoaringBitmap deserialized = new MutableRoaringBitmap();
  deserialized.deserialize(ByteStreams.newDataInput(out.toByteArray()));
  assertEquals(shifted, deserialized, failureMessage(bitmap));
}
Example 7
Source File: SlowMappedORaggregate1.java    From RoaringBitmap with Apache License 2.0 5 votes vote down vote up 
public List<ImmutableRoaringBitmap> convertToImmutableRoaring(List<MutableRoaringBitmap> source)
    throws IOException {
  File tmpfile = File.createTempFile("roaring", "bin");
  tmpfile.deleteOnExit();
  final FileOutputStream fos = new FileOutputStream(tmpfile);
  final DataOutputStream dos = new DataOutputStream(fos);

  for (MutableRoaringBitmap rb1 : source)
    rb1.serialize(dos);

  final long totalcount = fos.getChannel().position();
  dos.close();
  final RandomAccessFile memoryMappedFile = new RandomAccessFile(tmpfile, "r");
  ByteBuffer out =
      memoryMappedFile.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, totalcount);
  ArrayList<ImmutableRoaringBitmap> answer =
      new ArrayList<ImmutableRoaringBitmap>(source.size());
  while (out.position() < out.limit()) {
    final ByteBuffer bb = out.slice();
    MutableRoaringBitmap equiv = source.get(answer.size());
    ImmutableRoaringBitmap newbitmap = new ImmutableRoaringBitmap(bb);
    if (!equiv.equals(newbitmap))
      throw new RuntimeException("bitmaps do not match");
    answer.add(newbitmap);
    out.position(out.position() + newbitmap.serializedSizeInBytes());
  }
  memoryMappedFile.close();
  return answer;
}
Example 8
Source File: SlowMappedORaggregate2.java    From RoaringBitmap with Apache License 2.0 5 votes vote down vote up 
public List<ImmutableRoaringBitmap> convertToImmutableRoaring(List<MutableRoaringBitmap> source)
    throws IOException {
  File tmpfile = File.createTempFile("roaring", "bin");
  tmpfile.deleteOnExit();
  final FileOutputStream fos = new FileOutputStream(tmpfile);
  final DataOutputStream dos = new DataOutputStream(fos);

  for (MutableRoaringBitmap rb1 : source)
    rb1.serialize(dos);

  final long totalcount = fos.getChannel().position();
  dos.close();
  final RandomAccessFile memoryMappedFile = new RandomAccessFile(tmpfile, "r");
  ByteBuffer out =
      memoryMappedFile.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, totalcount);
  ArrayList<ImmutableRoaringBitmap> answer =
      new ArrayList<ImmutableRoaringBitmap>(source.size());
  while (out.position() < out.limit()) {
    final ByteBuffer bb = out.slice();
    MutableRoaringBitmap equiv = source.get(answer.size());
    ImmutableRoaringBitmap newbitmap = new ImmutableRoaringBitmap(bb);
    if (!equiv.equals(newbitmap))
      throw new RuntimeException("bitmaps do not match");
    answer.add(newbitmap);
    out.position(out.position() + newbitmap.serializedSizeInBytes());
  }
  memoryMappedFile.close();
  return answer;
}
Example 9
Source File: MappedRunContainerRealDataBenchmarkRunOptimize.java    From RoaringBitmap with Apache License 2.0 5 votes vote down vote up 
@Benchmark
public int mutable_serializeToBAOSFromClone(BenchmarkState benchmarkState) throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.mac.size(); i++) {
    MutableRoaringBitmap bitmap = benchmarkState.mac.get(i).clone();
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}
Example 10
Source File: MappedRunContainerRealDataBenchmarkRunOptimize.java    From RoaringBitmap with Apache License 2.0 5 votes vote down vote up 
@Benchmark
public int mutable_serializeToBAOSNoClone(BenchmarkState benchmarkState) throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.mac.size(); i++) {
    MutableRoaringBitmap bitmap = benchmarkState.mac.get(i);
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}
Example 11
Source File: MappedRunContainerRealDataBenchmarkRunOptimize.java    From RoaringBitmap with Apache License 2.0 5 votes vote down vote up 
@Benchmark
public int mutable_serializeToBAOSNoClonePreOpti(BenchmarkState benchmarkState)
    throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.mrc.size(); i++) {
    MutableRoaringBitmap bitmap = benchmarkState.mrc.get(i);
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}
Example 12
Source File: MappedRunContainerRealDataBenchmarkRunOptimize.java    From RoaringBitmap with Apache License 2.0 5 votes vote down vote up 
@Benchmark
public int mutable_runOptimizeAndserializeToBAOSFromClone(BenchmarkState benchmarkState)
    throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.mac.size(); i++) {
    MutableRoaringBitmap bitmap = benchmarkState.mac.get(i).clone();
    bitmap.runOptimize();
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}
Example 13
Source File: BasicBenchmark.java    From RoaringBitmap with Apache License 2.0 5 votes vote down vote up 
private ImmutableRoaringBitmap convertToMappedBitmap(MutableRoaringBitmap orig)
    throws IOException {
  File tmpfile = File.createTempFile("roaring", ".bin");
  tmpfile.deleteOnExit();
  final FileOutputStream fos = new FileOutputStream(tmpfile);
  orig.serialize(new DataOutputStream(fos));
  long totalcount = fos.getChannel().position();
  fos.close();
  RandomAccessFile memoryMappedFile = new RandomAccessFile(tmpfile, "r");
  ByteBuffer bb =
      memoryMappedFile.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, totalcount);
  memoryMappedFile.close();
  return new ImmutableRoaringBitmap(bb);
}
Example 14
Source File: BitmapBenchmark.java    From bytebuffer-collections with Apache License 2.0 5 votes vote down vote up 
protected static ImmutableRoaringBitmap writeImmutable(MutableRoaringBitmap r, ByteBuffer buf) throws IOException
{
  final ByteArrayOutputStream out = new ByteArrayOutputStream();
  r.serialize(new DataOutputStream(out));
  final byte[] bytes = out.toByteArray();
  Assert.assertEquals(buf.remaining(), bytes.length);
  buf.put(bytes);
  buf.rewind();
  return new ImmutableRoaringBitmap(buf.asReadOnlyBuffer());
}
Example 15
Source File: TestSerialization.java    From RoaringBitmap with Apache License 2.0 4 votes vote down vote up 
@Test
public void testMutableRunSerializationBasicDeserialization() throws java.io.IOException {
  final int[] data = takeSortedAndDistinct(new Random(07734), 100000);
  RoaringBitmap bitmap_a = RoaringBitmap.bitmapOf(data);
  RoaringBitmap bitmap_ar = RoaringBitmap.bitmapOf(data);

  MutableRoaringBitmap bitmap_am = MutableRoaringBitmap.bitmapOf(data);
  MutableRoaringBitmap bitmap_amr = MutableRoaringBitmap.bitmapOf(data);

  for (int k = 100000; k < 200000; ++k) {
    bitmap_a.add(3 * k); // bitmap density and too many little runs
    bitmap_ar.add(3 * k);
    bitmap_am.add(3 * k);
    bitmap_amr.add(3 * k);
  }

  for (int k = 700000; k < 800000; ++k) { // will choose a runcontainer on this
    bitmap_a.add(k); // bitmap density and too many little runs
    bitmap_ar.add(k);
    bitmap_am.add(k);
    bitmap_amr.add(k);    
  }

  bitmap_ar.runOptimize();
  bitmap_amr.runOptimize();
  assertEquals(bitmap_a, bitmap_ar);
  assertEquals(bitmap_am, bitmap_amr);
  assertEquals(bitmap_am.serializedSizeInBytes(), bitmap_a.serializedSizeInBytes());
  assertEquals(bitmap_amr.serializedSizeInBytes(), bitmap_ar.serializedSizeInBytes());

  ByteBuffer outbuf = ByteBuffer.allocate(2*(bitmap_a.serializedSizeInBytes() + bitmap_ar.serializedSizeInBytes()));
  DataOutputStream out = new DataOutputStream(new ByteBufferBackedOutputStream(outbuf));
  try {
    bitmap_a.serialize(out);
    bitmap_ar.serialize(out);
    bitmap_am.serialize(out);
    bitmap_amr.serialize(out);
  } catch (Exception e) {
    e.printStackTrace();
  }
  outbuf.flip();

  RoaringBitmap bitmap_c1 = new RoaringBitmap();
  RoaringBitmap bitmap_c2 = new RoaringBitmap();
  RoaringBitmap bitmap_c3 = new RoaringBitmap();
  RoaringBitmap bitmap_c4 = new RoaringBitmap();

  DataInputStream in = new DataInputStream(new ByteBufferBackedInputStream(outbuf));
  bitmap_c1.deserialize(in);
  bitmap_c2.deserialize(in);
  bitmap_c3.deserialize(in);
  bitmap_c4.deserialize(in);

  assertEquals(bitmap_a, bitmap_c1);
  assertEquals(bitmap_a, bitmap_c2);
  assertEquals(bitmap_a, bitmap_c3);
  assertEquals(bitmap_a, bitmap_c4);
  assertEquals(bitmap_ar, bitmap_c1);
  assertEquals(bitmap_ar, bitmap_c2);
  assertEquals(bitmap_ar, bitmap_c3);
  assertEquals(bitmap_ar, bitmap_c4);

}