Java Code Examples for org.roaringbitmap.buffer.MutableRoaringBitmap#serialize()

The following examples show how to use org.roaringbitmap.buffer.MutableRoaringBitmap#serialize() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: FixLenColInvertIndexWriter.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Override
public void write(OutputStream out) throws IOException {
    int cardinality = valueBitmaps.size();
    int footLen = II_DICT_TYPE_SORT_VAL_FOOT_LEN;
    int dictAndFootLen = cardinality * (valueLenInBytes + 4) + footLen;

    DataOutputStream bitmapOut = new DataOutputStream(out);
    ByteBuffer dictBuffer = ByteBuffer.allocate(dictAndFootLen);
    int offset = 0;
    for (Map.Entry<byte[], MutableRoaringBitmap> bitmapEntry : valueBitmaps.entrySet()) {
        byte[] colValue = bitmapEntry.getKey();
        MutableRoaringBitmap bitmap = bitmapEntry.getValue();
        bitmap.runOptimize();
        int bitmapSize = bitmap.serializedSizeInBytes();
        bitmap.serialize(bitmapOut);

        dictBuffer.put(colValue);
        dictBuffer.putInt(offset);
        offset += bitmapSize;
    }

    dictBuffer.putInt(cardinality);
    dictBuffer.putInt(valueLenInBytes);
    dictBuffer.putInt(II_DICT_TYPE_SORT_VAL); // type
    out.write(dictBuffer.array(), 0, dictBuffer.position());

    out.flush();
}

Example 2

Source File: SeqColInvertIndexWriter.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Override
public void write(OutputStream out) throws IOException {
    int cardinality = valueBitmaps.length;
    if (nullValueBitmap != null) {
        cardinality++;
    }
    int dictionaryPartLen = cardinality * 4;

    DataOutputStream bitmapOut = new DataOutputStream(out);
    int footAndDictLen = II_DICT_TYPE_SEQ_FOOT_LEN + dictionaryPartLen;
    ByteBuffer footBuffer = ByteBuffer.allocate(footAndDictLen);

    int offset = 0;
    for (MutableRoaringBitmap bitmap : valueBitmaps) {
        bitmap.runOptimize();
        int bitmapSize = bitmap.serializedSizeInBytes();
        bitmap.serialize(bitmapOut);

        footBuffer.putInt(offset);
        offset += bitmapSize;
    }
    if (nullValueBitmap != null) {
        nullValueBitmap.serialize(bitmapOut);
        footBuffer.putInt(offset);
    }

    footBuffer.putInt(cardinality); // cardinality
    footBuffer.putInt(minColVal);
    footBuffer.putInt(maxColVal);
    footBuffer.putInt(II_DICT_TYPE_SEQ);// type
    if (nullValueBitmap != null) {
        footBuffer.put((byte) 1);
    } else {
        footBuffer.put((byte) 0);
    }
    out.write(footBuffer.array(), 0, footBuffer.position());

    out.flush();
}

Example 3

Source File: RoaringBitMapUtils.java From yuvi with Apache License 2.0

5 votes

public static ByteBuffer toByteBuffer(MutableRoaringBitmap rb) {
  // we add tests
  ByteBuffer outbb = ByteBuffer.allocate(rb.serializedSizeInBytes());
  try {
    rb.serialize(new DataOutputStream(new ByteBufferBackedOutputStream(outbb)));
  } catch (IOException e) {
    e.printStackTrace();
  }
  //
  outbb.flip();
  outbb.order(ByteOrder.LITTLE_ENDIAN);
  return outbb;
}

Example 4

Source File: FixLenColInvertIndexWriter.java From kylin with Apache License 2.0

5 votes

@Override
public void write(OutputStream out) throws IOException {
    int cardinality = valueBitmaps.size();
    int footLen = II_DICT_TYPE_SORT_VAL_FOOT_LEN;
    int dictAndFootLen = cardinality * (valueLenInBytes + 4) + footLen;

    DataOutputStream bitmapOut = new DataOutputStream(out);
    ByteBuffer dictBuffer = ByteBuffer.allocate(dictAndFootLen);
    int offset = 0;
    for (Map.Entry<byte[], MutableRoaringBitmap> bitmapEntry : valueBitmaps.entrySet()) {
        byte[] colValue = bitmapEntry.getKey();
        MutableRoaringBitmap bitmap = bitmapEntry.getValue();
        bitmap.runOptimize();
        int bitmapSize = bitmap.serializedSizeInBytes();
        bitmap.serialize(bitmapOut);

        dictBuffer.put(colValue);
        dictBuffer.putInt(offset);
        offset += bitmapSize;
    }

    dictBuffer.putInt(cardinality);
    dictBuffer.putInt(valueLenInBytes);
    dictBuffer.putInt(II_DICT_TYPE_SORT_VAL); // type
    out.write(dictBuffer.array(), 0, dictBuffer.position());

    out.flush();
}

Example 5

Source File: SeqColInvertIndexWriter.java From kylin with Apache License 2.0

5 votes

@Override
public void write(OutputStream out) throws IOException {
    int cardinality = valueBitmaps.length;
    if (nullValueBitmap != null) {
        cardinality++;
    }
    int dictionaryPartLen = cardinality * 4;

    DataOutputStream bitmapOut = new DataOutputStream(out);
    int footAndDictLen = II_DICT_TYPE_SEQ_FOOT_LEN + dictionaryPartLen;
    ByteBuffer footBuffer = ByteBuffer.allocate(footAndDictLen);

    int offset = 0;
    for (MutableRoaringBitmap bitmap : valueBitmaps) {
        bitmap.runOptimize();
        int bitmapSize = bitmap.serializedSizeInBytes();
        bitmap.serialize(bitmapOut);

        footBuffer.putInt(offset);
        offset += bitmapSize;
    }
    if (nullValueBitmap != null) {
        nullValueBitmap.serialize(bitmapOut);
        footBuffer.putInt(offset);
    }

    footBuffer.putInt(cardinality); // cardinality
    footBuffer.putInt(minColVal);
    footBuffer.putInt(maxColVal);
    footBuffer.putInt(II_DICT_TYPE_SEQ);// type
    if (nullValueBitmap != null) {
        footBuffer.put((byte) 1);
    } else {
        footBuffer.put((byte) 0);
    }
    out.write(footBuffer.array(), 0, footBuffer.position());

    out.flush();
}

Example 6

Source File: TestConcatenation.java From RoaringBitmap with Apache License 2.0

5 votes

@ParameterizedTest(name = "{1}")
@MethodSource("params")
public void canSerializeAndDeserializeBuffer(RoaringBitmap bitmap, int offset) throws IOException {
  MutableRoaringBitmap shifted = MutableRoaringBitmap.addOffset(bitmap.toMutableRoaringBitmap(), offset);
  ByteArrayDataOutput out = ByteStreams.newDataOutput();
  shifted.serialize(out);
  MutableRoaringBitmap deserialized = new MutableRoaringBitmap();
  deserialized.deserialize(ByteStreams.newDataInput(out.toByteArray()));
  assertEquals(shifted, deserialized, failureMessage(bitmap));
}

Example 7

Source File: SlowMappedORaggregate1.java From RoaringBitmap with Apache License 2.0

5 votes

public List<ImmutableRoaringBitmap> convertToImmutableRoaring(List<MutableRoaringBitmap> source)
    throws IOException {
  File tmpfile = File.createTempFile("roaring", "bin");
  tmpfile.deleteOnExit();
  final FileOutputStream fos = new FileOutputStream(tmpfile);
  final DataOutputStream dos = new DataOutputStream(fos);

  for (MutableRoaringBitmap rb1 : source)
    rb1.serialize(dos);

  final long totalcount = fos.getChannel().position();
  dos.close();
  final RandomAccessFile memoryMappedFile = new RandomAccessFile(tmpfile, "r");
  ByteBuffer out =
      memoryMappedFile.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, totalcount);
  ArrayList<ImmutableRoaringBitmap> answer =
      new ArrayList<ImmutableRoaringBitmap>(source.size());
  while (out.position() < out.limit()) {
    final ByteBuffer bb = out.slice();
    MutableRoaringBitmap equiv = source.get(answer.size());
    ImmutableRoaringBitmap newbitmap = new ImmutableRoaringBitmap(bb);
    if (!equiv.equals(newbitmap))
      throw new RuntimeException("bitmaps do not match");
    answer.add(newbitmap);
    out.position(out.position() + newbitmap.serializedSizeInBytes());
  }
  memoryMappedFile.close();
  return answer;
}

Example 8

Source File: SlowMappedORaggregate2.java From RoaringBitmap with Apache License 2.0

5 votes

public List<ImmutableRoaringBitmap> convertToImmutableRoaring(List<MutableRoaringBitmap> source)
    throws IOException {
  File tmpfile = File.createTempFile("roaring", "bin");
  tmpfile.deleteOnExit();
  final FileOutputStream fos = new FileOutputStream(tmpfile);
  final DataOutputStream dos = new DataOutputStream(fos);

  for (MutableRoaringBitmap rb1 : source)
    rb1.serialize(dos);

  final long totalcount = fos.getChannel().position();
  dos.close();
  final RandomAccessFile memoryMappedFile = new RandomAccessFile(tmpfile, "r");
  ByteBuffer out =
      memoryMappedFile.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, totalcount);
  ArrayList<ImmutableRoaringBitmap> answer =
      new ArrayList<ImmutableRoaringBitmap>(source.size());
  while (out.position() < out.limit()) {
    final ByteBuffer bb = out.slice();
    MutableRoaringBitmap equiv = source.get(answer.size());
    ImmutableRoaringBitmap newbitmap = new ImmutableRoaringBitmap(bb);
    if (!equiv.equals(newbitmap))
      throw new RuntimeException("bitmaps do not match");
    answer.add(newbitmap);
    out.position(out.position() + newbitmap.serializedSizeInBytes());
  }
  memoryMappedFile.close();
  return answer;
}

Example 9

Source File: MappedRunContainerRealDataBenchmarkRunOptimize.java From RoaringBitmap with Apache License 2.0

5 votes

@Benchmark
public int mutable_serializeToBAOSFromClone(BenchmarkState benchmarkState) throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.mac.size(); i++) {
    MutableRoaringBitmap bitmap = benchmarkState.mac.get(i).clone();
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}

Example 10

Source File: MappedRunContainerRealDataBenchmarkRunOptimize.java From RoaringBitmap with Apache License 2.0

5 votes

@Benchmark
public int mutable_serializeToBAOSNoClone(BenchmarkState benchmarkState) throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.mac.size(); i++) {
    MutableRoaringBitmap bitmap = benchmarkState.mac.get(i);
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}

Example 11

Source File: MappedRunContainerRealDataBenchmarkRunOptimize.java From RoaringBitmap with Apache License 2.0

5 votes

@Benchmark
public int mutable_serializeToBAOSNoClonePreOpti(BenchmarkState benchmarkState)
    throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.mrc.size(); i++) {
    MutableRoaringBitmap bitmap = benchmarkState.mrc.get(i);
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}

Example 12

Source File: MappedRunContainerRealDataBenchmarkRunOptimize.java From RoaringBitmap with Apache License 2.0

5 votes

@Benchmark
public int mutable_runOptimizeAndserializeToBAOSFromClone(BenchmarkState benchmarkState)
    throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.mac.size(); i++) {
    MutableRoaringBitmap bitmap = benchmarkState.mac.get(i).clone();
    bitmap.runOptimize();
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}

Example 13

Source File: BasicBenchmark.java From RoaringBitmap with Apache License 2.0

5 votes

private ImmutableRoaringBitmap convertToMappedBitmap(MutableRoaringBitmap orig)
    throws IOException {
  File tmpfile = File.createTempFile("roaring", ".bin");
  tmpfile.deleteOnExit();
  final FileOutputStream fos = new FileOutputStream(tmpfile);
  orig.serialize(new DataOutputStream(fos));
  long totalcount = fos.getChannel().position();
  fos.close();
  RandomAccessFile memoryMappedFile = new RandomAccessFile(tmpfile, "r");
  ByteBuffer bb =
      memoryMappedFile.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, totalcount);
  memoryMappedFile.close();
  return new ImmutableRoaringBitmap(bb);
}

Example 14

Source File: BitmapBenchmark.java From bytebuffer-collections with Apache License 2.0

5 votes

protected static ImmutableRoaringBitmap writeImmutable(MutableRoaringBitmap r, ByteBuffer buf) throws IOException
{
  final ByteArrayOutputStream out = new ByteArrayOutputStream();
  r.serialize(new DataOutputStream(out));
  final byte[] bytes = out.toByteArray();
  Assert.assertEquals(buf.remaining(), bytes.length);
  buf.put(bytes);
  buf.rewind();
  return new ImmutableRoaringBitmap(buf.asReadOnlyBuffer());
}

Example 15

Source File: TestSerialization.java From RoaringBitmap with Apache License 2.0

4 votes

@Test
public void testMutableRunSerializationBasicDeserialization() throws java.io.IOException {
  final int[] data = takeSortedAndDistinct(new Random(07734), 100000);
  RoaringBitmap bitmap_a = RoaringBitmap.bitmapOf(data);
  RoaringBitmap bitmap_ar = RoaringBitmap.bitmapOf(data);

  MutableRoaringBitmap bitmap_am = MutableRoaringBitmap.bitmapOf(data);
  MutableRoaringBitmap bitmap_amr = MutableRoaringBitmap.bitmapOf(data);

  for (int k = 100000; k < 200000; ++k) {
    bitmap_a.add(3 * k); // bitmap density and too many little runs
    bitmap_ar.add(3 * k);
    bitmap_am.add(3 * k);
    bitmap_amr.add(3 * k);
  }

  for (int k = 700000; k < 800000; ++k) { // will choose a runcontainer on this
    bitmap_a.add(k); // bitmap density and too many little runs
    bitmap_ar.add(k);
    bitmap_am.add(k);
    bitmap_amr.add(k);    
  }

  bitmap_ar.runOptimize();
  bitmap_amr.runOptimize();
  assertEquals(bitmap_a, bitmap_ar);
  assertEquals(bitmap_am, bitmap_amr);
  assertEquals(bitmap_am.serializedSizeInBytes(), bitmap_a.serializedSizeInBytes());
  assertEquals(bitmap_amr.serializedSizeInBytes(), bitmap_ar.serializedSizeInBytes());

  ByteBuffer outbuf = ByteBuffer.allocate(2*(bitmap_a.serializedSizeInBytes() + bitmap_ar.serializedSizeInBytes()));
  DataOutputStream out = new DataOutputStream(new ByteBufferBackedOutputStream(outbuf));
  try {
    bitmap_a.serialize(out);
    bitmap_ar.serialize(out);
    bitmap_am.serialize(out);
    bitmap_amr.serialize(out);
  } catch (Exception e) {
    e.printStackTrace();
  }
  outbuf.flip();

  RoaringBitmap bitmap_c1 = new RoaringBitmap();
  RoaringBitmap bitmap_c2 = new RoaringBitmap();
  RoaringBitmap bitmap_c3 = new RoaringBitmap();
  RoaringBitmap bitmap_c4 = new RoaringBitmap();

  DataInputStream in = new DataInputStream(new ByteBufferBackedInputStream(outbuf));
  bitmap_c1.deserialize(in);
  bitmap_c2.deserialize(in);
  bitmap_c3.deserialize(in);
  bitmap_c4.deserialize(in);

  assertEquals(bitmap_a, bitmap_c1);
  assertEquals(bitmap_a, bitmap_c2);
  assertEquals(bitmap_a, bitmap_c3);
  assertEquals(bitmap_a, bitmap_c4);
  assertEquals(bitmap_ar, bitmap_c1);
  assertEquals(bitmap_ar, bitmap_c2);
  assertEquals(bitmap_ar, bitmap_c3);
  assertEquals(bitmap_ar, bitmap_c4);

}