Java Code Examples for org.roaringbitmap.RoaringBitmap#serialize()

public static void main(String[] args) {
  RoaringBitmap mrb = RoaringBitmap.bitmapOf(1, 2, 3, 1000);
  System.out.println("starting with  bitmap " + mrb);
  mrb.runOptimize(); // to improve compression
  byte[] array = new byte[mrb.serializedSizeInBytes()];
  mrb.serialize(ByteBuffer.wrap(array));
  RoaringBitmap ret = new RoaringBitmap();
  try {
    ret.deserialize(ByteBuffer.wrap(array));
  } catch(IOException ioe) {
    ioe.printStackTrace(); // should not happen
  }
  if (!ret.equals(mrb))
    throw new RuntimeException("bug");
  System.out.println("decoded from byte array : " + ret);

}
 

@Setup(Level.Trial)
public void setup() throws Exception {
  ZipRealDataRetriever dataRetriever = new ZipRealDataRetriever(dataset);
  RoaringBitmap[] bitmaps = StreamSupport.stream(dataRetriever.fetchBitPositions().spliterator(), false)
          .map(RoaringBitmap::bitmapOf)
          .toArray(RoaringBitmap[]::new);
  buffers = new byte[bitmaps.length][];
  int i = 0;
  for (RoaringBitmap bitmap : bitmaps) {
    if (runOptimise) {
      bitmap.runOptimize();
    }
    ByteArrayOutputStream bos = new ByteArrayOutputStream(bitmap.serializedSizeInBytes());
    bitmap.serialize(new DataOutputStream(bos));
    buffers[i++] = bos.toByteArray();
  }
}
 

public static ByteBuffer toByteBuffer(RoaringBitmap rb) {
  // we add tests
  ByteBuffer outbb = ByteBuffer.allocate(rb.serializedSizeInBytes());
  try {
    rb.serialize(new DataOutputStream(new ByteBufferBackedOutputStream(outbb)));
  } catch (IOException e) {
    e.printStackTrace();
  }
  //
  outbb.flip();
  outbb.order(ByteOrder.LITTLE_ENDIAN);
  return outbb;
}
 

@Nonnull
public static byte[] serializeRoaring(@Nonnull final RoaringBitmap r) {
    r.runOptimize(); // might improve compression
    // next we create the ByteBuffer where the data will be stored
    final byte[] buf = new byte[r.serializedSizeInBytes()];
    // then we can serialize on a custom OutputStream
    try {
        r.serialize(new DataOutputStream(new FastByteArrayOutputStream(buf)));
    } catch (IOException e) {
        throw new IllegalStateException("Failed to serialize RoaringBitmap", e);
    }
    return buf;
}
 

private byte[] serializeBitmap(RoaringBitmap bitmap) {
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    try (DataOutputStream dos = new DataOutputStream(bos)) {
        bitmap.serialize(dos);
    } catch (IOException e) {
        throw new RuntimeException("Failed to serialize roaring bitmap.");
    }
    return bos.toByteArray();
}
 

public static VertexManagerEvent generateVMEvent(OutputContext context,
    long[] sizePerPartition, boolean reportDetailedPartitionStats, Deflater deflater)
        throws IOException {
  ShuffleUserPayloads.VertexManagerEventPayloadProto.Builder vmBuilder =
      ShuffleUserPayloads.VertexManagerEventPayloadProto.newBuilder();

  long outputSize = context.getCounters().
      findCounter(TaskCounter.OUTPUT_BYTES).getValue();

  // Set this information only when required.  In pipelined shuffle,
  // multiple events would end up adding up to final output size.
  // This is needed for auto-reduce parallelism to work properly.
  vmBuilder.setOutputSize(outputSize);
  vmBuilder.setNumRecord(context.getCounters().findCounter(TaskCounter.OUTPUT_RECORDS).getValue()
   + context.getCounters().findCounter(TaskCounter.OUTPUT_LARGE_RECORDS).getValue());

  //set partition stats
  if (sizePerPartition != null && sizePerPartition.length > 0) {
    if (reportDetailedPartitionStats) {
      vmBuilder.setDetailedPartitionStats(
          getDetailedPartitionStatsForPhysicalOutput(sizePerPartition));
    } else {
      RoaringBitmap stats = getPartitionStatsForPhysicalOutput(
          sizePerPartition);
      DataOutputBuffer dout = new DataOutputBuffer();
      stats.serialize(dout);
      ByteString partitionStatsBytes =
          TezCommonUtils.compressByteArrayToByteString(dout.getData(), deflater);
      vmBuilder.setPartitionStats(partitionStatsBytes);
    }
  }

  VertexManagerEvent vmEvent = VertexManagerEvent.create(
      context.getDestinationVertexName(),
      vmBuilder.build().toByteString().asReadOnlyByteBuffer());
  return vmEvent;
}
 

public static ByteBuffer toByteBuffer(RoaringBitmap rb) {
  // we add tests
  ByteBuffer outbb = ByteBuffer.allocate(rb.serializedSizeInBytes());
  try {
    rb.serialize(new DataOutputStream(new ByteBufferBackedOutputStream(outbb)));
  } catch (IOException e) {
    e.printStackTrace();
  }
  //
  outbb.flip();
  outbb.order(ByteOrder.LITTLE_ENDIAN);
  return outbb;
}
 

@Benchmark
public int serializeToBAOSFromClone(BenchmarkState benchmarkState) throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.ac.size(); i++) {
    RoaringBitmap bitmap = benchmarkState.ac.get(i).clone();
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}
 

@Benchmark
public int serializeToBAOSNoClone(BenchmarkState benchmarkState) throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.ac.size(); i++) {
    RoaringBitmap bitmap = benchmarkState.ac.get(i);
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}
 

@Benchmark
public int serializeToBAOSFromClonePreOpti(BenchmarkState benchmarkState) throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.rc.size(); i++) {
    RoaringBitmap bitmap = benchmarkState.rc.get(i).clone();
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}
 

@Benchmark
public int serializeToBAOSNoClonePreOpti(BenchmarkState benchmarkState) throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.rc.size(); i++) {
    RoaringBitmap bitmap = benchmarkState.rc.get(i);
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}
 

@Benchmark
public int runOptimizeAndserializeToBAOSFromClone(BenchmarkState benchmarkState)
    throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.ac.size(); i++) {
    RoaringBitmap bitmap = benchmarkState.ac.get(i).clone();
    bitmap.runOptimize();
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}
 

VertexManagerEvent getVertexManagerEvent(long[] partitionSizes,
    long uncompressedTotalSize, String vertexName, boolean reportDetailedStats)
    throws IOException {
  ByteBuffer payload;
  long totalSize = 0;
  // Use partition sizes to compute the total size.
  if (partitionSizes != null) {
    totalSize = estimatedUncompressedSum(partitionSizes);
  } else {
    totalSize = uncompressedTotalSize;
  }
  if (partitionSizes != null) {
    RoaringBitmap partitionStats =
        ShuffleUtils.getPartitionStatsForPhysicalOutput(partitionSizes);
    DataOutputBuffer dout = new DataOutputBuffer();
    partitionStats.serialize(dout);
    ByteString
        partitionStatsBytes = TezCommonUtils.compressByteArrayToByteString(
            dout.getData());
    if (reportDetailedStats) {
      payload = VertexManagerEventPayloadProto.newBuilder()
          .setOutputSize(totalSize)
          .setDetailedPartitionStats(
              ShuffleUtils.getDetailedPartitionStatsForPhysicalOutput(
                  partitionSizes))
          .build().toByteString()
          .asReadOnlyByteBuffer();
    } else {
      payload = VertexManagerEventPayloadProto.newBuilder()
          .setOutputSize(totalSize)
          .setPartitionStats(partitionStatsBytes)
          .build().toByteString()
          .asReadOnlyByteBuffer();
    }

  } else {
    payload = VertexManagerEventPayloadProto.newBuilder()
        .setOutputSize(totalSize)
        .build().toByteString()
        .asReadOnlyByteBuffer();
  }
  TaskAttemptIdentifierImpl taId = new TaskAttemptIdentifierImpl("dag", vertexName,
      TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, taskId++), 0));
  VertexManagerEvent vmEvent = VertexManagerEvent.create(vertexName, payload);
  vmEvent.setProducerAttemptIdentifier(taId);
  return vmEvent;
}