Java Code Examples for org.roaringbitmap.buffer.MutableRoaringBitmap#runOptimize()

The following examples show how to use org.roaringbitmap.buffer.MutableRoaringBitmap#runOptimize() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FixLenColInvertIndexWriter.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public void write(OutputStream out) throws IOException {
    int cardinality = valueBitmaps.size();
    int footLen = II_DICT_TYPE_SORT_VAL_FOOT_LEN;
    int dictAndFootLen = cardinality * (valueLenInBytes + 4) + footLen;

    DataOutputStream bitmapOut = new DataOutputStream(out);
    ByteBuffer dictBuffer = ByteBuffer.allocate(dictAndFootLen);
    int offset = 0;
    for (Map.Entry<byte[], MutableRoaringBitmap> bitmapEntry : valueBitmaps.entrySet()) {
        byte[] colValue = bitmapEntry.getKey();
        MutableRoaringBitmap bitmap = bitmapEntry.getValue();
        bitmap.runOptimize();
        int bitmapSize = bitmap.serializedSizeInBytes();
        bitmap.serialize(bitmapOut);

        dictBuffer.put(colValue);
        dictBuffer.putInt(offset);
        offset += bitmapSize;
    }

    dictBuffer.putInt(cardinality);
    dictBuffer.putInt(valueLenInBytes);
    dictBuffer.putInt(II_DICT_TYPE_SORT_VAL); // type
    out.write(dictBuffer.array(), 0, dictBuffer.position());

    out.flush();
}
Example 2
Source File: SeqColInvertIndexWriter.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
public void write(OutputStream out) throws IOException {
    int cardinality = valueBitmaps.length;
    if (nullValueBitmap != null) {
        cardinality++;
    }
    int dictionaryPartLen = cardinality * 4;

    DataOutputStream bitmapOut = new DataOutputStream(out);
    int footAndDictLen = II_DICT_TYPE_SEQ_FOOT_LEN + dictionaryPartLen;
    ByteBuffer footBuffer = ByteBuffer.allocate(footAndDictLen);

    int offset = 0;
    for (MutableRoaringBitmap bitmap : valueBitmaps) {
        bitmap.runOptimize();
        int bitmapSize = bitmap.serializedSizeInBytes();
        bitmap.serialize(bitmapOut);

        footBuffer.putInt(offset);
        offset += bitmapSize;
    }
    if (nullValueBitmap != null) {
        nullValueBitmap.serialize(bitmapOut);
        footBuffer.putInt(offset);
    }

    footBuffer.putInt(cardinality); // cardinality
    footBuffer.putInt(minColVal);
    footBuffer.putInt(maxColVal);
    footBuffer.putInt(II_DICT_TYPE_SEQ);// type
    if (nullValueBitmap != null) {
        footBuffer.put((byte) 1);
    } else {
        footBuffer.put((byte) 0);
    }
    out.write(footBuffer.array(), 0, footBuffer.position());

    out.flush();
}
Example 3
Source File: FixLenColInvertIndexWriter.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void write(OutputStream out) throws IOException {
    int cardinality = valueBitmaps.size();
    int footLen = II_DICT_TYPE_SORT_VAL_FOOT_LEN;
    int dictAndFootLen = cardinality * (valueLenInBytes + 4) + footLen;

    DataOutputStream bitmapOut = new DataOutputStream(out);
    ByteBuffer dictBuffer = ByteBuffer.allocate(dictAndFootLen);
    int offset = 0;
    for (Map.Entry<byte[], MutableRoaringBitmap> bitmapEntry : valueBitmaps.entrySet()) {
        byte[] colValue = bitmapEntry.getKey();
        MutableRoaringBitmap bitmap = bitmapEntry.getValue();
        bitmap.runOptimize();
        int bitmapSize = bitmap.serializedSizeInBytes();
        bitmap.serialize(bitmapOut);

        dictBuffer.put(colValue);
        dictBuffer.putInt(offset);
        offset += bitmapSize;
    }

    dictBuffer.putInt(cardinality);
    dictBuffer.putInt(valueLenInBytes);
    dictBuffer.putInt(II_DICT_TYPE_SORT_VAL); // type
    out.write(dictBuffer.array(), 0, dictBuffer.position());

    out.flush();
}
Example 4
Source File: SeqColInvertIndexWriter.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
public void write(OutputStream out) throws IOException {
    int cardinality = valueBitmaps.length;
    if (nullValueBitmap != null) {
        cardinality++;
    }
    int dictionaryPartLen = cardinality * 4;

    DataOutputStream bitmapOut = new DataOutputStream(out);
    int footAndDictLen = II_DICT_TYPE_SEQ_FOOT_LEN + dictionaryPartLen;
    ByteBuffer footBuffer = ByteBuffer.allocate(footAndDictLen);

    int offset = 0;
    for (MutableRoaringBitmap bitmap : valueBitmaps) {
        bitmap.runOptimize();
        int bitmapSize = bitmap.serializedSizeInBytes();
        bitmap.serialize(bitmapOut);

        footBuffer.putInt(offset);
        offset += bitmapSize;
    }
    if (nullValueBitmap != null) {
        nullValueBitmap.serialize(bitmapOut);
        footBuffer.putInt(offset);
    }

    footBuffer.putInt(cardinality); // cardinality
    footBuffer.putInt(minColVal);
    footBuffer.putInt(maxColVal);
    footBuffer.putInt(II_DICT_TYPE_SEQ);// type
    if (nullValueBitmap != null) {
        footBuffer.put((byte) 1);
    } else {
        footBuffer.put((byte) 0);
    }
    out.write(footBuffer.array(), 0, footBuffer.position());

    out.flush();
}
Example 5
Source File: SlowMappedORaggregate1.java    From RoaringBitmap with Apache License 2.0 5 votes vote down vote up 
@Setup
public void setup() throws Exception {
  ZipRealDataRetriever dataRetriever = new ZipRealDataRetriever(dataset);
  System.out.println();
  System.out.println("Loading files from " + dataRetriever.getName());
  ArrayList<MutableRoaringBitmap> tmprc = new ArrayList<MutableRoaringBitmap>();

  for (int[] data : dataRetriever.fetchBitPositions()) {
    MutableRoaringBitmap basic = MutableRoaringBitmap.bitmapOf(data);
    basic.runOptimize();
    tmprc.add(basic);
  }
  rc = convertToImmutableRoaring(tmprc);
}
Example 6
Source File: SlowMappedORaggregate2.java    From RoaringBitmap with Apache License 2.0 5 votes vote down vote up 
@Setup
public void setup() throws Exception {
  ZipRealDataRetriever dataRetriever = new ZipRealDataRetriever(dataset);
  System.out.println();
  System.out.println("Loading files from " + dataRetriever.getName());
  ArrayList<MutableRoaringBitmap> tmprc = new ArrayList<MutableRoaringBitmap>();

  for (int[] data : dataRetriever.fetchBitPositions()) {
    MutableRoaringBitmap basic = MutableRoaringBitmap.bitmapOf(data);
    basic.runOptimize();
    tmprc.add(basic);
  }
  rc = convertToImmutableRoaring(tmprc);
}
Example 7
Source File: MappedRunContainerRealDataBenchmarkRunOptimize.java    From RoaringBitmap with Apache License 2.0 5 votes vote down vote up 
@Benchmark
public int mutable_cloneAndrunOptimize(BenchmarkState benchmarkState) {
  int total = 0;
  for (int i = 0; i < benchmarkState.mac.size(); i++) {
    MutableRoaringBitmap bitmap = benchmarkState.mac.get(i).clone();
    bitmap.runOptimize();
    total += bitmap.getCardinality();
  }
  return total;
}
Example 8
Source File: MappedRunContainerRealDataBenchmarkRunOptimize.java    From RoaringBitmap with Apache License 2.0 5 votes vote down vote up 
@Benchmark
public int mutable_runOptimizeAndserializeToBAOSFromClone(BenchmarkState benchmarkState)
    throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.mac.size(); i++) {
    MutableRoaringBitmap bitmap = benchmarkState.mac.get(i).clone();
    bitmap.runOptimize();
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}
Example 9
Source File: MappedRunContainerRealDataBenchmarkRunOptimize.java    From RoaringBitmap with Apache License 2.0 5 votes vote down vote up 
@Setup
public void setup() throws Exception {
  ZipRealDataRetriever dataRetriever = new ZipRealDataRetriever(dataset);
  System.out.println();
  System.out.println("Loading files from " + dataRetriever.getName());

  for (int[] data : dataRetriever.fetchBitPositions()) {
    MutableRoaringBitmap mbasic = MutableRoaringBitmap.bitmapOf(data);
    mac.add(mbasic);
    MutableRoaringBitmap mopti = mbasic.clone();
    mopti.runOptimize();
    mrc.add(mopti);
  }
}
Example 10
Source File: WrappedRoaringBitmap.java    From bytebuffer-collections with Apache License 2.0 5 votes vote down vote up 
ImmutableBitmap toImmutableBitmap()
{
  MutableRoaringBitmap mrb = bitmap.clone();
  if (compressRunOnSerialization) {
    mrb.runOptimize();
  }
  return new WrappedImmutableRoaringBitmap(mrb);
}
Example 11
Source File: TestSerialization.java    From RoaringBitmap with Apache License 2.0 4 votes vote down vote up 
@Test
public void testMutableRunSerializationBasicDeserialization() throws java.io.IOException {
  final int[] data = takeSortedAndDistinct(new Random(07734), 100000);
  RoaringBitmap bitmap_a = RoaringBitmap.bitmapOf(data);
  RoaringBitmap bitmap_ar = RoaringBitmap.bitmapOf(data);

  MutableRoaringBitmap bitmap_am = MutableRoaringBitmap.bitmapOf(data);
  MutableRoaringBitmap bitmap_amr = MutableRoaringBitmap.bitmapOf(data);

  for (int k = 100000; k < 200000; ++k) {
    bitmap_a.add(3 * k); // bitmap density and too many little runs
    bitmap_ar.add(3 * k);
    bitmap_am.add(3 * k);
    bitmap_amr.add(3 * k);
  }

  for (int k = 700000; k < 800000; ++k) { // will choose a runcontainer on this
    bitmap_a.add(k); // bitmap density and too many little runs
    bitmap_ar.add(k);
    bitmap_am.add(k);
    bitmap_amr.add(k);    
  }

  bitmap_ar.runOptimize();
  bitmap_amr.runOptimize();
  assertEquals(bitmap_a, bitmap_ar);
  assertEquals(bitmap_am, bitmap_amr);
  assertEquals(bitmap_am.serializedSizeInBytes(), bitmap_a.serializedSizeInBytes());
  assertEquals(bitmap_amr.serializedSizeInBytes(), bitmap_ar.serializedSizeInBytes());

  ByteBuffer outbuf = ByteBuffer.allocate(2*(bitmap_a.serializedSizeInBytes() + bitmap_ar.serializedSizeInBytes()));
  DataOutputStream out = new DataOutputStream(new ByteBufferBackedOutputStream(outbuf));
  try {
    bitmap_a.serialize(out);
    bitmap_ar.serialize(out);
    bitmap_am.serialize(out);
    bitmap_amr.serialize(out);
  } catch (Exception e) {
    e.printStackTrace();
  }
  outbuf.flip();

  RoaringBitmap bitmap_c1 = new RoaringBitmap();
  RoaringBitmap bitmap_c2 = new RoaringBitmap();
  RoaringBitmap bitmap_c3 = new RoaringBitmap();
  RoaringBitmap bitmap_c4 = new RoaringBitmap();

  DataInputStream in = new DataInputStream(new ByteBufferBackedInputStream(outbuf));
  bitmap_c1.deserialize(in);
  bitmap_c2.deserialize(in);
  bitmap_c3.deserialize(in);
  bitmap_c4.deserialize(in);

  assertEquals(bitmap_a, bitmap_c1);
  assertEquals(bitmap_a, bitmap_c2);
  assertEquals(bitmap_a, bitmap_c3);
  assertEquals(bitmap_a, bitmap_c4);
  assertEquals(bitmap_ar, bitmap_c1);
  assertEquals(bitmap_ar, bitmap_c2);
  assertEquals(bitmap_ar, bitmap_c3);
  assertEquals(bitmap_ar, bitmap_c4);

}