org.roaringbitmap.buffer.MutableRoaringBitmap Java Examples

/**
 * Get docIds from the text inverted index for a given raw value
 * @param value value to look for in the inverted index
 * @return docIDs in bitmap
 */
@Override
public MutableRoaringBitmap getDocIds(Object value) {
  String searchQuery = (String) value;
  MutableRoaringBitmap docIds = new MutableRoaringBitmap();
  Collector docIDCollector = new LuceneDocIdCollector(docIds, _docIdTranslator);
  try {
    // Lucene Query Parser is JavaCC based. It is stateful and should
    // be instantiated per query. Analyzer on the other hand is stateless
    // and can be created upfront.
    QueryParser parser = new QueryParser(_column, _standardAnalyzer);
    Query query = parser.parse(searchQuery);
    _indexSearcher.search(query, docIDCollector);
    return docIds;
  } catch (Exception e) {
    String msg = "Caught excepttion while searching the text index for column:" + _column + " search query:" + searchQuery;
    throw new RuntimeException(msg, e);
  }
}
 

public static <T> void verifyInvariance(String testName,
                                        BiPredicate<MutableRoaringBitmap, MutableRoaringBitmap> validity,
                                        int count,
                                        int maxKeys,
                                        BiPredicate<MutableRoaringBitmap, MutableRoaringBitmap> test) {
  IntStream.range(0, count)
          .parallel()
          .forEach(i -> {
            MutableRoaringBitmap one = randomBitmap(maxKeys);
            MutableRoaringBitmap two = randomBitmap(maxKeys);
            if (validity.test(one, two)) {
              try {
                assertTrue(test.test(one, two));
              } catch (Throwable t) {
                Reporter.report(testName, ImmutableMap.of(), t, one, two);
                throw t;
              }
            }
          });
}
 

private MutableRoaringBitmap getPinotDocIds(IndexSearcher indexSearcher, MutableRoaringBitmap luceneDocIds) {
  IntIterator luceneDocIDIterator = luceneDocIds.getIntIterator();
  MutableRoaringBitmap actualDocIDs = new MutableRoaringBitmap();
  try {
    while (luceneDocIDIterator.hasNext()) {
      int luceneDocId = luceneDocIDIterator.next();
      Document document = indexSearcher.doc(luceneDocId);
      int pinotDocId = Integer.valueOf(document.get(LuceneTextIndexCreator.LUCENE_INDEX_DOC_ID_COLUMN_NAME));
      actualDocIDs.add(pinotDocId);
    }
  } catch (Exception e) {
    LOGGER.error("Failure while retrieving document from index for column {}, exception {}", _column, e.getMessage());
    throw new RuntimeException(e);
  }
  return actualDocIDs;
}
 

@Test
public void testAndDocIdIterator() {
  // AND result: [2, 7, 13, 15, 16, 20]
  int[] docIds1 = new int[]{0, 1, 2, 3, 5, 7, 10, 12, 13, 15, 16, 18, 20};
  int[] docIds2 = new int[]{1, 2, 4, 5, 6, 7, 9, 11, 12, 13, 15, 16, 17, 19, 20};
  int[] docIds3 = new int[]{0, 2, 3, 4, 7, 8, 10, 11, 13, 15, 16, 19, 20};

  MutableRoaringBitmap bitmap1 = new MutableRoaringBitmap();
  bitmap1.add(docIds1);
  MutableRoaringBitmap bitmap2 = new MutableRoaringBitmap();
  bitmap2.add(docIds2);
  MutableRoaringBitmap bitmap3 = new MutableRoaringBitmap();
  bitmap3.add(docIds3);
  AndDocIdIterator andDocIdIterator = new AndDocIdIterator(
      new BlockDocIdIterator[]{new RangelessBitmapDocIdIterator(bitmap1), new RangelessBitmapDocIdIterator(
          bitmap2), new RangelessBitmapDocIdIterator(bitmap3)});

  assertEquals(andDocIdIterator.next(), 2);
  assertEquals(andDocIdIterator.next(), 7);
  assertEquals(andDocIdIterator.advance(10), 13);
  assertEquals(andDocIdIterator.advance(16), 16);
  assertEquals(andDocIdIterator.next(), 20);
  assertEquals(andDocIdIterator.next(), Constants.EOF);
}
 

public static void verifyInvariance(String testName,
                                    Predicate<MutableRoaringBitmap> validity,
                                    int count,
                                    int maxKeys,
                                    IntBitmapPredicate predicate) {
  IntStream.range(0, count)
          .parallel()
          .mapToObj(i -> randomBitmap(maxKeys))
          .filter(validity)
          .forEach(bitmap -> {
            for (int i = 0; i < bitmap.getCardinality(); ++i) {
              try {
                assertTrue(predicate.test(i, bitmap));
              } catch (Throwable t) {
                Reporter.report(testName, ImmutableMap.of(), t, bitmap);
                throw t;
              }
            }
          });
}
 

@Setup
public void setup() {
  bitmap1 = new MutableRoaringBitmap();
  bitmap2 = new MutableRoaringBitmap();
  int k = 1 << 16;
  int i = 0;
  for (; i < 10000; ++i) {
    bitmap1.add(i * k);
  }
  for (; i < 10050; ++i) {
    bitmap2.add(i * k);
    bitmap1.add(i * k + 13);
  }
  for (; i < 20000; ++i) {
    bitmap2.add(i * k);
  }
  bitmap1.add(i * k);
}
 

public ImmutableState() {

      /**
       * Mutable & Immutable
       */

      for (int k = 0; k < N; ++k) {
        ewah_mutable[k] = new MutableRoaringBitmap();
        for (int x = 0; x < M; ++x) {
          ewah_mutable[k].add(x * (N - k + 2));
        }
        ewah_mutable[k].trim();
        try {
          ewah_immutable[k] = convertToMappedBitmap(ewah_mutable[k]);
        } catch (IOException e) {
          throw new RuntimeException(e);
        }
      }

      ors_immutable = new ArrayList<ImmutableRoaringBitmap[]>();
      for (int k = 1; k < N; k += 10) {
        ors_immutable.add(Arrays.copyOf(ewah_immutable, k + 1));

      }

    }
 

@Test
public void testBitmapDocIdIterator() {
  int[] docIds = new int[]{1, 2, 4, 5, 6, 8, 12, 15, 16, 18, 20, 21};
  MutableRoaringBitmap bitmap = new MutableRoaringBitmap();
  bitmap.add(docIds);
  int numDocs = 25;
  BitmapDocIdIterator docIdIterator = new BitmapDocIdIterator(bitmap, numDocs);
  assertEquals(docIdIterator.advance(2), 2);
  assertEquals(docIdIterator.advance(3), 4);
  assertEquals(docIdIterator.next(), 5);
  assertEquals(docIdIterator.advance(6), 6);
  assertEquals(docIdIterator.next(), 8);
  assertEquals(docIdIterator.advance(13), 15);
  assertEquals(docIdIterator.advance(19), 20);
  assertEquals(docIdIterator.next(), 21);
  assertEquals(docIdIterator.next(), Constants.EOF);
}
 

public MutableState() {

      for (int k = 0; k < N; ++k) {
        ewah_mutable[k] = new MutableRoaringBitmap();
        for (int x = 0; x < M; ++x) {
          ewah_mutable[k].add(x * (N - k + 2));
        }
        ewah_mutable[k].trim();

      }

      ors_mutable = new ArrayList<MutableRoaringBitmap[]>();

      for (int k = 1; k < N; k += 10) {
        ors_mutable.add(Arrays.copyOf(ewah_mutable, k + 1));
      }

    }
 

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public MutableRoaringBitmap inplace_andNot() {
  MutableRoaringBitmap b1 = bitmap1.clone();
  b1.andNot(bitmap2);
  return b1;
}
 

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public MutableRoaringBitmap inplace_and() {
  MutableRoaringBitmap b1 = bitmap1.clone();
  b1.and(bitmap2);
  return b1;
}
 

@Setup
public void setup() {
  bitmap1 = new MutableRoaringBitmap();
  bitmap2 = new MutableRoaringBitmap();
  int k = 1 << 16;
  for (int i = 0; i < 10000; ++i) {
    bitmap1.add(2 * i * k);
    bitmap2.add(2 * i * k + 1);
  }
}
 

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public MutableRoaringBitmap inplace_andNot() {
  MutableRoaringBitmap b1 = bitmap1.clone();
  b1.andNot(bitmap2);
  return b1;
}
 

@Setup
public void setup() {
  bitmap1 = new MutableRoaringBitmap();
  bitmap2 = new MutableRoaringBitmap();
  int k = 1 << 16;
  for (int i = 0; i < 10000; ++i) {
    bitmap1.add(i * k);
    bitmap2.add(i * k);
  }
}
 

@Setup
public void setup() throws Exception {
  ZipRealDataRetriever dataRetriever = new ZipRealDataRetriever(dataset);
  System.out.println();
  System.out.println("Loading files from " + dataRetriever.getName());
  ArrayList<MutableRoaringBitmap> tmprc = new ArrayList<MutableRoaringBitmap>();

  for (int[] data : dataRetriever.fetchBitPositions()) {
    MutableRoaringBitmap basic = MutableRoaringBitmap.bitmapOf(data);
    basic.runOptimize();
    tmprc.add(basic);
  }
  rc = convertToImmutableRoaring(tmprc);
}
 

@Benchmark
public int mutable_serializeToBAOSNoClone(BenchmarkState benchmarkState) throws IOException {
  ByteArrayOutputStream bos = new ByteArrayOutputStream();
  DataOutputStream dos = new DataOutputStream(bos);
  for (int i = 0; i < benchmarkState.mac.size(); i++) {
    MutableRoaringBitmap bitmap = benchmarkState.mac.get(i);
    bitmap.serialize(dos);
  }
  dos.flush();
  return bos.size();
}
 

@Benchmark
public int mutable_clone(BenchmarkState benchmarkState) {
  int total = 0;
  for (int i = 0; i < benchmarkState.mac.size(); i++) {
    MutableRoaringBitmap bitmap = benchmarkState.mac.get(i).clone();
    total += bitmap.getCardinality();
  }
  return total;
}
 

@Setup
public void setup() {
  bitmap1 = new MutableRoaringBitmap();
  bitmap2 = new MutableRoaringBitmap();
  int k = 1 << 16;
  for (int i = 0; i < 10000; ++i) {
    bitmap1.add(2 * i * k);
    bitmap2.add(2 * i * k + 1);
  }
}
 

public List<ImmutableRoaringBitmap> convertToImmutableRoaring(List<MutableRoaringBitmap> source)
    throws IOException {
  File tmpfile = File.createTempFile("roaring", "bin");
  tmpfile.deleteOnExit();
  final FileOutputStream fos = new FileOutputStream(tmpfile);
  final DataOutputStream dos = new DataOutputStream(fos);

  for (MutableRoaringBitmap rb1 : source)
    rb1.serialize(dos);

  final long totalcount = fos.getChannel().position();
  dos.close();
  final RandomAccessFile memoryMappedFile = new RandomAccessFile(tmpfile, "r");
  ByteBuffer out =
      memoryMappedFile.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, totalcount);
  ArrayList<ImmutableRoaringBitmap> answer =
      new ArrayList<ImmutableRoaringBitmap>(source.size());
  while (out.position() < out.limit()) {
    final ByteBuffer bb = out.slice();
    MutableRoaringBitmap equiv = source.get(answer.size());
    ImmutableRoaringBitmap newbitmap = new ImmutableRoaringBitmap(bb);
    if (!equiv.equals(newbitmap))
      throw new RuntimeException("bitmaps do not match");
    answer.add(newbitmap);
    out.position(out.position() + newbitmap.serializedSizeInBytes());
  }
  memoryMappedFile.close();
  return answer;
}
 

public static void verifyInvariance(String testName, int maxKeys, Predicate<MutableRoaringBitmap> pred) {
  IntStream.range(0, ITERATIONS)
          .parallel()
          .mapToObj(i -> randomBitmap(maxKeys))
          .forEach(bitmap -> {
            try {
              assertTrue(pred.test(bitmap));
            } catch (Throwable e) {
              Reporter.report(testName, ImmutableMap.of(), e, bitmap);
              throw e;
            }
          });
}
 

@Benchmark
public MutableRoaringBitmap createBitmapOrdered_mutable() {

  MutableRoaringBitmap r = new MutableRoaringBitmap();

  for (int k = 0; k < 65536; k++) {
    r.add(k * 32);
  }
  return r;
}
 

@Override
public void and(MutableBitmap mutableBitmap)
{
  WrappedRoaringBitmap other = (WrappedRoaringBitmap) mutableBitmap;
  MutableRoaringBitmap unwrappedOtherBitmap = other.bitmap;
  bitmap.and(unwrappedOtherBitmap);
}
 

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public MutableRoaringBitmap inplace_andNot() {
  MutableRoaringBitmap b1 = bitmap1.clone();
  b1.andNot(bitmap2);
  return b1;
}
 

@ParameterizedTest(name = "{1}")
@MethodSource("params")
public void canSerializeAndDeserializeBuffer(RoaringBitmap bitmap, int offset) throws IOException {
  MutableRoaringBitmap shifted = MutableRoaringBitmap.addOffset(bitmap.toMutableRoaringBitmap(), offset);
  ByteArrayDataOutput out = ByteStreams.newDataOutput();
  shifted.serialize(out);
  MutableRoaringBitmap deserialized = new MutableRoaringBitmap();
  deserialized.deserialize(ByteStreams.newDataInput(out.toByteArray()));
  assertEquals(shifted, deserialized, failureMessage(bitmap));
}
 

/**
 * In-place bitwise ANDNOT (difference) operation. The current bitmap is modified.
 *
 * @param x2 other bitmap
 */
public void andNot(final Roaring64NavigableMap x2) {
  boolean firstBucket = true;

  Iterator<Entry<Integer, BitmapDataProvider>> thisIterator = highToBitmap.entrySet().iterator();
  while (thisIterator.hasNext()) {
    Entry<Integer, BitmapDataProvider> e1 = thisIterator.next();

    // Keep object to prevent auto-boxing
    Integer high = e1.getKey();

    BitmapDataProvider lowBitmap2 = x2.highToBitmap.get(high);

    if (lowBitmap2 != null) {
      BitmapDataProvider lowBitmap1 = e1.getValue();

      if (lowBitmap2 instanceof RoaringBitmap && lowBitmap1 instanceof RoaringBitmap) {
        ((RoaringBitmap) lowBitmap1).andNot((RoaringBitmap) lowBitmap2);
      } else if (lowBitmap2 instanceof MutableRoaringBitmap
          && lowBitmap1 instanceof MutableRoaringBitmap) {
        ((MutableRoaringBitmap) lowBitmap1).andNot((MutableRoaringBitmap) lowBitmap2);
      } else {
        throw new UnsupportedOperationException(
            ".and is not between " + this.getClass() + " and " + lowBitmap1.getClass());
      }
    }

    if (firstBucket) {
      firstBucket = false;

      // Invalidate the lowest high as lowest not valid
      firstHighNotValid = Math.min(firstHighNotValid, high);
      allValid = false;
    }
  }
}
 

/**
 * In-place bitwise AND (intersection) operation. The current bitmap is modified.
 *
 * @param x2 other bitmap
 */
public void and(final Roaring64NavigableMap x2) {
  boolean firstBucket = true;

  Iterator<Entry<Integer, BitmapDataProvider>> thisIterator = highToBitmap.entrySet().iterator();
  while (thisIterator.hasNext()) {
    Entry<Integer, BitmapDataProvider> e1 = thisIterator.next();

    // Keep object to prevent auto-boxing
    Integer high = e1.getKey();

    BitmapDataProvider lowBitmap2 = x2.highToBitmap.get(high);

    if (lowBitmap2 == null) {
      // None of given high values are present in x2
      thisIterator.remove();
    } else {
      BitmapDataProvider lowBitmap1 = e1.getValue();

      if (lowBitmap2 instanceof RoaringBitmap && lowBitmap1 instanceof RoaringBitmap) {
        ((RoaringBitmap) lowBitmap1).and((RoaringBitmap) lowBitmap2);
      } else if (lowBitmap2 instanceof MutableRoaringBitmap
          && lowBitmap1 instanceof MutableRoaringBitmap) {
        ((MutableRoaringBitmap) lowBitmap1).and((MutableRoaringBitmap) lowBitmap2);
      } else {
        throw new UnsupportedOperationException(
            ".and is not between " + this.getClass() + " and " + lowBitmap1.getClass());
      }
    }

    if (firstBucket) {
      firstBucket = false;

      // Invalidate the lowest high as lowest not valid
      firstHighNotValid = Math.min(firstHighNotValid, high);
      allValid = false;
    }
  }
}
 

@Test
public void testOrDocIdIterator() {
  // OR result: [0, 1, 2, 4, 5, 6, 8, 10, 13, 15, 16, 17, 18, 19, 20]
  int[] docIds1 = new int[]{1, 4, 6, 10, 15, 17, 18, 20};
  int[] docIds2 = new int[]{0, 1, 5, 8, 15, 18};
  int[] docIds3 = new int[]{1, 2, 6, 13, 16, 19};

  MutableRoaringBitmap bitmap1 = new MutableRoaringBitmap();
  bitmap1.add(docIds1);
  MutableRoaringBitmap bitmap2 = new MutableRoaringBitmap();
  bitmap2.add(docIds2);
  MutableRoaringBitmap bitmap3 = new MutableRoaringBitmap();
  bitmap3.add(docIds3);
  OrDocIdIterator andDocIdIterator = new OrDocIdIterator(
      new BlockDocIdIterator[]{new RangelessBitmapDocIdIterator(bitmap1), new RangelessBitmapDocIdIterator(
          bitmap2), new RangelessBitmapDocIdIterator(bitmap3)});

  assertEquals(andDocIdIterator.advance(1), 1);
  assertEquals(andDocIdIterator.next(), 2);
  assertEquals(andDocIdIterator.next(), 4);
  assertEquals(andDocIdIterator.advance(7), 8);
  assertEquals(andDocIdIterator.advance(13), 13);
  assertEquals(andDocIdIterator.next(), 15);
  assertEquals(andDocIdIterator.advance(18), 18);
  assertEquals(andDocIdIterator.next(), 19);
  assertEquals(andDocIdIterator.advance(21), Constants.EOF);
}
 

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public MutableRoaringBitmap inplace_and() {
  MutableRoaringBitmap b1 = bitmap1.clone();
  b1.and(bitmap2);
  return b1;
}
 

@Override
public ImmutableBitmap union(ImmutableBitmap otherBitmap)
{
  WrappedRoaringBitmap other = (WrappedRoaringBitmap) otherBitmap;
  MutableRoaringBitmap unwrappedOtherBitmap = other.bitmap;
  return new WrappedImmutableRoaringBitmap(MutableRoaringBitmap.or(bitmap, unwrappedOtherBitmap));
}
 

@Override
public void or(MutableBitmap mutableBitmap)
{
  WrappedRoaringBitmap other = (WrappedRoaringBitmap) mutableBitmap;
  MutableRoaringBitmap unwrappedOtherBitmap = other.bitmap;
  bitmap.or(unwrappedOtherBitmap);
}