org.roaringbitmap.RoaringBitmap Java Examples

@Setup(Level.Trial)
public void createBitmaps() {
  random = new SplittableRandom(seed);
  RoaringBitmapWriter<RoaringBitmap> bitmapWriter = constructionStrategy.create();
  bitmaps = new RoaringBitmap[count];
  bufferBitmaps = new ImmutableRoaringBitmap[count];
  double p = Math.pow(probability, 1D/count);
  for (int i = 0; i < count; ++i) {
    for (int j = (int)(Math.log(random.nextDouble())/Math.log(1-p));
         j < size;
         j += (int)(Math.log(random.nextDouble())/Math.log(1-p)) + 1) {
        bitmapWriter.add(j);
    }
    bitmaps[i] = bitmapWriter.get();
    bufferBitmaps[i] = bitmaps[i].toMutableRoaringBitmap();
    bitmapWriter.reset();
  }
}
 

@VisibleForTesting
void parsePartitionStats(SourceVertexInfo srcInfo,
    RoaringBitmap partitionStats) {
  Preconditions.checkState(srcInfo.statsInMB != null,
      "Stats should be initialized");
  Iterator<Integer> it = partitionStats.iterator();
  final DATA_RANGE_IN_MB[] RANGES = DATA_RANGE_IN_MB.values();
  final int RANGE_LEN = RANGES.length;
  while (it.hasNext()) {
    int pos = it.next();
    int index = ((pos) / RANGE_LEN);
    int rangeIndex = ((pos) % RANGE_LEN);
    //Add to aggregated stats and normalize to DATA_RANGE_IN_MB.
    if (RANGES[rangeIndex].getSizeInMB() > 0) {
      srcInfo.statsInMB[index] += RANGES[rangeIndex].getSizeInMB();
    }
  }
}
 

/**
 * Finds internal ids of routes.
 * @param tagsMetadata tags to match
 * @return bitmap of all internal ids of routes.
 */
RoaringBitmap find(TagsMetadata tagsMetadata) {
	RoaringBitmap found = new RoaringBitmap();
	if (tagsMetadata == null) {
		if (log.isDebugEnabled()) {
			log.debug("find() called with null TagsMetadata");
		}
		return found;
	}
	AtomicBoolean first = new AtomicBoolean(true);
	tagsMetadata.getTags().forEach((key, value) -> {
		TagKey tagKey = new TagKey(key, value);
		if (tagsToBitmaps.containsKey(tagKey)) {
			RoaringBitmap search = tagsToBitmaps.get(tagKey);
			if (first.get()) {
				// initiliaze found bitmap with current search
				found.or(search);
				first.compareAndSet(true, false);
			}
			else {
				found.and(search);
			}
		}
	});
	return found;
}
 

private void setupEntities(Entities entities) throws Exception {
  logger.debug("Initializing MilneWittenEntityEntitySimilarity for " + entities.size() + " entities");

  collectionSize = DataAccess.getCollectionSize();

  TIntObjectHashMap<int[]> entityInlinks = DataAccess.getInlinkNeighbors(entities);

  // inlinks are assumed to be pre-sorted.
  entity2vector = new TIntObjectHashMap<>();

  for (TIntObjectIterator<int[]> itr = entityInlinks.iterator(); itr.hasNext(); ) {
    itr.advance();
    int entity = itr.key();
    int[] inLinks = itr.value();

    RoaringBitmap bs = new RoaringBitmap();
    for (int l : inLinks) {
      bs.add(l);
    }
    entity2vector.put(entity, bs);
  }

  logger.debug("Done initializing MilneWittenEntityEntitySimilarity for " + entities.size() + " entities");
}
 

private void setupEntities(Entities entities) throws Exception {
  if (entities.size() == 0) {
    logger.debug("Skipping initialization of InlinkEntityEntitySimilarity for " + entities.size() + " entities");
    return;
  }

  logger.debug("Initializing InlinkEntityEntitySimilarity for " + entities.size() + " entities");

  entity2vector = new TIntObjectHashMap<>();

  TIntObjectHashMap<int[]> entityInlinks = DataAccess.getInlinkNeighbors(entities);

  for (TIntObjectIterator<int[]> itr = entityInlinks.iterator(); itr.hasNext(); ) {
    itr.advance();
    int entity = itr.key();
    int[] inLinks = itr.value();

    RoaringBitmap bs = new RoaringBitmap();
    for (int l : inLinks) {
      bs.add(l);
    }
    entity2vector.put(entity, bs);
  }

  logger.debug("Done initializing InlinkEntityEntitySimilarity");
}
 

private RSocket assertRegister(RoutingTable routingTable, TagsMetadata tagsMetadata,
		int internalId) {
	String routeId = tagsMetadata.getRouteId();
	RSocket rsocket = new TestRSocket(routeId);
	routingTable.register(tagsMetadata, rsocket);

	assertThat(routingTable.internalRouteId).hasValue(internalId);
	assertThat(routingTable.internalRouteIdToRouteId).containsEntry(internalId,
			routeId);
	assertThat(routingTable.routeEntries).containsKey(routeId);
	tagsMetadata.getTags().forEach((key, value) -> {
		RoutingTable.TagKey tagKey = new RoutingTable.TagKey(key, value);
		assertThat(routingTable.tagsToBitmaps).containsKey(tagKey);
		RoaringBitmap bitmap = routingTable.tagsToBitmaps.get(tagKey);
		assertThat(bitmap.contains(internalId));
	});

	return rsocket;
}
 

/**
 * Given a metric name, extract all the values associated for the given key.
 */
@VisibleForTesting
public Map<Integer, String> getValuesForMetricKey(String metricName, String key) {
  List<RoaringBitmap> andBitMaps = new ArrayList<>();

  if (metricIndex.containsKey(metricName)) {
    andBitMaps.add(lookupMetricIndex(metricName));
  } else {
    andBitMaps.add(EMPTY_BITMAP);
  }

  if (metricIndex.containsKey(key)) {
    andBitMaps.add(lookupMetricIndex(key));
  } else {
    andBitMaps.add(EMPTY_BITMAP);
  }

  RoaringBitmap resultBitMap = FastAggregation.and(andBitMaps.iterator());
  HashMap<Integer, String> resultMap = new HashMap<>();
  for (int i : resultBitMap.toArray()) {
    resultMap.put(i, extractTagValueForTagKey(metricIdMap.get(i), key));
  }
  return resultMap;
}
 

/**
 * @param opt command separated list of Q and C. Q for {@link NumericAttribute}, C for
 *        {@link NominalAttribute}.
 */
@Nonnull
public static RoaringBitmap resolveAttributes(@Nullable final String opt)
        throws UDFArgumentException {
    final RoaringBitmap attr = new RoaringBitmap();
    if (opt == null) {
        return attr;
    }
    final String[] opts = opt.split(",");
    final int size = opts.length;
    for (int i = 0; i < size; i++) {
        final String type = opts[i];
        if ("Q".equals(type)) {
            continue;
        } else if ("C".equals(type)) {
            attr.add(i);
        } else {
            throw new UDFArgumentException("Unsupported attribute type: " + type);
        }
    }
    return attr;
}
 

/**
 * @param opt comma separated list of zero-start indexes
 */
@Nonnull
public static RoaringBitmap parseNominalAttributeIndicies(@Nullable final String opt)
        throws UDFArgumentException {
    final RoaringBitmap attr = new RoaringBitmap();
    if (opt == null) {
        return attr;
    }
    for (String s : opt.split(",")) {
        if (NumberUtils.isDigits(s)) {
            int index = NumberUtils.parseInt(s);
            attr.add(index);
        } else {
            throw new UDFArgumentException("Expected integer but got " + s);
        }
    }
    return attr;
}
 

@Override
public Map<Integer, Integer> discnt(String distinctName, boolean isDim, String groupByDimName,
        Map<String, List<Integer>> filterDims) {
    
    try {
        Map<Integer, RoaringBitmap> distinct = distinct(distinctName, isDim, groupByDimName, filterDims);
        // Count it.
        Map<Integer, Integer> result = distinct.entrySet().stream().collect(
                Collectors.toMap(e -> e.getKey(), e -> e.getValue().getCardinality()));
        LOGGER.debug("Distinct {} on {} with filter {} results is {}", distinctName, AGG_CONTEXT.get(), filterDims, result);
        LOGGER.info("Distinct {} on {} with filter {} results size is {}", distinctName, AGG_CONTEXT.get(), filterDims, result.size());
        return result;
    } finally {
        AGG_CONTEXT.remove();
    }
}
 

public static void testSuperDense() {
  System.out.println("Sparse case... universe = [0,"+universe_size+")");
  RoaringBitmap r = new RoaringBitmap();
  int howmany = 100;
  int gap = universe_size / howmany;
  for (int i = 1; i < howmany; i++) {
    r.add(i * gap + 1,((i + 1) * gap));
  }
  System.out.println("Adding "+r.getCardinality()+" values partionned by "+howmany+" gaps of 1 ...");
  System.out.println("As a bitmap it would look like 01111...11011111... ");

  System.out.println("Bits used per value = "+F.format(r.getSizeInBytes()*8.0/r.getCardinality()));
  r.runOptimize();
  System.out.println("Bits used per value after run optimize = "+F.format(r.getSizeInBytes()*8.0/r.getCardinality()));
  System.out.println("Bits used per gap after run optimize = "+F.format(r.getSizeInBytes()*8.0/howmany));

  System.out.println("An uncompressed bitset might use "+F.format(universe_size*1.0/r.getCardinality())+" bits per value set");
  System.out.println();
}
 

private static String graphvizOutput(String datasetUrl, int responseIndex, int numLeafs,
        boolean dense, String[] featureNames, String[] classNames, String outputName)
        throws IOException, HiveException, ParseException {
    URL url = new URL(datasetUrl);
    InputStream is = new BufferedInputStream(url.openStream());

    ArffParser arffParser = new ArffParser();
    arffParser.setResponseIndex(responseIndex);

    AttributeDataset ds = arffParser.parse(is);
    double[][] x = ds.toArray(new double[ds.size()][]);
    int[] y = ds.toArray(new int[ds.size()]);

    RoaringBitmap attrs = SmileExtUtils.convertAttributeTypes(ds.attributes());
    DecisionTree tree = new DecisionTree(attrs, matrix(x, dense), y, numLeafs,
        RandomNumberGeneratorFactory.createPRNG(31));

    Text model = new Text(Base91.encode(tree.serialize(true)));

    Evaluator eval = new Evaluator(OutputType.graphviz, outputName, false);
    Text exported = eval.export(model, featureNames, classNames);

    return exported.toString();
}
 

/**
 * Test of learn method, of class DecisionTree.
 */
@Test
public void testIris() throws IOException, ParseException, HiveException {
    URL url = new URL(
        "https://gist.githubusercontent.com/myui/143fa9d05bd6e7db0114/raw/500f178316b802f1cade6e3bf8dc814a96e84b1e/iris.arff");
    InputStream is = new BufferedInputStream(url.openStream());

    ArffParser arffParser = new ArffParser();
    arffParser.setResponseIndex(4);
    AttributeDataset iris = arffParser.parse(is);
    double[][] x = iris.toArray(new double[iris.size()][]);
    int[] y = iris.toArray(new int[iris.size()]);

    int n = x.length;
    LOOCV loocv = new LOOCV(n);
    for (int i = 0; i < n; i++) {
        double[][] trainx = Math.slice(x, loocv.train[i]);
        int[] trainy = Math.slice(y, loocv.train[i]);

        RoaringBitmap attrs = SmileExtUtils.convertAttributeTypes(iris.attributes());
        DecisionTree tree = new DecisionTree(attrs,
            new RowMajorDenseMatrix2d(trainx, x[0].length), trainy, 4);
        assertEquals(tree.predict(x[loocv.test[i]]), evalPredict(tree, x[loocv.test[i]]));
    }
}
 

/**
 * Test of learn method, of class DecisionTree.
 */
@Test
public void testIris() throws IOException, ParseException, HiveException {
    URL url = new URL(
        "https://gist.githubusercontent.com/myui/143fa9d05bd6e7db0114/raw/500f178316b802f1cade6e3bf8dc814a96e84b1e/iris.arff");
    InputStream is = new BufferedInputStream(url.openStream());

    ArffParser arffParser = new ArffParser();
    arffParser.setResponseIndex(4);
    AttributeDataset iris = arffParser.parse(is);
    double[][] x = iris.toArray(new double[iris.size()][]);
    int[] y = iris.toArray(new int[iris.size()]);

    int n = x.length;
    LOOCV loocv = new LOOCV(n);
    for (int i = 0; i < n; i++) {
        double[][] trainx = Math.slice(x, loocv.train[i]);
        int[] trainy = Math.slice(y, loocv.train[i]);

        RoaringBitmap attrs = SmileExtUtils.convertAttributeTypes(iris.attributes());
        DecisionTree tree = new DecisionTree(attrs,
            new RowMajorDenseMatrix2d(trainx, x[0].length), trainy, 4);
        Assert.assertEquals(tree.predict(x[loocv.test[i]]),
            evalPredict(tree, x[loocv.test[i]]));
    }
}
 

private Iterator<RoaringBitmap> toRoaringBitmapIterator(final Iterable<Bitmap> bitmaps) {
  return new Iterator<RoaringBitmap>() {
    final Iterator<Bitmap> i = bitmaps.iterator();

    @Override
    public boolean hasNext() {
      return i.hasNext();
    }


    @Override
    public void remove() {
      throw new UnsupportedOperationException();
    }


    @Override
    public RoaringBitmap next() {
      return ((RoaringBitmapWrapper) i.next()).bitmap;
    }
  };
}
 

private static Bitmap newRoaringBitmap(int[] data, boolean optimize) {
  RoaringBitmap roaring = RoaringBitmap.bitmapOf(data);
  if (optimize) {
    roaring.runOptimize();
  }
  return new RoaringBitmapWrapper(roaring);
}
 

public static void main(String[] args) {
      // some bitmap
      RoaringBitmap rr = RoaringBitmap.bitmapOf(1,2,3,1000);

      // we want to check if it intersects a given range [10,1000]
      int low = 10;
      int high = 1000;
      RoaringBitmap range = new RoaringBitmap();
      range.add((long)low, (long)high + 1);
      //
      //

      System.out.println(RoaringBitmap.intersects(rr,range)); // prints true if they intersect
}
 

public BenchmarkState() {
  int N = 30;
  Random rand = new Random(1234);
  for (int k = 0; k < N; ++k) {
    RoaringBitmap rb = new RoaringBitmap();
    int start = rand.nextInt(10000);

    for (int z = 0; z < 50; ++z) {
      int end = start + rand.nextInt(10000);
      rb.add(start, end);
      start = end + rand.nextInt(1000);
    }
    ConciseSet ccs = toConcise(rb.toArray());
    cc.add(ccs);
    wah.add(toWAH(rb.toArray()));
    icc.add(ImmutableConciseSet.newImmutableFromMutable(ccs));

    ac.add(rb);

    rb = rb.clone();
    rb.runOptimize();
    rc.add(rb);
    ewah.add(EWAHCompressedBitmap.bitmapOf(rb.toArray()));
    ewah32.add(EWAHCompressedBitmap32.bitmapOf(rb.toArray()));

  }
}
 

@Benchmark
public int clone(BenchmarkState benchmarkState) {
  int total = 0;
  for (int i = 0; i < benchmarkState.ac.size(); i++) {
    RoaringBitmap bitmap = benchmarkState.ac.get(i).clone();
    total += bitmap.getCardinality();
  }
  return total;
}
 

@Override
public BitmapAggregator naiveOrAggregator() {
  return new BitmapAggregator() {
    @Override
    public Bitmap aggregate(final Iterable<Bitmap> bitmaps) {
      Iterator<RoaringBitmap> iterator = toRoaringBitmapIterator(bitmaps);
      return new RoaringBitmapWrapper(FastAggregation.naive_or(iterator));
    }
  };
}
 

@Benchmark
public int OrRunContainer(BenchmarkState benchmarkState) {
  int answer = 0;
  for (int k = 1; k < benchmarkState.rc.size(); ++k)
    answer +=
        RoaringBitmap.or(benchmarkState.rc.get(k - 1), benchmarkState.rc.get(k)).getCardinality();
  return answer;
}
 

/**
 * Finds routeIds of matching routes.
 * @param tagsMetadata tags to match.
 * @return all matching routeIds or empty list.
 */
public Set<String> findRouteIds(TagsMetadata tagsMetadata) {
	RoaringBitmap found = find(tagsMetadata);
	if (found.isEmpty()) {
		return Collections.emptySet();
	}
	HashSet<String> routeIds = new HashSet<>();
	found.forEach((IntConsumer) internalId -> {
		String routeId = internalRouteIdToRouteId.get(internalId);
		routeIds.add(routeId);
	});
	return routeIds;
}
 

public static ByteBuffer toByteBuffer(RoaringBitmap rb) {
  // we add tests
  ByteBuffer outbb = ByteBuffer.allocate(rb.serializedSizeInBytes());
  try {
    rb.serialize(new DataOutputStream(new ByteBufferBackedOutputStream(outbb)));
  } catch (IOException e) {
    e.printStackTrace();
  }
  //
  outbb.flip();
  outbb.order(ByteOrder.LITTLE_ENDIAN);
  return outbb;
}
 

@Setup(Level.Trial)
public void setup() throws Exception {
  bitmaps = DATASET_CACHE.get(dataset, () -> {
    System.out.println("Loading" + dataset);
    ZipRealDataRetriever dataRetriever = new ZipRealDataRetriever(dataset);
    return StreamSupport.stream(dataRetriever.fetchBitPositions().spliterator(), false)
            .map(RoaringBitmap::bitmapOf)
            .toArray(RoaringBitmap[]::new);
  });
  immutableRoaringBitmaps = Arrays.stream(bitmaps).map(RoaringBitmap::toMutableRoaringBitmap)
          .toArray(ImmutableRoaringBitmap[]::new);
}
 

@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public RoaringBitmap inplace_and() {
  RoaringBitmap b1 = bitmap1.clone();
  b1.and(bitmap2);
  return b1;
}
 

@Benchmark
public int XorBitmapContainer(BenchmarkState benchmarkState) {
  int answer = 0;
  for (int k = 1; k < benchmarkState.ac.size(); ++k)
    answer += RoaringBitmap.xor(benchmarkState.ac.get(k - 1), benchmarkState.ac.get(k))
        .getCardinality();
  return answer;
}
 

private static int run(String datasetUrl, int responseIndex, int numLeafs, boolean dense)
        throws IOException, ParseException {
    URL url = new URL(datasetUrl);
    InputStream is = new BufferedInputStream(url.openStream());

    ArffParser arffParser = new ArffParser();
    arffParser.setResponseIndex(responseIndex);

    AttributeDataset ds = arffParser.parse(is);
    double[][] x = ds.toArray(new double[ds.size()][]);
    int[] y = ds.toArray(new int[ds.size()]);

    int n = x.length;
    LOOCV loocv = new LOOCV(n);
    int error = 0;
    for (int i = 0; i < n; i++) {
        double[][] trainx = Math.slice(x, loocv.train[i]);
        int[] trainy = Math.slice(y, loocv.train[i]);

        RoaringBitmap attrs = SmileExtUtils.convertAttributeTypes(ds.attributes());
        DecisionTree tree = new DecisionTree(attrs, matrix(trainx, dense), trainy, numLeafs,
            RandomNumberGeneratorFactory.createPRNG(i));
        if (y[loocv.test[i]] != tree.predict(x[loocv.test[i]])) {
            error++;
        }
    }

    debugPrint("Decision Tree error = " + error);
    return error;
}
 

@Benchmark
public void pairwiseOrNotExternal(RealDataRoaringBitmaps state, Blackhole bh) {
  RoaringBitmap[] bitmaps = state.getBitmaps();
  for (int k = 0; k + 1 < bitmaps.length; ++k) {
    long limit = toUnsignedLong(bitmaps[k].last());
    RoaringBitmap range = new RoaringBitmap();
    range.add(0, limit);
    RoaringBitmap bitmap = RoaringBitmap.and(range, bitmaps[k+1]);
    bitmap.flip(0L, limit);
    bitmap.or(RoaringBitmap.and(range, bitmaps[k]));
    bh.consume(bitmap);
  }
}
 

@Benchmark
public void pairwiseOrNot(RealDataRoaringBitmaps state, Blackhole bh) {
  RoaringBitmap[] bitmaps = state.getBitmaps();
  for (int k = 0; k + 1 < bitmaps.length; ++k) {
    RoaringBitmap bitmap = bitmaps[k].clone();
    bitmap.orNot(bitmaps[k+1], bitmap.last());
    bh.consume(RoaringBitmap.orNot(bitmaps[k], bitmaps[k + 1], toUnsignedLong(bitmaps[k].last())));
  }
}
 

@Benchmark
public void bufferBackedDataInput(BenchmarkState state, Blackhole bh) throws IOException {
    byte[][] buffers = state.buffers;
    for (int i = 0; i < buffers.length; ++i) {
        RoaringBitmap bitmap = new RoaringBitmap();
        bitmap.deserialize(new BufferDataInput(ByteBuffer.wrap(state.buffers[i])));
        bh.consume(bitmap);
    }
}