com.mongodb.client.model.Aggregates Java Examples

private Single<Map<Object, Object>> usersStatusRepartition(AnalyticsQuery query) {
    return Observable.fromPublisher(usersCollection.aggregate(
            Arrays.asList(
                    Aggregates.match(and(eq(FIELD_REFERENCE_TYPE, DOMAIN.name()), eq(FIELD_REFERENCE_ID, query.getDomain()))),
                    Aggregates.group(
                            new BasicDBObject("_id", query.getField()),
                            Accumulators.sum("total", 1),
                            Accumulators.sum("disabled", new BasicDBObject("$cond", Arrays.asList(new BasicDBObject("$eq", Arrays.asList("$enabled", false)), 1, 0))),
                            Accumulators.sum("locked", new BasicDBObject("$cond", Arrays.asList(new BasicDBObject("$and", Arrays.asList(new BasicDBObject("$eq", Arrays.asList("$accountNonLocked", false)), new BasicDBObject("$gte", Arrays.asList("$accountLockedUntil", new Date())))), 1, 0))),
                            Accumulators.sum("inactive", new BasicDBObject("$cond", Arrays.asList(new BasicDBObject("$lte", Arrays.asList("$loggedAt", new Date(Instant.now().minus(90, ChronoUnit.DAYS).toEpochMilli()))), 1, 0)))
                    )
            )))
            .map(doc -> {
                Long nonActiveUsers = ((Number) doc.get("disabled")).longValue() + ((Number) doc.get("locked")).longValue() + ((Number) doc.get("inactive")).longValue();
                Long activeUsers = ((Number) doc.get("total")).longValue() - nonActiveUsers;
                Map<Object, Object> users = new HashMap<>();
                users.put("active", activeUsers);
                users.putAll(doc.entrySet()
                        .stream()
                        .filter(e -> !"_id".equals(e.getKey()) && !"total".equals(e.getKey()))
                        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
                return users;
            })
            .first(Collections.emptyMap());
}
 

@Test
public void testStream_whenWatchDatabase() {
    IList<Document> list = jet.getList("list");

    String connectionString = mongoContainer.connectionString();
    long value = startAtOperationTime.getValue();

    StreamSource<? extends Document> source = MongoDBSourceBuilder
            .streamDatabase(SOURCE_NAME, () -> MongoClients.create(connectionString))
            .databaseFn(client -> client.getDatabase(DB_NAME))
            .destroyFn(MongoClient::close)
            .searchFn(db -> {
                List<Bson> aggregates = new ArrayList<>();
                aggregates.add(Aggregates.match(new Document("fullDocument.val", new Document("$gte", 10))
                        .append("operationType", "insert")));

                aggregates.add(Aggregates.project(new Document("fullDocument.val", 1).append("_id", 1)));
                return db.watch(aggregates);
            })
            .mapFn(ChangeStreamDocument::getFullDocument)
            .startAtOperationTimeFn(client -> new BsonTimestamp(value))
            .build();


    Pipeline p = Pipeline.create();
    p.readFrom(source)
     .withNativeTimestamps(0)
     .writeTo(Sinks.list(list));

    Job job = jet.newJob(p);

    MongoCollection<Document> col1 = collection("col1");
    MongoCollection<Document> col2 = collection("col2");

    col1.insertOne(new Document("val", 1));
    col1.insertOne(new Document("val", 10).append("foo", "bar"));

    col2.insertOne(new Document("val", 2));
    col2.insertOne(new Document("val", 11).append("foo", "bar"));

    assertTrueEventually(() -> {
        assertEquals(2, list.size());
        list.forEach(document -> assertNull(document.get("foo")));

        assertEquals(10, list.get(0).get("val"));
        assertEquals(11, list.get(1).get("val"));

    });

    col1.insertOne(new Document("val", 3));
    col1.insertOne(new Document("val", 12).append("foo", "bar"));

    col2.insertOne(new Document("val", 4));
    col2.insertOne(new Document("val", 13).append("foo", "bar"));

    assertTrueEventually(() -> {
        assertEquals(4, list.size());
        list.forEach(document -> assertNull(document.get("foo")));

        assertEquals(12, list.get(2).get("val"));
        assertEquals(13, list.get(3).get("val"));
    });

    job.cancel();

}
 

private void submitBatchQuery() {
    int count = 0;
    executedRangeMap.clear();
    final List<Bson> pipeline = new ArrayList<>();
    final List<Bson> matches = new ArrayList<>();

    while (queryIterator.hasNext() && count < QUERY_BATCH_SIZE){
        count++;
        final RyaStatement query = queryIterator.next();
        executedRangeMap.putAll(query, rangeMap.get(query));
        final Document currentQuery = strategy.getQuery(query);
        matches.add(currentQuery);
    }

    final int numMatches = matches.size();
    if (numMatches > 1) {
        pipeline.add(Aggregates.match(Filters.or(matches)));
    } else if (numMatches == 1) {
        pipeline.add(Aggregates.match(matches.get(0)));
    } else {
        batchQueryResultsIterator = Iterators.emptyIterator();
        return;
    }

    // Executing redact aggregation to only return documents the user has access to.
    pipeline.addAll(AggregationUtil.createRedactPipeline(auths));
    log.trace(pipeline);

    final AggregateIterable<Document> aggIter = coll.aggregate(pipeline);
    aggIter.batchSize(1000);
    batchQueryResultsIterator = aggIter.iterator();
}
 

/**
 * Given that the current state of the pipeline produces data that can be
 * interpreted as triples, add a project step to map each result from the
 * intermediate result structure to a structure that can be stored in the
 * triple store. Does not modify the internal pipeline, which will still
 * produce intermediate results suitable for query evaluation.
 * @param timestamp Attach this timestamp to the resulting triples.
 * @param requireNew If true, add an additional step to check constructed
 *  triples against existing triples and only include new ones in the
 *  result. Adds a potentially expensive $lookup step.
 * @throws IllegalStateException if the results produced by the current
 *  pipeline do not have variable names allowing them to be interpreted as
 *  triples (i.e. "subject", "predicate", and "object").
 */
public List<Bson> getTriplePipeline(final long timestamp, final boolean requireNew) {
    if (!assuredBindingNames.contains(SUBJECT)
            || !assuredBindingNames.contains(PREDICATE)
            || !assuredBindingNames.contains(OBJECT)) {
        throw new IllegalStateException("Current pipeline does not produce "
                + "records that can be converted into triples.\n"
                + "Required variable names: <" + SUBJECT + ", " + PREDICATE
                + ", " + OBJECT + ">\nCurrent variable names: "
                + assuredBindingNames);
    }
    final List<Bson> triplePipeline = new LinkedList<>(pipeline);
    final List<Bson> fields = new LinkedList<>();
    fields.add(Projections.computed(SUBJECT, valueFieldExpr(SUBJECT)));
    fields.add(Projections.computed(SUBJECT_HASH, hashFieldExpr(SUBJECT)));
    fields.add(Projections.computed(PREDICATE, valueFieldExpr(PREDICATE)));
    fields.add(Projections.computed(PREDICATE_HASH, hashFieldExpr(PREDICATE)));
    fields.add(Projections.computed(OBJECT, valueFieldExpr(OBJECT)));
    fields.add(Projections.computed(OBJECT_HASH, hashFieldExpr(OBJECT)));
    fields.add(Projections.computed(OBJECT_TYPE,
            ConditionalOperators.ifNull(typeFieldExpr(OBJECT), DEFAULT_TYPE)));
    fields.add(Projections.computed(CONTEXT, DEFAULT_CONTEXT));
    fields.add(Projections.computed(STATEMENT_METADATA, DEFAULT_METADATA));
    fields.add(DEFAULT_DV);
    fields.add(Projections.computed(TIMESTAMP, new Document("$literal", timestamp)));
    fields.add(Projections.computed(LEVEL, new Document("$add", Arrays.asList("$" + LEVEL, 1))));
    triplePipeline.add(Aggregates.project(Projections.fields(fields)));
    if (requireNew) {
        // Prune any triples that already exist in the data store
        final String collectionName = collection.getNamespace().getCollectionName();
        final Bson includeAll = Projections.include(SUBJECT, SUBJECT_HASH,
                PREDICATE, PREDICATE_HASH, OBJECT, OBJECT_HASH,
                OBJECT_TYPE, CONTEXT, STATEMENT_METADATA,
                DOCUMENT_VISIBILITY, TIMESTAMP, LEVEL);
        final List<Bson> eqTests = new LinkedList<>();
        eqTests.add(new Document("$eq", Arrays.asList("$$this." + PREDICATE_HASH, "$" + PREDICATE_HASH)));
        eqTests.add(new Document("$eq", Arrays.asList("$$this." + OBJECT_HASH, "$" + OBJECT_HASH)));
        final Bson redundantFilter = new Document("$filter", new Document("input", "$" + JOINED_TRIPLE)
                .append("as", "this").append("cond", new Document("$and", eqTests)));
        triplePipeline.add(Aggregates.lookup(collectionName, SUBJECT_HASH,
                SUBJECT_HASH, JOINED_TRIPLE));
        final String numRedundant = "REDUNDANT";
        triplePipeline.add(Aggregates.project(Projections.fields(includeAll,
                Projections.computed(numRedundant, new Document("$size", redundantFilter)))));
        triplePipeline.add(Aggregates.match(Filters.eq(numRedundant, 0)));
        triplePipeline.add(Aggregates.project(Projections.fields(includeAll)));
    }
    return triplePipeline;
}
 

/**
 * Add a step to the end of the current pipeline which prunes the results
 * according to the timestamps of their sources. At least one triple that
 * was used to construct the result must have a timestamp at least as
 * recent as the parameter. Use in iterative applications to avoid deriving
 * solutions that would have been generated in an earlier iteration.
 * @param t Minimum required timestamp. Reject a solution to the query if
 *  all of the triples involved in producing that solution have an earlier
 *  timestamp than this.
 */
public void requireSourceTimestamp(final long t) {
    pipeline.add(Aggregates.match(Filters.gte(TIMESTAMP, t)));
}
 

/**
 * Add a step to the end of the current pipeline which prunes the results
 * according to the recorded derivation level of their sources. At least one
 * triple that was used to construct the result must have a derivation level
 * at least as high as the parameter, indicating that it was derived via
 * that many steps from the original data. (A value of zero is equivalent to
 * input data that was not derived at all.) Use in conjunction with
 * getTriplePipeline (which sets source level for generated triples) to
 * avoid repeatedly deriving the same results.
 * @param requiredLevel Required derivation depth. Reject a solution to the
 *  query if all of the triples involved in producing that solution have a
 *  lower derivation depth than this. If zero, does nothing.
 */
public void requireSourceDerivationDepth(final int requiredLevel) {
    if (requiredLevel > 0) {
        pipeline.add(Aggregates.match(Filters.gte(LEVEL, requiredLevel)));
    }
}
 

/**
 * Add a $group step to filter out redundant solutions.
 * @return True if the distinct operation was successfully appended.
 */
public boolean distinct() {
    final List<String> key = new LinkedList<>();
    for (final String varName : bindingNames) {
        key.add(hashFieldExpr(varName));
    }
    final List<BsonField> reduceOps = new LinkedList<>();
    for (final String field : FIELDS) {
        reduceOps.add(new BsonField(field, new Document("$first", "$" + field)));
    }
    pipeline.add(Aggregates.group(new Document("$concat", key), reduceOps));
    return true;
}
 

/**
 * Add a SPARQL filter to the pipeline, if possible. A filter eliminates
 * results that don't satisfy a given condition. Not all conditional
 * expressions are supported. If unsupported expressions are used in the
 * filter, the pipeline will remain unchanged and this method will return
 * false. Currently only supports binary {@link Compare} conditions among
 * variables and/or literals.
 * @param condition The filter condition
 * @return True if the filter was successfully converted into a pipeline
 *  step, false otherwise.
 */
public boolean filter(final ValueExpr condition) {
    if (condition instanceof Compare) {
        final Compare compare = (Compare) condition;
        final Compare.CompareOp operator = compare.getOperator();
        final Object leftArg = valueFieldExpr(compare.getLeftArg());
        final Object rightArg = valueFieldExpr(compare.getRightArg());
        if (leftArg == null || rightArg == null) {
            // unsupported value expression, can't convert filter
            return false;
        }
        final String opFunc;
        switch (operator) {
        case EQ:
            opFunc = "$eq";
            break;
        case NE:
            opFunc = "$ne";
            break;
        case LT:
            opFunc = "$lt";
            break;
        case LE:
            opFunc = "$le";
            break;
        case GT:
            opFunc = "$gt";
            break;
        case GE:
            opFunc = "$ge";
            break;
        default:
            // unrecognized comparison operator, can't convert filter
            return false;
        }
        final Document compareDoc = new Document(opFunc, Arrays.asList(leftArg, rightArg));
        pipeline.add(Aggregates.project(Projections.fields(
                Projections.computed("FILTER", compareDoc),
                Projections.include(VALUES, HASHES, TYPES, LEVEL, TIMESTAMP))));
        pipeline.add(Aggregates.match(new Document("FILTER", true)));
        pipeline.add(Aggregates.project(Projections.fields(
                Projections.include(VALUES, HASHES, TYPES, LEVEL, TIMESTAMP))));
        return true;
    }
    return false;
}
 

/**
 * Add a SPARQL projection or multi-projection operation to the pipeline.
 * The number of documents produced by the pipeline after this operation
 * will be the number of documents entering this stage (the number of
 * intermediate results) multiplied by the number of
 * {@link ProjectionElemList}s supplied here. Empty projections are
 * unsupported; if one or more projections given binds zero variables, then
 * the pipeline will be unchanged and the method will return false.
 * @param projections One or more projections, i.e. mappings from the result
 *  at this stage of the query into a set of variables.
 * @return true if the projection(s) were added to the pipeline.
 */
public boolean project(final Iterable<ProjectionElemList> projections) {
    if (projections == null || !projections.iterator().hasNext()) {
        return false;
    }
    final List<Bson> projectOpts = new LinkedList<>();
    final Set<String> bindingNamesUnion = new HashSet<>();
    Set<String> bindingNamesIntersection = null;
    for (final ProjectionElemList projection : projections) {
        if (projection.getElements().isEmpty()) {
            // Empty projections are unsupported -- fail when seen
            return false;
        }
        final Document valueDoc = new Document();
        final Document hashDoc = new Document();
        final Document typeDoc = new Document();
        final Set<String> projectionBindingNames = new HashSet<>();
        for (final ProjectionElem elem : projection.getElements()) {
            String to = elem.getTargetName();
            // If the 'to' name is invalid, replace it internally
            if (!isValidFieldName(to)) {
                to = replace(to);
            }
            String from = elem.getSourceName();
            // If the 'from' name is invalid, use the internal substitute
            if (varToOriginalName.containsValue(from)) {
                from = varToOriginalName.inverse().get(from);
            }
            projectionBindingNames.add(to);
            if (to.equals(from)) {
                valueDoc.append(to, 1);
                hashDoc.append(to, 1);
                typeDoc.append(to, 1);
            }
            else {
                valueDoc.append(to, valueFieldExpr(from));
                hashDoc.append(to, hashFieldExpr(from));
                typeDoc.append(to, typeFieldExpr(from));
            }
        }
        bindingNamesUnion.addAll(projectionBindingNames);
        if (bindingNamesIntersection == null) {
            bindingNamesIntersection = new HashSet<>(projectionBindingNames);
        }
        else {
            bindingNamesIntersection.retainAll(projectionBindingNames);
        }
        projectOpts.add(new Document()
                .append(VALUES, valueDoc)
                .append(HASHES, hashDoc)
                .append(TYPES, typeDoc)
                .append(LEVEL, "$" + LEVEL)
                .append(TIMESTAMP, "$" + TIMESTAMP));
    }
    if (projectOpts.size() == 1) {
        pipeline.add(Aggregates.project(projectOpts.get(0)));
    }
    else {
        final String listKey = "PROJECTIONS";
        final Bson projectIndividual = Projections.fields(
                Projections.computed(VALUES, "$" + listKey + "." + VALUES),
                Projections.computed(HASHES, "$" + listKey + "." + HASHES),
                Projections.computed(TYPES, "$" + listKey + "." + TYPES),
                Projections.include(LEVEL),
                Projections.include(TIMESTAMP));
        pipeline.add(Aggregates.project(Projections.computed(listKey, projectOpts)));
        pipeline.add(Aggregates.unwind("$" + listKey));
        pipeline.add(Aggregates.project(projectIndividual));
    }
    assuredBindingNames.clear();
    bindingNames.clear();
    assuredBindingNames.addAll(bindingNamesIntersection);
    bindingNames.addAll(bindingNamesUnion);
    return true;
}
 

/**
 * Add a join with an individual {@link StatementPattern} to the pipeline.
 * @param sp The statement pattern to join with
 * @return true if the join was successfully added to the pipeline.
 */
public boolean joinWith(final StatementPattern sp) {
    Preconditions.checkNotNull(sp);
    // 1. Determine shared variables and new variables
    final StatementVarMapping spMap = new StatementVarMapping(sp, varToOriginalName);
    final NavigableSet<String> sharedVars = new ConcurrentSkipListSet<>(spMap.varNames());
    sharedVars.retainAll(assuredBindingNames);
    // 2. Join on one shared variable
    final String joinKey =  sharedVars.pollFirst();
    final String collectionName = collection.getNamespace().getCollectionName();
    Bson join;
    if (joinKey == null) {
        return false;
    }
    else {
        join = Aggregates.lookup(collectionName,
                HASHES + "." + joinKey,
                spMap.hashField(joinKey),
                JOINED_TRIPLE);
    }
    pipeline.add(join);
    // 3. Unwind the joined triples so each document represents a binding
    //   set (solution) from the base branch and a triple that may match.
    pipeline.add(Aggregates.unwind("$" + JOINED_TRIPLE));
    // 4. (Optional) If there are any shared variables that weren't used as
    //   the join key, project all existing fields plus a new field that
    //   tests the equality of those shared variables.
    final Document matchOpts = getMatchExpression(sp, JOINED_TRIPLE);
    if (!sharedVars.isEmpty()) {
        final List<Bson> eqTests = new LinkedList<>();
        for (final String varName : sharedVars) {
            final String oldField = valueFieldExpr(varName);
            final String newField = joinFieldExpr(spMap.valueField(varName));
            final Bson eqTest = new Document("$eq", Arrays.asList(oldField, newField));
            eqTests.add(eqTest);
        }
        final Bson eqProjectOpts = Projections.fields(
                Projections.computed(FIELDS_MATCH, Filters.and(eqTests)),
                Projections.include(JOINED_TRIPLE, VALUES, HASHES, TYPES, LEVEL, TIMESTAMP));
        pipeline.add(Aggregates.project(eqProjectOpts));
        matchOpts.put(FIELDS_MATCH, true);
    }
    // 5. Filter for solutions whose triples match the joined statement
    //  pattern, and, if applicable, whose additional shared variables
    //  match the current solution.
    pipeline.add(Aggregates.match(matchOpts));
    // 6. Project the results to include variables from the new SP (with
    // appropriate renaming) and variables referenced only in the base
    // pipeline (with previous names).
    final Bson finalProjectOpts = new StatementVarMapping(sp, varToOriginalName)
            .getProjectExpression(assuredBindingNames,
                    str -> joinFieldExpr(str));
    assuredBindingNames.addAll(spMap.varNames());
    bindingNames.addAll(spMap.varNames());
    pipeline.add(Aggregates.project(finalProjectOpts));
    return true;
}
 

/**
 * Create a pipeline query node based on a StatementPattern.
 * @param collection The collection of triples to query.
 * @param baseSP The leaf node in the query tree.
 */
public AggregationPipelineQueryNode(final MongoCollection<Document> collection, final StatementPattern baseSP) {
    this.collection = Preconditions.checkNotNull(collection);
    Preconditions.checkNotNull(baseSP);
    this.varToOriginalName = HashBiMap.create();
    final StatementVarMapping mapping = new StatementVarMapping(baseSP, varToOriginalName);
    this.assuredBindingNames = new HashSet<>(mapping.varNames());
    this.bindingNames = new HashSet<>(mapping.varNames());
    this.pipeline = new LinkedList<>();
    this.pipeline.add(Aggregates.match(getMatchExpression(baseSP)));
    this.pipeline.add(Aggregates.project(mapping.getProjectExpression()));
}
 

@Override
public Map<String, Integer> getCardsCountGlobal(MagicCollection c) throws SQLException {
	Map<String, Integer> map = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
	List<Bson> aggr = Arrays.asList(Aggregates.match(Filters.eq(dbColIDField, c.getName())),
			Aggregates.group("$edition", Accumulators.sum("count", 1)));

	logger.trace(aggr.toString());

	db.getCollection(colCards, BasicDBObject.class).aggregate(aggr).forEach(
			(Consumer<BasicDBObject>) document -> map.put(document.getString("_id"), document.getInt("count")));
	return map;
}
 

/**
 * Transform a list of split keys as a list of filters containing corresponding range.
 *
 * <p>The list of split keys contains BSon Document basically containing for example:
 *
 * <ul>
 *   <li>_id: 56
 *   <li>_id: 109
 *   <li>_id: 256
 * </ul>
 *
 * <p>This method will generate a list of range filters performing the following splits:
 *
 * <ul>
 *   <li>from the beginning of the collection up to _id 56, so basically data with _id lower
 *       than 56
 *   <li>from _id 57 up to _id 109
 *   <li>from _id 110 up to _id 256
 *   <li>from _id 257 up to the end of the collection, so basically data with _id greater than
 *       257
 * </ul>
 *
 * @param splitKeys The list of split keys.
 * @return A list of filters containing the ranges.
 */
@VisibleForTesting
static List<BsonDocument> splitKeysToMatch(List<Document> splitKeys) {
  List<Bson> aggregates = new ArrayList<>();
  ObjectId lowestBound = null; // lower boundary (previous split in the iteration)
  for (int i = 0; i < splitKeys.size(); i++) {
    ObjectId splitKey = splitKeys.get(i).getObjectId("_id");
    String rangeFilter;
    if (i == 0) {
      aggregates.add(Aggregates.match(Filters.lte("_id", splitKey)));
      if (splitKeys.size() == 1) {
        aggregates.add(Aggregates.match(Filters.and(Filters.gt("_id", splitKey))));
      }
    } else if (i == splitKeys.size() - 1) {
      // this is the last split in the list, the filters define
      // the range from the previous split to the current split and also
      // the current split to the end
      aggregates.add(
          Aggregates.match(
              Filters.and(Filters.gt("_id", lowestBound), Filters.lte("_id", splitKey))));
      aggregates.add(Aggregates.match(Filters.and(Filters.gt("_id", splitKey))));
    } else {
      aggregates.add(
          Aggregates.match(
              Filters.and(Filters.gt("_id", lowestBound), Filters.lte("_id", splitKey))));
    }

    lowestBound = splitKey;
  }
  return aggregates.stream()
      .map(s -> s.toBsonDocument(BasicDBObject.class, MongoClient.getDefaultCodecRegistry()))
      .collect(Collectors.toList());
}
 

private Single<Map<Object, Object>> executeGroupBy(AuditReportableCriteria criteria, Bson query) {
    return Observable.fromPublisher(reportableCollection.aggregate(
            Arrays.asList(
                    Aggregates.match(query),
                    Aggregates.group(new BasicDBObject("_id", "$" + criteria.field()), Accumulators.sum("count", 1)),
                    Aggregates.limit(criteria.size() != null ? criteria.size() : 50))
    ))
            .toList()
            .map(docs -> docs.stream().collect(Collectors.toMap(d -> ((Document) d.get("_id")).get("_id"), d -> d.get("count"))));
}
 

private Single<Map<Object, Object>> executeHistogram(AuditReportableCriteria criteria, Bson query) {
    // NOTE : MongoDB does not return count : 0 if there is no matching document in the given time range, we need to add it by hand
    Map<Long, Long> intervals = intervals(criteria);
    String fieldSuccess = (criteria.types().get(0) + "_" + Status.SUCCESS).toLowerCase();
    String fieldFailure = (criteria.types().get(0) + "_" + Status.FAILURE).toLowerCase();
    return Observable.fromPublisher(reportableCollection.aggregate(Arrays.asList(
            Aggregates.match(query),
            Aggregates.group(
                    new BasicDBObject("_id",
                            new BasicDBObject("$subtract",
                                    Arrays.asList(
                                            new BasicDBObject("$subtract", Arrays.asList("$timestamp", new Date(0))),
                                            new BasicDBObject("$mod", Arrays.asList(new BasicDBObject("$subtract", Arrays.asList("$timestamp", new Date(0))), criteria.interval()))
                                    ))),
                    Accumulators.sum(fieldSuccess, new BasicDBObject("$cond", Arrays.asList(new BasicDBObject("$eq", Arrays.asList("$outcome.status", Status.SUCCESS)), 1, 0))),
                    Accumulators.sum(fieldFailure, new BasicDBObject("$cond", Arrays.asList(new BasicDBObject("$eq", Arrays.asList("$outcome.status", Status.FAILURE)), 1, 0)))))))
            .toList()
            .map(docs -> {
                Map<Long, Long> successResult = new HashMap<>();
                Map<Long, Long> failureResult = new HashMap<>();
                docs.forEach(document -> {
                    Long timestamp = ((Number) ((Document) document.get("_id")).get("_id")).longValue();
                    Long successAttempts = ((Number) document.get(fieldSuccess)).longValue();
                    Long failureAttempts = ((Number) document.get(fieldFailure)).longValue();
                    successResult.put(timestamp, successAttempts);
                    failureResult.put(timestamp, failureAttempts);
                });
                // complete result with remaining intervals
                intervals.forEach((k, v) -> {
                    successResult.putIfAbsent(k, v);
                    failureResult.putIfAbsent(k, v);
                });
                List<Long> successData = successResult.entrySet().stream().sorted(Map.Entry.comparingByKey()).map(e -> e.getValue()).collect(Collectors.toList());
                List<Long> failureData = failureResult.entrySet().stream().sorted(Map.Entry.comparingByKey()).map(e -> e.getValue()).collect(Collectors.toList());
                Map<Object, Object> result = new HashMap<>();
                result.put(fieldSuccess, successData);
                result.put(fieldFailure, failureData);
                return result;
            });
}
 

private Single<Map<Object, Object>> registrationsStatusRepartition(AnalyticsQuery query) {
    return Observable.fromPublisher(usersCollection.aggregate(
            Arrays.asList(
                    Aggregates.match(and(eq(FIELD_REFERENCE_TYPE, DOMAIN.name()), eq(FIELD_REFERENCE_ID, query.getDomain()), eq(FIELD_PRE_REGISTRATION, true))),
                    Aggregates.group(new BasicDBObject("_id", query.getField()),
                            Accumulators.sum("total", 1),
                            Accumulators.sum("completed", new BasicDBObject("$cond", Arrays.asList(new BasicDBObject("$eq", Arrays.asList("$registrationCompleted", true)), 1, 0))))
            )))
            .map(doc -> {
                Map<Object, Object> registrations = new HashMap<>();
                registrations.putAll(doc.entrySet()
                        .stream()
                        .filter(e -> !"_id".equals(e.getKey()))
                        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
                return registrations;
            })
            .first(Collections.emptyMap());
}
 

private Source<SnapshotBatch, NotUsed> listNewestActiveSnapshotsByBatch(
        final MongoCollection<Document> snapshotStore,
        final String start,
        final int batchSize,
        final String... snapshotFields) {

    final List<Bson> pipeline = new ArrayList<>(5);
    // optional match stage
    if (!start.isEmpty()) {
        pipeline.add(Aggregates.match(Filters.gt(PROCESSOR_ID, start)));
    }

    // sort stage
    pipeline.add(Aggregates.sort(Sorts.orderBy(Sorts.ascending(PROCESSOR_ID), Sorts.descending(SN))));

    // limit stage. It should come before group stage or MongoDB would scan the entire journal collection.
    pipeline.add(Aggregates.limit(batchSize));

    // group stage 1: by PID
    pipeline.add(Aggregates.group("$" + PROCESSOR_ID, asFirstSnapshotBsonFields(snapshotFields)));

    // sort stage 2 -- order after group stage is not defined
    pipeline.add(Aggregates.sort(Sorts.ascending(ID)));

    // group stage 2: filter out pids whose latest snapshot is a deleted snapshot, but retain max encountered pid
    final String maxPid = "m";
    final String items = "i";
    pipeline.add(Aggregates.group(null,
            Accumulators.max(maxPid, "$" + ID),
            Accumulators.push(
                    items,
                    new Document().append("$cond", new Document()
                            .append("if", new Document().append("$ne", Arrays.asList("$" + LIFECYCLE, "DELETED")))
                            .append("then", "$$CURRENT")
                            .append("else", null)
                    ))
    ));

    // remove null entries by projection
    pipeline.add(Aggregates.project(new Document()
            .append(maxPid, 1)
            .append(items, new Document()
                    .append("$setDifference", Arrays.asList("$" + items, Collections.singletonList(null)))
            )
    ));

    return Source.fromPublisher(snapshotStore.aggregate(pipeline))
            .flatMapConcat(document -> {
                final String theMaxPid = document.getString(maxPid);
                if (theMaxPid == null) {
                    return Source.empty();
                } else {
                    return Source.single(new SnapshotBatch(theMaxPid, document.getList(items, Document.class)));
                }
            });
}
 

private Source<String, NotUsed> listJournalPidsAbove(final MongoCollection<Document> journal, final String start,
        final int batchSize, final Duration maxBackOff, final int maxRestarts) {

    final List<Bson> pipeline = new ArrayList<>(5);
    // optional match stage
    if (!start.isEmpty()) {
        pipeline.add(Aggregates.match(Filters.gt(PROCESSOR_ID, start)));
    }

    // sort stage
    pipeline.add(Aggregates.sort(Sorts.ascending(PROCESSOR_ID)));

    // limit stage. It should come before group stage or MongoDB would scan the entire journal collection.
    pipeline.add(Aggregates.limit(batchSize));

    // group stage
    pipeline.add(Aggregates.group("$" + PROCESSOR_ID));

    // sort stage 2 -- order after group stage is not defined
    pipeline.add(Aggregates.sort(Sorts.ascending(ID)));

    final Duration minBackOff = Duration.ofSeconds(1L);
    final double randomFactor = 0.1;

    return RestartSource.onFailuresWithBackoff(minBackOff, maxBackOff, randomFactor, maxRestarts,
            () -> Source.fromPublisher(journal.aggregate(pipeline))
                    .filter(document -> document.containsKey(ID))
                    .map(document -> document.getString(ID)));
}
 

private void configQuery(MongoConnection connection, ConsumerRegistry registry, boolean updateLookUp) {
  List<Bson> pipeline = singletonList(Aggregates.match(
      Filters.and(
          Filters.in(NS, getNamespaces(connection, registry)),
          Filters.in("operationType", asList("insert", "delete", "replace", "update")))));
  changeStreamDocuments = client.watch(pipeline).batchSize(MONGO_CHANGE_STREAM_BATCH_SIZE);

  DocTimestamp docTimestamp = registry.votedMongoId(connection);
  if (DocTimestamp.earliest == docTimestamp) {
    MongoCursor<Document> firstLog = client.getDatabase(LOCAL).getCollection(OPLOG_RS).find(new Document()).limit(1).iterator();
    if (firstLog.hasNext()) {
      Document next = firstLog.next();
      logger.info("Connect to earliest oplog time: {}", next.get(TS));
      changeStreamDocuments.startAtOperationTime(((BsonTimestamp) next.get(TS)));
    } else {
      logger.info("Document not found in local.oplog.rs -- is this a new and empty db instance?");
      changeStreamDocuments.startAtOperationTime(docTimestamp.getTimestamp());
    }
  } else {
    /*
    Optional. The starting point for the change stream.
    If the specified starting point is in the past, it must be in the time range of the oplog.
    To check the time range of the oplog, see rs.printReplicationInfo().
     */
    changeStreamDocuments.startAtOperationTime(docTimestamp.getTimestamp());
  }
  // UPDATE_LOOKUP: return the most current majority-committed version of the updated document.
  // i.e. run at different time, will have different fullDocument
  changeStreamDocuments.fullDocument(updateLookUp ? FullDocument.UPDATE_LOOKUP : FullDocument.DEFAULT);
}
 

private StreamSource<? extends Document> streamSource(
        Document filter,
        Document projection,
        int connectionTimeoutSeconds
) {
    String connectionString = mongoContainer.connectionString();
    long value = startAtOperationTime.getValue();
    return MongoDBSourceBuilder
            .stream(SOURCE_NAME, () -> mongoClient(connectionString, connectionTimeoutSeconds))
            .databaseFn(client -> client.getDatabase(DB_NAME))
            .collectionFn(db -> db.getCollection(COL_NAME))
            .destroyFn(MongoClient::close)
            .searchFn(col -> {
                List<Bson> aggregates = new ArrayList<>();
                if (filter != null) {
                    aggregates.add(Aggregates.match(filter));
                }
                if (projection != null) {
                    aggregates.add(Aggregates.project(projection));
                }
                ChangeStreamIterable<? extends Document> watch;
                if (aggregates.isEmpty()) {
                    watch = col.watch();
                } else {
                    watch = col.watch(aggregates);
                }
                return watch;
            })
            .mapFn(ChangeStreamDocument::getFullDocument)
            .startAtOperationTimeFn(client -> new BsonTimestamp(value))
            .build();
}
 

@Test
public void testStream_whenWatchAll() {
    IList<Document> list = jet.getList("list");

    String connectionString = mongoContainer.connectionString();
    long value = startAtOperationTime.getValue();

    StreamSource<? extends Document> source = MongoDBSourceBuilder
            .streamAll(SOURCE_NAME, () -> MongoClients.create(connectionString))
            .destroyFn(MongoClient::close)
            .searchFn(client -> {
                List<Bson> aggregates = new ArrayList<>();
                aggregates.add(Aggregates.match(new Document("fullDocument.val", new Document("$gt", 10))
                        .append("operationType", "insert")));

                aggregates.add(Aggregates.project(new Document("fullDocument.val", 1).append("_id", 1)));
                return client.watch(aggregates);
            })
            .mapFn(ChangeStreamDocument::getFullDocument)
            .startAtOperationTimeFn(client -> new BsonTimestamp(value))
            .build();

    Pipeline p = Pipeline.create();
    p.readFrom(source)
     .withNativeTimestamps(0)
     .writeTo(Sinks.list(list));

    Job job = jet.newJob(p);

    MongoCollection<Document> col1 = collection("db1", "col1");
    MongoCollection<Document> col2 = collection("db1", "col2");
    MongoCollection<Document> col3 = collection("db2", "col3");

    col1.insertOne(new Document("val", 1));
    col1.insertOne(new Document("val", 11).append("foo", "bar"));
    col2.insertOne(new Document("val", 2));
    col2.insertOne(new Document("val", 12).append("foo", "bar"));
    col3.insertOne(new Document("val", 3));
    col3.insertOne(new Document("val", 13).append("foo", "bar"));

    assertTrueEventually(() -> {
        assertEquals(3, list.size());
        list.forEach(document -> assertNull(document.get("foo")));

        assertEquals(11, list.get(0).get("val"));
        assertEquals(12, list.get(1).get("val"));
        assertEquals(13, list.get(2).get("val"));
    });

    col1.insertOne(new Document("val", 4));
    col1.insertOne(new Document("val", 14).append("foo", "bar"));
    col2.insertOne(new Document("val", 5));
    col2.insertOne(new Document("val", 15).append("foo", "bar"));
    col2.insertOne(new Document("val", 6));
    col2.insertOne(new Document("val", 16).append("foo", "bar"));

    assertTrueEventually(() -> {
        assertEquals(6, list.size());
        list.forEach(document -> assertNull(document.get("foo")));

        assertEquals(14, list.get(3).get("val"));
        assertEquals(15, list.get(4).get("val"));
        assertEquals(16, list.get(5).get("val"));
    });

    job.cancel();

}
 

@Override
public void process(Consumer<Bson> consumer) {
  BsonDocument projections = new BsonDocument();
  naming.forEach((expr, name) -> {
    projections.put(name, new BsonString("$" + pathNaming.name(extractPath(expr))));
  });

  if (!projections.isEmpty()) {
    consumer.accept(Aggregates.project(projections));
  }
}
 

@Override
public void process(Consumer<Bson> consumer) {
  final Function<Collation, Bson> toSortFn = col -> {
    final String name = naming.get(col.path());
    return col.direction().isAscending() ? Sorts.ascending(name) : Sorts.descending(name);

  };

  BsonDocument sort = new BsonDocument();
  for (Collation collation: query.collations()) {
    sort.putAll(toSortFn.apply(collation).toBsonDocument(BsonDocument.class, codecRegistry));
  }

  if (!sort.isEmpty()) {
    consumer.accept(Aggregates.sort(sort));
  }
}
 

@Override
public void process(Consumer<Bson> consumer) {
  query.offset().ifPresent(offset -> consumer.accept(Aggregates.skip((int) offset)));
}
 

@Override
public void process(Consumer<Bson> consumer) {
  query.limit().ifPresent(limit -> consumer.accept(Aggregates.limit((int) limit)));
}
 

/**
 * Returns a MongoDB stream source which watches the changes on the
 * collection. The source applies the given {@code filter} and {@code
 * projection} on the change stream documents.
 * <p>
 * Change stream is available for replica sets and sharded clusters that
 * use WiredTiger storage engine and replica set protocol version 1 (pv1).
 * Change streams can also be used on deployments which employ MongoDB's
 * encryption-at-rest feature. You cannot watch on system collections and
 * collections in admin, local and config databases.
 * <p>
 * See {@link MongoDBSourceBuilder} for creating custom MongoDB sources.
 * <p>
 * Here's an example which streams inserts on a collection having the
 * field {@code age} with a value greater than {@code 10} and applies a
 * projection so that only the {@code age} field is returned in the
 * emitted document.
 *
 * <pre>{@code
 * StreamSource<? extends Document> streamSource =
 *         MongoDBSources.stream(
 *                 "stream-source",
 *                 "mongodb://127.0.0.1:27017",
 *                 "myDatabase",
 *                 "myCollection",
 *                 new Document("fullDocument.age", new Document("$gt", 10))
 *                         .append("operationType", "insert"),
 *                 new Document("fullDocument.age", 1)
 *         );
 *
 * Pipeline p = Pipeline.create();
 * StreamSourceStage<? extends Document> srcStage = p.readFrom(streamSource);
 * }</pre>
 *
 * @param name             a descriptive name for the source (diagnostic purposes)
 * @param connectionString a connection string URI to MongoDB for example:
 *                         {@code mongodb://127.0.0.1:27017}
 * @param database         the name of the database
 * @param collection       the name of the collection
 * @param filter           filter object as a {@link Document}
 * @param projection       projection object as a {@link Document}
 */
@Nonnull
public static StreamSource<? extends Document> stream(
        @Nonnull String name,
        @Nonnull String connectionString,
        @Nonnull String database,
        @Nonnull String collection,
        @Nullable Document filter,
        @Nullable Document projection
) {
    return MongoDBSourceBuilder
            .stream(name, () -> MongoClients.create(connectionString))
            .databaseFn(client -> client.getDatabase(database))
            .collectionFn(db -> db.getCollection(collection))
            .destroyFn(MongoClient::close)
            .searchFn(
                    col -> {
                        List<Bson> aggregates = new ArrayList<>();
                        if (filter != null) {
                            aggregates.add(Aggregates.match(filter));
                        }
                        if (projection != null) {
                            aggregates.add(Aggregates.project(projection));
                        }
                        ChangeStreamIterable<? extends Document> watch;
                        if (aggregates.isEmpty()) {
                            watch = col.watch();
                        } else {
                            watch = col.watch(aggregates);
                        }
                        return watch;
                    }
            )
            .mapFn(ChangeStreamDocument::getFullDocument)
            .build();
}