com.hazelcast.collection.IList Java Examples

@Test
public void test_influxDbSink_nonExistingDb() {
    IList<Integer> measurements = jet.getList("mem_usage");
    IntStream.range(0, VALUE_COUNT).forEach(measurements::add);
    influxdbContainer.getNewInfluxDB();

    Pipeline p = Pipeline.create();
    int startTime = 0;
    p.readFrom(Sources.list(measurements))
     .map(index -> Point.measurement("mem_usage")
                        .time(startTime + index, TimeUnit.MILLISECONDS)
                        .addField("value", index)
                        .build())
     .writeTo(InfluxDbSinks.influxDb(influxdbContainer.getUrl(), "non-existing", USERNAME, PASSWORD));

    expected.expectMessage("database not found: \"non-existing\"");
    jet.newJob(p).join();
}
 

@Test
public void stream() {
    IList<String> list = instance.getList("list");
    for (int i = 0; i < 10; i++) {
        list.add("key-" + i);
    }

    Pipeline p = Pipeline.create();
    p.readFrom(Sources.list(list))
            .writeTo(RedisSinks.stream("source", uri, "stream"));

    instance.newJob(p).join();

    RedisCommands<String, String> sync = connection.sync();
    List<StreamMessage<String, String>> messages = sync.xread(XReadArgs.StreamOffset.from("stream", "0"));
    assertEquals(list.size(), messages.size());
}
 

@Test
public void test() {
    IList<Integer> list = jet.getList("list");
    for (int i = 0; i < 100; i++) {
        list.add(i);
    }

    String connectionString = mongoContainer.connectionString();

    Pipeline p = Pipeline.create();
    p.readFrom(Sources.list(list))
     .map(i -> new Document("key", i))
     .writeTo(MongoDBSinks.mongodb(SINK_NAME, connectionString, DB_NAME, COL_NAME));

    jet.newJob(p).join();

    MongoCollection<Document> collection = collection();
    assertEquals(100, collection.countDocuments());
}
 

@Verify
public void globalVerify() {
    IList<Integer> results = targetInstance.getList(name + "max");
    int observedMaxSize = 0;
    for (int m : results) {
        if (observedMaxSize < m) {
            observedMaxSize = m;
        }
    }
    logger.info(name + ": cache " + cache.getName()
            + " toleranceFactor=" + toleranceFactor
            + " configuredMaxSize=" + configuredMaxSize
            + " estimatedMaxSize=" + estimatedMaxSize
            + " observedMaxSize=" + observedMaxSize
            + " size=" + cache.size()
    );

    IList<Counter> counters = targetInstance.getList(name + "counter");
    Counter total = new Counter();
    for (Counter c : counters) {
        total.add(c);
    }
    logger.info(name + ": " + total);
    logger.info(name + ": putAllMap size=" + putAllMap.size());
}
 

@Test
public void test_influxDbSink() {
    IList<Integer> measurements = jet.getList("mem_usage");
    for (int i = 0; i < VALUE_COUNT; i++) {
        measurements.add(i);
    }

    InfluxDB db = influxdbContainer.getNewInfluxDB();
    db.query(new Query("DROP SERIES FROM mem_usage"));

    Pipeline p = Pipeline.create();

    int startTime = 0;
    p.readFrom(Sources.list(measurements))
     .map(index -> Point.measurement("mem_usage")
                        .time(startTime + index, TimeUnit.MILLISECONDS)
                        .addField("value", index)
                        .build())
     .writeTo(InfluxDbSinks.influxDb(influxdbContainer.getUrl(), DATABASE_NAME, USERNAME, PASSWORD));

    jet.newJob(p).join();

    List<Result> results = db.query(new Query("SELECT * FROM mem_usage")).getResults();
    assertEquals(1, results.size());
    List<Series> seriesList = results.get(0).getSeries();
    assertEquals(1, seriesList.size());
    Series series = seriesList.get(0);
    assertEquals(SERIES, series.getName());
    assertEquals(VALUE_COUNT, series.getValues().size());
}
 

@Test
public void test_whenServerNotAvailable() {
    String connectionString = mongoContainer.connectionString();
    mongoContainer.close();

    IList<Integer> list = jet.getList("list");
    for (int i = 0; i < 100; i++) {
        list.add(i);
    }

    Sink<Document> sink = MongoDBSinks
            .<Document>builder(SINK_NAME, () -> mongoClient(connectionString, 3))
            .databaseFn(client -> client.getDatabase(DB_NAME))
            .collectionFn(db -> db.getCollection(COL_NAME))
            .destroyFn(MongoClient::close)
            .build();


    Pipeline p = Pipeline.create();
    p.readFrom(Sources.list(list))
     .map(i -> new Document("key", i))
     .writeTo(sink);

    try {
        jet.newJob(p).join();
        fail();
    } catch (CompletionException e) {
        assertTrue(e.getCause() instanceof JetException);
    }
}
 

@Test
public void testBatch() {

    IList<Document> list = jet.getList("list");

    List<Document> documents = new ArrayList<>();
    for (int i = 0; i < 100; i++) {
        documents.add(new Document("key", i).append("val", i));
    }
    collection().insertMany(documents);


    String connectionString = mongoContainer.connectionString();

    Pipeline p = Pipeline.create();
    p.readFrom(MongoDBSources.batch(SOURCE_NAME, connectionString, DB_NAME, COL_NAME,
            new Document("val", new Document("$gte", 10)),
            new Document("val", 1).append("_id", 0)))
     .writeTo(Sinks.list(list));

    jet.newJob(p).join();

    assertEquals(90, list.size());
    Document actual = list.get(0);
    assertNull(actual.get("key"));
    assertNull(actual.get("_id"));
    assertNotNull(actual.get("val"));
}
 

@Verify(global = false)
public void verify() {
    IList<TxnCounter> counts = targetInstance.getList(name + "count");
    TxnCounter total = new TxnCounter();
    for (TxnCounter c : counts) {
        total.add(c);
    }
    logger.info(name + ": " + total + " from " + counts.size() + " worker threads");

    IList<long[]> allIncrements = targetInstance.getList(name + "inc");
    long[] expected = new long[keyCount];
    for (long[] incs : allIncrements) {
        for (int i = 0; i < incs.length; i++) {
            expected[i] += incs[i];
        }
    }

    IMap<Integer, Long> map = targetInstance.getMap(name);
    int failures = 0;
    for (int i = 0; i < keyCount; i++) {
        if (expected[i] != map.get(i)) {
            failures++;
            logger.info(name + ": key=" + i + " expected " + expected[i] + " != " + "actual " + map.get(i));
        }
    }
    assertEquals(name + ": " + failures + " key=>values have been incremented unExpected", 0, failures);
}
 

@Verify(global = false)
public void verify() {
    IList<TxnCounter> counts = targetInstance.getList(name + "report");
    TxnCounter total = new TxnCounter();
    for (TxnCounter c : counts) {
        total.add(c);
    }
    logger.info(name + ": " + total + " from " + counts.size() + " workers");

    IList<long[]> allIncrements = targetInstance.getList(name + "res");
    long[] expected = new long[keyCount];
    for (long[] incs : allIncrements) {
        for (int i = 0; i < incs.length; i++) {
            expected[i] += incs[i];
        }
    }

    IMap<Integer, Long> map = targetInstance.getMap(name);

    int failures = 0;
    for (int i = 0; i < keyCount; i++) {
        if (expected[i] != map.get(i)) {
            failures++;
            logger.info(name + ": key=" + i + " expected " + expected[i] + " != " + "actual " + map.get(i));
        }
    }

    assertEquals(name + ": " + failures + " key=>values have been incremented unExpected", 0, failures);
}
 

@Verify
public void globalVerify() {
    IQueue queue = targetInstance.getQueue(name + 'q');
    IList<TxnCounter> results = targetInstance.getList(name + "results");

    TxnCounter total = new TxnCounter();
    for (TxnCounter counter : results) {
        total.add(counter);
    }

    logger.info(name + ": " + total + " from " + results.size() + " worker Threads  Queue size=" + queue.size());
    assertFalse(name + ": firstLock.isLocked()", firstLock.isLocked());
    assertFalse(name + ": secondLock.isLocked()", secondLock.isLocked());
    // TODO: check if this assert can be re-enabled: assertEquals(total.committed - total.rolled, queue.size())
}
 

@AfterRun
public void afterRun(ThreadState state) {
    IList<TxnCounter> results = targetInstance.getList(name + "results");
    results.add(state.counter);
}
 

@Test
public void testStream_whenWatchDatabase() {
    IList<Document> list = jet.getList("list");

    String connectionString = mongoContainer.connectionString();
    long value = startAtOperationTime.getValue();

    StreamSource<? extends Document> source = MongoDBSourceBuilder
            .streamDatabase(SOURCE_NAME, () -> MongoClients.create(connectionString))
            .databaseFn(client -> client.getDatabase(DB_NAME))
            .destroyFn(MongoClient::close)
            .searchFn(db -> {
                List<Bson> aggregates = new ArrayList<>();
                aggregates.add(Aggregates.match(new Document("fullDocument.val", new Document("$gte", 10))
                        .append("operationType", "insert")));

                aggregates.add(Aggregates.project(new Document("fullDocument.val", 1).append("_id", 1)));
                return db.watch(aggregates);
            })
            .mapFn(ChangeStreamDocument::getFullDocument)
            .startAtOperationTimeFn(client -> new BsonTimestamp(value))
            .build();


    Pipeline p = Pipeline.create();
    p.readFrom(source)
     .withNativeTimestamps(0)
     .writeTo(Sinks.list(list));

    Job job = jet.newJob(p);

    MongoCollection<Document> col1 = collection("col1");
    MongoCollection<Document> col2 = collection("col2");

    col1.insertOne(new Document("val", 1));
    col1.insertOne(new Document("val", 10).append("foo", "bar"));

    col2.insertOne(new Document("val", 2));
    col2.insertOne(new Document("val", 11).append("foo", "bar"));

    assertTrueEventually(() -> {
        assertEquals(2, list.size());
        list.forEach(document -> assertNull(document.get("foo")));

        assertEquals(10, list.get(0).get("val"));
        assertEquals(11, list.get(1).get("val"));

    });

    col1.insertOne(new Document("val", 3));
    col1.insertOne(new Document("val", 12).append("foo", "bar"));

    col2.insertOne(new Document("val", 4));
    col2.insertOne(new Document("val", 13).append("foo", "bar"));

    assertTrueEventually(() -> {
        assertEquals(4, list.size());
        list.forEach(document -> assertNull(document.get("foo")));

        assertEquals(12, list.get(2).get("val"));
        assertEquals(13, list.get(3).get("val"));
    });

    job.cancel();

}
 

@Test
public void testStream_whenWatchAll() {
    IList<Document> list = jet.getList("list");

    String connectionString = mongoContainer.connectionString();
    long value = startAtOperationTime.getValue();

    StreamSource<? extends Document> source = MongoDBSourceBuilder
            .streamAll(SOURCE_NAME, () -> MongoClients.create(connectionString))
            .destroyFn(MongoClient::close)
            .searchFn(client -> {
                List<Bson> aggregates = new ArrayList<>();
                aggregates.add(Aggregates.match(new Document("fullDocument.val", new Document("$gt", 10))
                        .append("operationType", "insert")));

                aggregates.add(Aggregates.project(new Document("fullDocument.val", 1).append("_id", 1)));
                return client.watch(aggregates);
            })
            .mapFn(ChangeStreamDocument::getFullDocument)
            .startAtOperationTimeFn(client -> new BsonTimestamp(value))
            .build();

    Pipeline p = Pipeline.create();
    p.readFrom(source)
     .withNativeTimestamps(0)
     .writeTo(Sinks.list(list));

    Job job = jet.newJob(p);

    MongoCollection<Document> col1 = collection("db1", "col1");
    MongoCollection<Document> col2 = collection("db1", "col2");
    MongoCollection<Document> col3 = collection("db2", "col3");

    col1.insertOne(new Document("val", 1));
    col1.insertOne(new Document("val", 11).append("foo", "bar"));
    col2.insertOne(new Document("val", 2));
    col2.insertOne(new Document("val", 12).append("foo", "bar"));
    col3.insertOne(new Document("val", 3));
    col3.insertOne(new Document("val", 13).append("foo", "bar"));

    assertTrueEventually(() -> {
        assertEquals(3, list.size());
        list.forEach(document -> assertNull(document.get("foo")));

        assertEquals(11, list.get(0).get("val"));
        assertEquals(12, list.get(1).get("val"));
        assertEquals(13, list.get(2).get("val"));
    });

    col1.insertOne(new Document("val", 4));
    col1.insertOne(new Document("val", 14).append("foo", "bar"));
    col2.insertOne(new Document("val", 5));
    col2.insertOne(new Document("val", 15).append("foo", "bar"));
    col2.insertOne(new Document("val", 6));
    col2.insertOne(new Document("val", 16).append("foo", "bar"));

    assertTrueEventually(() -> {
        assertEquals(6, list.size());
        list.forEach(document -> assertNull(document.get("foo")));

        assertEquals(14, list.get(3).get("val"));
        assertEquals(15, list.get(4).get("val"));
        assertEquals(16, list.get(5).get("val"));
    });

    job.cancel();

}