com.hazelcast.jet.pipeline.StreamSource Java Examples

@Test
public void testStream_withTermFilter() {
    Pipeline pipeline = Pipeline.create();
    List<String> terms = new ArrayList<String>(Arrays.asList("BTC", "ETH"));
    final StreamSource<String> twitterTestStream = TwitterSources.stream(
            credentials, () -> new StatusesFilterEndpoint().trackTerms(terms));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withoutTimestamps()
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));

    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 20 tweets in 1 min.", list.size(), 20)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

/**
 * Equivalent of {@link #stream(Properties, SupplierEx)} with the
 * additional option to specify the Twitter {@code host}.
 *
 * @param host a Twitter host URL to connect to. These hosts are defined in
 *          {@link com.twitter.hbc.core.Constants}.
 */
@Nonnull
public static StreamSource<String> stream(
        @Nonnull Properties credentials,
        @Nonnull String host,
        @Nonnull SupplierEx<? extends StreamingEndpoint> endpointSupplier
) {
    return SourceBuilder.stream("twitter-stream-source",
            ctx -> new TwitterStreamSourceContext(ctx.logger(), credentials, host, endpointSupplier))
                        .fillBufferFn(TwitterStreamSourceContext::fillBuffer)
                        .destroyFn(TwitterStreamSourceContext::close)
                        .build();
}
 

public static StreamSource<Aircraft> flightDataSource(String url, long pollIntervalMillis) {
    return SourceBuilder.timestampedStream("Flight Data Source",
            ctx -> new FlightDataSource(ctx.logger(), url, pollIntervalMillis))
            .fillBufferFn(FlightDataSource::fillBuffer)
            .build();

}
 

/**
 * Creates and returns the Pulsar Consumer {@link StreamSource} with using builder configurations set before.
 */
public StreamSource<T> build() {
    return SourceBuilder.timestampedStream("pulsar-consumer-source",
            ctx -> new PulsarConsumerBuilder.ConsumerContext<>(
                    ctx.logger(), connectionSupplier.get(), topics, consumerConfig,
                    schemaSupplier, batchReceivePolicySupplier, projectionFn))
            .<T>fillBufferFn(PulsarConsumerBuilder.ConsumerContext::fillBuffer)
            .destroyFn(PulsarConsumerBuilder.ConsumerContext::destroy)
            .distributed(2)
            .build();
}
 

@Nonnull
StreamSource<U> build(@Nonnull SupplierEx<StreamContext<T, U>> contextFn) {
    return SourceBuilder
            .timestampedStream(name, ctx -> contextFn.get())
            .<U>fillBufferFn(StreamContext::fillBuffer)
            .createSnapshotFn(StreamContext::snapshot)
            .restoreSnapshotFn(StreamContext::restore)
            .destroyFn(StreamContext::close)
            .build();
}
 

@Test
public void testTimestampedStream_userFilter() {
    Pipeline pipeline = Pipeline.create();
    List<Long> userIds = new ArrayList<Long>(
            Arrays.asList(612473L, 759251L, 1367531L, 34713362L, 51241574L, 87818409L));
    final StreamSource<String> twitterTestStream = TwitterSources.timestampedStream(
            credentials, () -> new StatusesFilterEndpoint().followings(userIds));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withNativeTimestamps(1)
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 15 tweets in 1 min.",
                    list.size(), 15)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void testTimestampedStream_termFilter() {
    Pipeline pipeline = Pipeline.create();
    List<String> terms = new ArrayList<String>(Arrays.asList("San Mateo", "Brno", "London", "Istanbul"));

    final StreamSource<String> twitterTestStream = TwitterSources.timestampedStream(
            credentials, () -> new StatusesFilterEndpoint().trackTerms(terms));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withNativeTimestamps(0)
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 20 tweets in 1 min.",
                    list.size(), 20)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void testStream_userFilter() {
    Pipeline pipeline = Pipeline.create();
    List<Long> userIds = new ArrayList<Long>(
            Arrays.asList(612473L, 759251L, 1367531L, 34713362L, 51241574L, 87818409L));
    final StreamSource<String> twitterTestStream = TwitterSources.stream(credentials,
            () -> new StatusesFilterEndpoint().followings(userIds));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withoutTimestamps()
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 15 tweets in 1 min.",
                    list.size(), 15)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

@Test
public void streamApiMockTest() {

    List<String> terms = new ArrayList<>(Arrays.asList("San Mateo", "Brno", "London", "Istanbul"));
    final StreamSource<String> twitterTestStream = TwitterSources.timestampedStream(getCredentials(),
            "http://" + server.getHostName() + ":" + server.getPort(),
            () -> new StatusesFilterEndpoint().trackTerms(terms));

    Pipeline pipeline = Pipeline.create();
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withNativeTimestamps(0)
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(10,
            list -> assertGreaterOrEquals("Emits at least 100 tweets in 1 min.",
                    list.size(), 100)));
    Job job = createJetMember().newJob(pipeline);

    sleepAtLeastSeconds(5);

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

/**
 * Equivalent for {@link #timestampedStream(Properties, SupplierEx)} with
 * the additional option to specify the Twitter {@code host}.
 *
 * @param host a Twitter host URL to connect to. These hosts are defined in
 *          {@link com.twitter.hbc.core.Constants}.
 */
@Nonnull
public static StreamSource<String> timestampedStream(
        @Nonnull Properties credentials,
        @Nonnull String host,
        @Nonnull SupplierEx<? extends StreamingEndpoint> endpointSupplier
) {
    return SourceBuilder.timestampedStream("twitter-timestamped-stream-source",
            ctx -> new TwitterStreamSourceContext(ctx.logger(), credentials, host, endpointSupplier))
                        .fillBufferFn(TwitterStreamSourceContext::fillTimestampedBuffer)
                        .destroyFn(TwitterStreamSourceContext::close)
                        .build();
}
 

public static StreamSource<Trade> tradeSource(List<String> tickers, int tradesPerSec) {
    return SourceBuilder.timestampedStream("trade-source", ctx -> {
        Map<Integer, List<String>> partitions = partitionTickers(tickers, ctx.totalParallelism());
        return new TradeGenerator(partitions.get(ctx.globalProcessorIndex()), tradesPerSec);
    })
            .fillBufferFn(TradeGenerator::fillBuffer)
            .distributed(1)
            .build();
}
 

private static Pipeline buildPipeline() {
    Pipeline p = Pipeline.create();

    StreamSource<Long> source = TestSources.itemStream(1, (ts, seq) -> seq);
    // StreamSource<String> source = Sources.fileWatcher(DIRECTORY);

    p.readFrom(source)
     .withoutTimestamps()
     // .map( line-> Long.valueOf(line))
     .filter(item -> (item % 2) == 0)
     .writeTo(Sinks.logger());

    return p;
}
 

private static Pipeline buildPipeline() {
    Pipeline p = Pipeline.create();

    StreamSource<Long> source = TestSources.itemStream(1, (ts, seq) -> seq);

    p.readFrom(source)
     .withoutTimestamps()
     .writeTo(Sinks.logger());

    // Run the code to see the results in the console
    // Stop it before leaving the lab

    return p;
}
 

/**
 * Creates and returns the MongoDB {@link StreamSource} which watches
 * all collections across all databases.
 */
@Nonnull
public StreamSource<U> build() {
    checkNotNull(connectionSupplier, "connectionSupplier must be set");
    checkNotNull(searchFn, "searchFn must be set");
    checkNotNull(mapFn, "mapFn must be set");

    return build(contextFn(connectionSupplier, destroyFn, searchFn, mapFn, startAtOperationTimeFn));
}
 

protected static StreamSource<String> setupReaderSource(String topicName,
                                                        FunctionEx<Message<byte[]>, String> projectionFn) {
    return PulsarSources.pulsarReader(
            topicName,
            () -> PulsarClient.builder().serviceUrl(getServiceUrl()).build(),
            () -> Schema.BYTES,
            projectionFn);
}
 

protected static StreamSource<String> setupConsumerSource(String topicName,
                                                          FunctionEx<Message<byte[]>, String> projectionFn) {
    return PulsarSources.pulsarConsumer(
            topicName,
            () -> PulsarClient.builder().serviceUrl(getServiceUrl()).build(),
            () -> Schema.BYTES,
            projectionFn);
}
 

@Test
public void when_readFromPulsarConsumer_then_jobGetsAllPublishedMessages() {
    JetInstance[] instances = new JetInstance[2];
    Arrays.setAll(instances, i -> createJetMember());

    String topicName = randomName();
    StreamSource<String> pulsarConsumerSrc = setupConsumerSource(topicName,
            x -> new String(x.getData(), StandardCharsets.UTF_8));

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(pulsarConsumerSrc)
            .withoutTimestamps()
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> {
                        assertEquals("# of Emitted items should be equal to # of published items",
                                ITEM_COUNT, list.size());
                        for (int i = 0; i < ITEM_COUNT; i++) {
                            String message = "hello-pulsar-" + i;
                            Assert.assertTrue("missing entry: " + message, list.contains(message));
                        }
                    })
            );
    Job job = instances[0].newJob(pipeline);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    produceMessages("hello-pulsar", topicName, ITEM_COUNT);

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
    for (JetInstance instance:instances) {
        instance.shutdown();
    }
}
 

@Test
public void when_projectionFunctionProvided_thenAppliedToReadRecords() {
    String topicName = randomName();
    // Add a suffix to messages so that this projectionFn does a bit more than byte->String conversion.
    StreamSource<String> pulsarConsumerSrc = setupConsumerSource(topicName,
            x -> new String(x.getData(), StandardCharsets.UTF_8) + "-suffix");

    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(pulsarConsumerSrc)
            .withoutTimestamps()
            .writeTo(AssertionSinks.assertCollectedEventually(60,
                    list -> {
                        assertEquals("# of Emitted items should be equal to # of published items",
                                ITEM_COUNT, list.size());
                        for (int i = 0; i < ITEM_COUNT; i++) {
                            String message = "hello-pulsar-" + i + "-suffix";
                            Assert.assertTrue("missing entry: " + message, list.contains(message));
                        }
                    })
            );
    Job job = createJetMember().newJob(pipeline);
    assertJobStatusEventually(job, JobStatus.RUNNING);

    produceMessages("hello-pulsar", topicName, ITEM_COUNT);

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 

/**
 * Creates and returns the Pulsar Reader {@link StreamSource} with using builder configurations set before.
 */
public StreamSource<T> build() {
    return SourceBuilder.timestampedStream("pulsar-reader-source", ctx -> new PulsarReaderBuilder.ReaderContext<>(
            ctx.logger(), connectionSupplier.get(), topic, readerConfig, schemaSupplier, projectionFn))
            .<T>fillBufferFn(PulsarReaderBuilder.ReaderContext::fillBuffer)
            .createSnapshotFn(PulsarReaderBuilder.ReaderContext::createSnapshot)
            .restoreSnapshotFn(PulsarReaderBuilder.ReaderContext::restoreSnapshot)
            .destroyFn(PulsarReaderBuilder.ReaderContext::destroy)
            .build();
}
 

/**
 * Creates and returns the MongoDB {@link StreamSource} which watches
 * all collections in the given database.
 */
@Nonnull
public StreamSource<U> build() {
    checkNotNull(connectionSupplier, "connectionSupplier must be set");
    checkNotNull(databaseFn, "databaseFn must be set");
    checkNotNull(searchFn, "searchFn must be set");
    checkNotNull(mapFn, "mapFn must be set");

    return build(contextFn(connectionSupplier, databaseFn, destroyFn, searchFn, mapFn, startAtOperationTimeFn));
}
 

/**
 * Creates and returns the MongoDB {@link StreamSource} which watches
 * the given collection.
 */
@Nonnull
public StreamSource<U> build() {
    checkNotNull(connectionSupplier, "connectionSupplier must be set");
    checkNotNull(databaseFn, "databaseFn must be set");
    checkNotNull(collectionFn, "collectionFn must be set");
    checkNotNull(searchFn, "searchFn must be set");
    checkNotNull(mapFn, "mapFn must be set");

    SupplierEx<StreamContext<T, U>> contextFn = contextFn(connectionSupplier, databaseFn,
            (FunctionEx<? super MongoDatabase, ? extends MongoCollection<? extends T>>) collectionFn, destroyFn,
            searchFn, mapFn, startAtOperationTimeFn);

    return build(contextFn);
}
 

private StreamSource<? extends Document> streamSource(int connectionTimeoutSeconds) {
    return streamSource(null, null, connectionTimeoutSeconds);
}
 

/**
 * Creates a {@link StreamSource} which reads all items from multiple Redis
 * Streams (starting from given indexes) and emits them as they arrive.
 * <p>
 * Here is an example which reads all elements from two different Redis
 * Streams, maps the objects received to a stream specific ID and drains
 * the results out to some generic sink.
 * <pre>{@code
 * Map<String, String> streamOffsets = new HashMap<>();
 * streamOffsets.put("streamA", "0");
 * streamOffsets.put("streamB", "0");
 *
 * RedisURI uri = RedisURI.create("redis://localhost/");
 *
 * Pipeline.create()
 *     .readFrom(RedisSources.stream("source", uri, streamOffsets,
 *                      StringCodec::new,
 *                      mes -> mes.getStream() + " - " + mes.getId()))
 *     .withoutTimestamps()
 *     .writeTo(sink);
 * }</pre>
 *
 * @param <K>           type of the stream identifier and of fields (keys of
 *                      the Redis Stream entry's body map)
 * @param <V>           type of the field values in the message body
 * @param <T>           type of data coming out of the stream, the result of
 *                      applying the projection function over {@link
 *                      StreamMessage} instances
 * @param name          name of the source being created
 * @param uri           URI of the Redis server
 * @param streamOffsets map keyed by stream identifiers, containing offset
 *                      values from which to start element retrieval of each
 *                      stream
 * @param codecFn       supplier of {@link RedisCodec} instances, used in
 *                      turn for serializing/deserializing keys and values
 * @param projectionFn  built in mapping function of the source which can be
 *                      used to map {@link StreamMessage} instances received
 *                      from Redis to an arbitrary type of output; this
 *                      could be done by an external mapping function in the
 *                      pipeline too, but it's included for convenience
 * @return source to use in {@link com.hazelcast.jet.pipeline.Pipeline#readFrom}
 */
@Nonnull
public static <K, V, T> StreamSource<T> stream(
        @Nonnull String name,
        @Nonnull RedisURI uri,
        @Nonnull Map<K, String> streamOffsets,
        @Nonnull SupplierEx<RedisCodec<K, V>> codecFn,
        @Nonnull FunctionEx<? super StreamMessage<K, V>, ? extends T> projectionFn
) {
    Objects.requireNonNull(name, "name");
    Objects.requireNonNull(uri, "uri");
    Objects.requireNonNull(streamOffsets, "streamOffsets");
    Objects.requireNonNull(codecFn, "codecFn");
    Objects.requireNonNull(projectionFn, "projectionFn");

    Util.checkSerializable(codecFn, "codecFn");
    Util.checkSerializable(projectionFn, "projectionFn");

    return streamFromProcessorWithWatermarks(name, false,
            w -> StreamRedisP.streamRedisP(uri, streamOffsets, w, codecFn, projectionFn));
}
 

public static StreamSource<BufferedImage> webcam(long pollIntervalMillis) {
    return SourceBuilder.timestampedStream("webcam", ctx -> new WebcamSource(pollIntervalMillis))
            .fillBufferFn(WebcamSource::addToBufferFn)
            .destroyFn(WebcamSource::close)
            .build();
}
 

public static StreamSource<Trade> tradeSource() {
    return tradeSource(1);
}
 

public static StreamSource<Trade> tradeSource(int tradesPerSec) {
    return SourceBuilder.timestampedStream("trade-source", x -> new TradeGenerator(SYMBOLS, tradesPerSec))
            .fillBufferFn(TradeGenerator::fillBuffer)
            .build();
}
 

private static Pipeline buildPipeline() {
    Pipeline p = Pipeline.create();

    StreamSource<Long> source = TestSources.itemStream(1, (ts, seq) -> seq);

    p.readFrom(source)
     .withoutTimestamps()
     .writeTo(Sinks.logger());

    // STEP 1: Filter out odd numbers from the stream

    // Add filter() to  your pipeline
    // - Use lambda to define the predicate

    // Stop the job before continuing to Step 2



    // STEP 2: Process data from a file instead of generated data

    // Create a directory somewhere in your computer and create an empty input.txt file in it

    // Replace itemStream with fileWatcher source from com.hazelcast.jet.pipeline.Sources
    // - (fileWatcher stream lines added to files in a directory.)
    // - Adjust source type - the generator was producing Longs, fileWatcher produces Strings

    // Add a mapping step before the filter to convert the stream from Strings to Longs

    // Run this pipeline to test it!
    // - Add text lines to the file.
    // - Use echo -- some text editors create a new file for every save. That results in replaying the file.
    //
    // echo "0" >> input.txt
    // echo "1" >> input.txt

    // Stop the job


    return p;
}
 

/**
 * Creates a {@link StreamSource} which reads all items from multiple Redis
 * Streams (starting from given indexes) and emits them as they arrive.
 * Assumes all keys and values are {@code String}s.
 *
 * @param <T>           type of data coming out of the stream, the result of
 *                      applying the projection function over {@link
 *                      StreamMessage} instances
 * @param name          name of the source being created
 * @param uri           URI of the Redis server
 * @param streamOffsets map keyed by stream identifiers, containing offset
 *                      values from which to start element retrieval of each
 *                      stream
 * @param projectionFn  built in mapping function of the source which can be
 *                      used to map {@link StreamMessage} instances received
 *                      from Redis to an arbitrary type of output; this
 *                      could be done by an external mapping function in the
 *                      pipeline too, but it's included for convenience
 * @return source to use in {@link com.hazelcast.jet.pipeline.Pipeline#readFrom}
 */
@Nonnull
public static <T> StreamSource<T> stream(
        @Nonnull String name,
        @Nonnull RedisURI uri,
        @Nonnull Map<String, String> streamOffsets,
        @Nonnull FunctionEx<? super StreamMessage<String, String>, ? extends T> projectionFn
) {
    return stream(name, uri, streamOffsets, StringCodec::new, projectionFn);
}
 

/**
 * Creates a {@link StreamSource} which reads all items from multiple Redis
 * Streams (starting from given indexes) and emits them as they arrive.
 * Assumes all keys and values are {@code String}s and assumes a projection
 * function which just emits message bodies from stream elements received.
 *
 * @param name          name of the source being created
 * @param uri           URI of the Redis server
 * @param streamOffsets map keyed by stream identifiers, containing offset
 *                      values from which to start element retrieval of each
 *                      stream
 * @return source to use in {@link com.hazelcast.jet.pipeline.Pipeline#readFrom}
 */
@Nonnull
public static StreamSource<Map<String, String>> stream(
        @Nonnull String name,
        @Nonnull RedisURI uri,
        @Nonnull Map<String, String> streamOffsets
) {
    return stream(name, uri, streamOffsets, StringCodec::new, StreamMessage::getBody);
}
 

/**
 * Variant of {@link #stream(Properties, SupplierEx)} which parses the
 * messages and used the {@code timestamp_ms} field as a native timestamp.
 * If you don't need this timestamp, prefer to use the other variant.
 * <p>
 * For parameter documentation {@linkplain #stream(Properties,
 * SupplierEx) see here}.
 */
@Nonnull
public static StreamSource<String> timestampedStream(
        @Nonnull Properties credentials,
        @Nonnull SupplierEx<? extends StreamingEndpoint> endpointSupplier
) {
    return timestampedStream(credentials, Constants.STREAM_HOST, endpointSupplier);
}