Java Code Examples for io.pravega.client.EventStreamClientFactory#createReader()

/**
 * Create a stream reader for reading Integer events.
 *
 * @param streamName    Name of the test stream.
 *
 * @return Stream reader instance.
 */
public EventStreamReader<Integer> getIntegerReader(final String streamName) {
    Preconditions.checkState(this.started.get(), "Services not yet started");
    Preconditions.checkNotNull(streamName);

    ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(this.scope, getClientConfig());
    final String readerGroup = "testReaderGroup" + this.scope + streamName;
    readerGroupManager.createReaderGroup(
            readerGroup,
            ReaderGroupConfig.builder().stream(Stream.of(this.scope, streamName)).build());

    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(this.scope, getClientConfig());
    final String readerGroupId = UUID.randomUUID().toString();
    return clientFactory.createReader(
            readerGroupId,
            readerGroup,
            new IntegerSerializer(),
            ReaderConfig.builder().build());
}
 

private <T extends Serializable> int createReaderAndReadEvents(ReaderGroupManager rgMgr, EventStreamClientFactory clientFactory,
                                                               String rGroupId, int readerIndex) {
    // create a reader.
    EventStreamReader<T> reader = clientFactory.createReader(rGroupId + "-" + readerIndex, rGroupId, new JavaSerializer<>(),
                                                             ReaderConfig.builder().build());
    EventRead<T> event = null;
    int validEvents = 0;
    AtomicBoolean sealedSegmentUpdated = new AtomicBoolean(false);
    try {
        do {
            try {
                event = reader.readNextEvent(READ_TIMEOUT);
                log.debug("Read event result in readEvents: {}.", event.getEvent());
                if (event.getEvent() == null && !event.isCheckpoint() && !sealedSegmentUpdated.get()) {
                    // initiate a checkpoint to ensure all sealed segments are acquired by the reader.
                    ReaderGroup readerGroup = rgMgr.getReaderGroup(rGroupId);
                    readerGroup.initiateCheckpoint("chkPoint", chkPointExecutor)
                               .whenComplete((checkpoint, t) -> {
                                   if (t != null) {
                                       log.error("Checkpoint operation failed", t);
                                   } else {
                                       log.info("Checkpoint {} completed", checkpoint);
                                       sealedSegmentUpdated.set(true);
                                   }
                               });
                }
                if (event.getEvent() != null) {
                    validEvents++;
                }
            } catch (ReinitializationRequiredException e) {
                log.error("Reinitialization Exception while reading event using readerId: {}", reader, e);
                fail("Reinitialization Exception is not expected");
            }
        } while (event.getEvent() != null || event.isCheckpoint() || !sealedSegmentUpdated.get());
    } finally {
        closeReader(reader);
    }
    return validEvents;
}
 

private EventStreamReader<Integer> getIntegerReader() {
    ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, clientConfig);
    final String readerGroup = "side-reader-" + this.scope + "-" + sideStream;
    readerGroupManager.createReaderGroup(
            readerGroup,
            ReaderGroupConfig.builder().stream(Stream.of(this.scope, sideStream)).build());

    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(this.scope, clientConfig);
    final String readerGroupId = UUID.randomUUID().toString();
    return clientFactory.createReader(
            readerGroupId,
            readerGroup,
            new IntegerSerializer(),
            ReaderConfig.builder().build());
}
 

@Test(timeout = 60000)
public void testReaderGroupWithSameBounds() throws Exception {
    createScope(SCOPE);
    createStream(STREAM1);

    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, ClientConfig.builder().controllerURI(controllerUri).build());
    @Cleanup
    EventStreamWriter<String> writer1 = clientFactory.createEventWriter(STREAM1, serializer,
                                                                        EventWriterConfig.builder().build());
    // 1. Prep the stream with data.
    // Write events with event size of 30
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(1)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(2)).get();

    // 2. Create a StreamCut Pointing to offset 30L
    StreamCut streamCut = getStreamCut(STREAM1, 30L, 0);

    // 3. Create a ReaderGroup where the lower and upper bound are the same.
    @Cleanup
    ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, controllerUri);
    groupManager.createReaderGroup("group", ReaderGroupConfig
            .builder().disableAutomaticCheckpoints()
            .stream(Stream.of(SCOPE, STREAM1), streamCut, streamCut)
            .build());

    // 4. Create a reader
    @Cleanup
    EventStreamReader<String> reader = clientFactory.createReader("readerId", "group", serializer,
                                                                  ReaderConfig.builder().build());

    // 5. Verify if configuration is enforced.
    Assert.assertNull("Null is expected", reader.readNextEvent(1000).getEvent());
}
 

public EventStreamReader<String> createReader(EventStreamClientFactory clientFactory) {
    String readerName = "Reader";

    //reusing a reader group name doesn't work (probably because the sequence is already consumed)
    //until we figure out how to manage this, use a random reader group name
    String readerGroup = UUID.randomUUID().toString().replace("-", "");
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder()
            .stream(Stream.of(scopeName, streamName))
            .build();
    readerGroupManager.createReaderGroup(readerGroup, groupConfig);
    ReaderConfig readerConfig = ReaderConfig.builder().build();
    return clientFactory.createReader(readerName, readerGroup, SERIALIZER, readerConfig);
}
 

/**
 * Compares reads and writes to verify that an in-process Pravega cluster responds properly with
 * with valid client configuration.
 *
 * Note:
 * Strictly speaking, this test is really an "integration test" and is a little time consuming. For now, its
 * intended to also run as a unit test, but it could be moved to an integration test suite if and when necessary.
 *
 */
@Test(timeout = 50000)
public void testWriteAndReadEventWithValidClientConfig() throws ExecutionException,
        InterruptedException, ReinitializationRequiredException {

    String scope = scopeName();
    String streamName = streamName();
    int numSegments = 1;
    String message = eventMessage();

    ClientConfig clientConfig = prepareValidClientConfig();

    @Cleanup
    StreamManager streamManager = StreamManager.create(clientConfig);
    assertNotNull(streamManager);

    boolean isScopeCreated = streamManager.createScope(scope);
    assertTrue("Failed to create scope", isScopeCreated);

    boolean isStreamCreated = streamManager.createStream(scope, streamName, StreamConfiguration.builder()
            .scalingPolicy(ScalingPolicy.fixed(numSegments))
            .build());
    Assert.assertTrue("Failed to create the stream ", isStreamCreated);

    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);

    // Write an event to the stream.

    @Cleanup
    EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName,
            new JavaSerializer<String>(),
            EventWriterConfig.builder().build());
    writer.writeEvent(message).get();
    log.debug("Done writing message '{}' to stream '{} / {}'", message, scope, streamName);

    // Now, read the event from the stream.

    String readerGroup = UUID.randomUUID().toString().replace("-", "");
    ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder()
            .stream(Stream.of(scope, streamName))
            .disableAutomaticCheckpoints()
            .build();

    @Cleanup
    ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, clientConfig);
    readerGroupManager.createReaderGroup(readerGroup, readerGroupConfig);

    @Cleanup
    EventStreamReader<String> reader = clientFactory.createReader(
            "readerId", readerGroup,
            new JavaSerializer<String>(), ReaderConfig.builder().initialAllocationDelay(0).build());

    // Keeping the read timeout large so that there is ample time for reading the event even in
    // case of abnormal delays in test environments.
    String readMessage = reader.readNextEvent(10000).getEvent();
    log.info("Done reading event [{}]", readMessage);

    assertEquals(message, readMessage);
}
 

@Test(timeout = 60000)
public void testStreamcutsUpdateInReaderGroup() throws Exception {
    final String scope = "testStreamcutsUpdateInReaderGroup";
    final String stream = "myStream";
    final String readerGroupName = "testStreamcutsUpdateInReaderGroupRG";
    final int checkpointingIntervalMs = 2000;
    final int readerSleepInterval = 250;
    final int numEvents = 100;

    // First, create the stream.
    StreamManager streamManager = StreamManager.create(controllerURI);
    Assert.assertTrue(streamManager.createScope(scope));
    StreamConfiguration streamConfiguration = StreamConfiguration.builder()
                                                                 .scalingPolicy(ScalingPolicy.fixed(2))
                                                                 .build();
    streamManager.createStream(scope, stream, streamConfiguration);

    // Write some events in the stream.
    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, ClientConfig.builder().controllerURI(controllerURI).build());
    writeEvents(clientFactory, stream, numEvents);

    // Read the events and test that positions are getting updated.
    ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder()
                                                           .stream(Stream.of(scope, stream))
                                                           .automaticCheckpointIntervalMillis(checkpointingIntervalMs)
                                                           .build();

    @Cleanup
    ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, controllerURI);
    readerGroupManager.createReaderGroup(readerGroupName, readerGroupConfig);
    ReaderGroup readerGroup = readerGroupManager.getReaderGroup(readerGroupName);
    @Cleanup
    EventStreamReader<Double> reader = clientFactory.createReader("myReader", readerGroupName,
            new JavaSerializer<>(), ReaderConfig.builder().build());

    Map<Stream, StreamCut> currentStreamcuts = readerGroup.getStreamCuts();
    EventRead eventRead;
    int lastIteration = 0, iteration = 0;
    int assertionFrequency = checkpointingIntervalMs / readerSleepInterval;
    do {
        eventRead = reader.readNextEvent(5000);

        // Check that the streamcuts are being updated periodically via automatic reader group checkpoints.
        if (iteration != lastIteration && iteration % assertionFrequency == 0) {
            log.info("Comparing streamcuts: {} / {} in iteration {}.", currentStreamcuts, readerGroup.getStreamCuts(), iteration);
            Assert.assertNotEquals(currentStreamcuts, readerGroup.getStreamCuts());
            currentStreamcuts = readerGroup.getStreamCuts();
            lastIteration = iteration;
        }

        Thread.sleep(readerSleepInterval);
        if (!eventRead.isCheckpoint()) {
            iteration++;
        }
    } while ((eventRead.isCheckpoint() || eventRead.getEvent() != null) && iteration < numEvents);
}
 

@Test(timeout = 50000)
public void testUnreadBytes() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder()
            .scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1))
            .build();

    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope("unreadbytes").get();
    controller.createStream("unreadbytes", "unreadbytes", config).get();

    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope("unreadbytes", ClientConfig.builder().controllerURI(controllerUri).build());
    @Cleanup
    EventStreamWriter<String> writer = clientFactory.createEventWriter("unreadbytes", new JavaSerializer<>(),
            EventWriterConfig.builder().build());

    @Cleanup
    ReaderGroupManager groupManager = ReaderGroupManager.withScope("unreadbytes",  ClientConfig.builder().controllerURI(controllerUri).build());
    groupManager.createReaderGroup("group", ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream("unreadbytes/unreadbytes").build());
    @Cleanup
    ReaderGroup readerGroup = groupManager.getReaderGroup("group");

    @Cleanup
    EventStreamReader<String> reader = clientFactory.createReader("readerId", "group", new JavaSerializer<>(),
            ReaderConfig.builder().build());
    long unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 0);

    writer.writeEvent("0", "data of size 30").get();
    writer.writeEvent("0", "data of size 30").get();

    EventRead<String> firstEvent = reader.readNextEvent(15000);
    EventRead<String> secondEvent = reader.readNextEvent(15000);
    assertNotNull(firstEvent);
    assertEquals("data of size 30", firstEvent.getEvent());
    assertNotNull(secondEvent);
    assertEquals("data of size 30", secondEvent.getEvent());

    // trigger a checkpoint.
    CompletableFuture<Checkpoint> chkPointResult = readerGroup.initiateCheckpoint("test", executor);
    EventRead<String> chkpointEvent = reader.readNextEvent(15000);
    assertEquals("test", chkpointEvent.getCheckpointName());
    
    EventRead<String> emptyEvent = reader.readNextEvent(100);
    assertEquals(false, emptyEvent.isCheckpoint());
    assertEquals(null, emptyEvent.getEvent());
    chkPointResult.join();

    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 0);

    writer.writeEvent("0", "data of size 30").get();
    unreadBytes = readerGroup.getMetrics().unreadBytes();
    assertTrue("Unread bytes: " + unreadBytes, unreadBytes == 30);
}
 

@Test(timeout = 50000)
public void testUnreadBytesWithEndStreamCuts() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder()
                                                    .scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1))
                                                    .build();

    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope("unreadbytes").get();
    controller.createStream("unreadbytes", "unreadbytes", config).get();

    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope("unreadbytes", ClientConfig.builder().controllerURI(controllerUri).build());
    @Cleanup
    EventStreamWriter<String> writer = clientFactory.createEventWriter("unreadbytes", new JavaSerializer<>(),
            EventWriterConfig.builder().build());
    //Write just 2 events to simplify simulating a checkpoint.
    writer.writeEvent("0", "data of size 30").get();
    writer.writeEvent("0", "data of size 30").get();

    @Cleanup
    ReaderGroupManager groupManager = ReaderGroupManager.withScope("unreadbytes",  ClientConfig.builder().controllerURI(controllerUri).build());
    //create a bounded reader group.
    groupManager.createReaderGroup("group", ReaderGroupConfig
            .builder().disableAutomaticCheckpoints().stream("unreadbytes/unreadbytes", StreamCut.UNBOUNDED,
                    getStreamCut("unreadbytes", 90L, 0)).build());

    ReaderGroup readerGroup = groupManager.getReaderGroup("group");
    @Cleanup
    EventStreamReader<String> reader = clientFactory.createReader("readerId", "group", new JavaSerializer<>(),
            ReaderConfig.builder().build());

    EventRead<String> firstEvent = reader.readNextEvent(15000);
    EventRead<String> secondEvent = reader.readNextEvent(15000);
    assertNotNull(firstEvent);
    assertEquals("data of size 30", firstEvent.getEvent());
    assertNotNull(secondEvent);
    assertEquals("data of size 30", secondEvent.getEvent());

    // trigger a checkpoint.
    CompletableFuture<Checkpoint> chkPointResult = readerGroup.initiateCheckpoint("test", executor);
    EventRead<String> chkpointEvent = reader.readNextEvent(15000);
    assertEquals("test", chkpointEvent.getCheckpointName());
    
    EventRead<String> emptyEvent = reader.readNextEvent(100);
    assertEquals(false, emptyEvent.isCheckpoint());
    assertEquals(null, emptyEvent.getEvent());
    
    chkPointResult.join();

    //Writer events, to ensure 120Bytes are written.
    writer.writeEvent("0", "data of size 30").get();
    writer.writeEvent("0", "data of size 30").get();

    long unreadBytes = readerGroup.getMetrics().unreadBytes();
    //Ensure the endoffset of 90 Bytes is taken into consideration when computing unread
    assertTrue("Unread bvtes: " + unreadBytes, unreadBytes == 30);
}
 

@Test(timeout = 40000)
@SuppressWarnings("deprecation")
public void testStreamRecreation() throws Exception {
    final String myScope = "myScope";
    final String myStream = "myStream";
    final String myReaderGroup = "myReaderGroup";
    final int numIterations = 10;

    // Create the scope and the stream.
    @Cleanup
    StreamManager streamManager = StreamManager.create(controllerURI);
    streamManager.createScope(myScope);
    @Cleanup
    ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(myScope, controllerURI);
    final ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder()
                                                                 .stream(Stream.of(myScope, myStream))
                                                                 .build();

    for (int i = 0; i < numIterations; i++) {
        log.info("Stream re-creation iteration {}.", i);
        final String eventContent = "myEvent" + String.valueOf(i);
        StreamConfiguration streamConfiguration = StreamConfiguration.builder()
                                                                     .scalingPolicy(ScalingPolicy.fixed(i + 1))
                                                                     .build();
        EventWriterConfig eventWriterConfig = EventWriterConfig.builder().build();
        streamManager.createStream(myScope, myStream, streamConfiguration);

        // Write a single event.
        @Cleanup
        EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(myScope, ClientConfig.builder().controllerURI(controllerURI).build());
        EventStreamWriter<String> writer = clientFactory.createEventWriter(myStream, new JavaSerializer<>(),
                                                                           eventWriterConfig);
        TransactionalEventStreamWriter<String> txnWriter = clientFactory.createTransactionalEventWriter(myStream,
                                                                                                        new JavaSerializer<>(),
                                                                                                        eventWriterConfig);

        // Write events regularly and with transactions.
        if (i % 2 == 0) {
            writer.writeEvent(eventContent).join();
        } else {
            Transaction<String> myTransaction = txnWriter.beginTxn();
            myTransaction.writeEvent(eventContent);
            myTransaction.commit();
            while (myTransaction.checkStatus() != Transaction.Status.COMMITTED) {
                Exceptions.handleInterrupted(() -> Thread.sleep(100));
            }
        }

        writer.close();

        // Read the event.
        readerGroupManager.createReaderGroup(myReaderGroup, readerGroupConfig);
        readerGroupManager.getReaderGroup(myReaderGroup).resetReaderGroup(readerGroupConfig);
        @Cleanup
        EventStreamReader<String> reader = clientFactory.createReader("myReader", myReaderGroup, new JavaSerializer<>(),
                ReaderConfig.builder().build());
        String readResult;
        do {
            readResult = reader.readNextEvent(1000).getEvent();
        } while (readResult == null);

        assertEquals("Wrong event read in re-created stream", eventContent, readResult);

        // Delete the stream.
        StreamInfo streamInfo = streamManager.getStreamInfo(myScope, myStream);
        assertFalse(streamInfo.isSealed());
        assertTrue("Unable to seal re-created stream.", streamManager.sealStream(myScope, myStream));
        streamInfo = streamManager.getStreamInfo(myScope, myStream);
        assertTrue(streamInfo.isSealed());
        assertTrue("Unable to delete re-created stream.", streamManager.deleteStream(myScope, myStream));
    }
}
 

@Test(timeout = 60000)
public void testBoundedStreamTest() throws Exception {
    createScope(SCOPE);
    createStream(STREAM1);
    createStream(STREAM2);

    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, ClientConfig.builder().controllerURI(controllerUri).build());
    @Cleanup
    EventStreamWriter<String> writer1 = clientFactory.createEventWriter(STREAM1, serializer,
            EventWriterConfig.builder().build());
    //Prep the stream with data.
    //1.Write events with event size of 30
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(1)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(2)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(3)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(4)).get();

    @Cleanup
    ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, controllerUri);
    groupManager.createReaderGroup("group", ReaderGroupConfig
            .builder().disableAutomaticCheckpoints()
            .stream(Stream.of(SCOPE, STREAM1),
                    //startStreamCut points to the current HEAD of stream
                    StreamCut.UNBOUNDED,
                    //endStreamCut points to the offset after two events.(i.e 2 * 30(event size) = 60)
                    getStreamCut(STREAM1, 60L, 0))
            .stream(Stream.of(SCOPE, STREAM2))
            .build());

    //Create a reader
    @Cleanup
    EventStreamReader<String> reader = clientFactory.createReader("readerId", "group", serializer,
            ReaderConfig.builder().build());

    //2. Verify if endStreamCut configuration is enforced.
    readAndVerify(reader, 1, 2);
    //The following read should not return events 3, 4 due to the endStreamCut configuration.
    Assert.assertNull("Null is expected", reader.readNextEvent(2000).getEvent());

    //3. Write events to the STREAM2.
    @Cleanup
    EventStreamWriter<String> writer2 = clientFactory.createEventWriter(STREAM2, serializer,
            EventWriterConfig.builder().build());
    writer2.writeEvent(keyGenerator.get(), getEventData.apply(5)).get();
    writer2.writeEvent(keyGenerator.get(), getEventData.apply(6)).get();

    //4. Verify that events can be read from STREAM2. (Events from STREAM1 are not read since endStreamCut is reached).
    readAndVerify(reader, 5, 6);
    Assert.assertNull("Null is expected", reader.readNextEvent(2000).getEvent());

}
 

public void read() throws ReinitializationRequiredException {

        /*
         * Note about setting the client config for HTTPS:
         *    - The client config below is configured to use an optional truststore. The truststore is expected to be
         *      the certificate of the certification authority (CA) that was used to sign the server certificates.
         *      If this is null or empty, the default JVM trust store is used. In this demo, we use a provided
         *      "cert.pem" as the CA certificate, which is also provided on the server-side. If the cluster uses a
         *      different CA (which it should), use that CA's certificate as the truststore instead.
         *
         *    - Also, the client config below disables host name verification. If the cluster's server certificates
         *      have DNS names / IP addresses of the servers specified in them, you may turn this on. In a production
         *      deployment, it is recommended to keep this on.
         *
         * Note about setting the client config for auth:
         *    - The client config below is configured with an object of DefaultCredentials class. The user name
         *      and password arguments passed to the object represent the credentials used for authentication
         *      and authorization. The assumption we are making here is that the username is valid on the server,
         *      the password is correct and the username has all the permissions necessary for performing the
         *      subsequent operations.
         */
        ClientConfig clientConfig = ClientConfig.builder()
                .controllerURI(this.controllerURI) // "tls://localhost:9090"

                // TLS-related client-side configuration
                .trustStore(this.truststorePath)
                .validateHostName(this.validateHostName)

                // Auth-related client-side configuration
                .credentials(new DefaultCredentials(this.password, this.username))
                .build();

        System.out.println("Done creating a client config.");

        // Everything below depicts the usual flow of reading events. All client-side security configuration is
        // done through the ClientConfig object as shown above.

        EventStreamClientFactory clientFactory = null;
        ReaderGroupManager readerGroupManager = null;
        EventStreamReader<String> reader = null;
        try {
            ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder()
                    .stream(Stream.of(this.scope, this.stream))
                    .disableAutomaticCheckpoints()
                    .build();
            System.out.println("Done creating a reader group config with specified scope: [" +
                    this.scope +"] and stream name: [" + this.stream + "].");

            String readerGroupName = UUID.randomUUID().toString().replace("-", "");
            readerGroupManager = ReaderGroupManager.withScope(this.scope, clientConfig);
            readerGroupManager.createReaderGroup(readerGroupName, readerGroupConfig);
            System.out.println("Done creating a reader group with specified name  and config.");

            clientFactory = EventStreamClientFactory.withScope(this.scope, clientConfig);
            System.out.println("Done creating a client factory with the specified scope and client config.");

            reader = clientFactory.createReader("readerId", readerGroupName,
                    new JavaSerializer<>(), ReaderConfig.builder().build());
            System.out.println("Done creating a reader.");

            String readMessage = reader.readNextEvent(2000).getEvent();
            System.out.println("Done reading an event: [" + readMessage + "].");

        } finally {
            if (reader != null) reader.close();
            if (clientFactory != null) clientFactory.close();
            if (readerGroupManager != null) readerGroupManager.close();
        }
        System.err.println("All done with reading! Exiting...");
    }
 

@Test(timeout = 60000)
public void testBoundedStreamWithScaleTest() throws Exception {
    createScope(SCOPE);
    createStream(STREAM1);

    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, ClientConfig.builder().controllerURI(controllerUri).build());
    @Cleanup
    EventStreamWriter<String> writer1 = clientFactory.createEventWriter(STREAM1, serializer,
            EventWriterConfig.builder().build());
    //Prep the stream with data.
    //1.Write events with event size of 30
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(1)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(2)).get();

    //2.Scale stream
    Map<Double, Double> newKeyRanges = new HashMap<>();
    newKeyRanges.put(0.0, 0.33);
    newKeyRanges.put(0.33, 0.66);
    newKeyRanges.put(0.66, 1.0);
    scaleStream(STREAM1, newKeyRanges);

    //3.Write three events with event size of 30
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(3)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(4)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(5)).get();

    @Cleanup
    ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, controllerUri);

    ReaderGroupConfig readerGroupCfg1 = ReaderGroupConfig.builder().disableAutomaticCheckpoints().groupRefreshTimeMillis(0)
            .stream(Stream.of(SCOPE, STREAM1),
                    //startStreamCut points to the current HEAD of stream
                    StreamCut.UNBOUNDED,
                    //endStreamCut points to the offset after two events.(i.e 2 * 30(event size) = 60)
                    getStreamCut(STREAM1, 60L, 0))
            .build();
    groupManager.createReaderGroup("group", readerGroupCfg1);
    ReaderGroup readerGroup = groupManager.getReaderGroup("group");
    //Create a reader
    @Cleanup
    EventStreamReader<String> reader1 = clientFactory.createReader("readerId1", "group", serializer,
            ReaderConfig.builder().build());

    //2. Verify if endStreamCut configuration is enforced.
    readAndVerify(reader1, 1, 2);
    //The following read should not return events 3, 4 due to the endStreamCut configuration.
    Assert.assertNull("Null is expected", reader1.readNextEvent(2000).getEvent());

    final ReaderGroupConfig readerGroupCfg2 = ReaderGroupConfig.builder().disableAutomaticCheckpoints().groupRefreshTimeMillis(0)
                                                         .stream(Stream.of(SCOPE, STREAM1),
                                                                 getStreamCut(STREAM1, 60L, 0),
                                                                 //endStreamCut points to the offset after two events.(i.e 2 * 30(event size) = 60)
                                                                 getStreamCut(STREAM1, 90L, 1, 2, 3))
                                                         .build();
    readerGroup.resetReaderGroup(readerGroupCfg2);
    verifyReinitializationRequiredException(reader1);

    //Create a reader
    @Cleanup
    EventStreamReader<String> reader2 = clientFactory.createReader("readerId2", "group", serializer,
            ReaderConfig.builder().build());
    assertNull(reader2.readNextEvent(100).getEvent());
    readerGroup.initiateCheckpoint("c1", executor);
    readAndVerify(reader2, 3, 4, 5);
    Assert.assertNull("Null is expected", reader2.readNextEvent(2000).getEvent());
}
 

@Test(timeout = 60000)
public void testBoundedStreamWithTruncationTest() throws Exception {
    createScope(SCOPE);
    createStream(STREAM3);

    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, ClientConfig.builder().controllerURI(controllerUri).build());
    @Cleanup
    EventStreamWriter<String> writer1 = clientFactory.createEventWriter(STREAM3, serializer,
            EventWriterConfig.builder().build());
    //Prep the stream with data.
    //1.Write events with event size of 30
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(1)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(2)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(3)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(4)).get();

    @Cleanup
    ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, controllerUri);
    StreamCut offset30SC = getStreamCut(STREAM3, 30L, 0); // Streamcut pointing to event 2.
    StreamCut offset60SC = getStreamCut(STREAM3, 60L, 0);
    groupManager.createReaderGroup("group", ReaderGroupConfig
            .builder().disableAutomaticCheckpoints()
            .stream(Stream.of(SCOPE, STREAM3),
                    //startStreamCut points to second event in the stream.
                    offset30SC,
                    //endStreamCut points to the offset after two events.(i.e 2 * 30(event size) = 60)
                    offset60SC)
            .build());

    final ReaderGroup rg = groupManager.getReaderGroup("group");

    //Create a reader
    @Cleanup
    EventStreamReader<String> reader = clientFactory.createReader("readerId", "group", serializer,
            ReaderConfig.builder().build());

    //2. Verify if endStreamCut configuration is enforced.
    readAndVerify(reader, 2);
    //The following read should not return events 3, 4 due to the endStreamCut configuration.
    Assert.assertNull("Null is expected", reader.readNextEvent(2000).getEvent());

    truncateStream(STREAM3, offset60SC);

    //Truncation should not affect the reader as it is already post the truncation point.
    Assert.assertNull("Null is expected", reader.readNextEvent(2000).getEvent());

    //Reset RG with startStreamCut which is already truncated.
    rg.resetReaderGroup(ReaderGroupConfig.builder().disableAutomaticCheckpoints()
                                         .stream(Stream.of(SCOPE, STREAM3), offset30SC, StreamCut.UNBOUNDED)
                                         .build());

    verifyReinitializationRequiredException(reader);

    //Create a reader
    @Cleanup
    EventStreamReader<String> reader2 = clientFactory.createReader("readerId2", "group", serializer,
            ReaderConfig.builder().build());

    assertThrows(TruncatedDataException.class, () -> reader2.readNextEvent(10000));
    //subsequent read should return data present post truncation, Event3 is returned here since stream was truncated @ offset 30 * 2.
    readAndVerify(reader2, 3);
}
 

@Test(timeout = 50000)
public void testStreamSeek() throws Exception {

    createScope(SCOPE);
    createStream(STREAM1);
    createStream(STREAM2);

    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, ClientConfig.builder().controllerURI(controllerUri).build());
    @Cleanup
    EventStreamWriter<String> writer1 = clientFactory.createEventWriter(STREAM1, serializer,
            EventWriterConfig.builder().build());

    @Cleanup
    ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, controllerUri);
    groupManager.createReaderGroup("group",
                                   ReaderGroupConfig.builder()
                                                    .disableAutomaticCheckpoints()
                                                    .groupRefreshTimeMillis(0)
                                                    .stream(Stream.of(SCOPE, STREAM1))
                                                    .stream(Stream.of(SCOPE, STREAM2))
                                                    .build());
    @Cleanup
    ReaderGroup readerGroup = groupManager.getReaderGroup("group");

    //Prep the stream with data.
    //1.Write two events with event size of 30
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(1)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(2)).get();

    //2.Scale stream
    Map<Double, Double> newKeyRanges = new HashMap<>();
    newKeyRanges.put(0.0, 0.33);
    newKeyRanges.put(0.33, 0.66);
    newKeyRanges.put(0.66, 1.0);
    scaleStream(STREAM1, newKeyRanges);

    //3.Write three events with event size of 30
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(3)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(4)).get();
    writer1.writeEvent(keyGenerator.get(), getEventData.apply(5)).get();

    //Create a reader
    @Cleanup
    EventStreamReader<String> reader = clientFactory.createReader("readerId", "group", serializer,
            ReaderConfig.builder().build());

    //Offset of a streamCut is always set to zero.
    Map<Stream, StreamCut> streamCut1 = readerGroup.getStreamCuts(); //Stream cut 1
    readAndVerify(reader, 1, 2);
    assertNull(reader.readNextEvent(100).getEvent()); //Sees the segments are empty prior to scaling
    readerGroup.initiateCheckpoint("cp1", executor); //Checkpoint to move past the scale
    readAndVerify(reader, 3, 4, 5); // Old segments are released and new ones can be read
    Map<Stream, StreamCut> streamCut2 = readerGroup.getStreamCuts(); //Stream cut 2

    readerGroup.resetReaderGroup(ReaderGroupConfig.builder().startFromStreamCuts(streamCut1).build()); //reset the readers to offset 0.
    verifyReinitializationRequiredException(reader);

    @Cleanup
    EventStreamReader<String> reader1 = clientFactory.createReader("readerId", "group", serializer,
            ReaderConfig.builder().build());

    //verify that we are at streamCut1
    readAndVerify(reader1, 1, 2);

    readerGroup.resetReaderGroup(ReaderGroupConfig.builder().startFromStreamCuts(streamCut2).build()); // reset readers to post scale offset 0
    verifyReinitializationRequiredException(reader1);

    @Cleanup
    EventStreamReader<String> reader2 = clientFactory.createReader("readerId", "group", serializer,
            ReaderConfig.builder().build());

    //verify that we are at streamCut2
    readAndVerify(reader2, 3, 4, 5);
}
 

/**
 * This test verifies that a event stream reader continues to read events as a result of automatic delegation token
 * renewal, after the initial delegation token it uses expires.
 *
 * We use an extraordinarily high test timeout and read timeouts to account for any inordinate delays that may be
 * encountered in testing environments.
 */
@Test(timeout = 50000)
public void testDelegationTokenGetsRenewedAfterExpiry() throws InterruptedException {
    // Delegation token renewal threshold is 5 seconds, so we are using 6 seconds as Token TTL so that token doesn't
    // get renewed before each use.
    ClusterWrapper pravegaCluster = new ClusterWrapper(true, 6);
    try {
        pravegaCluster.initialize();

        final String scope = "testscope";
        final String streamName = "teststream";
        final int numSegments = 1;

        final ClientConfig clientConfig = ClientConfig.builder()
                .controllerURI(URI.create(pravegaCluster.controllerUri()))
                .credentials(new DefaultCredentials("1111_aaaa", "admin"))
                .build();
        log.debug("Done creating client config.");

        createScopeStream(scope, streamName, numSegments, clientConfig);

        @Cleanup
        final EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);

        // Perform writes on a separate thread.
        Runnable runnable = () -> {
            @Cleanup
            EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName,
                    new JavaSerializer<String>(),
                    EventWriterConfig.builder().build());

            for (int i = 0; i < 10; i++) {
                String msg = "message: " + i;
                writer.writeEvent(msg).join();
                log.debug("Done writing message '{}' to stream '{} / {}'", msg, scope, streamName);
            }
        };
        Thread writerThread = new Thread(runnable);
        writerThread.start();

        // Now, read the events from the stream.

        String readerGroup = UUID.randomUUID().toString().replace("-", "");
        ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder()
                .stream(Stream.of(scope, streamName))
                .disableAutomaticCheckpoints()
                .build();

        @Cleanup
        ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, clientConfig);
        readerGroupManager.createReaderGroup(readerGroup, readerGroupConfig);

        @Cleanup
        EventStreamReader<String> reader = clientFactory.createReader(
                "readerId", readerGroup,
                new JavaSerializer<String>(), ReaderConfig.builder().build());

        int j = 0;
        EventRead<String> event = null;
        do {
            event = reader.readNextEvent(2000);
            if (event.getEvent() != null) {
                log.info("Done reading event: {}", event.getEvent());
                j++;
            }

            // We are keeping sleep time relatively large, just to make sure that the delegation token expires
            // midway.
            Thread.sleep(500);
        } while (event.getEvent() != null);

        // Assert that we end up reading 10 events even though delegation token must have expired midway.
        //
        // To look for evidence of delegation token renewal check the logs for the following message:
        // - "Token is nearing expiry, so refreshing it"
        assertSame(10, j);
    } finally {
        pravegaCluster.close();
    }
}
 

private void testReadWrite() {
    List<URI> ctlURIs = this.controllerInstance.getServiceDetails();
    URI controllerUri = ctlURIs.get(0);

    String scope = "testscope" + RandomStringUtils.randomAlphanumeric(10);
    String stream = "teststream" + RandomStringUtils.randomAlphanumeric(10);

    ClientConfig clientConfig = Utils.buildClientConfig(controllerUri);
    @Cleanup
    StreamManager streamManager = StreamManager.create(clientConfig);
    Assert.assertTrue(streamManager.createScope(scope));

    // Create stream with large number of segments so that most segment containers are used.
    Assert.assertTrue(streamManager.createStream(scope, stream, StreamConfiguration.builder()
            .scalingPolicy(ScalingPolicy.fixed(10))
            .build()));

    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope,
            clientConfig);

    log.info("Invoking writer with controller URI: {}", controllerUri);
    @Cleanup
    EventStreamWriter<Serializable> writer = clientFactory.createEventWriter(stream,
            new JavaSerializer<>(),
            EventWriterConfig.builder().build());

    final int numEvents = 1000;
    final String fixedEvent = "testevent";
    for (int i = 0; i < numEvents; i++) {
        log.debug("Producing event: {} ", fixedEvent);
        writer.writeEvent(String.valueOf(i), fixedEvent);
    }
    writer.flush();

    log.info("Invoking reader with controller URI: {}", controllerUri);
    final String readerGroup = "testreadergroup" + RandomStringUtils.randomAlphanumeric(10);
    ReaderGroupManager groupManager = ReaderGroupManager.withScope(scope, clientConfig);
    groupManager.createReaderGroup(readerGroup,
            ReaderGroupConfig.builder().disableAutomaticCheckpoints().stream(Stream.of(scope, stream)).build());

    @Cleanup
    EventStreamReader<String> reader = clientFactory.createReader(UUID.randomUUID().toString(),
            readerGroup,
            new JavaSerializer<>(),
            ReaderConfig.builder().build());
    for (int i = 0; i < numEvents; i++) {
        try {
            String event = reader.readNextEvent(60000).getEvent();
            Assert.assertEquals(fixedEvent, event);
        } catch (ReinitializationRequiredException e) {
            log.error("Unexpected request to reinitialize {}", e);
            throw new IllegalStateException("Unexpected request to reinitialize");
        }
    }
}
 

/**
 * This test checks the behavior of a reader (or group of readers) that gets a delete event while reading. While the
 * client is reading events (Segment Store) the test deletes the Stream (Controller and metadata). Once the client
 * reads all the events and reaches the end of segment, it contacts the Controller to retrieve subsequent segments
 * (if any). However, the Stream-related metadata to answer this request has been previously deleted.
 */
//@Ignore //TODO: The controller does not currently handle the stream being deleted properly.
//Once it does so the client will need to throw an appropriate exception, and this test should reflect it.
@Test(timeout = 20000)
public void testDeleteStreamWhileReading() {
    final String scope = "truncationTests";
    final String streamName = "testDeleteStreamWhileReading";
    final String readerGroup = "RGTestDeleteStreamWhileReading";
    final int totalEvents = 100;
    final int parallelism = 1;

    StreamConfiguration streamConfiguration = StreamConfiguration.builder()
                                                                 .scalingPolicy(ScalingPolicy.fixed(parallelism))
                                                                 .build();
    StreamManager streamManager = StreamManager.create(controllerURI);
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, streamConfiguration);
    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, ClientConfig.builder().controllerURI(controllerURI).build());

    // Write totalEvents to the Stream.
    writeEvents(clientFactory, streamName, totalEvents);

    // Instantiate readers to consume from Stream.
    @Cleanup
    ReaderGroupManager groupManager = ReaderGroupManager.withScope(scope, controllerURI);
    groupManager.createReaderGroup(readerGroup, ReaderGroupConfig.builder().automaticCheckpointIntervalMillis(500).stream(Stream.of(scope, streamName)).build());
    EventStreamReader<String> reader = clientFactory.createReader(String.valueOf(0), readerGroup, new UTF8StringSerializer(), ReaderConfig.builder().build());
    assertEquals(totalEvents / 2, ReadWriteUtils.readEvents(reader, totalEvents / 2, 0));
    reader.close();
    
    val readerRecreated = clientFactory.createReader(String.valueOf(0), readerGroup, new JavaSerializer<>(), ReaderConfig.builder().build());
    
    assertTrue(streamManager.sealStream(scope, streamName));
    assertTrue(streamManager.deleteStream(scope, streamName));

    assertThrows(InvalidStreamException.class, () -> 
        clientFactory.createReader(String.valueOf(1), readerGroup, new JavaSerializer<>(), ReaderConfig.builder().build())
    );

    // At the control plane, we expect a RetriesExhaustedException as readers try to get successor segments from a deleted stream.
    assertThrows(TruncatedDataException.class,
                 () -> ReadWriteUtils.readEvents(readerRecreated, totalEvents / 2, 0));
    assertTrue(!streamManager.deleteStream(scope, streamName));
}
 

@Test(timeout = 30000)
public void testGenerateStreamCuts() throws Exception {
    final Stream stream = Stream.of(SCOPE, STREAM);
    final String group = "group";

    createScope(SCOPE);
    createStream(SCOPE, STREAM, ScalingPolicy.fixed(1));

    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, ClientConfig.builder().controllerURI(controllerURI).build());
    @Cleanup
    EventStreamWriter<String> writer = clientFactory.createEventWriter(STREAM, serializer,
                                                                       EventWriterConfig.builder().build());
    //Prep the stream with data.
    //1.Write events with event size of 30
    writer.writeEvent(randomKeyGenerator.get(), getEventData.apply(1)).join();
    writer.writeEvent(randomKeyGenerator.get(), getEventData.apply(2)).join();
    writer.writeEvent(randomKeyGenerator.get(), getEventData.apply(3)).join();
    writer.writeEvent(randomKeyGenerator.get(), getEventData.apply(4)).join();

    @Cleanup
    ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, controllerURI);
    groupManager.createReaderGroup(group, ReaderGroupConfig
            .builder().disableAutomaticCheckpoints().groupRefreshTimeMillis(1000)
            .stream(stream)
            .build());

    ReaderGroup readerGroup = groupManager.getReaderGroup(group);

    //Create a reader
    @Cleanup
    EventStreamReader<String> reader = clientFactory.createReader("readerId", group, serializer,
                                                                  ReaderConfig.builder().build());

    readAndVerify(reader, 1);
    @Cleanup("shutdown")
    InlineExecutor backgroundExecutor = new InlineExecutor();
    CompletableFuture<Map<Stream, StreamCut>> sc = readerGroup.generateStreamCuts(backgroundExecutor);
    // The reader group state will be updated after 1 second.
    TimeUnit.SECONDS.sleep(1);
    EventRead<String> data = reader.readNextEvent(15000);
    assertTrue(Futures.await(sc)); // wait until the streamCut is obtained.

    //expected segment 0 offset is 30L.
    Map<Segment, Long> expectedOffsetMap = ImmutableMap.of(getSegment(0, 0), 30L);
    Map<Stream, StreamCut> scMap = sc.join();

    assertEquals("StreamCut for a single stream expected", 1, scMap.size());
    assertEquals("StreamCut pointing ot offset 30L expected", new StreamCutImpl(stream, expectedOffsetMap), scMap.get(stream));
}
 

@Test(timeout = 30000)
public void testReaderOfflineWithSilentCheckpoint() throws Exception {
    final Stream stream = Stream.of(SCOPE, STREAM);
    final String group = "group";

    @Cleanup("shutdown")
    InlineExecutor backgroundExecutor = new InlineExecutor();

    createScope(SCOPE);
    createStream(SCOPE, STREAM, ScalingPolicy.fixed(1));

    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(SCOPE, ClientConfig.builder().controllerURI(controllerURI).build());
    @Cleanup
    EventStreamWriter<String> writer = clientFactory.createEventWriter(STREAM, serializer,
                                                                       EventWriterConfig.builder().build());
    //Prep the stream with data.
    //1.Write events with event size of 30
    writer.writeEvent(randomKeyGenerator.get(), getEventData.apply(1)).join();
    writer.writeEvent(randomKeyGenerator.get(), getEventData.apply(2)).join();
    writer.writeEvent(randomKeyGenerator.get(), getEventData.apply(3)).join();
    writer.writeEvent(randomKeyGenerator.get(), getEventData.apply(4)).join();

    @Cleanup
    ReaderGroupManager groupManager = ReaderGroupManager.withScope(SCOPE, controllerURI);
    groupManager.createReaderGroup(group, ReaderGroupConfig
            .builder().disableAutomaticCheckpoints().groupRefreshTimeMillis(1000)
            .stream(stream)
            .build());

    ReaderGroup readerGroup = groupManager.getReaderGroup(group);

    //Create a reader
    @Cleanup
    EventStreamReader<String> reader = clientFactory.createReader("readerId", group, serializer,
                                                                  ReaderConfig.builder().build());

    //2. Read an event.
    readAndVerify(reader, 1);

    //3. Trigger a checkpoint and verify it is completed.
    CompletableFuture<Checkpoint> checkpoint = readerGroup.initiateCheckpoint("chk1", backgroundExecutor);
    // The reader group state will be updated after 1 second.
    TimeUnit.SECONDS.sleep(1);
    EventRead<String> data = reader.readNextEvent(15000);
    assertTrue(data.isCheckpoint());
    readAndVerify(reader, 2);
    assertTrue("Checkpointing should complete successfully", Futures.await(checkpoint));

    //4. GenerateStreamCuts and validate the offset of stream cut.
    CompletableFuture<Map<Stream, StreamCut>> sc = readerGroup.generateStreamCuts(backgroundExecutor);
    // The reader group state will be updated after 1 second.
    TimeUnit.SECONDS.sleep(1);
    data = reader.readNextEvent(15000);
    assertTrue("StreamCut generation should complete successfully", Futures.await(sc));
    //expected segment 0 offset is 60L, since 2 events are read.
    Map<Segment, Long> expectedOffsetMap = ImmutableMap.of(getSegment(0, 0), 60L);
    Map<Stream, StreamCut> scMap = sc.join();
    assertEquals("StreamCut for a single stream expected", 1, scMap.size());
    assertEquals("StreamCut pointing ot offset 30L expected", new StreamCutImpl(stream, expectedOffsetMap),
                 scMap.get(stream));

    //5. Invoke readerOffline with last position as null. The newer readers should start reading
    //from the last checkpointed position
    readerGroup.readerOffline("readerId", null);
    @Cleanup
    EventStreamReader<String> reader1 = clientFactory.createReader("readerId", group, serializer,
                                                                   ReaderConfig.builder().build());
    readAndVerify(reader1, 2);
}