Java Code Examples for io.pravega.client.stream.EventRead#getEvent()

@Override
public T map(T value) throws Exception {
    if (this.sideStreamReader == null) {
        this.sideStreamReader = getIntegerReader();
    }
    EventRead<Integer> rule = sideStreamReader.readNextEvent(50);
    if (rule.getEvent() != null) {
        log.info("Mapper: received side stream event: {}", rule.getEvent());
        /*
         * Event == 1, continue process original events
         * Event == 2, trigger an exception (simulate failure) and reset the writer thread and start processing all the records
         */
        if (rule.getEvent() == 2) {
            RESUME_WRITE_HANDLER.get().run();
            throw new IntentionalException("artificial test failure");
        }
    }
    return value;
}
 

@SneakyThrows
private void readNextItem(Consumer<ReadItem> eventHandler) {
    int readAttempts = MAX_READ_ATTEMPTS;
    long timeoutMillis = ClientReader.this.testConfig.getTimeout().toMillis();
    while (readAttempts-- > 0) {
        EventRead<byte[]> readResult = READ_RETRY.run(() -> getReader().readNextEvent(timeoutMillis));
        if (readResult.getEvent() == null && readAttempts > 0) {
            // EventStreamReader.readNextEvent() will return null if we get no new events within the given timeout.
            // Retry the read up to the maximum allowed number of times before giving up.
            Thread.sleep(timeoutMillis / MAX_READ_ATTEMPTS);
        } else if (readResult.getEvent() == null) {
            // We are done.
            close();
            return;
        } else {
            StreamReadItem readItem = toReadItem(readResult.getEvent(), readResult.getEventPointer());
            eventHandler.accept(readItem);
            return;
        }
    }

}
 

private <T extends Serializable> int createReaderAndReadEvents(ReaderGroupManager rgMgr, EventStreamClientFactory clientFactory,
                                                               String rGroupId, int readerIndex) {
    // create a reader.
    EventStreamReader<T> reader = clientFactory.createReader(rGroupId + "-" + readerIndex, rGroupId, new JavaSerializer<>(),
                                                             ReaderConfig.builder().build());
    EventRead<T> event = null;
    int validEvents = 0;
    AtomicBoolean sealedSegmentUpdated = new AtomicBoolean(false);
    try {
        do {
            try {
                event = reader.readNextEvent(READ_TIMEOUT);
                log.debug("Read event result in readEvents: {}.", event.getEvent());
                if (event.getEvent() == null && !event.isCheckpoint() && !sealedSegmentUpdated.get()) {
                    // initiate a checkpoint to ensure all sealed segments are acquired by the reader.
                    ReaderGroup readerGroup = rgMgr.getReaderGroup(rGroupId);
                    readerGroup.initiateCheckpoint("chkPoint", chkPointExecutor)
                               .whenComplete((checkpoint, t) -> {
                                   if (t != null) {
                                       log.error("Checkpoint operation failed", t);
                                   } else {
                                       log.info("Checkpoint {} completed", checkpoint);
                                       sealedSegmentUpdated.set(true);
                                   }
                               });
                }
                if (event.getEvent() != null) {
                    validEvents++;
                }
            } catch (ReinitializationRequiredException e) {
                log.error("Reinitialization Exception while reading event using readerId: {}", reader, e);
                fail("Reinitialization Exception is not expected");
            }
        } while (event.getEvent() != null || event.isCheckpoint() || !sealedSegmentUpdated.get());
    } finally {
        closeReader(reader);
    }
    return validEvents;
}
 

private <T> int readEvents(EventStreamReader<T> reader, int limit, boolean reinitializationExpected) {
    EventRead<T> event = null;
    int validEvents = 0;
    boolean reinitializationRequired = false;
    try {
        do {
            try {
                event = reader.readNextEvent(READ_TIMEOUT);
                log.debug("Read event result in readEvents: {}.", event.getEvent());
                if (event.getEvent() != null) {
                    validEvents++;
                }
                reinitializationRequired = false;
            } catch (ReinitializationRequiredException e) {
                log.error("Exception while reading event using readerId: {}", reader, e);
                if (reinitializationExpected) {
                    reinitializationRequired = true;
                } else {
                    fail("Reinitialization Exception is not expected");
                }
            }
        } while (reinitializationRequired || ((event.getEvent() != null || event.isCheckpoint()) && validEvents < limit));
    } finally {
        closeReader(reader);
    }

    return validEvents;
}
 

/**
 * Reads events and puts them in a queue for later checking (potentially by another thread).
 */
private void readAndQueueEvents(EventStreamReader<String> reader, int eventsToWrite, Queue<Entry<Integer, PositionImpl>> readEventsPositions) {
    int eventCount = 1;
    for (int i = 0; i < eventsToWrite; i++) {
        final EventRead<String> event = reader.readNextEvent(1000);
        if (event.getEvent() != null && !event.isCheckpoint()) {
            // The reader should own only 1 segment.
            readEventsPositions.add(new AbstractMap.SimpleEntry<>(eventCount, (PositionImpl) event.getPosition()));
            eventCount++;
        }
    }
}
 

@Test(timeout = 10000)
public void testEndOfSegmentWithoutSuccessors() throws SegmentSealedException, ReaderNotInReaderGroupException {
    AtomicLong clock = new AtomicLong();
    MockSegmentStreamFactory segmentStreamFactory = new MockSegmentStreamFactory();
    Orderer orderer = new Orderer();
    ReaderGroupStateManager groupState = Mockito.mock(ReaderGroupStateManager.class);
    EventStreamReaderImpl<byte[]> reader = new EventStreamReaderImpl<>(segmentStreamFactory, segmentStreamFactory,
                                                                       new ByteArraySerializer(), groupState,
                                                                       orderer, clock::get,
                                                                       ReaderConfig.builder().build(),
                                                                       createWatermarkReaders(),
                                                                       Mockito.mock(Controller.class));
    Segment segment = Segment.fromScopedName("Foo/Bar/0");
    Mockito.when(groupState.acquireNewSegmentsIfNeeded(eq(0L), any()))
           .thenReturn(ImmutableMap.of(new SegmentWithRange(segment, 0, 1), 0L))
           .thenReturn(Collections.emptyMap());
    Mockito.when(groupState.getEndOffsetForSegment(any(Segment.class))).thenReturn(Long.MAX_VALUE);
    Mockito.when(groupState.handleEndOfSegment(any())).thenReturn(true);
    SegmentOutputStream stream = segmentStreamFactory.createOutputStreamForSegment(segment, segmentSealedCallback, writerConfig,
            DelegationTokenProviderFactory.createWithEmptyToken());
    ByteBuffer buffer = writeInt(stream, 1);
    EventRead<byte[]> read = reader.readNextEvent(0);
    byte[] event = read.getEvent();
    assertEquals(buffer, ByteBuffer.wrap(event));
    read = reader.readNextEvent(0);
    assertNull(read.getEvent());
    read = reader.readNextEvent(0);
    assertNull(read.getEvent());
    assertEquals(0, reader.getReaders().size());
    assertEquals(1, reader.getRanges().size());
    Mockito.when(groupState.getCheckpoint()).thenReturn("CP1");
    read = reader.readNextEvent(0);
    assertTrue(read.isCheckpoint());
    read = reader.readNextEvent(0);
    assertNull(read.getEvent());
    assertEquals(0, reader.getRanges().size());
    reader.close();
}
 

/**
 * This method continuously performs two tasks: first, it reads events that are being written by console writer
 * or by any other process in that stream. Second, it creates a new StreamCut after every read event. The new
 * {@link StreamCut} represents the current tail of the {@link Stream} and it may be used to read events to or from
 * that position in the {@link Stream}.
 */
public void run() {
    final ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder().disableAutomaticCheckpoints()
                                                                 .stream(Stream.of(scope, streamName)).build();

    try (ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, controllerURI);
         EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope,
                 ClientConfig.builder().controllerURI(controllerURI).build())) {

        // Create the ReaderGroup to which readers will belong to.
        readerGroupManager.createReaderGroup(readerGroupName, readerGroupConfig);
        @Cleanup
        ReaderGroup readerGroup = readerGroupManager.getReaderGroup(readerGroupName);

        EventStreamReader<String> reader = clientFactory.createReader("backgroundReader", readerGroupName,
                new JavaSerializer<>(), ReaderConfig.builder().build());
        EventRead<String> event;

        // Start main loop to continuously read and display events written to the scope/stream.
        log.info("Start reading events from {}/{}.", scope, streamName);
        do {
            event = reader.readNextEvent(READER_TIMEOUT_MS);
            if (event.getEvent() != null) {
                // TODO: Problem finding logback.xml in Pravega example applications (Issue #87).
                System.out.println("[BackgroundReader] Read event: " + event.getEvent());
                log.info("[BackgroundReader] Read event: {}.", event.getEvent());
            }

            // Update the StreamCut after every event read, just in case the user wants to use it.
            if (!event.isCheckpoint()) {
                readerGroup.initiateCheckpoint("myCheckpoint" + System.nanoTime(), executor)
                           .thenAccept(checkpoint -> lastStreamCut.set(checkpoint.asImpl().getPositions()));
            }
        } while (!end.get());
    } catch (ReinitializationRequiredException e) {
        // We do not expect this Exception from the reader in this situation, so we leave.
        log.error("Non-expected reader re-initialization.");
    }
}
 

public void run() {
    setRunning(true);
    final String readerGroup = UUID.randomUUID().toString().replace("-", "");
    final ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder()
            .stream(Stream.of(scope, streamName))
            .build();

    try (ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, controllerURI)) {
        readerGroupManager.createReaderGroup(readerGroup, readerGroupConfig);
    }

    try (EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, ClientConfig.builder().controllerURI(controllerURI).build());
         EventStreamReader<T> reader = clientFactory.createReader("reader",
                 readerGroup, serializer, ReaderConfig.builder().build())) {

        while (isRunning()) {
            try {
                EventRead<T> event = reader.readNextEvent(READER_TIMEOUT_MS);
                T eventData = event.getEvent();
                if (eventData != null) {
                    onNext.accept(event.getEvent());
                }
            }
            catch (ReinitializationRequiredException e) {
                onError.accept(e);
            }
        }
    }
}
 

/**
 * This method is an example of bounded processing in Pravega with {@link StreamCut}s. {@link ReaderGroupConfig}
 * contains the information related to the {@link Stream}s to be read as well as the (optional) user-defined
 * boundaries in the form of {@link StreamCut}s that will limit the events to be read by reader processes. Note that
 * event readers (i.e., {@link EventStreamReader}) are agnostic to any notion of boundaries and they do not interact
 * with {@link StreamCut}s; they only consume events, which will be bounded within specific {@link Stream} slices as
 * configured in {@link ReaderGroupConfig}. The method basically creates a string representation of the events read
 * from {@link Stream}s within the bounds defined in the configuration parameter.
 *
 * @param config Configuration for the {@link ReaderGroup}, possibly containing {@link StreamCut} boundaries for
 *               limiting the number of events to read.
 * @return String representation of the events read by the reader.
 */
public String printBoundedStreams(ReaderGroupConfig config) {
    StringBuilder result = new StringBuilder();
    final String randomId = String.valueOf(new Random(System.nanoTime()).nextInt());
    try (ReaderGroupManager manager = ReaderGroupManager.withScope(scope, controllerURI);
         EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope,
                 ClientConfig.builder().controllerURI(controllerURI).build())) {
        final String readerGroupName = "RG" + randomId;
        manager.createReaderGroup(readerGroupName, config);
        @Cleanup
        EventStreamReader<String> reader = clientFactory.createReader(randomId, readerGroupName,
                new JavaSerializer<>(), ReaderConfig.builder().build());

        // Write dummy events that identify each Stream.
        EventRead<String> event;
        do {
            event = reader.readNextEvent(1000);
            if (event.getEvent() != null) {
                result = result.append(event.getEvent()).append('|');
            }

        } while (event.isCheckpoint() || event.getEvent() != null);

        result = result.append('\n');
    } catch (ReinitializationRequiredException e) {
        // We do not expect this Exception from the reader in this situation, so we leave.
        log.error("Non-expected reader re-initialization.");
    }
    return result.toString();
}
 

private <T extends Serializable> List<EventRead<T>> readEvents(final String scope, final String readerId) {
    List<EventRead<T>> events = new ArrayList<>();

    final ClientConfig clientConfig = Utils.buildClientConfig(controllerURI);

    try (EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);
         EventStreamReader<T> reader = clientFactory.createReader(readerId,
                 READER_GROUP_NAME,
                 new JavaSerializer<T>(),
                 readerConfig)) {
        log.info("Reading events from {}/{} with readerId: {}", scope, STREAM, readerId);
        EventRead<T> event = null;
        do {
            try {
                event = reader.readNextEvent(READ_TIMEOUT);
                if (event.getEvent() != null) {
                    log.info("Read event {}", event.getEvent());
                    events.add(event);
                }
                if (event.isCheckpoint()) {
                    log.info("Read a check point event, checkpointName: {}", event.getCheckpointName());
                }
            } catch (ReinitializationRequiredException e) {
                log.error("Exception while reading event using readerId: {}", readerId, e);
                fail("Reinitialization Exception is not expected");
            }
        } while (event.isCheckpoint() || event.getEvent() != null);
        //stop reading if event read(non-checkpoint) is null.
        log.info("No more events from {}/{} for readerId: {}", scope, STREAM, readerId);
    } //reader.close() will automatically invoke ReaderGroup#readerOffline(String, Position)
    return events;
}
 

@Override
public IntegerWithEventPointer extractEvent(EventRead<IntegerWithEventPointer> eventRead) {
    if (!includeMetadata) {
        return super.extractEvent(eventRead);
    }

    IntegerWithEventPointer event = eventRead.getEvent();
    event.setEventPointer(eventRead.getEventPointer());
    return event;
}
 

/**
 * A {@link StreamCut} is a collection of offsets, one for each open segment of a set of {@link Stream}s, which
 * indicates an event boundary. With a {@link StreamCut}, users can instruct readers to read from and/or up to a
 * particular event boundary (e.g., read events from 100 to 200, events created since Tuesday) on multiple
 * {@link Stream}s. To this end, Pravega allows us to create {@link StreamCut}s while readers are reading. In this
 * method, we read create two {@link StreamCut}s for a {@link Stream} according to the initial and final event
 * indexes passed by parameter.
 *
 * @param streamName Name of the {@link Stream} from which {@link StreamCut}s will be created.
 * @param iniEventIndex Index of the initial boundary for the {@link Stream} slice to process.
 * @param endEventIndex Index of the final boundary for the {@link Stream} slice to process.
 * @return Initial and final {@link Stream} boundaries represented as {@link StreamCut}s.
 */
public List<StreamCut> createStreamCutsByIndexFor(String streamName, int iniEventIndex, int endEventIndex) {
    // Create the StreamCuts for the streams.
    final List<StreamCut> streamCuts = new ArrayList<>();
    final String randomId = String.valueOf(new Random(System.nanoTime()).nextInt());

    // Free resources after execution.
    try (ReaderGroupManager manager = ReaderGroupManager.withScope(scope, controllerURI);
         EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope,
                 ClientConfig.builder().controllerURI(controllerURI).build())) {

        // Create a reader group and a reader to read from the stream.
        final String readerGroupName = streamName + randomId;
        ReaderGroupConfig config = ReaderGroupConfig.builder().stream(Stream.of(scope, streamName)).build();
        manager.createReaderGroup(readerGroupName, config);
        @Cleanup
        ReaderGroup readerGroup = manager.getReaderGroup(readerGroupName);
        @Cleanup
        EventStreamReader<String> reader = clientFactory.createReader(randomId, readerGroup.getGroupName(),
                new JavaSerializer<>(), ReaderConfig.builder().build());

        // Read streams and create the StreamCuts during the read process.
        int eventIndex = 0;
        EventRead<String> event;
        do {
            // Here is where we create a StreamCut that points to the event indicated by the user.
            if (eventIndex == iniEventIndex || eventIndex == endEventIndex) {
                reader.close();
                streamCuts.add(readerGroup.getStreamCuts().get(Stream.of(scope, streamName)));
                reader = clientFactory.createReader(randomId, readerGroup.getGroupName(),
                        new JavaSerializer<>(), ReaderConfig.builder().build());
            }

            event = reader.readNextEvent(1000);
            eventIndex++;
        } while (event.isCheckpoint() || event.getEvent() != null);

        // If there is only the initial StreamCut, this means that the final one is the tail of the stream.
        if (streamCuts.size() == 1) {
            streamCuts.add(StreamCut.UNBOUNDED);
        }
    } catch (ReinitializationRequiredException e) {
        // We do not expect this Exception from the reader in this situation, so we leave.
        log.error("Non-expected reader re-initialization.");
    }
    return streamCuts;
}
 

private int readInt(EventRead<byte[]> eventRead) {
    byte[] event = eventRead.getEvent();
    assertNotNull(event);
    return ByteBuffer.wrap(event).getInt();
}
 

@Test(timeout = 60000)
public void testStreamcutsUpdateInReaderGroup() throws Exception {
    final String scope = "testStreamcutsUpdateInReaderGroup";
    final String stream = "myStream";
    final String readerGroupName = "testStreamcutsUpdateInReaderGroupRG";
    final int checkpointingIntervalMs = 2000;
    final int readerSleepInterval = 250;
    final int numEvents = 100;

    // First, create the stream.
    StreamManager streamManager = StreamManager.create(controllerURI);
    Assert.assertTrue(streamManager.createScope(scope));
    StreamConfiguration streamConfiguration = StreamConfiguration.builder()
                                                                 .scalingPolicy(ScalingPolicy.fixed(2))
                                                                 .build();
    streamManager.createStream(scope, stream, streamConfiguration);

    // Write some events in the stream.
    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, ClientConfig.builder().controllerURI(controllerURI).build());
    writeEvents(clientFactory, stream, numEvents);

    // Read the events and test that positions are getting updated.
    ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder()
                                                           .stream(Stream.of(scope, stream))
                                                           .automaticCheckpointIntervalMillis(checkpointingIntervalMs)
                                                           .build();

    @Cleanup
    ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, controllerURI);
    readerGroupManager.createReaderGroup(readerGroupName, readerGroupConfig);
    ReaderGroup readerGroup = readerGroupManager.getReaderGroup(readerGroupName);
    @Cleanup
    EventStreamReader<Double> reader = clientFactory.createReader("myReader", readerGroupName,
            new JavaSerializer<>(), ReaderConfig.builder().build());

    Map<Stream, StreamCut> currentStreamcuts = readerGroup.getStreamCuts();
    EventRead eventRead;
    int lastIteration = 0, iteration = 0;
    int assertionFrequency = checkpointingIntervalMs / readerSleepInterval;
    do {
        eventRead = reader.readNextEvent(5000);

        // Check that the streamcuts are being updated periodically via automatic reader group checkpoints.
        if (iteration != lastIteration && iteration % assertionFrequency == 0) {
            log.info("Comparing streamcuts: {} / {} in iteration {}.", currentStreamcuts, readerGroup.getStreamCuts(), iteration);
            Assert.assertNotEquals(currentStreamcuts, readerGroup.getStreamCuts());
            currentStreamcuts = readerGroup.getStreamCuts();
            lastIteration = iteration;
        }

        Thread.sleep(readerSleepInterval);
        if (!eventRead.isCheckpoint()) {
            iteration++;
        }
    } while ((eventRead.isCheckpoint() || eventRead.getEvent() != null) && iteration < numEvents);
}
 

public static void main(String[] args) throws Exception {
    // Initialize the parameter utility tool in order to retrieve input parameters.
    ParameterTool params = ParameterTool.fromArgs(args);

    // The writer will contact with the Pravega controller to get information about streams.
    URI pravegaControllerURI = URI.create(params.get(Constants.CONTROLLER_ADDRESS_PARAM, Constants.CONTROLLER_ADDRESS));
    PravegaConfig pravegaConfig = PravegaConfig
            .fromParams(params)
            .withControllerURI(pravegaControllerURI)
            .withDefaultScope(Constants.DEFAULT_SCOPE);

    // Create the scope if it is not present.
    StreamManager streamManager = StreamManager.create(pravegaControllerURI);
    streamManager.createScope(Constants.DEFAULT_SCOPE);

    // We will read from the stream slices published by StreamBookmarker.
    ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder()
                                                           .stream(Stream.of(Constants.DEFAULT_SCOPE, Constants.STREAMCUTS_STREAM))
                                                           .build();

    // Instantiate the reader group manager to create the reader group and the client factory to create readers.
    try (ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(Constants.DEFAULT_SCOPE, pravegaControllerURI);
         EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(Constants.DEFAULT_SCOPE,
                 ClientConfig.builder().controllerURI(pravegaControllerURI).build())) {

        // Create the reader group to read the stream slices.
        readerGroupManager.createReaderGroup(READER_GROUP_NAME, readerGroupConfig);
        EventStreamReader<SensorStreamSlice> sliceReader = clientFactory.createReader("sliceReader", READER_GROUP_NAME,
                new JavaSerializer<>(), ReaderConfig.builder().build());

        // The application locally executes bounded batch jobs for every slice received from StreamBookmarker. Note
        // that this is only for simplifying the demo and have the three processes working in a loop; but, in a real
        // setting, we could pass a representation of the SensorStreamSlice object as input argument for a batch job.
        EventRead<SensorStreamSlice> sliceToAnalyze;
        do {
            sliceToAnalyze = sliceReader.readNextEvent(READER_TIMEOUT_MS);

            // If we got a new stream slice to process, run a new batch job on it.
            if (sliceToAnalyze.getEvent() != null) {
                LOG.warn("Running batch job for slice: {}.", sliceToAnalyze.getEvent());
                triggerBatchJobOnSlice(pravegaConfig, sliceToAnalyze.getEvent());
            }
        } while (sliceToAnalyze.isCheckpoint() || sliceToAnalyze.getEvent() != null);

        sliceReader.close();
    }
}
 

/**
 * This test verifies that a event stream reader continues to read events as a result of automatic delegation token
 * renewal, after the initial delegation token it uses expires.
 *
 * We use an extraordinarily high test timeout and read timeouts to account for any inordinate delays that may be
 * encountered in testing environments.
 */
@Test(timeout = 50000)
public void testDelegationTokenGetsRenewedAfterExpiry() throws InterruptedException {
    // Delegation token renewal threshold is 5 seconds, so we are using 6 seconds as Token TTL so that token doesn't
    // get renewed before each use.
    ClusterWrapper pravegaCluster = new ClusterWrapper(true, 6);
    try {
        pravegaCluster.initialize();

        final String scope = "testscope";
        final String streamName = "teststream";
        final int numSegments = 1;

        final ClientConfig clientConfig = ClientConfig.builder()
                .controllerURI(URI.create(pravegaCluster.controllerUri()))
                .credentials(new DefaultCredentials("1111_aaaa", "admin"))
                .build();
        log.debug("Done creating client config.");

        createScopeStream(scope, streamName, numSegments, clientConfig);

        @Cleanup
        final EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scope, clientConfig);

        // Perform writes on a separate thread.
        Runnable runnable = () -> {
            @Cleanup
            EventStreamWriter<String> writer = clientFactory.createEventWriter(streamName,
                    new JavaSerializer<String>(),
                    EventWriterConfig.builder().build());

            for (int i = 0; i < 10; i++) {
                String msg = "message: " + i;
                writer.writeEvent(msg).join();
                log.debug("Done writing message '{}' to stream '{} / {}'", msg, scope, streamName);
            }
        };
        Thread writerThread = new Thread(runnable);
        writerThread.start();

        // Now, read the events from the stream.

        String readerGroup = UUID.randomUUID().toString().replace("-", "");
        ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder()
                .stream(Stream.of(scope, streamName))
                .disableAutomaticCheckpoints()
                .build();

        @Cleanup
        ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, clientConfig);
        readerGroupManager.createReaderGroup(readerGroup, readerGroupConfig);

        @Cleanup
        EventStreamReader<String> reader = clientFactory.createReader(
                "readerId", readerGroup,
                new JavaSerializer<String>(), ReaderConfig.builder().build());

        int j = 0;
        EventRead<String> event = null;
        do {
            event = reader.readNextEvent(2000);
            if (event.getEvent() != null) {
                log.info("Done reading event: {}", event.getEvent());
                j++;
            }

            // We are keeping sleep time relatively large, just to make sure that the delegation token expires
            // midway.
            Thread.sleep(500);
        } while (event.getEvent() != null);

        // Assert that we end up reading 10 events even though delegation token must have expired midway.
        //
        // To look for evidence of delegation token renewal check the logs for the following message:
        // - "Token is nearing expiry, so refreshing it"
        assertSame(10, j);
    } finally {
        pravegaCluster.close();
    }
}
 

@Test(timeout = 100000)
public void testOrder() throws Exception {
    final AtomicBoolean done = new AtomicBoolean(false);

    CompletableFuture<Void> writer1 = startWriter("1", clientFactory, done);
    CompletableFuture<Void> writer2 = startWriter("2", clientFactory, done);
    CompletableFuture<Void> writer3 = startWriter("3", clientFactory, done);
    CompletableFuture<Void> writer4 = startWriter("4", clientFactory, done);

    // perform multiple scale stream operations so that rolling transactions may happen
    Stream s = new StreamImpl("test", "test");
    Map<Double, Double> map = new HashMap<>();
    map.put(0.0, 1.0);
    ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();

    controller.scaleStream(s, Collections.singletonList(0L), map, executor).getFuture().get();

    controller.scaleStream(s, Collections.singletonList(NameUtils.computeSegmentId(1, 1)), map, executor).getFuture().get();

    controller.scaleStream(s, Collections.singletonList(NameUtils.computeSegmentId(2, 2)), map, executor).getFuture().get();

    // stop writers
    done.set(true);

    CompletableFuture.allOf(writer1, writer2, writer3, writer4).join();

    // wait for all transactions to commit
    Futures.allOf(eventToTxnMap.entrySet().stream()
                               .map(x -> waitTillCommitted(controller, s, x.getValue(), uncommitted)).collect(Collectors.toList())).join();

    assertTrue(uncommitted.isEmpty());
    // read all events using a single reader and verify the order

    List<Triple<Integer, UUID, String>> eventOrder = new LinkedList<>();
    // create a reader
    while (!eventToTxnMap.isEmpty()) {
        EventRead<Integer> integerEventRead = reader.readNextEvent(SECONDS.toMillis(60));
        if (integerEventRead.getEvent() != null) {
            int event1 = integerEventRead.getEvent();
            UUID txnId = eventToTxnMap.remove(event1);
            String writerId = txnToWriter.get(txnId);
            UUID first = writersList.get(writerId).remove(0);
            eventOrder.add(new ImmutableTriple<>(event1, txnId, writerId));
            assertEquals(first, txnId);
        }
    }
}
 

@Test(timeout = 30000)
public void testCheckpointsOccur() throws ReinitializationRequiredException, DurableDataLogException {
    String endpoint = "localhost";
    String streamName = "abc";
    String readerName = "reader";
    String readerGroup = "group";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world: ";
    String scope = "Scope1";
    @Cleanup
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    @Cleanup
    PravegaConnectionListener server = new PravegaConnectionListener(false, port, store, mock(TableStore.class),
            serviceBuilder.getLowPriorityExecutor());
    server.startListening();
    @Cleanup
    MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
    MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder()
                                                     .automaticCheckpointIntervalMillis(10000)
                                                     .stream(Stream.of(scope, streamName))
                                                     .build();
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, null);
    streamManager.createReaderGroup(readerGroup, groupConfig);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    populateEvents(streamName, testString, clientFactory, serializer);

    AtomicLong fakeClock = new AtomicLong(0);
    @Cleanup
    EventStreamReader<String> reader = clientFactory.createReader(readerName, readerGroup, serializer,
                                                                  ReaderConfig.builder().build(),
                                                                  () -> fakeClock.get(),
                                                                  () -> fakeClock.get() / NANOS_PER_SECOND);
    int numRead = 0;
    int checkpointCount = 0;
    while (numRead < 100) {
        fakeClock.addAndGet(NANOS_PER_SECOND);
        EventRead<String> event = reader.readNextEvent(1000);
        if (event.isCheckpoint()) {
            checkpointCount++;
        } else {
            String message = event.getEvent();
            assertEquals(testString + numRead, message);
            numRead++;
        }
    }
    assertTrue("Count was " + checkpointCount, checkpointCount > 5);
    assertTrue("Count was " + checkpointCount, checkpointCount < 20);
}
 

@Test(timeout = 30000)
public void testOnlyOneOutstanding() throws ReinitializationRequiredException, DurableDataLogException {
    String endpoint = "localhost";
    String streamName = "abc";
    String readerGroup = "group";
    int port = TestUtils.getAvailableListenPort();
    String testString = "Hello world: ";
    String scope = "Scope1";
    @Cleanup
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    @Cleanup
    PravegaConnectionListener server = new PravegaConnectionListener(false, port, store, mock(TableStore.class),
            serviceBuilder.getLowPriorityExecutor());
    server.startListening();
    @Cleanup
    MockStreamManager streamManager = new MockStreamManager(scope, endpoint, port);
    MockClientFactory clientFactory = streamManager.getClientFactory();
    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder()
                                                     .automaticCheckpointIntervalMillis(1000)
                                                     .stream(Stream.of(scope, streamName))
                                                     .build();
    streamManager.createScope(scope);
    streamManager.createStream(scope, streamName, null);
    streamManager.createReaderGroup(readerGroup, groupConfig);
    JavaSerializer<String> serializer = new JavaSerializer<>();
    populateEvents(streamName, testString, clientFactory, serializer);
    AtomicLong fakeClock = new AtomicLong(0);
    @Cleanup
    EventStreamReader<String> reader1 = clientFactory.createReader("reader1", readerGroup, serializer,
                                                                   ReaderConfig.builder().build(),
                                                                   () -> fakeClock.get(),
                                                                   () -> fakeClock.get() / NANOS_PER_SECOND);
    @Cleanup
    EventStreamReader<String> reader2 = clientFactory.createReader("reader2", readerGroup, serializer,
                                                                   ReaderConfig.builder().build(),
                                                                   () -> fakeClock.get(),
                                                                   () -> fakeClock.get() / NANOS_PER_SECOND);
    int numRead = 0;
    int checkpointCount = 0;
    while (numRead < 100) {
        fakeClock.addAndGet(NANOS_PER_SECOND);
        EventRead<String> event = reader1.readNextEvent(1000);
        if (event.isCheckpoint()) {
            checkpointCount++;
        } else {
            String message = event.getEvent();
            assertEquals(testString + numRead, message);
            numRead++;
        }
    }
    assertEquals("As there is a second reader that does not pass the checkpoint, only one should occur", 1,
                 checkpointCount);
}
 

/**
 * An method for applying the metadata in deserialization.
 * Override it in the custom extended {@link PravegaDeserializationSchema} if the Pravega metadata is needed.
 *
 * @param eventRead The EventRead structure the client returns which contains metadata
 *
 * @return the deserialized event with metadata
 */
public T extractEvent(EventRead<T> eventRead) {
    return eventRead.getEvent();
}