com.couchbase.client.dcp.StreamTo Java Examples

@Test
public void testStartStopStreaming() throws InterruptedException {
    Mockito.when(client.initializeState(StreamFrom.BEGINNING, StreamTo.NOW)).thenReturn(Completable.complete());
    Mockito.when(client.startStreaming(Mockito.<Short[]>anyVararg())).thenReturn(Completable.complete());
    Mockito.when(client.stopStreaming(Mockito.<Short[]>anyVararg())).thenReturn(Completable.complete());
    SessionState sessionState = Mockito.mock(SessionState.class);
    Mockito.when(sessionState.isAtEnd()).thenReturn(false, true);
    Mockito.when(client.sessionState()).thenReturn(sessionState);

    BlockingQueue<ByteBuf> resultsQueue = new ArrayBlockingQueue<>(4);
    resultsQueue.put(Mockito.mock(ByteBuf.class));
    resultsQueue.put(Mockito.mock(ByteBuf.class));
    resultsQueue.put(Mockito.mock(ByteBuf.class));
    resultsQueue.put(Mockito.mock(ByteBuf.class));
    Mockito.when(client.disconnect()).thenReturn(Completable.complete());


    streamingConnection.startStreaming(resultsQueue);
    streamingConnection.stopStreaming();

    Thread.sleep(1500);
    Mockito.verify(client, Mockito.times(2)).sessionState();
    Mockito.verify(client, Mockito.times(1)).disconnect();
}
 

@Test
public void testStartStreaming() throws InterruptedException {
    Mockito.when(client.initializeState(StreamFrom.BEGINNING, StreamTo.NOW)).thenReturn(Completable.complete());
    Mockito.when(client.startStreaming(Mockito.<Short[]>anyVararg())).thenReturn(Completable.complete());
    SessionState sessionState = Mockito.mock(SessionState.class);
    Mockito.when(sessionState.isAtEnd()).thenReturn(false, false, true);
    Mockito.when(client.sessionState()).thenReturn(sessionState);

    BlockingQueue<ByteBuf> resultsQueue = new ArrayBlockingQueue<>(3);

    streamingConnection.startStreaming(resultsQueue);

    Assert.assertTrue(streamingConnection.isStreaming());

    Thread.sleep(2000);
    Mockito.verify(client, Mockito.times(3)).sessionState();
}
 

@Override
public void run() {
  try {
    client.connect().await();

    // Apply the fallback state to all partitions. As of DCP client version 0.12.0,
    // this is the only way to set the sequence number to "now".
    StreamFrom fallbackStreamFrom = streamFrom.withoutSavedOffset();
    client.initializeState(fallbackStreamFrom.asDcpStreamFrom(), StreamTo.INFINITY).await();

    // Overlay any saved offsets (might have saved offsets for only some partitions).
    if (streamFrom.isSavedOffset()) {
      // As of DCP client version 0.12.0, Client.initializeState(BEGINNING, INFINITY)
      // doesn't fetch the failover logs. Do it ourselves to avoid a spurious rollback :-/
      initFailoverLogs();

      // Then restore the partition state and use saved vBucket UUIDs to overlay synthetic failover log entries.
      restoreSavedOffsets();
    }

    client.startStreaming(partitions).await();

  } catch (Throwable t) {
    errorQueue.offer(t);
  }
}
 

private static void runTest(final Client client, Args args) throws InterruptedException {
  final CountDownLatch latch = new CountDownLatch(args.dcpMessageCount);

  final boolean highLevelApi = Boolean.parseBoolean(args.settings.getProperty("highLevelApi", "false"));
  if (highLevelApi) {
    System.out.println("Using high-level API. Won't be collecting compression metrics.");
    registerHighLevelListeners(latch, client);
  } else {
    System.out.println("Using low-level API.");
    registerLowLevelListeners(latch, client);
  }

  long startNanos = System.nanoTime();
  client.connect().await();
  client.initializeState(StreamFrom.BEGINNING, StreamTo.INFINITY).await();
  client.startStreaming().await();

  latch.await();
  System.out.println("Received at least " + args.dcpMessageCount + " messages. Done!");
  long elapsedNanos = System.nanoTime() - startNanos;

  client.disconnect().await();

  System.out.println("Shutdown complete. Receiving " + args.dcpMessageCount + " DCP events took " +
      TimeUnit.NANOSECONDS.toMillis(elapsedNanos) + " ms");
}
 

@Test
public void canStreamFromBeginningToNow() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    bucket.createScopes(NUMBER_OF_SCOPES, "S").size();
    List<String> collections = bucket.createCollections(NUMBER_OF_COLLECTIONS, "C", "S0");
    Thread.sleep(1000);
    int documentsSize = bucket.upsertOneDocumentToEachCollection(collections, "D", "S0").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.BEGINNING, StreamTo.NOW)
        .collectionAware()
        .start()) {

      //Expect to see all events
      assertStatus(streamer.awaitStreamEnd(), documentsSize, 0, 0);
    }
  }


}
 

@Test
public void canStreamFromNowToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    bucket.createScopes(NUMBER_OF_SCOPES, "S");
    List<String> collections = bucket.createCollections(NUMBER_OF_COLLECTIONS, "C", "S0");
    Thread.sleep(1000);
    bucket.upsertOneDocumentToEachCollection(collections, "D-a", "S0").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.NOW, StreamTo.INFINITY)
        .collectionAware()
        .start()) {
      //Expect to not see any events
      streamer.assertStateCount(0, DcpStreamer.State.SCOPE_CREATIONS);
      streamer.assertStateCount(0, DcpStreamer.State.COLLECTION_CREATIONS);
      streamer.assertStateCount(0, DcpStreamer.State.MUTATIONS);

      int documentsSize = bucket.upsertOneDocumentToEachCollection(collections, "D-b", "S0").size();
      //See only new mutations
      streamer.assertStateCount(documentsSize, DcpStreamer.State.MUTATIONS);
    }
  }
}
 

@Test
public void scopesReconnectsAfterServerRestart() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.BEGINNING, StreamTo.INFINITY)
        .collectionAware()
        .start()) {
      int scopesA = bucket.createScopes(NUMBER_OF_SCOPES, "A").size();

      streamer.assertStateCount(scopesA * VB, DcpStreamer.State.SCOPE_CREATIONS);

      agent().resetCluster();
      couchbase().restart();

      int scopesB = bucket.createScopes(NUMBER_OF_SCOPES, "B").size();
      //See both A and B scopes
      streamer.assertStateCount((scopesA + scopesB) * VB, DcpStreamer.State.SCOPE_CREATIONS);
    }
  }
}
 

@Test
public void collectionsStreamFromNowToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    bucket.createScopes(NUMBER_OF_SCOPES, "S-a");
    bucket.createCollections(NUMBER_OF_COLLECTIONS, "C-a", "S-a0");

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.NOW, StreamTo.INFINITY)
        .collectionAware()
        .start()) {
      streamer.assertStateCount(0, DcpStreamer.State.SCOPE_CREATIONS);
      streamer.assertStateCount(0, DcpStreamer.State.COLLECTION_CREATIONS);

      int scopesB = bucket.createScopes(NUMBER_OF_SCOPES, "S-b").size();
      int collectionsB = bucket.createCollections(NUMBER_OF_COLLECTIONS, "C-b", "S-b0").size();

      streamer.assertStateCount(scopesB * VB, DcpStreamer.State.SCOPE_CREATIONS);
      streamer.assertStateCount(collectionsB * VB, DcpStreamer.State.COLLECTION_CREATIONS);
    }
  }
}
 

@Override
public String start(String bucket, @Default("[]") List<Short> vbuckets, StreamFrom from, StreamTo to, boolean mitigateRollbacks, boolean collectionAware) {
  String streamerId = "dcp-test-streamer-" + nextStreamerId.getAndIncrement();

  Client client = Client.builder()
      .bucket(bucket)
      .credentials(username, password)
      .mitigateRollbacks(mitigateRollbacks ? 100 : 0, TimeUnit.MILLISECONDS)
      .flowControl(1024 * 128)
      .seedNodes(nodes.split(","))
      .userAgent("dcp-integration-test", "0", streamerId)
      .collectionsAware(collectionAware)
      .build();

  idToStreamer.put(streamerId, new DcpStreamer(client, vbuckets, from, to));
  return streamerId;
}
 

@Test
public void scopesStreamFromBeginningToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    int scopesA = bucket.createScopes(NUMBER_OF_SCOPES, "A").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.BEGINNING, StreamTo.INFINITY)
        .collectionAware()
        .start()) {
      streamer.assertStateCount(scopesA * VB, DcpStreamer.State.SCOPE_CREATIONS);

      int scopesB = bucket.createScopes(1, "B").size();
      //See both A and B scopes
      streamer.assertStateCount((scopesA + scopesB) * VB, DcpStreamer.State.SCOPE_CREATIONS);
    }
  }
}
 

@Test
public void scopesStreamFromNowToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    int scopesA = bucket.createScopes(NUMBER_OF_SCOPES, "A").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.NOW, StreamTo.INFINITY)
        .collectionAware()
        .start()) {
      streamer.assertStateCount(0, DcpStreamer.State.SCOPE_CREATIONS);

      int scopesB = bucket.createScopes(NUMBER_OF_SCOPES, "B").size();
      //See only B scopes
      streamer.assertStateCount(scopesB * VB, DcpStreamer.State.SCOPE_CREATIONS);
    }
  }
}
 

@Test
public void collectionsStreamFromBeginningToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    int scopesA = bucket.createScopes(NUMBER_OF_SCOPES, "S-a").size();
    int collectionsA = bucket.createCollections(NUMBER_OF_COLLECTIONS, "C-a", "S-a0").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .collectionAware()
        .range(StreamFrom.BEGINNING, StreamTo.INFINITY)
        .start()) {
      streamer.assertStateCount(scopesA * VB, DcpStreamer.State.SCOPE_CREATIONS);
      streamer.assertStateCount(collectionsA * VB, DcpStreamer.State.COLLECTION_CREATIONS);
    }
  }
}
 

public void startStreaming(final BlockingQueue<ByteBuf> resultsQueue) {
    if (streaming) {
        LOG.warn("This connection already in streaming mode, create another one.");
        return;
    }
    streaming = true;
    this.resultsQueue = resultsQueue;
    client.initializeState(StreamFrom.BEGINNING, StreamTo.NOW).await();
    new Thread(new Runnable() {
        @Override
        public void run() {
            int i = 0;
            try {
                client.startStreaming(partitionsToStream()).await();
                while (true) {
                    if (client.sessionState().isAtEnd()) {
                        break;
                    }
                    try {
                        Thread.sleep(500);
                    } catch (InterruptedException e) {
                        break;
                    }
                }
            } finally {
                streaming = false;
            }
        }
    }, "CouchbaseStreaming-" + threadId.incrementAndGet())
            .start();
}
 

@Test
public void scopesStreamFromBeginningToNow() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    int scopes = bucket.createScopes(NUMBER_OF_SCOPES, "S").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.BEGINNING, StreamTo.NOW)
        .collectionAware()
        .start()) {
      assertStatus(streamer.awaitStreamEnd(), 0, 0, 0, scopes * VB, 0, 0, 0, 0);
    }
  }

}
 

@Test
public void collectionsStreamFromBeginningToNow() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    int scopes = bucket.createScopes(NUMBER_OF_SCOPES, "S").size();
    int collections = bucket.createCollections(NUMBER_OF_COLLECTIONS, "C", "S0").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .collectionAware()
        .range(StreamFrom.BEGINNING, StreamTo.NOW)
        .start()) {
      assertStatus(streamer.awaitStreamEnd(), 0, 0, 0, scopes * VB, 0, collections * VB, 0, 0);
    }
  }
}
 

@Test
public void canStreamFromNowToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    final int batchSize = bucket.createOneDocumentInEachVbucket("a").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.NOW, StreamTo.INFINITY)
        .start()) {

      streamer.assertStateCount(0, DcpStreamer.State.MUTATIONS);
      bucket.createOneDocumentInEachVbucket("b").size();
      streamer.assertStateCount(batchSize, DcpStreamer.State.MUTATIONS);
    }
  }
}
 

@Test
public void canStreamFromBeginningToNow() throws Exception {
  couchbase().loadSampleBucket("beer-sample");

  try (RemoteDcpStreamer streamer = newStreamer("beer-sample")
      .range(StreamFrom.BEGINNING, StreamTo.NOW)
      .start()) {

    assertStatus(streamer.awaitStreamEnd(), 7303, 0, 0);
  }
}
 

public Status awaitStreamEnd(long timeout, TimeUnit unit) throws TimeoutException {
  if (this.streamTo == StreamTo.INFINITY) {
    throw new IllegalStateException("Streaming to infinity; can't wait for that!");
  }

  poll().atInterval(100, MILLISECONDS)
      .withTimeout(timeout, unit)
      .until(() -> client.sessionState().isAtEnd());

  return status();
}
 

public StreamBuilder range(StreamFrom from, StreamTo to) {
  this.from = requireNonNull(from);
  this.to = requireNonNull(to);
  return this;
}
 

public DcpStreamer(final Client client, final List<Short> vbuckets,
                   final StreamFrom from, final StreamTo to) {
  this.client = requireNonNull(client);
  this.streamTo = requireNonNull(to);

  client.listener(new DatabaseChangeListener() {
    @Override
    public void onFailure(StreamFailure streamFailure) {
      LOGGER.error("stream failure", streamFailure.getCause());
    }

    @Override
    public void onMutation(Mutation mutation) {
      mutations.getAndIncrement();
      mutation.flowControlAck();
    }

    @Override
    public void onDeletion(Deletion deletion) {
      (deletion.isDueToExpiration() ? expirations : deletions).getAndIncrement();
      deletion.flowControlAck();
    }

    @Override
    public void onScopeCreated(ScopeCreated scopeCreated) {
      scopeCreations.getAndIncrement();
    }

    @Override
    public void onScopeDropped(ScopeDropped scopeDropped) {
      scopeDrops.getAndIncrement();
    }

    @Override
    public void onCollectionCreated(CollectionCreated collectionCreated) {
      collectionCreations.getAndIncrement();
    }

    @Override
    public void onCollectionDropped(CollectionDropped collectionDropped) {
      collectionDrops.getAndIncrement();
    }

    @Override
    public void onCollectionFlushed(CollectionFlushed collectionFlushed) {
      collectionFlushes.getAndIncrement();
    }
  }, FlowControlMode.AUTOMATIC);

  client.connect().await(30, TimeUnit.SECONDS);
  try {
    client.initializeState(from, to).await();
    client.startStreaming(vbuckets.toArray(new Short[0])).await();
  } catch (Throwable t) {
    stop();
    throw t;
  }
}
 

/**
 * @param bucket Name of the bucket to stream from.
 * @param vbuckets List of vBuckets to stream from, or empty list for all vBuckets.
 * @return ID of the new streamer.
 */
String start(String bucket, @Default("[]") List<Short> vbuckets, StreamFrom from, StreamTo to, boolean mitigateRollbacks, boolean collectionAware);