Java Code Examples for org.apache.pulsar.client.api.Consumer#close()

@Test
public void upgradeToReplicatedSubscriptionAfterRestart() throws Exception {
    String topic = "upgradeToReplicatedSubscriptionAfterRestart-" + System.nanoTime();

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("sub")
            .replicateSubscriptionState(false)
            .subscribe();

    TopicStats stats = admin.topics().getStats(topic);
    assertFalse(stats.subscriptions.get("sub").isReplicated);
    consumer.close();

    admin.topics().unload(topic);

    consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("sub")
            .replicateSubscriptionState(true)
            .subscribe();

    stats = admin.topics().getStats(topic);
    assertTrue(stats.subscriptions.get("sub").isReplicated);
    consumer.close();
}
 

@Test
public void testGC() throws Exception {
    // 1. Simple successful GC
    String topicName = "persistent://prop/ns-abc/topic-10";
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    producer.close();

    assertTrue(pulsar.getBrokerService().getTopicReference(topicName).isPresent());
    runGC();
    assertFalse(pulsar.getBrokerService().getTopicReference(topicName).isPresent());

    // 2. Topic is not GCed with live connection
    String subName = "sub1";
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();

    runGC();
    assertTrue(pulsar.getBrokerService().getTopicReference(topicName).isPresent());

    // 3. Topic with subscription is not GCed even with no connections
    consumer.close();

    runGC();
    assertTrue(pulsar.getBrokerService().getTopicReference(topicName).isPresent());

    // 4. Topic can be GCed after unsubscribe
    admin.topics().deleteSubscription(topicName, subName);

    runGC();
    assertFalse(pulsar.getBrokerService().getTopicReference(topicName).isPresent());
}
 

@Test
public void testSimpleCloseTopic() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic5";
    final String subName = "sub5";

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    assertNotNull(subRef);

    Message<byte[]> msg;
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }

    producer.close();
    consumer.close();

    topicRef.close().get();
    assertFalse(pulsar.getBrokerService().getTopicReference(topicName).isPresent());
}
 

@Test
public void testEnabledChecksumClient() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int totalMsg = 10;

    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name")
            .subscribe();

    final int batchMessageDelayMs = 300;
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .enableBatching(true)
            .batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
            .batchingMaxMessages(5)
            .create();

    for (int i = 0; i < totalMsg; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message<byte[]>msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < totalMsg; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

/**
 * A topic that has retention policy set to -1, should not be GCed until it has been inactive for at least the
 * retention time and the data should never be deleted
 */
@Test
public void testInfiniteRetentionPolicy() throws Exception {
    // Retain data forever
    admin.namespaces().setRetention("prop/ns-abc", new RetentionPolicies(-1, -1));

    // 1. Simple successful GC
    String topicName = "persistent://prop/ns-abc/topic-10";
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    producer.close();

    assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
    runGC();
    // Should not have been deleted, since we have retention
    assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));

    // Remove retention
    admin.namespaces().setRetention("prop/ns-abc", new RetentionPolicies(0, 10));
    Thread.sleep(300);

    // 2. Topic is not GCed with live connection
    String subName = "sub1";
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();

    runGC();
    assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));

    // 3. Topic with subscription is not GCed even with no connections
    consumer.close();

    runGC();
    assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));

    // 4. Topic can be GCed after unsubscribe
    admin.topics().deleteSubscription(topicName, subName);

    runGC();
    assertFalse(pulsar.getBrokerService().getTopicReference(topicName).isPresent());
}
 

@Test(timeOut=20000)
public void testTopicWithWildCardChar() throws Exception {
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(pulsar.getWebServiceAddress())
            .statsInterval(0, TimeUnit.SECONDS)
            .build();

    final String ns1 = "prop/usc/topicWithSpecialChar";
    try {
        admin.namespaces().createNamespace(ns1);
    } catch (Exception e) {

    }

    final String topic1 = "persistent://"+ns1+"/`~!@#$%^&*()-_+=[]://{}|\\;:'\"<>,./?-30e04524";
    final String subName1 = "c1";
    final byte[] content = "test".getBytes();

    Consumer<byte[]> consumer = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
    org.apache.pulsar.client.api.Producer<byte[]> producer = client.newProducer().topic(topic1).create();

    producer.send(content);
    Message<byte[]> msg = consumer.receive();
    Assert.assertEquals(msg.getData(), content);
    consumer.close();
    producer.close();
    client.close();
}
 

@Test
public void testProducerAndConsumerPassed() throws Exception {
    log.info("-- {} -- start", methodName);

    Consumer<byte[]> consumer = pulsarClient.newConsumer()
        .topic("persistent://my-property/my-ns/my-topic")
        .subscriptionName("my-subscriber-name")
        .subscribe();

    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer()
        .topic("persistent://my-property/my-ns/my-topic")
        .enableBatching(false);

    Producer<byte[]> producer = producerBuilder.create();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        log.info("Produced message: [{}]", message);
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.info("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();

    log.info("-- {} -- end", methodName);
}
 

@Test
public void testConsumerFlowControl() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic2";
    final String subName = "sub2";

    Message<byte[]> msg;
    int recvQueueSize = 4;

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .receiverQueueSize(recvQueueSize).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    assertNotNull(subRef);

    // 1. initial receive queue size recorded
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), recvQueueSize);

    for (int i = 0; i < recvQueueSize / 2; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }

    // 2. queue size re-adjusted after successful receive of half of window size
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), recvQueueSize);

    consumer.close();
    assertFalse(subRef.getDispatcher().isConsumerConnected());
}
 

@Test
public void testZeroQueueSizeMessageRedeliveryForAsyncReceive() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "persistent://prop/ns-abc/testZeroQueueSizeMessageRedeliveryForAsyncReceive";
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .receiverQueueSize(0)
        .subscriptionName("sub")
        .subscriptionType(SubscriptionType.Shared)
        .ackTimeout(1, TimeUnit.SECONDS)
        .subscribe();

    final int messages = 10;
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .enableBatching(false)
        .create();

    for (int i = 0; i < messages; i++) {
        producer.send(i);
    }

    Set<Integer> receivedMessages = new HashSet<>();
    List<CompletableFuture<Message<Integer>>> futures = new ArrayList<>(20);
    for (int i = 0; i < messages * 2; i++) {
        futures.add(consumer.receiveAsync());
    }
    for (CompletableFuture<Message<Integer>> future : futures) {
        receivedMessages.add(future.get().getValue());
    }

    Assert.assertEquals(receivedMessages.size(), messages);

    consumer.close();
    producer.close();
}
 

@Test
public void testMultiTopicSetSchemaProvider() throws Exception {
    final String tenant = PUBLIC_TENANT;
    final String namespace = "test-namespace-" + randomName(16);
    final String topicOne = "test-multi-version-schema-one";
    final String topicTwo = "test-multi-version-schema-two";
    final String fqtnOne = TopicName.get(
            TopicDomain.persistent.value(),
            tenant,
            namespace,
            topicOne
    ).toString();

    final String fqtnTwo = TopicName.get(
            TopicDomain.persistent.value(),
            tenant,
            namespace,
            topicTwo
    ).toString();


    admin.namespaces().createNamespace(
            tenant + "/" + namespace,
            Sets.newHashSet(CLUSTER_NAME)
    );

    admin.topics().createPartitionedTopic(fqtnOne, 3);
    admin.topics().createPartitionedTopic(fqtnTwo, 3);

    admin.schemas().createSchema(fqtnOne, Schema.AVRO(
            SchemaDefinition.<Schemas.PersonOne>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonOne.class).build()).getSchemaInfo());

    admin.schemas().createSchema(fqtnOne, Schema.AVRO(
            SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo());

    admin.schemas().createSchema(fqtnTwo, Schema.AVRO(
            SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo());

    Producer<Schemas.PersonTwo> producer = pulsarClient.newProducer(Schema.AVRO(
            SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .topic(fqtnOne)
            .create();

    Schemas.PersonTwo personTwo = new Schemas.PersonTwo();
    personTwo.setId(1);
    personTwo.setName("Tom");


    Consumer<Schemas.PersonTwo> consumer = pulsarClient.newConsumer(Schema.AVRO(
            SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .subscriptionName("test")
            .topic(fqtnOne, fqtnTwo)
            .subscribe();

    producer.send(personTwo);

    Schemas.PersonTwo personConsume = consumer.receive().getValue();
    assertEquals("Tom", personConsume.getName());
    assertEquals(1, personConsume.getId());

    producer.close();
    consumer.close();
}
 

@Test
public void testBatchMessageIndexAckForExclusiveSubscription() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "testBatchMessageIndexAckForExclusiveSubscription";

    @Cleanup
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    @Cleanup
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .batchingMaxPublishDelay(50, TimeUnit.MILLISECONDS)
        .create();

    final int messages = 100;
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < messages; i++) {
        if (i == 49) {
            consumer.acknowledgeCumulative(consumer.receive());
        } else {
            consumer.receive();
        }
    }

    //Wait ack send.
    Thread.sleep(1000);
    consumer.close();
    consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    List<Message<Integer>> received = new ArrayList<>(100);
    for (int i = 0; i < messages; i++) {
        received.add(consumer.receive());
    }

    Assert.assertEquals(received.size(), 100);
}
 

@Test
public void testPublishConsumerStats() throws Exception {
    final String topicName = "statTopic";
    final String subscriberName = topicName + "-my-sub-1";
    final String topic = "persistent://prop-xyz/use/ns1/" + topicName;
    final String producerName = "myProducer";

    @Cleanup
    PulsarClient client = PulsarClient.builder().serviceUrl(pulsar.getWebServiceAddress()).build();
    Consumer<byte[]> consumer = client.newConsumer().topic(topic).subscriptionName(subscriberName)
            .subscriptionType(SubscriptionType.Shared).subscribe();
    Producer<byte[]> producer = client.newProducer()
        .topic(topic)
        .producerName(producerName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    retryStrategically((test) -> {
        TopicStats stats;
        try {
            stats = admin.topics().getStats(topic);
            return stats.publishers.size() > 0 && stats.subscriptions.get(subscriberName) != null
                    && stats.subscriptions.get(subscriberName).consumers.size() > 0;
        } catch (PulsarAdminException e) {
            return false;
        }
    }, 5, 200);

    TopicStats topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.publishers.size(), 1);
    assertNotNull(topicStats.publishers.get(0).getAddress());
    assertNotNull(topicStats.publishers.get(0).getClientVersion());
    assertNotNull(topicStats.publishers.get(0).getConnectedSince());
    assertNotNull(topicStats.publishers.get(0).getProducerName());
    assertEquals(topicStats.publishers.get(0).getProducerName(), producerName);

    SubscriptionStats subscriber = topicStats.subscriptions.get(subscriberName);
    assertNotNull(subscriber);
    assertEquals(subscriber.consumers.size(), 1);
    ConsumerStats consumerStats = subscriber.consumers.get(0);
    assertNotNull(consumerStats.getAddress());
    assertNotNull(consumerStats.getClientVersion());
    assertNotNull(consumerStats.getConnectedSince());

    producer.close();
    consumer.close();
}
 

/**
 * Test topic partitions auto subscribed.
 *
 * Steps:
 * 1. Create a consumer with 2 topics, and each topic has 2 partitions: xx-partition-0, xx-partition-1.
 * 2. update topics to have 3 partitions.
 * 3. trigger partitionsAutoUpdate. this should be done automatically, this is to save time to manually trigger.
 * 4. produce message to xx-partition-2 again,  and verify consumer could receive message.
 *
 */
@Test(timeOut = 30000)
public void testTopicAutoUpdatePartitions() throws Exception {
    String key = "TestTopicAutoUpdatePartitions";
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 6;

    final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" + key;
    final String topicName2 = "persistent://prop/use/ns-abc/topic-2-" + key;
    List<String> topicNames = Lists.newArrayList(topicName1, topicName2);

    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("prop", tenantInfo);
    admin.topics().createPartitionedTopic(topicName1, 2);
    admin.topics().createPartitionedTopic(topicName2, 2);

    // 1. Create a  consumer
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topics(topicNames)
            .subscriptionName(subscriptionName)
            .subscriptionType(SubscriptionType.Shared)
            .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS)
            .receiverQueueSize(4)
            .autoUpdatePartitions(true)
            .subscribe();
    assertTrue(consumer instanceof MultiTopicsConsumerImpl);

    MultiTopicsConsumerImpl topicsConsumer = (MultiTopicsConsumerImpl) consumer;

    // 2. update to 3 partitions
    admin.topics().updatePartitionedTopic(topicName1, 3);
    admin.topics().updatePartitionedTopic(topicName2, 3);

    // 3. trigger partitionsAutoUpdate. this should be done automatically in 1 minutes,
    // this is to save time to manually trigger.
    log.info("trigger partitionsAutoUpdateTimerTask");
    Timeout timeout = topicsConsumer.getPartitionsAutoUpdateTimeout();
    timeout.task().run(timeout);
    Thread.sleep(200);

    // 4. produce message to xx-partition-2,  and verify consumer could receive message.
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1 + "-partition-2")
            .enableBatching(false)
            .create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2 + "-partition-2")
            .enableBatching(false)
            .create();
    for (int i = 0; i < totalMessages; i++) {
        producer1.send((messagePredicate + "topic1-partition-2 index:" + i).getBytes());
        producer2.send((messagePredicate + "topic2-partition-2 index:" + i).getBytes());
        log.info("produce message to partition-2 again. messageindex: {}", i);
    }
    int messageSet = 0;
    Message<byte[]> message = consumer.receive();
    do {
        messageSet ++;
        consumer.acknowledge(message);
        log.info("4 Consumer acknowledged : " + new String(message.getData()));
        message = consumer.receive(200, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(messageSet, 2 * totalMessages);

    consumer.close();
}
 

@Test(dataProvider =  "AllCheckSchemaCompatibilityStrategy")
public void testIsAutoUpdateSchema(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception {
    final String tenant = PUBLIC_TENANT;
    final String topic = "test-consumer-compatibility";

    String namespace = "test-namespace-" + randomName(16);
    String fqtn = TopicName.get(
            TopicDomain.persistent.value(),
            tenant,
            namespace,
            topic
    ).toString();

    NamespaceName namespaceName = NamespaceName.get(tenant, namespace);

    admin.namespaces().createNamespace(
            tenant + "/" + namespace,
            Sets.newHashSet(CLUSTER_NAME)
    );

    assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(namespaceName.toString()),
            SchemaCompatibilityStrategy.FULL);
    
    admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy);
    admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo());

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false);
    ProducerBuilder<Schemas.PersonTwo> producerThreeBuilder = pulsarClient
            .newProducer(Schema.AVRO(SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .topic(fqtn);
    try {
        producerThreeBuilder.create();
    } catch (Exception e) {
        Assert.assertTrue(e.getMessage().contains("Schema not found and schema auto updating is disabled."));
    }

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), true);
    ConsumerBuilder<Schemas.PersonTwo> comsumerBuilder = pulsarClient.newConsumer(Schema.AVRO(
            SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .subscriptionName("test")
            .topic(fqtn);

    Producer<Schemas.PersonTwo> producer = producerThreeBuilder.create();
    Consumer<Schemas.PersonTwo> consumerTwo = comsumerBuilder.subscribe();

    producer.send(new Schemas.PersonTwo(2, "Lucy"));
    Message<Schemas.PersonTwo> message = consumerTwo.receive();

    Schemas.PersonTwo personTwo = message.getValue();
    consumerTwo.acknowledge(message);

    assertEquals(personTwo.getId(), 2);
    assertEquals(personTwo.getName(), "Lucy");

    producer.close();
    consumerTwo.close();

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false);

    producer = producerThreeBuilder.create();
    consumerTwo = comsumerBuilder.subscribe();

    producer.send(new Schemas.PersonTwo(2, "Lucy"));
    message = consumerTwo.receive();

    personTwo = message.getValue();
    consumerTwo.acknowledge(message);

    assertEquals(personTwo.getId(), 2);
    assertEquals(personTwo.getName(), "Lucy");

    consumerTwo.close();
    producer.close();
}
 

/**
 * Verify: 1. Broker should not replay already acknowledged messages 2. Dispatcher should not stuck while
 * dispatching new messages due to previous-replay of invalid/already-acked messages
 *
 * @throws Exception
 */
@Test
public void testMessageReplay() throws Exception {

    final String topicName = "persistent://prop/ns-abc/topic2";
    final String subName = "sub2";

    Message<byte[]> msg;
    int totalMessages = 10;
    int replayIndex = totalMessages / 2;

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .subscriptionType(SubscriptionType.Shared).receiverQueueSize(1).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subRef
            .getDispatcher();
    Field replayMap = PersistentDispatcherMultipleConsumers.class.getDeclaredField("messagesToRedeliver");
    replayMap.setAccessible(true);
    ConcurrentLongPairSet messagesToReplay = new ConcurrentLongPairSet(64, 1);

    assertNotNull(subRef);

    // (1) Produce messages
    for (int i = 0; i < totalMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    MessageIdImpl firstAckedMsg = null;
    // (2) Consume and ack messages except first message
    for (int i = 0; i < totalMessages; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
        MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
        if (i == 0) {
            firstAckedMsg = msgId;
        }
        if (i < replayIndex) {
            // (3) accumulate acked messages for replay
            messagesToReplay.add(msgId.getLedgerId(), msgId.getEntryId());
        }
    }

    // (4) redelivery : should redeliver only unacked messages
    Thread.sleep(1000);

    replayMap.set(dispatcher, messagesToReplay);
    // (a) redelivery with all acked-message should clear messageReply bucket
    dispatcher.redeliverUnacknowledgedMessages(dispatcher.getConsumers().get(0));
    assertEquals(messagesToReplay.size(), 0);

    // (b) fill messageReplyBucket with already acked entry again: and try to publish new msg and read it
    messagesToReplay.add(firstAckedMsg.getLedgerId(), firstAckedMsg.getEntryId());
    replayMap.set(dispatcher, messagesToReplay);
    // send new message
    final String testMsg = "testMsg";
    producer.send(testMsg.getBytes());
    // consumer should be able to receive only new message and not the
    dispatcher.consumerFlow(dispatcher.getConsumers().get(0), 1);
    msg = consumer.receive(1, TimeUnit.SECONDS);
    assertNotNull(msg);
    assertEquals(msg.getData(), testMsg.getBytes());

    consumer.close();
    producer.close();
}
 

@Test(timeOut = testTimeout)
public void testSyncProducerAndConsumer() throws Exception {
    String key = "TopicsConsumerSyncTest";
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 30;

    final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" + key;
    final String topicName2 = "persistent://prop/use/ns-abc/topic-2-" + key;
    final String topicName3 = "persistent://prop/use/ns-abc/topic-3-" + key;
    List<String> topicNames = Lists.newArrayList(topicName1, topicName2, topicName3);

    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("prop", tenantInfo);
    admin.topics().createPartitionedTopic(topicName2, 2);
    admin.topics().createPartitionedTopic(topicName3, 3);

    // 1. producer connect
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2)
        .enableBatching(false)
        .messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
        .create();
    Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3)
        .enableBatching(false)
        .messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
        .create();

    // 2. Create consumer
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
        .topics(topicNames)
        .subscriptionName(subscriptionName)
        .subscriptionType(SubscriptionType.Shared)
        .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS)
        .receiverQueueSize(4)
        .subscribe();
    assertTrue(consumer instanceof MultiTopicsConsumerImpl);

    // 3. producer publish messages
    for (int i = 0; i < totalMessages / 3; i++) {
        producer1.send((messagePredicate + "producer1-" + i).getBytes());
        producer2.send((messagePredicate + "producer2-" + i).getBytes());
        producer3.send((messagePredicate + "producer3-" + i).getBytes());
    }

    int messageSet = 0;
    Message<byte[]> message = consumer.receive();
    do {
        assertTrue(message instanceof TopicMessageImpl);
        messageSet ++;
        consumer.acknowledge(message);
        log.debug("Consumer acknowledged : " + new String(message.getData()));
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(messageSet, totalMessages);

    consumer.unsubscribe();
    consumer.close();
    producer1.close();
    producer2.close();
    producer3.close();
}
 

@Test
public void testReplayOnConsumerDisconnect() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/shared-topic3";
    final String subName = "sub3";
    final int numMsgs = 100;

    final List<String> messagesProduced = Lists.newArrayListWithCapacity(numMsgs);
    final List<String> messagesConsumed = new BlockingArrayQueue<>(numMsgs);

    Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .subscriptionType(SubscriptionType.Shared).messageListener((consumer, msg) -> {
                try {
                    consumer.acknowledge(msg);
                    messagesConsumed.add(new String(msg.getData()));
                } catch (Exception e) {
                    fail("Should not fail");
                }
            }).subscribe();

    // consumer2 does not ack messages
    PulsarClient newPulsarClient = newPulsarClient(lookupUrl.toString(), 0);// Creates new client connection
    Consumer<byte[]> consumer2 = newPulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .subscriptionType(SubscriptionType.Shared).messageListener((consumer, msg) -> {
                // do notthing
            }).subscribe();

    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs * 2);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    for (int i = 0; i < numMsgs; i++) {
        String message = "msg-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
        messagesProduced.add(message);
    }
    FutureUtil.waitForAll(futures).get();
    producer.close();

    consumer2.close();

    for (int n = 0; n < 10 && messagesConsumed.size() < numMsgs; n++) {
        Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    }

    // 1. consumer1 gets all messages
    assertTrue(CollectionUtils.subtract(messagesProduced, messagesConsumed).isEmpty());

    consumer1.close();
    newPulsarClient.close();

    deleteTopic(topicName);
}
 

@Test(timeOut = 2000)
public void testAsyncProducerAndConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int totalMsg = 100;
    final Set<String> produceMsgs = Sets.newHashSet();
    final Set<String> consumeMsgs = Sets.newHashSet();

    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name")
            .subscriptionType(SubscriptionType.Exclusive)
            .subscribe();

    // produce message
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .create();
    for (int i = 0; i < totalMsg; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        produceMsgs.add(message);
    }

    log.info(" start receiving messages :");
    CountDownLatch latch = new CountDownLatch(totalMsg);
    // receive messages
    ExecutorService executor = Executors.newFixedThreadPool(1);
    receiveAsync(consumer, totalMsg, 0, latch, consumeMsgs, executor);

    latch.await();

    // verify message produced correctly
    assertEquals(produceMsgs.size(), totalMsg);
    // verify produced and consumed messages must be exactly same
    produceMsgs.removeAll(consumeMsgs);
    assertTrue(produceMsgs.isEmpty());

    producer.close();
    consumer.close();
    executor.shutdownNow();
    log.info("-- Exiting {} test --", methodName);
}
 

@Test
public void testPerConsumerStats() throws Exception {
    Producer<byte[]> p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
    Producer<byte[]> p2 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic2").create();

    Consumer<byte[]> c1 = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("test")
            .subscribe();

    Consumer<byte[]> c2 = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic2")
            .subscriptionName("test")
            .subscribe();

    final int messages = 10;

    for (int i = 0; i < messages; i++) {
        String message = "my-message-" + i;
        p1.send(message.getBytes());
        p2.send(message.getBytes());
    }

    for (int i = 0; i < messages; i++) {
        c1.acknowledge(c1.receive());
        c2.acknowledge(c2.receive());
    }

    ByteArrayOutputStream statsOut = new ByteArrayOutputStream();
    PrometheusMetricsGenerator.generate(pulsar, true, true, statsOut);
    String metricsStr = new String(statsOut.toByteArray());

    Multimap<String, Metric> metrics = parseMetrics(metricsStr);

    metrics.entries().forEach(e -> {
        System.out.println(e.getKey() + ": " + e.getValue());
    });

    // There should be 1 metric aggregated per namespace
    List<Metric> cm = (List<Metric>) metrics.get("pulsar_out_bytes_total");
    assertEquals(cm.size(), 4);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2");
    assertEquals(cm.get(0).tags.get("subscription"), "test");

    assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2");
    assertEquals(cm.get(1).tags.get("subscription"), "test");
    assertEquals(cm.get(1).tags.get("consumer_id"), "1");

    assertEquals(cm.get(2).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(2).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1");
    assertEquals(cm.get(2).tags.get("subscription"), "test");

    assertEquals(cm.get(3).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(3).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1");
    assertEquals(cm.get(3).tags.get("subscription"), "test");
    assertEquals(cm.get(3).tags.get("consumer_id"), "0");

    cm = (List<Metric>) metrics.get("pulsar_out_messages_total");
    assertEquals(cm.size(), 4);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(0).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2");
    assertEquals(cm.get(0).tags.get("subscription"), "test");

    assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(1).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic2");
    assertEquals(cm.get(1).tags.get("subscription"), "test");
    assertEquals(cm.get(1).tags.get("consumer_id"), "1");

    assertEquals(cm.get(2).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(2).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1");
    assertEquals(cm.get(2).tags.get("subscription"), "test");

    assertEquals(cm.get(3).tags.get("namespace"), "my-property/use/my-ns");
    assertEquals(cm.get(3).tags.get("topic"), "persistent://my-property/use/my-ns/my-topic1");
    assertEquals(cm.get(3).tags.get("subscription"), "test");
    assertEquals(cm.get(3).tags.get("consumer_id"), "0");

    p1.close();
    p2.close();
    c1.close();
    c2.close();
}
 

@Test
public void testPerNamespaceStats() throws Exception {
    Producer<byte[]> p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
    Producer<byte[]> p2 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic2").create();

    Consumer<byte[]> c1 = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("test")
            .subscribe();

    Consumer<byte[]> c2 = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic2")
            .subscriptionName("test")
            .subscribe();

    final int messages = 10;

    for (int i = 0; i < messages; i++) {
        String message = "my-message-" + i;
        p1.send(message.getBytes());
        p2.send(message.getBytes());
    }

    for (int i = 0; i < messages; i++) {
        c1.acknowledge(c1.receive());
        c2.acknowledge(c2.receive());
    }

    ByteArrayOutputStream statsOut = new ByteArrayOutputStream();
    PrometheusMetricsGenerator.generate(pulsar, false, false, statsOut);
    String metricsStr = new String(statsOut.toByteArray());

    Multimap<String, Metric> metrics = parseMetrics(metricsStr);

    metrics.entries().forEach(e -> {
        System.out.println(e.getKey() + ": " + e.getValue());
    });

    // There should be 1 metric aggregated per namespace
    List<Metric> cm = (List<Metric>) metrics.get("pulsar_storage_write_latency_le_1");
    assertEquals(cm.size(), 1);
    assertNull(cm.get(0).tags.get("topic"));
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_producers_count");
    assertEquals(cm.size(), 2);
    assertNull(cm.get(1).tags.get("topic"));
    assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_in_bytes_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_in_messages_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_out_bytes_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_out_messages_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    p1.close();
    p2.close();
    c1.close();
    c2.close();
}