Java Code Examples for org.apache.pulsar.client.api.Consumer#acknowledge()

@Test
public void testConsumerBacklogEvictionWithAck() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/ns-quota"),
            ConfigHelper.backlogQuotaMap(config));
    admin.namespaces().setBacklogQuota("prop/ns-quota",
            new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).build();

    final String topic1 = "persistent://prop/ns-quota/topic11";
    final String subName1 = "c11";
    final String subName2 = "c21";
    final int numMsgs = 20;

    Consumer<byte[]> consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
    Consumer<byte[]> consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe();
    org.apache.pulsar.client.api.Producer<byte[]> producer = client.newProducer().topic(topic1).create();
    byte[] content = new byte[1024];
    for (int i = 0; i < numMsgs; i++) {
        producer.send(content);
        // only one consumer acknowledges the message
        consumer1.acknowledge(consumer1.receive());
        consumer2.receive();
    }

    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();

    TopicStats stats = admin.topics().getStats(topic1);
    assertTrue(stats.backlogSize <= 10 * 1024, "Storage size is [" + stats.storageSize + "]");
    client.close();
}
 

protected <T> void receiveMessagesCheckStickyKeyAndDuplicate
        (List<Consumer<T>> consumerList, int messagesToReceive) throws PulsarClientException {
    Map<String, Set<String>> consumerKeys = Maps.newHashMap();
    Set<T> messagesReceived = Sets.newHashSet();
    for (Consumer<T> consumer : consumerList) {
        Message<T> currentReceived;
        while (true) {
            try {
                currentReceived = consumer.receive(3, TimeUnit.SECONDS);
            } catch (PulsarClientException e) {
                log.info("no more messages to receive for consumer {}", consumer.getConsumerName());
                break;
            }
            if (currentReceived != null) {
                consumer.acknowledge(currentReceived);
                assertNotNull(currentReceived.getKey());
                consumerKeys.putIfAbsent(consumer.getConsumerName(), Sets.newHashSet());
                consumerKeys.get(consumer.getConsumerName()).add(currentReceived.getKey());
                // Make sure that there are no duplicates
                assertTrue(messagesReceived.add(currentReceived.getValue()),
                        "Received duplicate message " + currentReceived.getValue());
            } else {
                break;
            }
        }
    }
    // Make sure key will not be distributed to multiple consumers
    Set<String> allKeys = Sets.newHashSet();
    consumerKeys.forEach((k, v) -> v.forEach(key -> {
        assertTrue(allKeys.add(key),
                "Key "+ key +  "is distributed to multiple consumers" );
    }));
    assertEquals(messagesReceived.size(), messagesToReceive);
}
 

/**
 * Validates proxy connection stats api.
 * 
 * @throws Exception
 */
@Test
public void testConnectionsStats() throws Exception {
    final String topicName1 = "persistent://sample/test/local/connections-stats";
    PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()).build();
    Producer<byte[]> producer = client.newProducer(Schema.BYTES).topic(topicName1).enableBatching(false)
            .messageRoutingMode(MessageRoutingMode.SinglePartition).create();

    // Create a consumer directly attached to broker
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName1).subscriptionName("my-sub").subscribe();

    int totalMessages = 10;
    for (int i = 0; i < totalMessages; i++) {
        producer.send("test".getBytes());
    }

    for (int i = 0; i < totalMessages; i++) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        checkNotNull(msg);
        consumer.acknowledge(msg);
    }

    Client httpClient = ClientBuilder.newClient(new ClientConfig().register(LoggingFeature.class));
    Response r = httpClient.target(proxyWebServer.getServiceUri()).path("/proxy-stats/connections").request()
            .get();
    Assert.assertEquals(r.getStatus(), Response.Status.OK.getStatusCode());
    String response = r.readEntity(String.class).trim();
    List<ConnectionStats> connectionStats = new Gson().fromJson(response, new TypeToken<List<ConnectionStats>>() {
    }.getType());

    assertNotNull(connectionStats);

    consumer.close();
    client.close();
}
 

@Test
public void testSimpleCloseTopic() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic5";
    final String subName = "sub5";

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    assertNotNull(subRef);

    Message<byte[]> msg;
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }

    producer.close();
    consumer.close();

    topicRef.close().get();
    assertFalse(pulsar.getBrokerService().getTopicReference(topicName).isPresent());
}
 

public void validateSourceResultAvro(Consumer<KeyValue<GenericRecord, GenericRecord>> consumer,
                                 int number, String eventType) throws Exception {
    int recordsNumber = 0;
    Message<KeyValue<GenericRecord, GenericRecord>> msg = consumer.receive(2, TimeUnit.SECONDS);
    while(msg != null) {
        recordsNumber ++;
        GenericRecord keyRecord = msg.getValue().getKey();
        Assert.assertNotNull(keyRecord.getFields());
        Assert.assertTrue(keyRecord.getFields().size() > 0);

        GenericRecord valueRecord = msg.getValue().getValue();
        Assert.assertNotNull(valueRecord.getFields());
        Assert.assertTrue(valueRecord.getFields().size() > 0);
        for (Field field : valueRecord.getFields()) {
            Assert.assertTrue(DEBEZIUM_FIELD_SET.contains(field.getName()));
        }

        if (eventType != null) {
            String op = valueRecord.getField("op").toString();
            Assert.assertEquals(this.eventContains(eventType, false), op);
        }
        consumer.acknowledge(msg);
        msg = consumer.receive(1, TimeUnit.SECONDS);
    }

    Assert.assertEquals(recordsNumber, number);
    log.info("Stop {} server container. topic: {} has {} records.", getSourceType(), consumer.getTopic(), recordsNumber);
}
 

public static void main(String[] args) throws IOException {
    // Create a Pulsar client instance. A single instance can be shared across many
    // producers and consumer within the same application
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(SERVICE_URL)
            .build();

    //Configure consumer specific settings.
    Consumer<byte[]> consumer = client.newConsumer()
            .topic(TOPIC_NAME)
            // Allow multiple consumers to attach to the same subscription
            // and get messages dispatched as a queue
            .subscriptionType(SubscriptionType.Shared)
            .subscriptionName(SUBSCRIPTION_NAME)
            .subscribe();


    // Once the consumer is created, it can be used for the entire application lifecycle
    System.out.println("Created consumer for the topic "+ TOPIC_NAME);

    do {
        // Wait until a message is available
        Message<byte[]> msg = consumer.receive();

        // Extract the message as a printable string and then log
        String content = new String(msg.getData());
        System.out.println("Received message '"+content+"' with ID "+msg.getMessageId());

        // Acknowledge processing of the message so that it can be deleted
        consumer.acknowledge(msg);
    } while (true);
}
 

@Test
public void testBatchMessageIndexAckForSharedSubscription() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "testBatchMessageIndexAckForSharedSubscription";

    @Cleanup
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscriptionType(SubscriptionType.Shared)
        .ackTimeout(1, TimeUnit.SECONDS)
        .subscribe();

    @Cleanup
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .batchingMaxPublishDelay(50, TimeUnit.MILLISECONDS)
        .create();

    final int messages = 100;
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < messages; i++) {
        if (i % 2 == 0) {
            consumer.acknowledge(consumer.receive());
        }
    }

    List<Message<Integer>> received = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        received.add(consumer.receive());
    }

    Assert.assertEquals(received.size(), 100);
}
 

@Test(timeOut = testTimeout)
public void multiTopicsInDifferentNameSpace() throws PulsarAdminException, PulsarClientException {
    List<String> topics = new ArrayList<>();
    topics.add("persistent://prop/use/ns-abc/topic-1");
    topics.add("persistent://prop/use/ns-abc/topic-2");
    topics.add("persistent://prop/use/ns-abc1/topic-3");
    admin.clusters().createCluster("use", new ClusterData(brokerUrl.toString()));
    admin.tenants().createTenant("prop", new TenantInfo(null, Sets.newHashSet("use")));
    admin.namespaces().createNamespace("prop/use/ns-abc");
    admin.namespaces().createNamespace("prop/use/ns-abc1");
    Consumer consumer = pulsarClient.newConsumer()
            .topics(topics)
            .subscriptionName("multiTopicSubscription")
            .subscriptionType(SubscriptionType.Exclusive)
            .subscribe();
    // create Producer
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
            .topic("persistent://prop/use/ns-abc/topic-1")
            .producerName("producer")
            .create();
    Producer<String> producer1 = pulsarClient.newProducer(Schema.STRING)
            .topic("persistent://prop/use/ns-abc/topic-2")
            .producerName("producer1")
            .create();
    Producer<String> producer2 = pulsarClient.newProducer(Schema.STRING)
            .topic("persistent://prop/use/ns-abc1/topic-3")
            .producerName("producer2")
            .create();
    //send message
    producer.send("ns-abc/topic-1-Message1");

    producer1.send("ns-abc/topic-2-Message1");

    producer2.send("ns-abc1/topic-3-Message1");

    int messageSet = 0;
    Message<byte[]> message = consumer.receive();
    do {
        messageSet ++;
        consumer.acknowledge(message);
        log.info("Consumer acknowledged : " + new String(message.getData()));
        message = consumer.receive(200, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(messageSet, 3);

    consumer.unsubscribe();
    consumer.close();
    producer.close();
    producer1.close();
    producer2.close();
}
 

@Test(dataProvider =  "AllCheckSchemaCompatibilityStrategy")
public void testIsAutoUpdateSchema(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception {
    final String tenant = PUBLIC_TENANT;
    final String topic = "test-consumer-compatibility";

    String namespace = "test-namespace-" + randomName(16);
    String fqtn = TopicName.get(
            TopicDomain.persistent.value(),
            tenant,
            namespace,
            topic
    ).toString();

    NamespaceName namespaceName = NamespaceName.get(tenant, namespace);

    admin.namespaces().createNamespace(
            tenant + "/" + namespace,
            Sets.newHashSet(CLUSTER_NAME)
    );

    assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(namespaceName.toString()),
            SchemaCompatibilityStrategy.FULL);
    
    admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy);
    admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo());

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false);
    ProducerBuilder<Schemas.PersonTwo> producerThreeBuilder = pulsarClient
            .newProducer(Schema.AVRO(SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .topic(fqtn);
    try {
        producerThreeBuilder.create();
    } catch (Exception e) {
        Assert.assertTrue(e.getMessage().contains("Schema not found and schema auto updating is disabled."));
    }

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), true);
    ConsumerBuilder<Schemas.PersonTwo> comsumerBuilder = pulsarClient.newConsumer(Schema.AVRO(
            SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .subscriptionName("test")
            .topic(fqtn);

    Producer<Schemas.PersonTwo> producer = producerThreeBuilder.create();
    Consumer<Schemas.PersonTwo> consumerTwo = comsumerBuilder.subscribe();

    producer.send(new Schemas.PersonTwo(2, "Lucy"));
    Message<Schemas.PersonTwo> message = consumerTwo.receive();

    Schemas.PersonTwo personTwo = message.getValue();
    consumerTwo.acknowledge(message);

    assertEquals(personTwo.getId(), 2);
    assertEquals(personTwo.getName(), "Lucy");

    producer.close();
    consumerTwo.close();

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false);

    producer = producerThreeBuilder.create();
    consumerTwo = comsumerBuilder.subscribe();

    producer.send(new Schemas.PersonTwo(2, "Lucy"));
    message = consumerTwo.receive();

    personTwo = message.getValue();
    consumerTwo.acknowledge(message);

    assertEquals(personTwo.getId(), 2);
    assertEquals(personTwo.getName(), "Lucy");

    consumerTwo.close();
    producer.close();
}
 

@Test
public void testPerNamespaceStats() throws Exception {
    Producer<byte[]> p1 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1").create();
    Producer<byte[]> p2 = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic2").create();

    Consumer<byte[]> c1 = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("test")
            .subscribe();

    Consumer<byte[]> c2 = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic2")
            .subscriptionName("test")
            .subscribe();

    final int messages = 10;

    for (int i = 0; i < messages; i++) {
        String message = "my-message-" + i;
        p1.send(message.getBytes());
        p2.send(message.getBytes());
    }

    for (int i = 0; i < messages; i++) {
        c1.acknowledge(c1.receive());
        c2.acknowledge(c2.receive());
    }

    ByteArrayOutputStream statsOut = new ByteArrayOutputStream();
    PrometheusMetricsGenerator.generate(pulsar, false, false, statsOut);
    String metricsStr = new String(statsOut.toByteArray());

    Multimap<String, Metric> metrics = parseMetrics(metricsStr);

    metrics.entries().forEach(e -> {
        System.out.println(e.getKey() + ": " + e.getValue());
    });

    // There should be 1 metric aggregated per namespace
    List<Metric> cm = (List<Metric>) metrics.get("pulsar_storage_write_latency_le_1");
    assertEquals(cm.size(), 1);
    assertNull(cm.get(0).tags.get("topic"));
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_producers_count");
    assertEquals(cm.size(), 2);
    assertNull(cm.get(1).tags.get("topic"));
    assertEquals(cm.get(1).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_in_bytes_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_in_messages_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_out_bytes_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    cm = (List<Metric>) metrics.get("pulsar_out_messages_total");
    assertEquals(cm.size(), 1);
    assertEquals(cm.get(0).tags.get("namespace"), "my-property/use/my-ns");

    p1.close();
    p2.close();
    c1.close();
    c2.close();
}
 

@Test
public void persistentTopicsInvalidCursorReset() throws Exception {
    admin.namespaces().setRetention("prop-xyz/ns1", new RetentionPolicies(10, 10));

    assertEquals(admin.topics().getList("prop-xyz/ns1"), Lists.newArrayList());

    String topicName = "persistent://prop-xyz/ns1/invalidcursorreset";
    // Force to create a topic
    publishMessagesOnPersistentTopic(topicName, 0);
    assertEquals(admin.topics().getList("prop-xyz/ns1"), Lists.newArrayList(topicName));

    // create consumer and subscription
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(pulsar.getWebServiceAddress())
            .statsInterval(0, TimeUnit.SECONDS)
            .build();
    Consumer<byte[]> consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub")
            .subscriptionType(SubscriptionType.Exclusive).subscribe();

    assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));

    publishMessagesOnPersistentTopic(topicName, 10);

    List<Message<byte[]>> messages = admin.topics().peekMessages(topicName, "my-sub", 10);
    assertEquals(messages.size(), 10);

    for (int i = 0; i < 10; i++) {
        Message<byte[]> message = consumer.receive();
        consumer.acknowledge(message);
    }
    // use invalid timestamp
    try {
        admin.topics().resetCursor(topicName, "my-sub", System.currentTimeMillis() - 190000);
    } catch (Exception e) {
        // fail the test
        throw e;
    }

    admin.topics().resetCursor(topicName, "my-sub", System.currentTimeMillis() + 90000);
    consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
    consumer.close();
    client.close();

    admin.topics().deleteSubscription(topicName, "my-sub");

    assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList());
    admin.topics().delete(topicName);
}
 

@Test
public void persistentTopicsInvalidCursorReset() throws Exception {
    admin.namespaces().setRetention("prop-xyz/use/ns1", new RetentionPolicies(10, 10));

    assertEquals(admin.topics().getList("prop-xyz/use/ns1"), Lists.newArrayList());

    String topicName = "persistent://prop-xyz/use/ns1/invalidcursorreset";
    // Force to create a topic
    publishMessagesOnPersistentTopic(topicName, 0);
    assertEquals(admin.topics().getList("prop-xyz/use/ns1"), Lists.newArrayList(topicName));

    // create consumer and subscription
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(pulsar.getWebServiceAddress())
            .statsInterval(0, TimeUnit.SECONDS)
            .build();
    Consumer<byte[]> consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub")
            .subscriptionType(SubscriptionType.Exclusive).subscribe();

    assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));

    publishMessagesOnPersistentTopic(topicName, 10);

    List<Message<byte[]>> messages = admin.topics().peekMessages(topicName, "my-sub", 10);
    assertEquals(messages.size(), 10);

    for (int i = 0; i < 10; i++) {
        Message<byte[]> message = consumer.receive();
        consumer.acknowledge(message);
    }
    // use invalid timestamp
    try {
        admin.topics().resetCursor(topicName, "my-sub", System.currentTimeMillis() - 190000);
    } catch (Exception e) {
        // fail the test
        throw e;
    }

    admin.topics().resetCursor(topicName, "my-sub", System.currentTimeMillis() + 90000);
    consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
    consumer.close();
    client.close();

    admin.topics().deleteSubscription(topicName, "my-sub");

    assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList());
    admin.topics().delete(topicName);
}
 

@Test
public void testCurrentLedgerRolloverIfFull() throws Exception {
    super.baseSetup();
    final String topicName = "persistent://prop/ns-abc/CurrentLedgerRolloverIfFullTest";

    @Cleanup
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topicName)
            .producerName("CurrentLedgerRolloverIfFullTest-producer-name")
            .create();

    @Cleanup
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic(topicName)
            .subscriptionName("CurrentLedgerRolloverIfFullTest-subscriber-name")
            .subscribe();

    Topic topicRef = pulsar.getBrokerService().getTopicReference(topicName).get();
    Assert.assertNotNull(topicRef);
    PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();

    ManagedLedgerConfig managedLedgerConfig = persistentTopic.getManagedLedger().getConfig();
    managedLedgerConfig.setRetentionTime(1, TimeUnit.SECONDS);
    managedLedgerConfig.setMaxEntriesPerLedger(2);
    managedLedgerConfig.setMinimumRolloverTime(1, TimeUnit.MILLISECONDS);
    managedLedgerConfig.setMaximumRolloverTime(5, TimeUnit.MILLISECONDS);

    int msgNum = 10;
    for (int i = 0; i < msgNum; i++) {
        producer.send(new byte[1024 * 1024]);
    }

    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger();
    Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), msgNum / 2);

    for (int i = 0; i < msgNum; i++) {
        Message<byte[]> msg = consumer.receive(2, TimeUnit.SECONDS);
        Assert.assertTrue(msg != null);
        consumer.acknowledge(msg);
    }

    // all the messages have been acknowledged
    // and all the ledgers have been removed except the the last ledger
    Thread.sleep(500);
    Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), 1);
    Assert.assertNotEquals(managedLedger.getCurrentLedgerSize(), 0);

    // trigger a ledger rollover
    // and now we have two ledgers, one with expired data and one for empty
    managedLedger.rollCurrentLedgerIfFull();
    Thread.sleep(1000);
    Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), 2);

    // trigger a ledger trimming
    // and now we only have the empty ledger
    managedLedger.trimConsumedLedgersInBackground(Futures.NULL_PROMISE);
    Assert.assertEquals(managedLedger.getCurrentLedgerSize(), 0);
}
 

@Test(timeOut = testTimeout)
public void testStartEmptyPatternConsumer() throws Exception {
    String key = "StartEmptyPatternConsumerTest";
    String subscriptionName = "my-ex-subscription-" + key;
    String topicName1 = "persistent://my-property/my-ns/pattern-topic-1-" + key;
    String topicName2 = "persistent://my-property/my-ns/pattern-topic-2-" + key;
    String topicName3 = "persistent://my-property/my-ns/pattern-topic-3-" + key;
    Pattern pattern = Pattern.compile("persistent://my-property/my-ns/pattern-topic.*");

    // 1. create partition
    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("prop", tenantInfo);
    admin.topics().createPartitionedTopic(topicName2, 2);
    admin.topics().createPartitionedTopic(topicName3, 3);

    // 2. Create consumer, this should success, but with empty sub-consumser internal
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
        .topicsPattern(pattern)
        .patternAutoDiscoveryPeriod(2)
        .subscriptionName(subscriptionName)
        .subscriptionType(SubscriptionType.Shared)
        .ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS)
        .receiverQueueSize(4)
        .subscribe();

    // 3. verify consumer get methods, to get 5 number of partitions and topics.
    assertSame(pattern, ((PatternMultiTopicsConsumerImpl<?>) consumer).getPattern());
    assertEquals(((PatternMultiTopicsConsumerImpl<?>) consumer).getPartitionedTopics().size(), 5);
    assertEquals(((PatternMultiTopicsConsumerImpl<?>) consumer).getConsumers().size(), 5);
    assertEquals(((PatternMultiTopicsConsumerImpl<?>) consumer).getTopics().size(), 2);

    // 4. create producer
    String messagePredicate = "my-message-" + key + "-";
    int totalMessages = 30;

    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2)
        .enableBatching(false)
        .messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
        .create();
    Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3)
        .enableBatching(false)
        .messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition)
        .create();

    // 5. call recheckTopics to subscribe each added topics above
    log.debug("recheck topics change");
    PatternMultiTopicsConsumerImpl<byte[]> consumer1 = ((PatternMultiTopicsConsumerImpl<byte[]>) consumer);
    consumer1.run(consumer1.getRecheckPatternTimeout());
    Thread.sleep(100);

    // 6. verify consumer get methods, to get number of partitions and topics, value 6=1+2+3.
    assertSame(pattern, ((PatternMultiTopicsConsumerImpl<?>) consumer).getPattern());
    assertEquals(((PatternMultiTopicsConsumerImpl<?>) consumer).getPartitionedTopics().size(), 6);
    assertEquals(((PatternMultiTopicsConsumerImpl<?>) consumer).getConsumers().size(), 6);
    assertEquals(((PatternMultiTopicsConsumerImpl<?>) consumer).getTopics().size(), 3);


    // 7. produce data
    for (int i = 0; i < totalMessages / 3; i++) {
        producer1.send((messagePredicate + "producer1-" + i).getBytes());
        producer2.send((messagePredicate + "producer2-" + i).getBytes());
        producer3.send((messagePredicate + "producer3-" + i).getBytes());
    }

    // 8. should receive all the message
    int messageSet = 0;
    Message<byte[]> message = consumer.receive();
    do {
        assertTrue(message instanceof TopicMessageImpl);
        messageSet ++;
        consumer.acknowledge(message);
        log.debug("Consumer acknowledged : " + new String(message.getData()));
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(messageSet, totalMessages);

    consumer.unsubscribe();
    consumer.close();
    producer1.close();
    producer2.close();
    producer3.close();
}
 

@Test
public void TestConsumedLedgersTrim() throws Exception {
    conf.setRetentionCheckIntervalInSeconds(1);
    super.baseSetup();
    final String topicName = "persistent://prop/ns-abc/TestConsumedLedgersTrim";
    final String subscriptionName = "my-subscriber-name";

    @Cleanup
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topicName)
            .producerName("producer-name")
            .create();
    @Cleanup
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
            .subscribe();
    Topic topicRef = pulsar.getBrokerService().getTopicReference(topicName).get();
    Assert.assertNotNull(topicRef);
    PersistentTopic persistentTopic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();

    ManagedLedgerConfig managedLedgerConfig = persistentTopic.getManagedLedger().getConfig();
    managedLedgerConfig.setRetentionSizeInMB(1L);
    managedLedgerConfig.setRetentionTime(1, TimeUnit.SECONDS);
    managedLedgerConfig.setMaxEntriesPerLedger(2);
    managedLedgerConfig.setMinimumRolloverTime(1, TimeUnit.MILLISECONDS);

    int msgNum = 10;
    for (int i = 0; i < msgNum; i++) {
        producer.send(new byte[1024 * 1024]);
    }

    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) persistentTopic.getManagedLedger();
    Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), msgNum / 2);

    //no traffic, unconsumed ledger will be retained
    Thread.sleep(1200);
    Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), msgNum / 2);

    for (int i = 0; i < msgNum; i++) {
        Message<byte[]> msg = consumer.receive(2, TimeUnit.SECONDS);
        Assert.assertTrue(msg != null);
        consumer.acknowledge(msg);
    }
    Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), msgNum / 2);

    //no traffic, but consumed ledger will be cleaned
    Thread.sleep(1500);
    Assert.assertEquals(managedLedger.getLedgersInfoAsList().size(), 1);
}
 

private int consume(String topic) {
    int numMessagesConsumed = 0;
    int returnCode = 0;

    try {
        PulsarClient client = clientBuilder.build();
        ConsumerBuilder<byte[]> builder = client.newConsumer()
                .subscriptionName(this.subscriptionName)
                .subscriptionType(subscriptionType)
                .subscriptionInitialPosition(subscriptionInitialPosition);

        if (isRegex) {
            builder.topicsPattern(Pattern.compile(topic));
        } else {
            builder.topic(topic);
        }

        ConsumerBuilder<byte[]> consumerBuilder = client.newConsumer().topic(topic);
        if (this.maxPendingChuckedMessage > 0) {
            consumerBuilder.maxPendingChuckedMessage(this.maxPendingChuckedMessage);
        }
        if (this.receiverQueueSize > 0) {
            consumerBuilder.maxPendingChuckedMessage(this.receiverQueueSize);
        }
        Consumer<byte[]> consumer = consumerBuilder.subscriptionName(this.subscriptionName)
                .autoAckOldestChunkedMessageOnQueueFull(this.autoAckOldestChunkedMessageOnQueueFull)
                .subscriptionType(subscriptionType).subscribe();

        RateLimiter limiter = (this.consumeRate > 0) ? RateLimiter.create(this.consumeRate) : null;
        while (this.numMessagesToConsume == 0 || numMessagesConsumed < this.numMessagesToConsume) {
            if (limiter != null) {
                limiter.acquire();
            }

            Message<byte[]> msg = consumer.receive(5, TimeUnit.SECONDS);
            if (msg == null) {
                LOG.debug("No message to consume after waiting for 5 seconds.");
            } else {
                numMessagesConsumed += 1;
                System.out.println(MESSAGE_BOUNDARY);
                String output = this.interpretMessage(msg, displayHex);
                System.out.println(output);
                consumer.acknowledge(msg);
            }
        }
        client.close();
    } catch (Exception e) {
        LOG.error("Error while consuming messages");
        LOG.error(e.getMessage(), e);
        returnCode = -1;
    } finally {
        LOG.info("{} messages successfully consumed", numMessagesConsumed);
    }

    return returnCode;

}
 

/**
 * Validation: 1. validates active-cursor after active subscription 2. validate active-cursor with subscription 3.
 * unconsumed messages should be present into cache 4. cache and active-cursor should be empty once subscription is
 * closed
 *
 * @throws Exception
 */
@Test
public void testActiveSubscriptionWithCache() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic2";
    final String subName = "sub2";

    Message<byte[]> msg;
    int recvQueueSize = 4;

    // (1) Create subscription
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .receiverQueueSize(recvQueueSize).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
        .topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();

    // (2) Produce Messages
    for (int i = 0; i < recvQueueSize / 2; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();

    // (3) Get Entry cache
    ManagedLedgerImpl ledger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
    cacheField.setAccessible(true);
    EntryCacheImpl entryCache = (EntryCacheImpl) cacheField.get(ledger);

    /************* Validation on non-empty active-cursor **************/
    // (4) Get ActiveCursor : which is list of active subscription
    Iterable<ManagedCursor> activeCursors = ledger.getActiveCursors();
    ManagedCursor curosr = activeCursors.iterator().next();
    // (4.1) Validate: active Cursor must be non-empty
    assertNotNull(curosr);
    // (4.2) Validate: validate cursor name
    assertEquals(subName, curosr.getName());

    /************* Validation on empty active-cursor **************/
    // (5) Close consumer: which (1)removes activeConsumer and (2)clears the entry-cache
    consumer.close();
    Thread.sleep(1000);
    // (5.1) Validate: active-consumer must be empty
    assertFalse(ledger.getActiveCursors().iterator().hasNext());
    // (5.2) Validate: Entry-cache must be cleared
    assertEquals(entryCache.getSize(), 0);

}
 

@Test
public void testBrokerServicePersistentRedeliverTopicStats() throws Exception {
    final String topicName = "persistent://prop/ns-abc/successSharedTopic";
    final String subName = "successSharedSub";

    TopicStats stats;
    SubscriptionStats subStats;

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .subscriptionType(SubscriptionType.Shared).acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();

    // subscription stats
    assertEquals(stats.subscriptions.keySet().size(), 1);
    assertEquals(subStats.msgBacklog, 0);
    assertEquals(subStats.consumers.size(), 1);

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();

    // publisher stats
    assertEquals(subStats.msgBacklog, 10);
    assertEquals(stats.publishers.size(), 1);
    assertTrue(stats.publishers.get(0).msgRateIn > 0.0);
    assertTrue(stats.publishers.get(0).msgThroughputIn > 0.0);
    assertTrue(stats.publishers.get(0).averageMsgSize > 0.0);

    // aggregated publish stats
    assertEquals(stats.msgRateIn, stats.publishers.get(0).msgRateIn);
    assertEquals(stats.msgThroughputIn, stats.publishers.get(0).msgThroughputIn);
    double diff = stats.averageMsgSize - stats.publishers.get(0).averageMsgSize;
    assertTrue(Math.abs(diff) < 0.000001);

    // consumer stats
    assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
    assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
    assertEquals(subStats.msgRateRedeliver, 0.0);
    assertEquals(subStats.consumers.get(0).unackedMessages, 10);

    // aggregated consumer stats
    assertEquals(subStats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(subStats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);
    assertEquals(stats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(stats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);
    assertEquals(subStats.unackedMessages, subStats.consumers.get(0).unackedMessages);

    consumer.redeliverUnacknowledgedMessages();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();
    assertTrue(subStats.msgRateRedeliver > 0.0);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);

    Message<byte[]> msg;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    consumer.close();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();

    assertEquals(subStats.msgBacklog, 0);
}
 

/**
 * Validate proxy topic stats api
 * 
 * @throws Exception
 */
@Test
public void testTopicStats() throws Exception {
    proxyService.setProxyLogLevel(2);
    final String topicName = "persistent://sample/test/local/topic-stats";
    final String topicName2 = "persistent://sample/test/local/topic-stats-2";

    PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()).build();
    Producer<byte[]> producer1 = client.newProducer(Schema.BYTES).topic(topicName).enableBatching(false)
            .producerName("producer1").messageRoutingMode(MessageRoutingMode.SinglePartition).create();

    Producer<byte[]> producer2 = client.newProducer(Schema.BYTES).topic(topicName2).enableBatching(false)
            .producerName("producer2").messageRoutingMode(MessageRoutingMode.SinglePartition).create();

    // Create a consumer directly attached to broker
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
    Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName2).subscriptionName("my-sub")
            .subscribe();

    int totalMessages = 10;
    for (int i = 0; i < totalMessages; i++) {
        producer1.send("test".getBytes());
        producer2.send("test".getBytes());
    }

    for (int i = 0; i < totalMessages; i++) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        checkNotNull(msg);
        consumer.acknowledge(msg);
        msg = consumer2.receive(1, TimeUnit.SECONDS);
    }

    Client httpClient = ClientBuilder.newClient(new ClientConfig().register(LoggingFeature.class));
    Response r = httpClient.target(proxyWebServer.getServiceUri()).path("/proxy-stats/topics").request()
            .get();
    Assert.assertEquals(r.getStatus(), Response.Status.OK.getStatusCode());
    String response = r.readEntity(String.class).trim();
    Map<String, TopicStats> topicStats = new Gson().fromJson(response, new TypeToken<Map<String, TopicStats>>() {
    }.getType());

    assertNotNull(topicStats.get(topicName));

    consumer.close();
    consumer2.close();
    client.close();
}
 

@Test(timeOut = 20000, dataProvider = "partitionsAndBatch")
public void testKafkaProducePulsarConsume(int partitionNumber, boolean isBatch) throws Exception {
    String kafkaTopicName = "kopKafkaProducePulsarConsume" + partitionNumber;
    String pulsarTopicName = "persistent://public/default/" + kafkaTopicName;
    String key1 = "header_key1_";
    String key2 = "header_key2_";
    String value1 = "header_value1_";
    String value2 = "header_value2_";

    // create partitioned topic.
    admin.topics().createPartitionedTopic(kafkaTopicName, partitionNumber);

    @Cleanup
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
        .topic(pulsarTopicName)
        .subscriptionName("test_k_producer_k_pconsumer_sub")
        .subscribe();

    // 1. produce message with Kafka producer.
    @Cleanup
    KProducer kProducer = new KProducer(kafkaTopicName, false, getKafkaBrokerPort());

    int totalMsgs = 10;
    String messageStrPrefix = "Message_Kop_KafkaProducePulsarConsume_"  + partitionNumber + "_";

    for (int i = 0; i < totalMsgs; i++) {
        String messageStr = messageStrPrefix + i;
        ProducerRecord record = new ProducerRecord<>(
            kafkaTopicName,
            i,
            messageStr);

        record.headers()
            .add(key1 + i, (value1 + i).getBytes(UTF_8))
            .add(key2 + i, (value2 + i).getBytes(UTF_8));

        if (isBatch) {
            kProducer.getProducer()
                .send(record);
        } else {
            kProducer.getProducer()
                .send(record)
                .get();
        }
        if (log.isDebugEnabled()) {
            log.debug("Kafka Producer Sent message with header: ({}, {})", i, messageStr);
        }
    }

    // 2. Consume messages use Pulsar client Consumer. verify content and key and headers
    Message<byte[]> msg = null;
    for (int i = 0; i < totalMsgs; i++) {
        msg = consumer.receive(100, TimeUnit.MILLISECONDS);
        assertNotNull(msg);
        Integer key = kafkaIntDeserialize(Base64.getDecoder().decode(msg.getKey()));
        assertEquals(messageStrPrefix + key.toString(), new String(msg.getValue()));

        // verify added 2 key-value pair
        Map<String, String> properties = msg.getProperties();
        assertEquals(properties.size(), 2);
        for (Map.Entry<String, String> kv: properties.entrySet()) {
            String k = kv.getKey();
            String v = kv.getValue();

            if (log.isDebugEnabled()) {
                log.debug("headers key: {}, value:{}", k, v);
            }

            assertTrue(k.contains(key1) || k.contains(key2));
            assertTrue(v.contains(value1) || v.contains(value2));
        }
        if (log.isDebugEnabled()) {
            log.debug("Pulsar consumer get message: {}, key: {}",
                new String(msg.getData()),
                kafkaIntDeserialize(Base64.getDecoder().decode(msg.getKey())).toString());
        }
        consumer.acknowledge(msg);
    }

    // verify have received all messages
    msg = consumer.receive(100, TimeUnit.MILLISECONDS);
    assertNull(msg);
}