Java Code Examples for org.apache.pulsar.client.api.Consumer#receive()

public void validateSourceResultJson(Consumer<KeyValue<byte[], byte[]>> consumer, int number, String eventType) throws Exception {
    int recordsNumber = 0;
    Message<KeyValue<byte[], byte[]>> msg = consumer.receive(2, TimeUnit.SECONDS);
    while(msg != null) {
        recordsNumber ++;
        final String key = new String(msg.getValue().getKey());
        final String value = new String(msg.getValue().getValue());
        log.info("Received message: key = {}, value = {}.", key, value);
        Assert.assertTrue(key.contains(this.keyContains()));
        Assert.assertTrue(value.contains(this.valueContains()));
        if (eventType != null) {
            Assert.assertTrue(value.contains(this.eventContains(eventType, true)));
        }
        consumer.acknowledge(msg);
        msg = consumer.receive(1, TimeUnit.SECONDS);
    }

    Assert.assertEquals(recordsNumber, number);
    log.info("Stop {} server container. topic: {} has {} records.", getSourceType(), consumer.getTopic(), recordsNumber);
}
 

/**
 * verifies that messages whose size is larger than 2^14 bytes (max size of single TLS chunk) can be
 * produced/consumed
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testTlsLargeSizeMessage() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int MESSAGE_SIZE = 16 * 1024 + 1;
    log.info("-- message size --", MESSAGE_SIZE);
    String topicName = "persistent://my-property/use/my-ns/testTlsLargeSizeMessage"
                       + System.currentTimeMillis();

    internalSetUpForClient(true, pulsar.getBrokerServiceUrlTls());
    internalSetUpForNamespace();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
            .create();
    for (int i = 0; i < 10; i++) {
        byte[] message = new byte[MESSAGE_SIZE];
        Arrays.fill(message, (byte) i);
        producer.send(message);
    }

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        byte[] expected = new byte[MESSAGE_SIZE];
        Arrays.fill(expected, (byte) i);
        Assert.assertEquals(expected, msg.getData());
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

/**
 * verifies that messages whose size is larger than 2^14 bytes (max size of single TLS chunk) can be
 * produced/consumed
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testTlsLargeSizeMessage() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final int MESSAGE_SIZE = 16 * 1024 + 1;
    log.info("-- message size --", MESSAGE_SIZE);

    internalSetUpForClient(true, pulsar.getBrokerServiceUrlTls());
    internalSetUpForNamespace();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic1")
            .create();
    for (int i = 0; i < 10; i++) {
        byte[] message = new byte[MESSAGE_SIZE];
        Arrays.fill(message, (byte) i);
        producer.send(message);
    }

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        byte[] expected = new byte[MESSAGE_SIZE];
        Arrays.fill(expected, (byte) i);
        Assert.assertEquals(expected, msg.getData());
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

/**
 * Test that sequence id from a producer is correct when there are send errors
 */
@Test
public void testCheckSequenceId() throws Exception {
    admin.namespaces().createNamespace("prop/my-test", Collections.singleton("usc"));

    String topicName = "prop/my-test/my-topic";
    int N = 10;

    PulsarClient client = PulsarClient.builder().serviceUrl(pulsar.getBrokerServiceUrl()).build();

    // Create consumer
    Consumer<String> consumer = client.newConsumer(Schema.STRING).topic(topicName).subscriptionName("sub")
            .subscribe();

    // Fence the topic by opening the ManagedLedger for the topic outside the Pulsar broker. This will cause the
    // broker to fail subsequent send operation and it will trigger a recover
    ManagedLedgerClientFactory clientFactory = new ManagedLedgerClientFactory(pulsar.getConfiguration(),
            pulsar.getZkClient(), pulsar.getBookKeeperClientFactory());
    ManagedLedgerFactory mlFactory = clientFactory.getManagedLedgerFactory();
    ManagedLedger ml = mlFactory.open(TopicName.get(topicName).getPersistenceNamingEncoding());
    ml.close();
    clientFactory.close();

    // Create a producer
    Producer<String> producer = client.newProducer(Schema.STRING).topic(topicName).create();

    for (int i = 0; i < N; i++) {
        producer.send("Hello-" + i);
    }

    for (int i = 0; i < N; i++) {
        Message<String> msg = consumer.receive();
        assertEquals(msg.getValue(), "Hello-" + i);
        assertEquals(msg.getSequenceId(), i);
        consumer.acknowledge(msg);
    }

    client.close();
}
 

@Test(dataProvider = "containerBuilder")
public void testRetrieveSequenceIdGenerated(BatcherBuilder builder) throws Exception {

    int numMsgs = 10;
    final String topicName = "persistent://prop/ns-abc/testRetrieveSequenceIdGenerated-" + UUID.randomUUID();
    final String subscriptionName = "sub-1";

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
            .subscriptionType(SubscriptionType.Shared).subscribe();

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
            .batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgs).enableBatching(true)
            .batcherBuilder(builder)
            .create();

    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("my-message-" + i).getBytes();
        sendFutureList.add(producer.sendAsync(message));
    }
    FutureUtil.waitForAll(sendFutureList).get();

    for (int i = 0; i < numMsgs; i++) {
        Message<byte[]> received = consumer.receive();
        Assert.assertEquals(received.getSequenceId(), i);
        consumer.acknowledge(received);
    }

    producer.close();
    consumer.close();
}
 

@Test
public void testPartitions() throws Exception {
    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("sample", tenantInfo);
    @Cleanup
    PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl())
            .build();
    admin.topics().createPartitionedTopic("persistent://sample/test/local/partitioned-topic", 2);

    @Cleanup
    Producer<byte[]> producer = client.newProducer(Schema.BYTES)
        .topic("persistent://sample/test/local/partitioned-topic")
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();

    // Create a consumer directly attached to broker
    @Cleanup
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://sample/test/local/partitioned-topic")
            .subscriptionName("my-sub").subscribe();

    for (int i = 0; i < 10; i++) {
        producer.send("test".getBytes());
    }

    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        checkNotNull(msg);
    }
}
 

@Test
public void testSingleMessageBatch() throws Exception {
    String topicName = "prop/ns-abc/topic-estSingleMessageBatch";

    Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
            .topic(topicName)
            .enableBatching(true)
            .batchingMaxPublishDelay(10, TimeUnit.SECONDS)
            .create();

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topicName)
            .subscriptionName("subscription")
            .ackTimeout(1, TimeUnit.HOURS)
            .subscribe();

    // Force the creation of a batch with a single message
    producer.sendAsync("hello");
    producer.flush();

    Message<String> message = consumer.receive();

    assertFalse(((ConsumerImpl<?>) consumer).getUnAckedMessageTracker().isEmpty());

    consumer.acknowledge(message);

    assertTrue(((ConsumerImpl<?>) consumer).getUnAckedMessageTracker().isEmpty());
}
 

@Test
public void testPartitions() throws Exception {
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(proxyService.getServiceUrlTls())
            .allowTlsInsecureConnection(false).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH).build();
    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("sample", tenantInfo);
    admin.topics().createPartitionedTopic("persistent://sample/test/local/partitioned-topic", 2);

    Producer<byte[]> producer = client.newProducer(Schema.BYTES).topic("persistent://sample/test/local/partitioned-topic")
            .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();

    // Create a consumer directly attached to broker
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://sample/test/local/partitioned-topic")
            .subscriptionName("my-sub").subscribe();

    for (int i = 0; i < 10; i++) {
        producer.send("test".getBytes());
    }

    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        checkNotNull(msg);
    }

    client.close();
}
 

/**
 * Publish from Java and consume from Python
 */
@Test(dataProvider = "ServiceUrls")
public void testPythonPublishJavaConsume(String serviceUrl) throws Exception {
    String nsName = generateNamespaceName();
    pulsarCluster.createNamespace(nsName);

    String topicName = generateTopicName(nsName, "testPythonPublishJavaConsume", true);

    @Cleanup
    PulsarClient client = PulsarClient.builder()
            .serviceUrl(serviceUrl)
            .build();

    @Cleanup
    Consumer<Example2> consumer = client.newConsumer(Schema.AVRO(Example2.class))
            .topic(topicName)
            .subscriptionName("test-sub")
            .subscribe();

    // Verify Python can receive the typed message

    ContainerExecResult res = pulsarCluster.getAnyBroker()
            .execCmd("/pulsar/examples/python-examples/producer_schema.py", "pulsar://localhost:6650", topicName);
    assertEquals(res.getExitCode(), 0);

    Message<Example2> msg = consumer.receive();
    Example2 e2 = msg.getValue();
    assertEquals(e2.a, "Hello");
    assertEquals(e2.b, 1);
}
 

@Test
public void keyValueNullInlineTest() throws PulsarClientException {
    String topic = "persistent://prop/ns-abc/kv-null-value-test";

    @Cleanup
    Producer<KeyValue<String, String>> producer = pulsarClient
            .newProducer(KeyValueSchema.of(Schema.STRING, Schema.STRING))
            .topic(topic)
            .create();

    @Cleanup
    Consumer<KeyValue<String, String>> consumer = pulsarClient
            .newConsumer(KeyValueSchema.of(Schema.STRING, Schema.STRING))
            .topic(topic)
            .subscriptionName("test")
            .subscribe();

    int numMessage = 10;
    for (int i = 0; i < numMessage; i++) {
        producer.newMessage().value(new KeyValue<>(null, "test")).send();
        producer.newMessage().value(new KeyValue<>("test", null)).send();
        producer.newMessage().value(new KeyValue<>(null, null)).send();
    }

    Message<KeyValue<String, String>> message;
    KeyValue<String, String> keyValue;
    for (int i = 0; i < numMessage; i++) {
        message = consumer.receive();
        keyValue = message.getValue();
        Assert.assertNull(keyValue.getKey());
        Assert.assertEquals("test", keyValue.getValue());

        message = consumer.receive();
        keyValue = message.getValue();
        Assert.assertEquals("test", keyValue.getKey());
        Assert.assertNull(keyValue.getValue());

        message = consumer.receive();
        keyValue = message.getValue();
        Assert.assertNull(keyValue.getKey());
        Assert.assertNull(keyValue.getValue());
    }

}
 

private static void publishAndConsumeMessages(String inputTopic,
                                              String outputTopic,
                                              int numMessages) throws Exception {
    @Cleanup PulsarClient client = PulsarClient.builder()
        .serviceUrl(pulsarCluster.getPlainTextServiceUrl())
        .build();

    @Cleanup Consumer<String> consumer = client.newConsumer(Schema.STRING)
        .topic(outputTopic)
        .subscriptionType(SubscriptionType.Exclusive)
        .subscriptionName("test-sub")
        .subscribe();

    if (inputTopic.endsWith(".*")) {
        @Cleanup Producer<String> producer1 = client.newProducer(Schema.STRING)
                .topic(inputTopic.substring(0, inputTopic.length() - 2) + "1")
                .create();

        @Cleanup Producer<String> producer2 = client.newProducer(Schema.STRING)
                .topic(inputTopic.substring(0, inputTopic.length() - 2) + "2")
                .create();

        for (int i = 0; i < numMessages / 2; i++) {
            producer1.send("message-" + i);
        }

        for (int i = numMessages / 2; i < numMessages; i++) {
            producer2.send("message-" + i);
        }
    } else {
        @Cleanup Producer<String> producer = client.newProducer(Schema.STRING)
                .topic(inputTopic)
                .create();

        for (int i = 0; i < numMessages; i++) {
            producer.send("message-" + i);
        }
    }

    Set<String> expectedMessages = new HashSet<>();
    for (int i = 0; i < numMessages; i++) {
        expectedMessages.add("message-" + i + "!");
    }

    for (int i = 0; i < numMessages; i++) {
        Message<String> msg = consumer.receive(30, TimeUnit.SECONDS);
        log.info("Received: {}", msg.getValue());
        assertTrue(expectedMessages.contains(msg.getValue()));
        expectedMessages.remove(msg.getValue());
    }
}
 

@Test(timeOut = testTimeout)
public void testFailoverInactiveConsumer() throws Exception {
    String key = "testFailoverInactiveConsumer";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-failover-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);

    // 2. Create consumer
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName(subscriptionName).receiverQueueSize(10).subscriptionType(SubscriptionType.Failover);
    Consumer<byte[]> consumer1 = consumerBuilder.clone().consumerName("Consumer-1").subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("Consumer-2").subscribe();

    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }

    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message<byte[]> message1;
    Message<byte[]> message2;
    do {
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }
    } while (message1 != null);
    log.info("Consumer 1 receives = " + receivedConsumer1);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // Consumer 2 is on Stand By
    assertEquals(receivedConsumer2, 0);

    // 5. Consumer 2 asks for a redelivery but the request is ignored
    log.info("Consumer 2 asks for resend");
    consumer2.redeliverUnacknowledgedMessages();
    Thread.sleep(1000);

    message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
    message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    assertNull(message1);
    assertNull(message2);
}
 

@Test(dataProvider =  "CanReadLastSchemaCompatibilityStrategy")
public void testConsumerCompatibilityCheckCanReadLastTest(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception {
    final String tenant = PUBLIC_TENANT;
    final String topic = "test-consumer-compatibility";

        String namespace = "test-namespace-" + randomName(16);
        String fqtn = TopicName.get(
                TopicDomain.persistent.value(),
                tenant,
                namespace,
                topic
        ).toString();

        NamespaceName namespaceName = NamespaceName.get(tenant, namespace);

        admin.namespaces().createNamespace(
                tenant + "/" + namespace,
                Sets.newHashSet(CLUSTER_NAME)
        );

        admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy);
        admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo());
        admin.schemas().createSchema(fqtn, Schema.AVRO(SchemaDefinition.builder()
                .withAlwaysAllowNull(false).withPojo(Schemas.PersonTwo.class).build()).getSchemaInfo());

        Consumer<Schemas.PersonThree> consumerThree = pulsarClient.newConsumer(Schema.AVRO(
                SchemaDefinition.<Schemas.PersonThree>builder().withAlwaysAllowNull
                        (false).withSupportSchemaVersioning(true).
                        withPojo(Schemas.PersonThree.class).build()))
                .subscriptionName("test")
                .topic(fqtn)
                .subscribe();

        Producer<Schemas.PersonOne> producerOne = pulsarClient
                .newProducer(Schema.AVRO(Schemas.PersonOne.class))
                .topic(fqtn)
                .create();


        Schemas.PersonOne personOne = new Schemas.PersonOne();
        personOne.setId(1);

        producerOne.send(personOne);
        Message<Schemas.PersonThree> message = null;

        try {
            message = consumerThree.receive();
            message.getValue();
        } catch (Exception e) {
            Assert.assertTrue(e instanceof SchemaSerializationException);
            consumerThree.acknowledge(message);
        }

        Producer<Schemas.PersonTwo> producerTwo = pulsarClient
                .newProducer(Schema.AVRO(SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                        (false).withSupportSchemaVersioning(true).
                        withPojo(Schemas.PersonTwo.class).build()))
                .topic(fqtn)
                .create();

        Schemas.PersonTwo personTwo = new Schemas.PersonTwo();
        personTwo.setId(1);
        personTwo.setName("Jerry");
        producerTwo.send(personTwo);

        message = consumerThree.receive();
        Schemas.PersonThree personThree = message.getValue();
        consumerThree.acknowledge(message);

        assertEquals(personThree.getId(), 1);
        assertEquals(personThree.getName(), "Jerry");

        consumerThree.close();
        producerOne.close();
        producerTwo.close();
}
 

@Test
public void testUnackedCountWithRedeliveries() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/testUnackedCountWithRedeliveries";
    final String subName = "sub3";
    final int numMsgs = 10;

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();

    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .receiverQueueSize(10).subscriptionType(SubscriptionType.Shared)
            .acknowledgmentGroupTime(0, TimeUnit.SECONDS);
    ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();

    for (int i = 0; i < numMsgs; i++) {
        producer.send(("hello-" + i).getBytes());
    }

    Set<MessageId> c1_receivedMessages = new HashSet<>();

    // C-1 gets all messages but doesn't ack
    for (int i = 0; i < numMsgs; i++) {
        c1_receivedMessages.add(consumer1.receive().getMessageId());
    }

    // C-2 will not get any message initially, since everything went to C-1 already
    Consumer<byte[]> consumer2 = consumerBuilder.subscribe();

    // Trigger C-1 to redeliver everything, half will go C-1 again and the other half to C-2
    consumer1.redeliverUnacknowledgedMessages(c1_receivedMessages);

    // Consumer 2 will also receive all message but not ack
    for (int i = 0; i < numMsgs; i++) {
        consumer2.receive();
    }

    for (MessageId msgId : c1_receivedMessages) {
        consumer1.acknowledge(msgId);
    }

    TopicStats stats = admin.topics().getStats(topicName);

    // Unacked messages count should be 0 for both consumers at this point
    SubscriptionStats subStats = stats.subscriptions.get(subName);
    assertEquals(subStats.msgBacklog, 0);

    for (ConsumerStats cs : subStats.consumers) {
        assertEquals(cs.unackedMessages, 0);
    }

    producer.close();
    consumer1.close();
    consumer2.close();

    deleteTopic(topicName);
}
 

@Test(dataProvider = "batch")
public void testBackoffAndReconnect(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    // Create consumer and producer
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
            .topic("persistent://my-property/use/my-ns/my-topic4")
            .subscriptionName("my-subscriber-name")
            .subscriptionType(SubscriptionType.Exclusive)
            .startMessageIdInclusive()
            .subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic("persistent://my-property/use/my-ns/my-topic4")
            .batchingMaxMessages(5)
            .batchingMaxPublishDelay(BATCHING_MAX_PUBLISH_DELAY_THRESHOLD, TimeUnit.MILLISECONDS)
            .enableBatching(batchMessageDelayMs != 0)
            .create();

    // Produce messages
    CompletableFuture<MessageId> lastFuture = null;
    for (int i = 0; i < 10; i++) {
        lastFuture = producer.sendAsync(("my-message-" + i).getBytes()).thenApply(msgId -> {
            log.info("Published message id: {}", msgId);
            return msgId;
        });
    }

    lastFuture.get();

    Message<byte[]> msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        log.info("Received: [{}]", new String(msg.getData()));
    }

    // Restart the broker and wait for the backoff to kick in. The client library will try to reconnect, and once
    // the broker is up, the consumer should receive the duplicate messages.
    log.info("-- Restarting broker --");
    restartBroker();

    msg = null;
    log.info("Receiving duplicate messages..");
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        log.info("Received: [{}]", new String(msg.getData()));
        Assert.assertNotNull(msg, "Message cannot be null");
    }
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

@Test
public void testBatchMessageIndexAckForExclusiveSubscription() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "testBatchMessageIndexAckForExclusiveSubscription";

    @Cleanup
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    @Cleanup
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .batchingMaxPublishDelay(50, TimeUnit.MILLISECONDS)
        .create();

    final int messages = 100;
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < messages; i++) {
        if (i == 49) {
            consumer.acknowledgeCumulative(consumer.receive());
        } else {
            consumer.receive();
        }
    }

    //Wait ack send.
    Thread.sleep(1000);
    consumer.close();
    consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    List<Message<Integer>> received = new ArrayList<>(50);
    for (int i = 0; i < 50; i++) {
        received.add(consumer.receive());
    }

    Assert.assertEquals(received.size(), 50);

    Message<Integer> moreMessage = consumer.receive(1, TimeUnit.SECONDS);
    Assert.assertNull(moreMessage);

    futures.clear();
    for (int i = 0; i < 50; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < 50; i++) {
        received.add(consumer.receive());
    }

    // Ensure the flow permit is work well since the client skip the acked batch index,
    // broker also need to handle the available permits.
    Assert.assertEquals(received.size(), 100);
}
 

/**
 * <pre>
 * It verifies e2e tls + Authentication + Authorization (client -> proxy -> broker>
 *
 * 1. client connects to proxy over tls and pass auth-data
 * 2. proxy authenticate client and retrieve client-role
 *    and send it to broker as originalPrincipal over tls
 * 3. client creates producer/consumer via proxy
 * 4. broker authorize producer/consumer create request using originalPrincipal
 *
 * </pre>
 *
 * @throws Exception
 */
@SuppressWarnings("deprecation")
@Test
public void testTlsSyncProducerAndConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);

    final String proxyServiceUrl = proxyService.getServiceUrlTls();
    Map<String, String> authParams = Maps.newHashMap();
    authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
    authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);
    Authentication authTls = new AuthenticationTls();
    authTls.configure(authParams);
    // create a client which connects to proxy over tls and pass authData
    PulsarClient proxyClient = createPulsarClient(authTls, proxyServiceUrl);

    admin.clusters().createCluster(configClusterName, new ClusterData(brokerUrl.toString(), brokerUrlTls.toString(),
            pulsar.getBrokerServiceUrl(), pulsar.getBrokerServiceUrlTls()));
    admin.tenants().createTenant("my-property",
            new TenantInfo(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet("test")));
    admin.namespaces().createNamespace("my-property/my-ns", Sets.newHashSet("test"));

    Consumer<byte[]> consumer = proxyClient.newConsumer().topic("persistent://my-property/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();
    Producer<byte[]> producer = proxyClient.newProducer(Schema.BYTES).topic("persistent://my-property/my-ns/my-topic1")
            .create();
    final int msgs = 10;
    for (int i = 0; i < msgs; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    int count = 0;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        count++;
    }
    // Acknowledge the consumption of all messages at once
    Assert.assertEquals(msgs, count);
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

@Test
public void testBatchMessageIndexAckForExclusiveSubscription() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "testBatchMessageIndexAckForExclusiveSubscription";

    @Cleanup
    Consumer<Integer> consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    @Cleanup
    Producer<Integer> producer = pulsarClient.newProducer(Schema.INT32)
        .topic(topic)
        .batchingMaxPublishDelay(50, TimeUnit.MILLISECONDS)
        .create();

    final int messages = 100;
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.sendAsync(i));
    }
    FutureUtil.waitForAll(futures).get();

    for (int i = 0; i < messages; i++) {
        if (i == 49) {
            consumer.acknowledgeCumulative(consumer.receive());
        } else {
            consumer.receive();
        }
    }

    //Wait ack send.
    Thread.sleep(1000);
    consumer.close();
    consumer = pulsarClient.newConsumer(Schema.INT32)
        .topic(topic)
        .subscriptionName("sub")
        .receiverQueueSize(100)
        .subscribe();

    List<Message<Integer>> received = new ArrayList<>(100);
    for (int i = 0; i < messages; i++) {
        received.add(consumer.receive());
    }

    Assert.assertEquals(received.size(), 100);
}
 

@Test(timeOut = 30000)
private void testPulsarSinkDLQ() throws Exception {
    final String namespacePortion = "io";
    final String replNamespace = tenant + "/" + namespacePortion;
    final String sourceTopic = "persistent://" + replNamespace + "/input";
    final String dlqTopic = sourceTopic+"-DLQ";
    final String sinkName = "PulsarSink-test";
    final String propertyKey = "key";
    final String propertyValue = "value";
    final String subscriptionName = "test-sub";
    admin.namespaces().createNamespace(replNamespace);
    Set<String> clusters = Sets.newHashSet(Lists.newArrayList("use"));
    admin.namespaces().setNamespaceReplicationClusters(replNamespace, clusters);
    // 1 create producer、DLQ consumer
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic(sourceTopic).create();
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic(dlqTopic).subscriptionName(subscriptionName).subscribe();

    // 2 setup sink
    SinkConfig sinkConfig = createSinkConfig(tenant, namespacePortion, sinkName, sourceTopic, subscriptionName);
    sinkConfig.setNegativeAckRedeliveryDelayMs(1001L);
    sinkConfig.setProcessingGuarantees(FunctionConfig.ProcessingGuarantees.ATLEAST_ONCE);
    sinkConfig.setMaxMessageRetries(2);
    sinkConfig.setDeadLetterTopic(dlqTopic);
    sinkConfig.setInputSpecs(Collections.singletonMap(sourceTopic, ConsumerConfig.builder().receiverQueueSize(1000).build()));
    sinkConfig.setClassName(SinkForTest.class.getName());
    LocalRunner localRunner = LocalRunner.builder()
            .sinkConfig(sinkConfig)
            .clientAuthPlugin(AuthenticationTls.class.getName())
            .clientAuthParams(String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH))
            .useTls(true)
            .tlsTrustCertFilePath(TLS_TRUST_CERT_FILE_PATH)
            .tlsAllowInsecureConnection(true)
            .tlsHostNameVerificationEnabled(false)
            .brokerServiceUrl(pulsar.getBrokerServiceUrlTls()).build();

    localRunner.start(false);

    retryStrategically((test) -> {
        try {
            TopicStats topicStats = admin.topics().getStats(sourceTopic);

            return topicStats.subscriptions.containsKey(subscriptionName)
                    && topicStats.subscriptions.get(subscriptionName).consumers.size() == 1
                    && topicStats.subscriptions.get(subscriptionName).consumers.get(0).availablePermits == 1000;

        } catch (PulsarAdminException e) {
            return false;
        }
    }, 50, 150);

    // 3 send message
    int totalMsgs = 10;
    for (int i = 0; i < totalMsgs; i++) {
        producer.newMessage().property(propertyKey, propertyValue).value("fail" + i).sendAsync();
    }

    //4 All messages should enter DLQ
    for (int i = 0; i < totalMsgs; i++) {
        Message<String> message = consumer.receive(10, TimeUnit.SECONDS);
        assertNotNull(message);
        assertEquals(message.getValue(), "fail" + i);
    }

    //clean up
    producer.close();
    consumer.close();
}
 

@Test(timeOut = 20000)
public void testConsumerListOffsetLatest() throws Exception {
    String topicName = "testConsumerListOffsetLatest";
    TopicPartition tp = new TopicPartition(topicName, 0);

    // use producer to create some message to get Limit Offset.
    String pulsarTopicName = "persistent://public/default/" + topicName;

    // create partitioned topic.
    admin.topics().createPartitionedTopic(topicName, 1);

    // 1. prepare topic:
    //    use kafka producer to produce 10 messages.
    //    use pulsar consumer to get message offset.
    @Cleanup
    KProducer kProducer = new KProducer(topicName, false, getKafkaBrokerPort());
    int totalMsgs = 10;
    String messageStrPrefix = topicName + "_message_";

    for (int i = 0; i < totalMsgs; i++) {
        String messageStr = messageStrPrefix + i;
        kProducer.getProducer()
            .send(new ProducerRecord<>(
                topicName,
                i,
                messageStr))
            .get();
        log.debug("Kafka Producer Sent message: ({}, {})", i, messageStr);
    }

    @Cleanup
    Consumer<byte[]> consumer = pulsarClient.newConsumer()
        .topic(pulsarTopicName)
        .subscriptionName(topicName + "_sub")
        .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest)
        .subscribe();
    Message<byte[]> msg = consumer.receive(100, TimeUnit.MILLISECONDS);
    assertNotNull(msg);
    MessageIdImpl messageId = (MessageIdImpl) ((TopicMessageIdImpl) msg.getMessageId()).getInnerMessageId();
    // LAC entry should be the limit offset.
    long limitOffset = MessageIdUtils.getOffset(messageId.getLedgerId(), totalMsgs - 1);
    log.info("After create {} messages, get messageId: {} expected latest limit: {}",
        totalMsgs, messageId, limitOffset);

    // 2. real test, for ListOffset request verify Earliest get earliest
    Map<TopicPartition, Long> targetTimes = Maps.newHashMap();
    targetTimes.put(tp, ListOffsetRequest.LATEST_TIMESTAMP);

    ListOffsetRequest.Builder builder = ListOffsetRequest.Builder
        .forConsumer(true, IsolationLevel.READ_UNCOMMITTED)
        .setTargetTimes(targetTimes);

    KafkaHeaderAndRequest request = buildRequest(builder);
    CompletableFuture<AbstractResponse> responseFuture = new CompletableFuture<>();
    kafkaRequestHandler
        .handleListOffsetRequest(request, responseFuture);

    AbstractResponse response = responseFuture.get();
    ListOffsetResponse listOffsetResponse = (ListOffsetResponse) response;
    assertEquals(listOffsetResponse.responseData().get(tp).error, Errors.NONE);
    assertEquals(listOffsetResponse.responseData().get(tp).offset, Long.valueOf(limitOffset));
    assertEquals(listOffsetResponse.responseData().get(tp).timestamp, Long.valueOf(0));
}