Java Code Examples for org.apache.pulsar.client.api.Consumer#redeliverUnacknowledgedMessages()

@Test
public void testBrokerServicePersistentRedeliverTopicStats() throws Exception {
    final String topicName = "persistent://prop/ns-abc/successSharedTopic";
    final String subName = "successSharedSub";

    TopicStats stats;
    SubscriptionStats subStats;

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName)
            .subscriptionType(SubscriptionType.Shared).acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();

    // subscription stats
    assertEquals(stats.subscriptions.keySet().size(), 1);
    assertEquals(subStats.msgBacklog, 0);
    assertEquals(subStats.consumers.size(), 1);

    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();

    // publisher stats
    assertEquals(subStats.msgBacklog, 10);
    assertEquals(stats.publishers.size(), 1);
    assertTrue(stats.publishers.get(0).msgRateIn > 0.0);
    assertTrue(stats.publishers.get(0).msgThroughputIn > 0.0);
    assertTrue(stats.publishers.get(0).averageMsgSize > 0.0);

    // aggregated publish stats
    assertEquals(stats.msgRateIn, stats.publishers.get(0).msgRateIn);
    assertEquals(stats.msgThroughputIn, stats.publishers.get(0).msgThroughputIn);
    double diff = stats.averageMsgSize - stats.publishers.get(0).averageMsgSize;
    assertTrue(Math.abs(diff) < 0.000001);

    // consumer stats
    assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
    assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
    assertEquals(subStats.msgRateRedeliver, 0.0);
    assertEquals(subStats.consumers.get(0).unackedMessages, 10);

    // aggregated consumer stats
    assertEquals(subStats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(subStats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);
    assertEquals(stats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(stats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);
    assertEquals(subStats.unackedMessages, subStats.consumers.get(0).unackedMessages);

    consumer.redeliverUnacknowledgedMessages();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();
    assertTrue(subStats.msgRateRedeliver > 0.0);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);

    Message<byte[]> msg;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    consumer.close();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);

    rolloverPerIntervalStats();
    stats = topicRef.getStats(false);
    subStats = stats.subscriptions.values().iterator().next();

    assertEquals(subStats.msgBacklog, 0);
}
 

/**
 * Verify: Broker should not replay already acknowledged messages again and should clear them from messageReplay
 * bucket
 *
 * 1. produce messages 2. consume messages and ack all except 1 msg 3. Verification: should replay only 1 unacked
 * message
 */
@Test
public void testMessageRedelivery() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic2";
    final String subName = "sub2";

    Message<String> msg;
    List<Message<String>> unackedMessages = new ArrayList<>();
    int totalMessages = 20;

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topicName)
            .subscriptionName(subName)
            .subscriptionType(SubscriptionType.Shared)
            .acknowledgmentGroupTime(0, TimeUnit.SECONDS)
            .subscribe();
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
            .topic(topicName)
            .enableBatching(false)
            .messageRoutingMode(MessageRoutingMode.SinglePartition)
            .create();

    // (1) Produce messages
    for (int i = 0; i < totalMessages; i++) {
        producer.send("my-message-" + i);
    }

    // (2) Consume and only ack last 10 messages
    for (int i = 0; i < totalMessages; i++) {
        msg = consumer.receive();
        if (i >= 10) {
            unackedMessages.add(msg);
        } else {
            consumer.acknowledge(msg);
        }
    }

    consumer.redeliverUnacknowledgedMessages();

    for (int i = 0; i < 10; i++) {
        // Verify: msg [L:0] must be redelivered
        try {
            final Message<String> redeliveredMsg = consumer.receive(1, TimeUnit.SECONDS);
            unackedMessages.removeIf(unackedMessage -> unackedMessage.getValue().equals(redeliveredMsg.getValue()));
        } catch (Exception e) {
            fail("msg should be redelivered ", e);
        }
    }
    // Make sure that first 10 messages that we didn't acknowledge get redelivered.
    assertEquals(unackedMessages.size(), 0);

    // Verify no other messages are redelivered
    msg = consumer.receive(100, TimeUnit.MILLISECONDS);
    assertNull(msg);

    consumer.close();
    producer.close();
}
 

@Test(timeOut = testTimeout)
public void testSharedSingleAckedNormalTopic() throws Exception {
    String key = "testSharedSingleAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);

    // 2. Create consumer
    Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName(subscriptionName).receiverQueueSize(totalMessages / 2)
            .subscriptionType(SubscriptionType.Shared).subscribe();

    PulsarClient newPulsarClient = newPulsarClient(lookupUrl.toString(), 0);// Creates new client connection
    Consumer<byte[]> consumer2 = newPulsarClient.newConsumer().topic(topicName)
            .subscriptionName(subscriptionName).receiverQueueSize(totalMessages / 2)
            .subscriptionType(SubscriptionType.Shared).subscribe();

    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }

    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message<byte[]> message1 = consumer1.receive();
    Message<byte[]> message2 = consumer2.receive();
    do {
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }

        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
        }
        message1 = consumer1.receive(100, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(100, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);

    log.info("Consumer 1 receives = " + receivedConsumer1);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);

    // 5. Send a resend request from Consumer 1
    log.info("Consumer 1 sent a resend request");
    consumer1.redeliverUnacknowledgedMessages();

    // 6. Consumer 1's unAcked messages should be sent to Consumer 1 or 2
    int receivedMessagesAfterRedelivery = 0;
    receivedConsumer1 = 0;
    message1 = consumer1.receive(100, TimeUnit.MILLISECONDS);
    message2 = consumer2.receive(100, TimeUnit.MILLISECONDS);
    do {
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
            ++receivedMessagesAfterRedelivery;
        }

        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
            ++receivedMessagesAfterRedelivery;
        }
        message1 = consumer1.receive(200, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(200, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info("Additional received = " + receivedMessagesAfterRedelivery);
    newPulsarClient.close();
    assertTrue(receivedMessagesAfterRedelivery > 0);

    assertEquals(receivedConsumer1 + receivedConsumer2, totalMessages);
}
 

@Test(timeOut = testTimeout)
public void testSharedSingleAckedPartitionedTopic() throws Exception {
    String key = "testSharedSingleAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    final int numberOfPartitions = 3;
    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("prop", tenantInfo);
    admin.topics().createPartitionedTopic(topicName, numberOfPartitions);
    Random rn = new Random();
    // Special step to create partitioned topic

    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();

    // 2. Create consumer
    Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
            .receiverQueueSize(7).subscriptionType(SubscriptionType.Shared).subscribe();

    PulsarClient newPulsarClient = newPulsarClient(lookupUrl.toString(), 0);// Creates new client connection
    Consumer<byte[]> consumer2 = newPulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName)
            .receiverQueueSize(7).subscriptionType(SubscriptionType.Shared).subscribe();

    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        log.info("Message produced: " + message);
        producer.send(message.getBytes());
    }

    // 4. Receive messages
    Message<byte[]> message1 = consumer1.receive();
    Message<byte[]> message2 = consumer2.receive();
    int messageCount1 = 0;
    int messageCount2 = 0;
    int ackCount1 = 0;
    int ackCount2 = 0;
    do {
        if (message1 != null) {
            log.info("Consumer1 received " + new String(message1.getData()));
            messageCount1 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer1.acknowledge(message1);
                log.info("Consumer1 acked " + new String(message1.getData()));
                ackCount1 += 1;
            }
        }
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer2.acknowledge(message2);
                log.info("Consumer2 acked " + new String(message2.getData()));
                ackCount2 += 1;
            }
        }
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);

    // 5. Ask for redeliver
    log.info(key + ": Sent a Redeliver Message Request");
    consumer1.redeliverUnacknowledgedMessages();
    if ((ackCount1 + ackCount2) == totalMessages) {
        return;
    }

    // 6. Check if Messages redelivered again
    message1 = consumer1.receive(5000, TimeUnit.MILLISECONDS);
    message2 = consumer2.receive(5000, TimeUnit.MILLISECONDS);
    messageCount1 = 0;
    do {
        if (message1 != null) {
            log.info("Consumer1 received " + new String(message1.getData()));
            messageCount1 += 1;
        }
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
        }
        message1 = consumer1.receive(1000, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(1000, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);

    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    newPulsarClient.close();
    assertEquals(messageCount1 + messageCount2 + ackCount1, totalMessages);
}
 

@Test(timeOut = testTimeout)
public void testFailoverSingleAckedPartitionedTopic() throws Exception {
    String key = "testFailoverSingleAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-failover-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    final int numberOfPartitions = 3;
    TenantInfo tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("prop", tenantInfo);
    admin.topics().createPartitionedTopic(topicName, numberOfPartitions);
    Random rn = new Random();
    // Special step to create partitioned topic

    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.RoundRobinPartition)
        .create();

    // 2. Create consumer
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName(subscriptionName).receiverQueueSize(7).subscriptionType(SubscriptionType.Failover);
    Consumer<byte[]> consumer1 = consumerBuilder.clone().consumerName("Consumer-1").subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("Consumer-2").subscribe();

    Thread.sleep(1000);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        log.info("Message produced: " + message);
        producer.send(message.getBytes());
    }

    // 4. Receive messages
    Message<byte[]> message1 = consumer1.receive();
    Message<byte[]> message2 = consumer2.receive();
    int messageCount1 = 0;
    int messageCount2 = 0;
    int ackCount1 = 0;
    int ackCount2 = 0;
    do {
        if (message1 != null) {
            log.info("Consumer1 received " + new String(message1.getData()));
            messageCount1 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer1.acknowledge(message1);
                ackCount1 += 1;
            }
        }
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer2.acknowledge(message2);
                ackCount2 += 1;
            }
        }
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);
    if ((ackCount1 + ackCount2) == totalMessages) {
        return;
    }
    // 5. Ask for redeliver
    log.info(key + ": Sent a Redeliver Message Request");
    consumer1.redeliverUnacknowledgedMessages();
    consumer1.close();

    // 6. Check if Messages redelivered again
    message2 = consumer2.receive();
    messageCount1 = 0;
    do {
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
        }
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount2 + ackCount1, totalMessages);
}
 

@Test(timeOut = testTimeout)
public void testFailoverInactiveConsumer() throws Exception {
    String key = "testFailoverInactiveConsumer";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-failover-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName)
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition)
        .create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);

    // 2. Create consumer
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName)
            .subscriptionName(subscriptionName).receiverQueueSize(10).subscriptionType(SubscriptionType.Failover);
    Consumer<byte[]> consumer1 = consumerBuilder.clone().consumerName("Consumer-1").subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("Consumer-2").subscribe();

    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }

    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message<byte[]> message1;
    Message<byte[]> message2;
    do {
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }
    } while (message1 != null);
    log.info("Consumer 1 receives = " + receivedConsumer1);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // Consumer 2 is on Stand By
    assertEquals(receivedConsumer2, 0);

    // 5. Consumer 2 asks for a redelivery but the request is ignored
    log.info("Consumer 2 asks for resend");
    consumer2.redeliverUnacknowledgedMessages();
    Thread.sleep(1000);

    message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
    message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    assertNull(message1);
    assertNull(message2);
}