Java Code Examples for org.apache.pulsar.client.api.Producer#flush()

public void producerFlush() throws Exception {
    if (singleProducer != null) {
        singleProducer.flush();
    } else {
        if (topic2Producer != null) {
            for (Producer<?> p : topic2Producer.values()) {
                p.flush();
            }
        }
    }
    synchronized (pendingRecordsLock) {
        while (pendingRecords > 0) {
            try {
                pendingRecordsLock.wait();
            } catch (InterruptedException e) {
                // this can be interrupted when the Task has been cancelled.
                // by throwing an exception, we ensure that this checkpoint doesn't get confirmed
                throw new RuntimeException("Flushing got interrupted while checkpointing", e);
            }
        }
    }
}
 

@Test
public void testSingleMessageBatch() throws Exception {
    String topicName = "prop/ns-abc/topic-estSingleMessageBatch";

    Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
            .topic(topicName)
            .enableBatching(true)
            .batchingMaxPublishDelay(10, TimeUnit.SECONDS)
            .create();

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topicName)
            .subscriptionName("subscription")
            .ackTimeout(1, TimeUnit.HOURS)
            .subscribe();

    // Force the creation of a batch with a single message
    producer.sendAsync("hello");
    producer.flush();

    Message<String> message = consumer.receive();

    assertFalse(((ConsumerImpl<?>) consumer).getUnAckedMessageTracker().isEmpty());

    consumer.acknowledge(message);

    assertTrue(((ConsumerImpl<?>) consumer).getUnAckedMessageTracker().isEmpty());
}
 

public static <T> List<MessageId> sendTypedMessages(
        String topic,
        SchemaType type,
        List<T> messages,
        Optional<Integer> partition,
        Class<T> tClass) throws PulsarClientException {

    String topicName;
    if (partition.isPresent()) {
        topicName = topic + PulsarOptions.PARTITION_SUFFIX + partition.get();
    } else {
        topicName = topic;
    }

    Producer<T> producer = null;
    PulsarClient client = null;
    List<MessageId> mids = new ArrayList<>();

    try {
        client = PulsarClient.builder().serviceUrl(getServiceUrl()).build();

        switch (type) {
            case BOOLEAN:
                producer = (Producer<T>) client.newProducer(Schema.BOOL).topic(topicName).create();
                break;
            case BYTES:
                producer = (Producer<T>) client.newProducer(Schema.BYTES).topic(topicName).create();
                break;
            case DATE:
                producer = (Producer<T>) client.newProducer(Schema.DATE).topic(topicName).create();
                break;
            case STRING:
                producer = (Producer<T>) client.newProducer(Schema.STRING).topic(topicName).create();
                break;
            case TIMESTAMP:
                producer = (Producer<T>) client.newProducer(Schema.TIMESTAMP).topic(topicName).create();
                break;
            case INT8:
                producer = (Producer<T>) client.newProducer(Schema.INT8).topic(topicName).create();
                break;
            case DOUBLE:
                producer = (Producer<T>) client.newProducer(Schema.DOUBLE).topic(topicName).create();
                break;
            case FLOAT:
                producer = (Producer<T>) client.newProducer(Schema.FLOAT).topic(topicName).create();
                break;
            case INT32:
                producer = (Producer<T>) client.newProducer(Schema.INT32).topic(topicName).create();
                break;
            case INT16:
                producer = (Producer<T>) client.newProducer(Schema.INT16).topic(topicName).create();
                break;
            case INT64:
                producer = (Producer<T>) client.newProducer(Schema.INT64).topic(topicName).create();
                break;
            case AVRO:
                producer = (Producer<T>) client.newProducer(Schema.AVRO(tClass)).topic(topicName).create();
                break;
            case JSON:
                producer = (Producer<T>) client.newProducer(Schema.JSON(tClass)).topic(topicName).create();
                break;

            default:
                throw new NotImplementedException("Unsupported type " + type);
        }

        for (T message : messages) {
            MessageId mid = producer.send(message);
            log.info("Sent {} of mid: {}", message.toString(), mid.toString());
            mids.add(mid);
        }

    } finally {
        producer.flush();
        producer.close();
        client.close();
    }
    return mids;
}
 

@Test(timeOut = 20000)
public void testBatchAndNonBatchWithoutEmptyPayload() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "persistent://my-property/use/my-ns/testBatchAndNonBatchWithoutEmptyPayload" + UUID.randomUUID().toString();

    // 1.create producer and publish message to the topic.
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topic)
            .enableBatching(true)
            .batchingMaxPublishDelay(1, TimeUnit.DAYS)
            .create();

    final String k1 = "k1";
    final String k2 = "k2";
    producer.newMessage().key(k1).value("0".getBytes()).send();
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(7);
    for (int i = 0; i < 2; i++) {
        futures.add(producer.newMessage().key(k1).value((i + 1 + "").getBytes()).sendAsync());
    }
    producer.flush();
    producer.newMessage().key(k1).value("3".getBytes()).send();
    for (int i = 0; i < 2; i++) {
        futures.add(producer.newMessage().key(k1).value((i + 4 + "").getBytes()).sendAsync());
    }
    producer.flush();

    for (int i = 0; i < 3; i++) {
        futures.add(producer.newMessage().key(k2).value((i + "").getBytes()).sendAsync());
    }

    producer.newMessage().key(k2).value("3".getBytes()).send();
    producer.flush();
    FutureUtil.waitForAll(futures).get();

    // 2.compact the topic.
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();

    // consumer with readCompacted enabled only get compacted entries
    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
            .readCompacted(true).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
        Message<byte[]> m1 = consumer.receive(2, TimeUnit.SECONDS);
        Message<byte[]> m2 = consumer.receive(2, TimeUnit.SECONDS);
        assertNotNull(m1);
        assertNotNull(m2);
        assertEquals(m1.getKey(), k1);
        assertEquals(new String(m1.getValue()), "5");
        assertEquals(m2.getKey(), k2);
        assertEquals(new String(m2.getValue()), "3");
        Message<byte[]> none = consumer.receive(2, TimeUnit.SECONDS);
        assertNull(none);
    }
}
 

@Test(timeOut = 20000)
public void testBatchAndNonBatchWithEmptyPayload() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "persistent://my-property/use/my-ns/testBatchAndNonBatchWithEmptyPayload" + UUID.randomUUID().toString();

    // 1.create producer and publish message to the topic.
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topic)
            .enableBatching(true)
            .batchingMaxPublishDelay(1, TimeUnit.DAYS)
            .create();

    final String k1 = "k1";
    final String k2 = "k2";
    final String k3 = "k3";
    producer.newMessage().key(k1).value("0".getBytes()).send();
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(7);
    for (int i = 0; i < 2; i++) {
        futures.add(producer.newMessage().key(k1).value((i + 1 + "").getBytes()).sendAsync());
    }
    producer.flush();
    producer.newMessage().key(k1).value("3".getBytes()).send();
    for (int i = 0; i < 2; i++) {
        futures.add(producer.newMessage().key(k1).value((i + 4 + "").getBytes()).sendAsync());
    }
    producer.flush();

    for (int i = 0; i < 3; i++) {
        futures.add(producer.newMessage().key(k2).value((i + 10 + "").getBytes()).sendAsync());
    }
    producer.flush();

    producer.newMessage().key(k2).value("".getBytes()).send();

    producer.newMessage().key(k3).value("0".getBytes()).send();

    FutureUtil.waitForAll(futures).get();

    // 2.compact the topic.
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();

    // consumer with readCompacted enabled only get compacted entries
    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
            .readCompacted(true).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
        Message<byte[]> m1 = consumer.receive();
        Message<byte[]> m2 = consumer.receive();
        assertNotNull(m1);
        assertNotNull(m2);
        assertEquals(m1.getKey(), k1);
        assertEquals(m2.getKey(), k3);
        assertEquals(new String(m1.getValue()), "5");
        assertEquals(new String(m2.getValue()), "0");
        Message<byte[]> none = consumer.receive(2, TimeUnit.SECONDS);
        assertNull(none);
    }
}
 

@Test(timeOut = 20000)
public void testBatchAndNonBatchEndOfEmptyPayload() throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "persistent://my-property/use/my-ns/testBatchAndNonBatchWithEmptyPayload" + UUID.randomUUID().toString();

    // 1.create producer and publish message to the topic.
    Producer<byte[]> producer = pulsarClient.newProducer()
            .topic(topic)
            .enableBatching(true)
            .batchingMaxPublishDelay(1, TimeUnit.DAYS)
            .create();

    final String k1 = "k1";
    final String k2 = "k2";
    producer.newMessage().key(k1).value("0".getBytes()).send();
    List<CompletableFuture<MessageId>> futures = new ArrayList<>(7);
    for (int i = 0; i < 2; i++) {
        futures.add(producer.newMessage().key(k1).value((i + 1 + "").getBytes()).sendAsync());
    }
    producer.flush();
    producer.newMessage().key(k1).value("3".getBytes()).send();
    for (int i = 0; i < 2; i++) {
        futures.add(producer.newMessage().key(k1).value((i + 4 + "").getBytes()).sendAsync());
    }
    producer.flush();

    for (int i = 0; i < 3; i++) {
        futures.add(producer.newMessage().key(k2).value((i + 10 + "").getBytes()).sendAsync());
    }
    producer.flush();

    producer.newMessage().key(k2).value("".getBytes()).send();

    FutureUtil.waitForAll(futures).get();

    // 2.compact the topic.
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();

    // consumer with readCompacted enabled only get compacted entries
    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
            .readCompacted(true).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
        Message<byte[]> m1 = consumer.receive();
        assertNotNull(m1);
        assertEquals(m1.getKey(), k1);
        assertEquals(new String(m1.getValue()), "5");
        Message<byte[]> none = consumer.receive(2, TimeUnit.SECONDS);
        assertNull(none);
    }
}
 

@Test
public void testPeekMessages() throws Exception {
    @Cleanup
    PulsarClient newPulsarClient = PulsarClient.builder()
        .serviceUrl(lookupUrl.toString())
        .build();

    final String topic = "persistent://my-property/my-ns/test-publish-timestamp";

    @Cleanup
    Consumer<byte[]> consumer = newPulsarClient.newConsumer()
        .topic(topic)
        .subscriptionName("my-sub")
        .subscribe();

    final int numMessages = 5;

    @Cleanup
    Producer<byte[]> producer = newPulsarClient.newProducer()
            .topic(topic)
            .enableBatching(true)
            .batchingMaxPublishDelay(3, TimeUnit.SECONDS)
            .batchingMaxMessages(5)
            .create();

    for (int i = 0; i < numMessages; i++) {
        producer.newMessage()
            .value(("value-" + i).getBytes(UTF_8))
            .sendAsync();
    }
    producer.flush();

    for (int i = 0; i < numMessages; i++) {
        Message<byte[]> msg = consumer.receive();
        log.info("Received message '{}'.", new String(msg.getValue(), UTF_8));
    }
    List<Message<byte[]>> messages = admin.topics().peekMessages(topic, "my-sub", 5);
    Assert.assertEquals(new String(messages.get(0).getValue(), UTF_8), "value-0");
    Assert.assertEquals(new String(messages.get(1).getValue(), UTF_8), "value-1");
    Assert.assertEquals(new String(messages.get(2).getValue(), UTF_8), "value-2");
    Assert.assertEquals(new String(messages.get(3).getValue(), UTF_8), "value-3");
    Assert.assertEquals(new String(messages.get(4).getValue(), UTF_8), "value-4");
}
 

@Test
public void testDelayedDelivery()
        throws Exception {
    String topic = "testNegativeAcks-" + System.nanoTime();

    @Cleanup
    Consumer<String> failoverConsumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("failover-sub")
            .subscriptionType(SubscriptionType.Failover)
            .subscribe();

    @Cleanup
    Consumer<String> sharedConsumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("shared-sub")
            .subscriptionType(SubscriptionType.Shared)
            .subscribe();

    @Cleanup
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
            .topic(topic)
            .create();

    for (int i = 0; i < 10; i++) {
        producer.newMessage()
                .value("msg-" + i)
                .deliverAfter(5, TimeUnit.SECONDS)
                .sendAsync();
    }

    producer.flush();

    // Failover consumer will receive the messages immediately while
    // the shared consumer will get them after the delay
    Message<String> msg = sharedConsumer.receive(100, TimeUnit.MILLISECONDS);
    assertNull(msg);

    for (int i = 0; i < 10; i++) {
        msg = failoverConsumer.receive(100, TimeUnit.MILLISECONDS);
        assertEquals(msg.getValue(), "msg-" + i);
    }

    Set<String> receivedMsgs = new TreeSet<>();
    for (int i = 0; i < 10; i++) {
        msg = sharedConsumer.receive(10, TimeUnit.SECONDS);
        receivedMsgs.add(msg.getValue());
    }

    assertEquals(receivedMsgs.size(), 10);
    for (int i = 0; i < 10; i++) {
        assertTrue(receivedMsgs.contains("msg-" + i));
    }
}
 

@Test
public void testDelayedDeliveryWithMultipleConcurrentReadEntries()
        throws Exception {
    String topic = "persistent://public/default/testDelayedDelivery-" + System.nanoTime();

    @Cleanup
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
            .topic(topic)
            .subscriptionName("shared-sub")
            .subscriptionType(SubscriptionType.Shared)
            .receiverQueueSize(1) // Use small prefecthing to simulate the multiple read batches
            .subscribe();

    // Simulate race condition with high frequency of calls to dispatcher.readMoreEntries()
    PersistentDispatcherMultipleConsumers d = (PersistentDispatcherMultipleConsumers) ((PersistentTopic) pulsar
            .getBrokerService().getTopicReference(topic).get()).getSubscription("shared-sub").getDispatcher();
    Thread t = new Thread(() -> {
        while (true) {
            synchronized (d) {
                d.readMoreEntries();
            }

            try {
                Thread.sleep(1);
            } catch (InterruptedException e) {
                return;
            }
        }
    });
    t.start();

    @Cleanup
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING)
            .topic(topic)
            .create();

    final int N = 1000;

    for (int i = 0; i < N; i++) {
        producer.newMessage()
                .value("msg-" + i)
                .deliverAfter(5, TimeUnit.SECONDS)
                .sendAsync();
    }

    producer.flush();

    Message<String> msg = consumer.receive(100, TimeUnit.MILLISECONDS);
    assertNull(msg);

    Set<String> receivedMsgs = new TreeSet<>();
    for (int i = 0; i < N; i++) {
        msg = consumer.receive(10, TimeUnit.SECONDS);
        receivedMsgs.add(msg.getValue());
    }

    assertEquals(receivedMsgs.size(), N);
    for (int i = 0; i < N; i++) {
        assertTrue(receivedMsgs.contains("msg-" + i));
    }
    t.interrupt();
}