Java Code Examples for org.apache.pulsar.client.api.ProducerBuilder#create()

@Test
public void testCreateProducerWithSameName() throws Exception {
    String topic = "persistent://prop/ns-abc/testCreateProducerWithSameName";

    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer()
        .topic(topic)
        .producerName("test-producer-a")
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition);
    Producer<byte[]> p1 = producerBuilder.create();

    try {
        producerBuilder.create();
        fail("Should have thrown ProducerBusyException");
    } catch (ProducerBusyException e) {
        // Expected
    }

    p1.close();

    // Now p2 should succeed
    Producer<byte[]> p2 = producerBuilder.create();

    p2.close();
}
 

private Topic createTopicAndPublish(PulsarClient pulsarClient, String ns, String topicLocalName, int totalPublish)
        throws Exception {
    final String topicName = String.format("persistent://%s/%s", ns, topicLocalName);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name")
            .subscribe();
    consumer.close();

    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer().topic(topicName).sendTimeout(5, TimeUnit.SECONDS);

    Producer<byte[]> producer = producerBuilder.create();
    for (int i = 0; i < totalPublish; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    producer.close();

    return pulsarService.getBrokerService().getTopicReference(topicName).get();
}
 

public void startup() throws Exception {
    try {
        client = PULSAR_CLIENT_BUILDER.get()
            .serviceUrl(serviceUrl)
            .build();
        ProducerBuilder<byte[]> producerBuilder = client.newProducer()
            .topic(topic)
            .producerName("pulsar-log4j2-appender-" + topic)
            .blockIfQueueFull(false);
        if (syncSend) {
            // disable batching for sync send
            producerBuilder = producerBuilder.enableBatching(false);
        } else {
            // enable batching in 10 ms for async send
            producerBuilder = producerBuilder
                .enableBatching(true)
                .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS);
        }
        producer = producerBuilder.create();
    } catch (Exception t) {
        LOGGER.error("Failed to start pulsar manager", t);
        throw t;
    }
}
 

@Test(timeOut = 20000, dataProvider = "lastDeletedBatching")
public void testAllEmptyCompactionLedger(boolean batchEnabled) throws Exception {
    final String topic = "persistent://my-property/use/my-ns/testAllEmptyCompactionLedger" + UUID.randomUUID().toString();

    final int messages = 10;

    // 1.create producer and publish message to the topic.
    ProducerBuilder<byte[]> builder = pulsarClient.newProducer().topic(topic);
    if (!batchEnabled) {
        builder.enableBatching(false);
    } else {
        builder.batchingMaxMessages(messages / 5);
    }

    Producer<byte[]> producer = builder.create();

    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.newMessage().keyBytes("1".getBytes()).value("".getBytes()).sendAsync());
    }

    FutureUtil.waitForAll(futures).get();

    // 2.compact the topic.
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();

    // consumer with readCompacted enabled only get compacted entries
    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
            .readCompacted(true).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
        Message<byte[]> m = consumer.receive(2, TimeUnit.SECONDS);
        assertNull(m);
    }
}
 

@Test(dataProvider = "batch")
public void testSyncProducerAndConsumer(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
                .topic("persistent://my-property/use/my-ns/my-topic1")
                .subscriptionName("my-subscriber-name")
                .subscriptionType(SubscriptionType.Exclusive)
                .subscribe();

    ProducerBuilder<String> producerBuilder = pulsarClient.newProducer(Schema.STRING)
            .topic("persistent://my-property/use/my-ns/my-topic1");

    if (batchMessageDelayMs != 0) {
        producerBuilder.enableBatching(true)
            .batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
            .batchingMaxMessages(5);
    } else {
        producerBuilder.enableBatching(false);
    }

    Producer<String> producer = producerBuilder.create();
    for (int i = 0; i < 10; i++) {
        producer.send("my-message-" + i);
    }

    Message<String> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = msg.getValue();
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 

@Test()
public void testFailedZeroQueueSizeBatchMessage() throws PulsarClientException {

    int batchMessageDelayMs = 100;
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns-abc/topic1")
            .subscriptionName("my-subscriber-name").subscriptionType(SubscriptionType.Shared).receiverQueueSize(0)
            .subscribe();

    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer()
        .topic("persistent://prop-xyz/use/ns-abc/topic1")
        .messageRoutingMode(MessageRoutingMode.SinglePartition);

    if (batchMessageDelayMs != 0) {
        producerBuilder.enableBatching(true).batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
                .batchingMaxMessages(5);
    } else {
        producerBuilder.enableBatching(false);
    }

    Producer<byte[]> producer = producerBuilder.create();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    try {
        consumer.receiveAsync().handle((ok, e) -> {
            if (e == null) {
                // as zero receiverQueueSize doesn't support batch message, must receive exception at callback.
                Assert.fail();
            }
            return null;
        });
    } finally {
        consumer.close();
    }
}
 

private Set<String> publishMessages(String topic, int count, boolean enableBatch) throws Exception {
    Set<String> keys = new HashSet<>();
    ProducerBuilder<byte[]> builder = pulsarClient.newProducer();
    builder.messageRoutingMode(MessageRoutingMode.SinglePartition);
    builder.maxPendingMessages(count);
    // disable periodical flushing
    builder.batchingMaxPublishDelay(1, TimeUnit.DAYS);
    builder.topic(topic);
    if (enableBatch) {
        builder.enableBatching(true);
        builder.batchingMaxMessages(count);
    } else {
        builder.enableBatching(false);
    }
    try (Producer<byte[]> producer = builder.create()) {
        Future<?> lastFuture = null;
        for (int i = 0; i < count; i++) {
            String key = "key"+i;
            byte[] data = ("my-message-" + i).getBytes();
            lastFuture = producer.newMessage().key(key).value(data).sendAsync();
            keys.add(key);
        }
        producer.flush();
        lastFuture.get();
    }
    return keys;
}
 

MessageProducer(URL url, final TopicName dest, boolean batch) throws Exception {
    this.url = url;
    this.namespace = dest.getNamespace();
    this.topicName = dest.toString();
    client = PulsarClient.builder().serviceUrl(url.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    ProducerBuilder<byte[]> producerBuilder = client.newProducer()
        .topic(topicName)
        .enableBatching(batch)
        .batchingMaxPublishDelay(1, TimeUnit.SECONDS)
        .batchingMaxMessages(5);
    producer = producerBuilder.create();

}
 

@Test
public void testProducerAndConsumerPassed() throws Exception {
    log.info("-- {} -- start", methodName);

    Consumer<byte[]> consumer = pulsarClient.newConsumer()
        .topic("persistent://my-property/my-ns/my-topic")
        .subscriptionName("my-subscriber-name")
        .subscribe();

    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer()
        .topic("persistent://my-property/my-ns/my-topic")
        .enableBatching(false);

    Producer<byte[]> producer = producerBuilder.create();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        log.info("Produced message: [{}]", message);
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.info("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();

    log.info("-- {} -- end", methodName);
}
 

@Test(timeOut = 20000, dataProvider = "lastDeletedBatching")
public void testCompactMultipleTimesWithoutEmptyMessage(boolean batchEnabled) throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "persistent://my-property/use/my-ns/testCompactMultipleTimesWithoutEmptyMessage" + UUID.randomUUID().toString();

    final int messages = 10;
    final String key = "1";

    // 1.create producer and publish message to the topic.
    ProducerBuilder<byte[]> builder = pulsarClient.newProducer().topic(topic);
    if (!batchEnabled) {
        builder.enableBatching(false);
    } else {
        builder.batchingMaxMessages(messages / 5);
    }

    Producer<byte[]> producer = builder.create();

    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.newMessage().key(key).value((i + "").getBytes()).sendAsync());
    }

    FutureUtil.waitForAll(futures).get();

    // 2.compact the topic.
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();

    // 3. Send more ten messages
    futures.clear();
    for (int i = 0; i < messages; i++) {
        futures.add(producer.newMessage().key(key).value((i + 10 + "").getBytes()).sendAsync());
    }
    FutureUtil.waitForAll(futures).get();

    // 4.compact again.
    compactor.compact(topic).get();

    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
            .readCompacted(true).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
        Message<byte[]> m1 = consumer.receive();
        assertNotNull(m1);
        assertEquals(m1.getKey(), key);
        assertEquals(new String(m1.getValue()), "19");
        Message<byte[]> none = consumer.receive(2, TimeUnit.SECONDS);
        assertNull(none);
    }
}
 

@Test(dataProvider =  "AllCheckSchemaCompatibilityStrategy")
public void testIsAutoUpdateSchema(SchemaCompatibilityStrategy schemaCompatibilityStrategy) throws Exception {
    final String tenant = PUBLIC_TENANT;
    final String topic = "test-consumer-compatibility";

    String namespace = "test-namespace-" + randomName(16);
    String fqtn = TopicName.get(
            TopicDomain.persistent.value(),
            tenant,
            namespace,
            topic
    ).toString();

    NamespaceName namespaceName = NamespaceName.get(tenant, namespace);

    admin.namespaces().createNamespace(
            tenant + "/" + namespace,
            Sets.newHashSet(CLUSTER_NAME)
    );

    assertEquals(admin.namespaces().getSchemaCompatibilityStrategy(namespaceName.toString()),
            SchemaCompatibilityStrategy.FULL);
    
    admin.namespaces().setSchemaCompatibilityStrategy(namespaceName.toString(), schemaCompatibilityStrategy);
    admin.schemas().createSchema(fqtn, Schema.AVRO(Schemas.PersonOne.class).getSchemaInfo());

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false);
    ProducerBuilder<Schemas.PersonTwo> producerThreeBuilder = pulsarClient
            .newProducer(Schema.AVRO(SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .topic(fqtn);
    try {
        producerThreeBuilder.create();
    } catch (Exception e) {
        Assert.assertTrue(e.getMessage().contains("Schema not found and schema auto updating is disabled."));
    }

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), true);
    ConsumerBuilder<Schemas.PersonTwo> comsumerBuilder = pulsarClient.newConsumer(Schema.AVRO(
            SchemaDefinition.<Schemas.PersonTwo>builder().withAlwaysAllowNull
                    (false).withSupportSchemaVersioning(true).
                    withPojo(Schemas.PersonTwo.class).build()))
            .subscriptionName("test")
            .topic(fqtn);

    Producer<Schemas.PersonTwo> producer = producerThreeBuilder.create();
    Consumer<Schemas.PersonTwo> consumerTwo = comsumerBuilder.subscribe();

    producer.send(new Schemas.PersonTwo(2, "Lucy"));
    Message<Schemas.PersonTwo> message = consumerTwo.receive();

    Schemas.PersonTwo personTwo = message.getValue();
    consumerTwo.acknowledge(message);

    assertEquals(personTwo.getId(), 2);
    assertEquals(personTwo.getName(), "Lucy");

    producer.close();
    consumerTwo.close();

    admin.namespaces().setIsAllowAutoUpdateSchema(namespaceName.toString(), false);

    producer = producerThreeBuilder.create();
    consumerTwo = comsumerBuilder.subscribe();

    producer.send(new Schemas.PersonTwo(2, "Lucy"));
    message = consumerTwo.receive();

    personTwo = message.getValue();
    consumerTwo.acknowledge(message);

    assertEquals(personTwo.getId(), 2);
    assertEquals(personTwo.getName(), "Lucy");

    consumerTwo.close();
    producer.close();
}
 

private int publish(String topic) {
    int numMessagesSent = 0;
    int returnCode = 0;

    try {
        PulsarClient client = clientBuilder.build();
        ProducerBuilder<byte[]> producerBuilder = client.newProducer().topic(topic);
        if (this.chunkingAllowed) {
            producerBuilder.enableChunking(true);
            producerBuilder.enableBatching(false);
        }
        Producer<byte[]> producer = producerBuilder.create();

        List<byte[]> messageBodies = generateMessageBodies(this.messages, this.messageFileNames);
        RateLimiter limiter = (this.publishRate > 0) ? RateLimiter.create(this.publishRate) : null;

        Map<String, String> kvMap = new HashMap<>();
        for (String property : properties) {
            String [] kv = property.split("=");
            kvMap.put(kv[0], kv[1]);
        }

        for (int i = 0; i < this.numTimesProduce; i++) {
            for (byte[] content : messageBodies) {
                if (limiter != null) {
                    limiter.acquire();
                }

                TypedMessageBuilder<byte[]> message = producer.newMessage();

                if (!kvMap.isEmpty()) {
                    message.properties(kvMap);
                }

                if (key != null && !key.isEmpty()) {
                    message.key(key);
                }

                message.value(content).send();

                numMessagesSent++;
            }
        }
        client.close();
    } catch (Exception e) {
        LOG.error("Error while producing messages");
        LOG.error(e.getMessage(), e);
        returnCode = -1;
    } finally {
        LOG.info("{} messages successfully produced", numMessagesSent);
    }

    return returnCode;
}