Java Code Examples for org.apache.pulsar.client.api.ProducerBuilder#batchingMaxMessages()

public static ProducerBuilder<byte[]> getProducerBuilder(PulsarClient client, Properties properties) {
    ProducerBuilder<byte[]> producerBuilder = client.newProducer();

    if (properties.containsKey(PRODUCER_NAME)) {
        producerBuilder.producerName(properties.getProperty(PRODUCER_NAME));
    }

    if (properties.containsKey(INITIAL_SEQUENCE_ID)) {
        producerBuilder.initialSequenceId(Long.parseLong(properties.getProperty(INITIAL_SEQUENCE_ID)));
    }

    if (properties.containsKey(MAX_PENDING_MESSAGES)) {
        producerBuilder.maxPendingMessages(Integer.parseInt(properties.getProperty(MAX_PENDING_MESSAGES)));
    }

    if (properties.containsKey(MAX_PENDING_MESSAGES_ACROSS_PARTITIONS)) {
        producerBuilder.maxPendingMessagesAcrossPartitions(
                Integer.parseInt(properties.getProperty(MAX_PENDING_MESSAGES_ACROSS_PARTITIONS)));
    }

    producerBuilder.enableBatching(Boolean.parseBoolean(properties.getProperty(BATCHING_ENABLED, "true")));

    if (properties.containsKey(BATCHING_MAX_MESSAGES)) {
        producerBuilder.batchingMaxMessages(Integer.parseInt(properties.getProperty(BATCHING_MAX_MESSAGES)));
    }

    return producerBuilder;
}
 

public static ProducerBuilder<byte[]> getProducerBuilder(PulsarClient client, Properties properties) {
    ProducerBuilder<byte[]> producerBuilder = client.newProducer();

    if (properties.containsKey(PRODUCER_NAME)) {
        producerBuilder.producerName(properties.getProperty(PRODUCER_NAME));
    }

    if (properties.containsKey(INITIAL_SEQUENCE_ID)) {
        producerBuilder.initialSequenceId(Long.parseLong(properties.getProperty(INITIAL_SEQUENCE_ID)));
    }

    if (properties.containsKey(MAX_PENDING_MESSAGES)) {
        producerBuilder.maxPendingMessages(Integer.parseInt(properties.getProperty(MAX_PENDING_MESSAGES)));
    }

    if (properties.containsKey(MAX_PENDING_MESSAGES_ACROSS_PARTITIONS)) {
        producerBuilder.maxPendingMessagesAcrossPartitions(
                Integer.parseInt(properties.getProperty(MAX_PENDING_MESSAGES_ACROSS_PARTITIONS)));
    }

    producerBuilder.enableBatching(Boolean.parseBoolean(properties.getProperty(BATCHING_ENABLED, "true")));

    if (properties.containsKey(BATCHING_MAX_MESSAGES)) {
        producerBuilder.batchingMaxMessages(Integer.parseInt(properties.getProperty(BATCHING_MAX_MESSAGES)));
    }

    return producerBuilder;
}
 

@Test(timeOut = 20000, dataProvider = "lastDeletedBatching")
public void testAllEmptyCompactionLedger(boolean batchEnabled) throws Exception {
    final String topic = "persistent://my-property/use/my-ns/testAllEmptyCompactionLedger" + UUID.randomUUID().toString();

    final int messages = 10;

    // 1.create producer and publish message to the topic.
    ProducerBuilder<byte[]> builder = pulsarClient.newProducer().topic(topic);
    if (!batchEnabled) {
        builder.enableBatching(false);
    } else {
        builder.batchingMaxMessages(messages / 5);
    }

    Producer<byte[]> producer = builder.create();

    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.newMessage().keyBytes("1".getBytes()).value("".getBytes()).sendAsync());
    }

    FutureUtil.waitForAll(futures).get();

    // 2.compact the topic.
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();

    // consumer with readCompacted enabled only get compacted entries
    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
            .readCompacted(true).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
        Message<byte[]> m = consumer.receive(2, TimeUnit.SECONDS);
        assertNull(m);
    }
}
 

private Set<String> publishMessages(String topic, int count, boolean enableBatch) throws Exception {
    Set<String> keys = new HashSet<>();
    ProducerBuilder<byte[]> builder = pulsarClient.newProducer();
    builder.messageRoutingMode(MessageRoutingMode.SinglePartition);
    builder.maxPendingMessages(count);
    // disable periodical flushing
    builder.batchingMaxPublishDelay(1, TimeUnit.DAYS);
    builder.topic(topic);
    if (enableBatch) {
        builder.enableBatching(true);
        builder.batchingMaxMessages(count);
    } else {
        builder.enableBatching(false);
    }
    try (Producer<byte[]> producer = builder.create()) {
        Future<?> lastFuture = null;
        for (int i = 0; i < count; i++) {
            String key = "key"+i;
            byte[] data = ("my-message-" + i).getBytes();
            lastFuture = producer.newMessage().key(key).value(data).sendAsync();
            keys.add(key);
        }
        producer.flush();
        lastFuture.get();
    }
    return keys;
}
 

private ProducerBuilder<byte[]> getProducerBuilder(PulsarClient client) {
    ProducerBuilder<byte[]> builder = client.newProducer()
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition);

    // Set to false to prevent the server thread from being blocked if a lot of messages are pending.
    builder.blockIfQueueFull(false);

    if (queryParams.containsKey("producerName")) {
        builder.producerName(queryParams.get("producerName"));
    }

    if (queryParams.containsKey("initialSequenceId")) {
        builder.initialSequenceId(Long.parseLong("initialSequenceId"));
    }

    if (queryParams.containsKey("hashingScheme")) {
        builder.hashingScheme(HashingScheme.valueOf(queryParams.get("hashingScheme")));
    }

    if (queryParams.containsKey("sendTimeoutMillis")) {
        builder.sendTimeout(Integer.parseInt(queryParams.get("sendTimeoutMillis")), TimeUnit.MILLISECONDS);
    }

    if (queryParams.containsKey("batchingEnabled")) {
        builder.enableBatching(Boolean.parseBoolean(queryParams.get("batchingEnabled")));
    }

    if (queryParams.containsKey("batchingMaxMessages")) {
        builder.batchingMaxMessages(Integer.parseInt(queryParams.get("batchingMaxMessages")));
    }

    if (queryParams.containsKey("maxPendingMessages")) {
        builder.maxPendingMessages(Integer.parseInt(queryParams.get("maxPendingMessages")));
    }

    if (queryParams.containsKey("batchingMaxPublishDelay")) {
        builder.batchingMaxPublishDelay(Integer.parseInt(queryParams.get("batchingMaxPublishDelay")),
                TimeUnit.MILLISECONDS);
    }

    if (queryParams.containsKey("messageRoutingMode")) {
        checkArgument(
                Enums.getIfPresent(MessageRoutingMode.class, queryParams.get("messageRoutingMode")).isPresent(),
                "Invalid messageRoutingMode %s", queryParams.get("messageRoutingMode"));
        MessageRoutingMode routingMode = MessageRoutingMode.valueOf(queryParams.get("messageRoutingMode"));
        if (!MessageRoutingMode.CustomPartition.equals(routingMode)) {
            builder.messageRoutingMode(routingMode);
        }
    }

    if (queryParams.containsKey("compressionType")) {
        checkArgument(Enums.getIfPresent(CompressionType.class, queryParams.get("compressionType")).isPresent(),
                "Invalid compressionType %s", queryParams.get("compressionType"));
        builder.compressionType(CompressionType.valueOf(queryParams.get("compressionType")));
    }

    return builder;
}
 

@Test(timeOut = 20000, dataProvider = "lastDeletedBatching")
public void testCompactMultipleTimesWithoutEmptyMessage(boolean batchEnabled) throws PulsarClientException, ExecutionException, InterruptedException {
    final String topic = "persistent://my-property/use/my-ns/testCompactMultipleTimesWithoutEmptyMessage" + UUID.randomUUID().toString();

    final int messages = 10;
    final String key = "1";

    // 1.create producer and publish message to the topic.
    ProducerBuilder<byte[]> builder = pulsarClient.newProducer().topic(topic);
    if (!batchEnabled) {
        builder.enableBatching(false);
    } else {
        builder.batchingMaxMessages(messages / 5);
    }

    Producer<byte[]> producer = builder.create();

    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.newMessage().key(key).value((i + "").getBytes()).sendAsync());
    }

    FutureUtil.waitForAll(futures).get();

    // 2.compact the topic.
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();

    // 3. Send more ten messages
    futures.clear();
    for (int i = 0; i < messages; i++) {
        futures.add(producer.newMessage().key(key).value((i + 10 + "").getBytes()).sendAsync());
    }
    FutureUtil.waitForAll(futures).get();

    // 4.compact again.
    compactor.compact(topic).get();

    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
            .readCompacted(true).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
        Message<byte[]> m1 = consumer.receive();
        assertNotNull(m1);
        assertEquals(m1.getKey(), key);
        assertEquals(new String(m1.getValue()), "19");
        Message<byte[]> none = consumer.receive(2, TimeUnit.SECONDS);
        assertNull(none);
    }
}