org.apache.pulsar.client.api.ProducerBuilder Java Examples

The following examples show how to use org.apache.pulsar.client.api.ProducerBuilder. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: GroupMetadataManager.java    From kop with Apache License 2.0 6 votes vote down vote up
public GroupMetadataManager(OffsetConfig offsetConfig,
                            ProducerBuilder<ByteBuffer> metadataTopicProducerBuilder,
                            ReaderBuilder<ByteBuffer> metadataTopicReaderBuilder,
                            ScheduledExecutorService scheduler,
                            Time time) {
    this(
        offsetConfig,
        metadataTopicProducerBuilder,
        metadataTopicReaderBuilder,
        scheduler,
        time,
        // Be same with kafka: abs(groupId.hashCode) % groupMetadataTopicPartitionCount
        // return a partitionId
        groupId -> MathUtils.signSafeMod(
            groupId.hashCode(),
            offsetConfig.offsetsTopicNumPartitions()
        )
    );
}
 
Example #2
Source File: GroupMetadataManager.java    From kop with Apache License 2.0 6 votes vote down vote up
GroupMetadataManager(OffsetConfig offsetConfig,
                     ProducerBuilder<ByteBuffer> metadataTopicProducerBuilder,
                     ReaderBuilder<ByteBuffer> metadataTopicConsumerBuilder,
                     ScheduledExecutorService scheduler,
                     Time time,
                     Function<String, Integer> partitioner) {
    this.offsetConfig = offsetConfig;
    this.compressionType = offsetConfig.offsetsTopicCompressionType();
    this.groupMetadataCache = new ConcurrentHashMap<>();
    this.groupMetadataTopicPartitionCount = offsetConfig.offsetsTopicNumPartitions();
    this.metadataTopicProducerBuilder = metadataTopicProducerBuilder;
    this.metadataTopicReaderBuilder = metadataTopicConsumerBuilder;
    this.scheduler = scheduler;
    this.time = time;
    this.partitioner = partitioner;
}
 
Example #3
Source File: PulsarSink.java    From pulsar with Apache License 2.0 6 votes vote down vote up
public Producer<T> createProducer(PulsarClient client, String topic, String producerName, Schema<T> schema)
        throws PulsarClientException {
    ProducerBuilder<T> builder = client.newProducer(schema)
            .blockIfQueueFull(true)
            .enableBatching(true)
            .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS)
            .compressionType(CompressionType.LZ4)
            .hashingScheme(HashingScheme.Murmur3_32Hash) //
            .messageRoutingMode(MessageRoutingMode.CustomPartition)
            .messageRouter(FunctionResultRouter.of())
            // set send timeout to be infinity to prevent potential deadlock with consumer
            // that might happen when consumer is blocked due to unacked messages
            .sendTimeout(0, TimeUnit.SECONDS)
            .topic(topic);
    if (producerName != null) {
        builder.producerName(producerName);
    }

    return builder.properties(properties).create();
}
 
Example #4
Source File: PulsarManager.java    From pulsar with Apache License 2.0 6 votes vote down vote up
public void startup() throws Exception {
    try {
        client = PULSAR_CLIENT_BUILDER.get()
            .serviceUrl(serviceUrl)
            .build();
        ProducerBuilder<byte[]> producerBuilder = client.newProducer()
            .topic(topic)
            .producerName("pulsar-log4j2-appender-" + topic)
            .blockIfQueueFull(false);
        if (syncSend) {
            // disable batching for sync send
            producerBuilder = producerBuilder.enableBatching(false);
        } else {
            // enable batching in 10 ms for async send
            producerBuilder = producerBuilder
                .enableBatching(true)
                .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS);
        }
        producer = producerBuilder.create();
    } catch (Exception t) {
        LOGGER.error("Failed to start pulsar manager", t);
        throw t;
    }
}
 
Example #5
Source File: BrokerBookieIsolationTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
private Topic createTopicAndPublish(PulsarClient pulsarClient, String ns, String topicLocalName, int totalPublish)
        throws Exception {
    final String topicName = String.format("persistent://%s/%s", ns, topicLocalName);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscriber-name")
            .subscribe();
    consumer.close();

    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer().topic(topicName).sendTimeout(5, TimeUnit.SECONDS);

    Producer<byte[]> producer = producerBuilder.create();
    for (int i = 0; i < totalPublish; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    producer.close();

    return pulsarService.getBrokerService().getTopicReference(topicName).get();
}
 
Example #6
Source File: PersistentTopicE2ETest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateProducerWithSameName() throws Exception {
    String topic = "persistent://prop/ns-abc/testCreateProducerWithSameName";

    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer()
        .topic(topic)
        .producerName("test-producer-a")
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.SinglePartition);
    Producer<byte[]> p1 = producerBuilder.create();

    try {
        producerBuilder.create();
        fail("Should have thrown ProducerBusyException");
    } catch (ProducerBusyException e) {
        // Expected
    }

    p1.close();

    // Now p2 should succeed
    Producer<byte[]> p2 = producerBuilder.create();

    p2.close();
}
 
Example #7
Source File: MessageChunkingTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
/**
 * Validate that chunking is not supported with batching and non-persistent topic
 * 
 * @throws Exception
 */
@Test
public void testInvalidUseCaseForChunking() throws Exception {

    log.info("-- Starting {} test --", methodName);
    this.conf.setMaxMessageSize(5);
    final String topicName = "persistent://my-property/my-ns/my-topic1";

    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer().topic(topicName);

    try {
        Producer<byte[]> producer = producerBuilder.enableChunking(true).enableBatching(true).create();
        fail("it should have failied because chunking can't be used with batching enabled");
    } catch (IllegalArgumentException ie) {
        // Ok
    }

    log.info("-- Exiting {} test --", methodName);
}
 
Example #8
Source File: MessageChunkingTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@Test
public void testPublishWithFailure() throws Exception {
    log.info("-- Starting {} test --", methodName);
    this.conf.setMaxMessageSize(5);
    final String topicName = "persistent://my-property/my-ns/my-topic1";

    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer().topic(topicName);

    Producer<byte[]> producer = producerBuilder.enableChunking(true).enableBatching(false)
            .create();

    stopBroker();

    try {
        producer.send(createMessagePayload(100).getBytes());
        fail("should have failed with timeout exception");
    } catch (PulsarClientException.TimeoutException e) {
        // Ok
    }
    producer.close();
}
 
Example #9
Source File: SaslAuthenticateTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testProducerAndConsumerPassed() throws Exception {
    log.info("-- {} -- start", methodName);

    Consumer<byte[]> consumer = pulsarClient.newConsumer()
        .topic("persistent://my-property/my-ns/my-topic")
        .subscriptionName("my-subscriber-name")
        .subscribe();

    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer()
        .topic("persistent://my-property/my-ns/my-topic")
        .enableBatching(false);

    Producer<byte[]> producer = producerBuilder.create();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        log.info("Produced message: [{}]", message);
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.info("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();

    log.info("-- {} -- end", methodName);
}
 
Example #10
Source File: ProducerBuilderImpl.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Override
@Deprecated
public ProducerBuilder<T> intercept(org.apache.pulsar.client.api.ProducerInterceptor<T>... interceptors) {
    if (interceptorList == null) {
        interceptorList = new ArrayList<>();
    }
    interceptorList.addAll(Arrays.stream(interceptors).map(ProducerInterceptorWrapper::new)
                                 .collect(Collectors.toList()));
    return this;
}
 
Example #11
Source File: ProducerBuilderImpl.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Override
public ProducerBuilder<T> intercept(ProducerInterceptor... interceptors) {
    if (interceptorList == null) {
        interceptorList = new ArrayList<>();
    }
    interceptorList.addAll(Arrays.asList(interceptors));
    return this;
}
 
Example #12
Source File: ProducerBuilderImpl.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Override
public ProducerBuilder<T> property(String key, String value) {
    checkArgument(StringUtils.isNotBlank(key) && StringUtils.isNotBlank(value),
            "property key/value cannot be blank");
    conf.getProperties().put(key, value);
    return this;
}
 
Example #13
Source File: ReplicatorTestBase.java    From pulsar with Apache License 2.0 5 votes vote down vote up
MessageProducer(URL url, final TopicName dest, boolean batch) throws Exception {
    this.url = url;
    this.namespace = dest.getNamespace();
    this.topicName = dest.toString();
    client = PulsarClient.builder().serviceUrl(url.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    ProducerBuilder<byte[]> producerBuilder = client.newProducer()
        .topic(topicName)
        .enableBatching(batch)
        .batchingMaxPublishDelay(1, TimeUnit.SECONDS)
        .batchingMaxMessages(5);
    producer = producerBuilder.create();

}
 
Example #14
Source File: ReaderTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
private Set<String> publishMessages(String topic, int count, boolean enableBatch) throws Exception {
    Set<String> keys = new HashSet<>();
    ProducerBuilder<byte[]> builder = pulsarClient.newProducer();
    builder.messageRoutingMode(MessageRoutingMode.SinglePartition);
    builder.maxPendingMessages(count);
    // disable periodical flushing
    builder.batchingMaxPublishDelay(1, TimeUnit.DAYS);
    builder.topic(topic);
    if (enableBatch) {
        builder.enableBatching(true);
        builder.batchingMaxMessages(count);
    } else {
        builder.enableBatching(false);
    }
    try (Producer<byte[]> producer = builder.create()) {
        Future<?> lastFuture = null;
        for (int i = 0; i < count; i++) {
            String key = "key"+i;
            byte[] data = ("my-message-" + i).getBytes();
            lastFuture = producer.newMessage().key(key).value(data).sendAsync();
            keys.add(key);
        }
        producer.flush();
        lastFuture.get();
    }
    return keys;
}
 
Example #15
Source File: ZeroQueueSizeTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test()
public void testFailedZeroQueueSizeBatchMessage() throws PulsarClientException {

    int batchMessageDelayMs = 100;
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://prop-xyz/use/ns-abc/topic1")
            .subscriptionName("my-subscriber-name").subscriptionType(SubscriptionType.Shared).receiverQueueSize(0)
            .subscribe();

    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer()
        .topic("persistent://prop-xyz/use/ns-abc/topic1")
        .messageRoutingMode(MessageRoutingMode.SinglePartition);

    if (batchMessageDelayMs != 0) {
        producerBuilder.enableBatching(true).batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
                .batchingMaxMessages(5);
    } else {
        producerBuilder.enableBatching(false);
    }

    Producer<byte[]> producer = producerBuilder.create();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    try {
        consumer.receiveAsync().handle((ok, e) -> {
            if (e == null) {
                // as zero receiverQueueSize doesn't support batch message, must receive exception at callback.
                Assert.fail();
            }
            return null;
        });
    } finally {
        consumer.close();
    }
}
 
Example #16
Source File: V1_ProducerConsumerTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test(dataProvider = "batch")
public void testSyncProducerAndConsumer(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);

    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING)
                .topic("persistent://my-property/use/my-ns/my-topic1")
                .subscriptionName("my-subscriber-name")
                .subscriptionType(SubscriptionType.Exclusive)
                .subscribe();

    ProducerBuilder<String> producerBuilder = pulsarClient.newProducer(Schema.STRING)
            .topic("persistent://my-property/use/my-ns/my-topic1");

    if (batchMessageDelayMs != 0) {
        producerBuilder.enableBatching(true)
            .batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS)
            .batchingMaxMessages(5);
    } else {
        producerBuilder.enableBatching(false);
    }

    Producer<String> producer = producerBuilder.create();
    for (int i = 0; i < 10; i++) {
        producer.send("my-message-" + i);
    }

    Message<String> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = msg.getValue();
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
 
Example #17
Source File: MessageChunkingTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testInvalidConfig() throws Exception {
    final String topicName = "persistent://my-property/my-ns/my-topic1";
    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer().topic(topicName);
    // batching and chunking can't be enabled together
    try {
        Producer<byte[]> producer = producerBuilder.enableChunking(true).enableBatching(true).create();
        fail("producer creation should have fail");
    } catch (IllegalArgumentException ie) {
        // Ok
    }
}
 
Example #18
Source File: CompactionTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test(timeOut = 20000, dataProvider = "lastDeletedBatching")
public void testAllEmptyCompactionLedger(boolean batchEnabled) throws Exception {
    final String topic = "persistent://my-property/use/my-ns/testAllEmptyCompactionLedger" + UUID.randomUUID().toString();

    final int messages = 10;

    // 1.create producer and publish message to the topic.
    ProducerBuilder<byte[]> builder = pulsarClient.newProducer().topic(topic);
    if (!batchEnabled) {
        builder.enableBatching(false);
    } else {
        builder.batchingMaxMessages(messages / 5);
    }

    Producer<byte[]> producer = builder.create();

    List<CompletableFuture<MessageId>> futures = new ArrayList<>(messages);
    for (int i = 0; i < messages; i++) {
        futures.add(producer.newMessage().keyBytes("1".getBytes()).value("".getBytes()).sendAsync());
    }

    FutureUtil.waitForAll(futures).get();

    // 2.compact the topic.
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();

    // consumer with readCompacted enabled only get compacted entries
    try (Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1")
            .readCompacted(true).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
        Message<byte[]> m = consumer.receive(2, TimeUnit.SECONDS);
        assertNull(m);
    }
}
 
Example #19
Source File: PulsarKafkaProducerTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testPulsarKafkaProducer() {
    ClientBuilder mockClientBuilder = mock(ClientBuilder.class);
    ProducerBuilder mockProducerBuilder = mock(ProducerBuilder.class);
    doAnswer(invocation -> {
        Assert.assertEquals((int)invocation.getArguments()[0], 1000000, "Send time out is suppose to be 1000.");
        return mockProducerBuilder;
    }).when(mockProducerBuilder).sendTimeout(anyInt(), any(TimeUnit.class));
    doReturn(mockClientBuilder).when(mockClientBuilder).serviceUrl(anyString());
    doAnswer(invocation -> {
        Assert.assertEquals((int)invocation.getArguments()[0], 1000, "Keep alive interval is suppose to be 1000.");
        return mockClientBuilder;
    }).when(mockClientBuilder).keepAliveInterval(anyInt(), any(TimeUnit.class));

    PowerMockito.mockStatic(PulsarClientKafkaConfig.class);
    PowerMockito.mockStatic(PulsarProducerKafkaConfig.class);
    when(PulsarClientKafkaConfig.getClientBuilder(any(Properties.class))).thenReturn(mockClientBuilder);
    when(PulsarProducerKafkaConfig.getProducerBuilder(any(), any())).thenReturn(mockProducerBuilder);

    Properties properties = new Properties();
    properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    properties.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, DefaultPartitioner.class);
    properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, Arrays.asList("pulsar://localhost:6650"));
    properties.put(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, "1000000");
    properties.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "1000000");

    new PulsarKafkaProducer<>(properties);

    verify(mockClientBuilder, times(1)).keepAliveInterval(1000, TimeUnit.SECONDS);
    verify(mockProducerBuilder, times(1)).sendTimeout(1000000, TimeUnit.MILLISECONDS);
}
 
Example #20
Source File: PulsarProducerKafkaConfig.java    From pulsar with Apache License 2.0 5 votes vote down vote up
public static ProducerBuilder<byte[]> getProducerBuilder(PulsarClient client, Properties properties) {
    ProducerBuilder<byte[]> producerBuilder = client.newProducer();

    if (properties.containsKey(PRODUCER_NAME)) {
        producerBuilder.producerName(properties.getProperty(PRODUCER_NAME));
    }

    if (properties.containsKey(INITIAL_SEQUENCE_ID)) {
        producerBuilder.initialSequenceId(Long.parseLong(properties.getProperty(INITIAL_SEQUENCE_ID)));
    }

    if (properties.containsKey(MAX_PENDING_MESSAGES)) {
        producerBuilder.maxPendingMessages(Integer.parseInt(properties.getProperty(MAX_PENDING_MESSAGES)));
    }

    if (properties.containsKey(MAX_PENDING_MESSAGES_ACROSS_PARTITIONS)) {
        producerBuilder.maxPendingMessagesAcrossPartitions(
                Integer.parseInt(properties.getProperty(MAX_PENDING_MESSAGES_ACROSS_PARTITIONS)));
    }

    producerBuilder.enableBatching(Boolean.parseBoolean(properties.getProperty(BATCHING_ENABLED, "true")));

    if (properties.containsKey(BATCHING_MAX_MESSAGES)) {
        producerBuilder.batchingMaxMessages(Integer.parseInt(properties.getProperty(BATCHING_MAX_MESSAGES)));
    }

    return producerBuilder;
}
 
Example #21
Source File: PulsarProducerKafkaConfig.java    From pulsar with Apache License 2.0 5 votes vote down vote up
public static ProducerBuilder<byte[]> getProducerBuilder(PulsarClient client, Properties properties) {
    ProducerBuilder<byte[]> producerBuilder = client.newProducer();

    if (properties.containsKey(PRODUCER_NAME)) {
        producerBuilder.producerName(properties.getProperty(PRODUCER_NAME));
    }

    if (properties.containsKey(INITIAL_SEQUENCE_ID)) {
        producerBuilder.initialSequenceId(Long.parseLong(properties.getProperty(INITIAL_SEQUENCE_ID)));
    }

    if (properties.containsKey(MAX_PENDING_MESSAGES)) {
        producerBuilder.maxPendingMessages(Integer.parseInt(properties.getProperty(MAX_PENDING_MESSAGES)));
    }

    if (properties.containsKey(MAX_PENDING_MESSAGES_ACROSS_PARTITIONS)) {
        producerBuilder.maxPendingMessagesAcrossPartitions(
                Integer.parseInt(properties.getProperty(MAX_PENDING_MESSAGES_ACROSS_PARTITIONS)));
    }

    producerBuilder.enableBatching(Boolean.parseBoolean(properties.getProperty(BATCHING_ENABLED, "true")));

    if (properties.containsKey(BATCHING_MAX_MESSAGES)) {
        producerBuilder.batchingMaxMessages(Integer.parseInt(properties.getProperty(BATCHING_MAX_MESSAGES)));
    }

    return producerBuilder;
}
 
Example #22
Source File: PulsarKafkaProducerTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testPulsarKafkaProducer() {
    ClientBuilder mockClientBuilder = mock(ClientBuilder.class);
    ProducerBuilder mockProducerBuilder = mock(ProducerBuilder.class);
    doAnswer(invocation -> {
        Assert.assertEquals((int)invocation.getArguments()[0], 1000000, "Send time out is suppose to be 1000.");
        return mockProducerBuilder;
    }).when(mockProducerBuilder).sendTimeout(anyInt(), any(TimeUnit.class));
    doReturn(mockClientBuilder).when(mockClientBuilder).serviceUrl(anyString());
    doAnswer(invocation -> {
        Assert.assertEquals((int)invocation.getArguments()[0], 1000, "Keep alive interval is suppose to be 1000.");
        return mockClientBuilder;
    }).when(mockClientBuilder).keepAliveInterval(anyInt(), any(TimeUnit.class));

    PowerMockito.mockStatic(PulsarClientKafkaConfig.class);
    PowerMockito.mockStatic(PulsarProducerKafkaConfig.class);
    when(PulsarClientKafkaConfig.getClientBuilder(any(Properties.class))).thenReturn(mockClientBuilder);
    when(PulsarProducerKafkaConfig.getProducerBuilder(any(), any())).thenReturn(mockProducerBuilder);

    Properties properties = new Properties();
    properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    properties.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, DefaultPartitioner.class);
    properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, Arrays.asList("pulsar://localhost:6650"));
    properties.put(ProducerConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG, "1000000");
    properties.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "1000000");
    properties.put(PulsarProducerKafkaConfig.BLOCK_IF_PRODUCER_QUEUE_FULL, Boolean.FALSE.toString());

    new PulsarKafkaProducer<>(properties);

    verify(mockClientBuilder, times(1)).keepAliveInterval(1000, TimeUnit.SECONDS);
    verify(mockProducerBuilder, times(1)).sendTimeout(1000000, TimeUnit.MILLISECONDS);
    verify(mockProducerBuilder, times(1)).blockIfQueueFull(false);
}
 
Example #23
Source File: ProducerBuilderImpl.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Override
public ProducerBuilder<T> enableChunking(boolean chunkingEnabled) {
    conf.setChunkingEnabled(chunkingEnabled);
    return this;
}
 
Example #24
Source File: PulsarMessageProducerImpl.java    From datacollector with Apache License 2.0 4 votes vote down vote up
@Override
public List<Stage.ConfigIssue> init(Target.Context context) {
  List<Stage.ConfigIssue> issues = new ArrayList<>();

  issues.addAll(pulsarConfig.init(context));

  if (issues.isEmpty()) {
    // pulsar client
    pulsarClient = pulsarConfig.getClient();

    // pulsar message producers
    messageProducers = CacheBuilder.newBuilder()
                                   .expireAfterAccess(15, TimeUnit.MINUTES)
                                   .removalListener((RemovalListener<String, Producer>) removalNotification -> {
                                     try {
                                       removalNotification.getValue().close();
                                     } catch (PulsarClientException e) {
                                       LOG.warn("Exception when trying to remove pulsar message producer {}. " +
                                           "Exception: {}", removalNotification.getValue().getProducerName(), e);
                                     }
                                   })
                                   .build(new CacheLoader<String, Producer>() {
                                     @Override
                                     public Producer load(String key) throws Exception {
                                       ProducerBuilder producerBuilder = pulsarClient.newProducer()
                                                                     .properties(pulsarConfig.properties)
                                                                     .topic(key)
                                                                     .messageRoutingMode(pulsarConfig.partitionType
                                                                         .getMessageRoutingMode())
                                                                     .hashingScheme(pulsarConfig.hashingScheme
                                                                         .getHashingScheme())
                                                                     .compressionType(pulsarConfig.compressionType
                                                                         .getCompressionType())
                                                                     .blockIfQueueFull(true);
                                       if (pulsarConfig.asyncSend) {
                                         producerBuilder.maxPendingMessages(pulsarConfig.maxPendingMessages)
                                                        .enableBatching(pulsarConfig.enableBatching)
                                                        .batchingMaxMessages(pulsarConfig.batchMaxMessages)
                                                        .batchingMaxPublishDelay(pulsarConfig.batchMaxPublishDelay,
                                                            TimeUnit.MILLISECONDS);

                                       }

                                       return producerBuilder.create();
                                     }
                                   });
  }

  destinationEval = context.createELEval("destinationTopic");
  messageKeyEval = context.createELEval("messageKey");
  errorHandler = new DefaultErrorRecordHandler(context);

  return issues;
}
 
Example #25
Source File: ProducerBuilderImpl.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Override
public ProducerBuilder<T> addEncryptionKey(String key) {
    checkArgument(StringUtils.isNotBlank(key), "Encryption key cannot be blank");
    conf.getEncryptionKeys().add(key);
    return this;
}
 
Example #26
Source File: ProducerBuilderImpl.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Override
public ProducerBuilder<T> cryptoFailureAction(@NonNull ProducerCryptoFailureAction action) {
    conf.setCryptoFailureAction(action);
    return this;
}
 
Example #27
Source File: ProducerBuilderImpl.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Override
public ProducerBuilder<T> batchingMaxPublishDelay(long batchDelay, @NonNull TimeUnit timeUnit) {
    conf.setBatchingMaxPublishDelayMicros(batchDelay, timeUnit);
    return this;
}
 
Example #28
Source File: ProducerBuilderImpl.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Override
public ProducerBuilder<T> roundRobinRouterBatchingPartitionSwitchFrequency(int frequency) {
    conf.setBatchingPartitionSwitchFrequencyByPublishDelay(frequency);
    return this;
}
 
Example #29
Source File: ProducerBuilderImpl.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Override
public ProducerBuilder<T> batchingMaxMessages(int batchMessagesMaxMessagesPerBatch) {
    conf.setBatchingMaxMessages(batchMessagesMaxMessagesPerBatch);
    return this;
}
 
Example #30
Source File: PulsarKafkaProducer.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
public ProducerBuilder<byte[]> getPulsarProducerBuilder() {
    return pulsarProducerBuilder;
}