org.apache.kafka.clients.producer.Producer Java Examples

@Before
public void init() throws TimeoutException {
    // there may exists other topics than these build in(configured in pom.xml) topics, but ignore them

    // ----------------- Send sample data to TOPIC_IN start --------------------
    String testID = "sampleTest" + new Random().nextInt();

    List<Person> expectedPersons = Person.genRandomList(testID, 10);

    Properties props = new Properties();
    props.put("bootstrap.servers", BOOTSTRAP_HOST);
    props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");

    Producer<Void, String> producer = new KafkaProducer<>(props);
    for (Person person : expectedPersons) {
        ProducerRecord<Void, String> message = new ProducerRecord<>(TOPIC_IN, person.toCSV(fieldDelimiter));
        producer.send(message);
    }
    producer.close();
    // ----------------- Send sample data to TOPIC_IN end --------------------
}
 

@Test
void testTransactionalProducerCreation() {
    assumeFalse(
            System.getProperty("os.name").contains("Windows"),
            "Transactional producers not supported on Windows"
    );

    ProducerFactory<String, String> producerFactory = transactionalProducerFactory(kafkaBroker, "xyz");
    Producer<String, String> testProducer = producerFactory.createProducer();

    testProducer.beginTransaction();
    testProducer.commitTransaction();
    assertFalse(testProducer.metrics().isEmpty());

    cleanup(producerFactory, testProducer);
}
 

public static Producer buildProducer() {
	// 1. 指定生产者的配置
	Properties properties = new Properties();
	properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, HOST);
	properties.put(ProducerConfig.ACKS_CONFIG, "all");
	properties.put(ProducerConfig.RETRIES_CONFIG, 1);
	properties.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
	properties.put(ProducerConfig.LINGER_MS_CONFIG, 1);
	properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432);
	properties.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "first-transactional");
	properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
		"org.apache.kafka.common.serialization.StringSerializer");
	properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
		"org.apache.kafka.common.serialization.StringSerializer");

	// 2. 使用配置初始化 Kafka 生产者
	Producer<String, String> producer = new KafkaProducer<>(properties);
	return producer;
}
 

@Before
public void setup() {
    // Make sure the topic that the change log uses exists.
    final String changeLogTopic = KafkaTopics.queryChangeLogTopic("" + ryaInstance);
    kafka.createTopic(changeLogTopic);

    // Setup the QueryRepository used by the test.
    final Producer<?, QueryChange> queryProducer = KafkaTestUtil.makeProducer(kafka, StringSerializer.class, QueryChangeSerializer.class);
    final Consumer<?, QueryChange> queryConsumer = KafkaTestUtil.fromStartConsumer(kafka, StringDeserializer.class, QueryChangeDeserializer.class);
    final QueryChangeLog changeLog = new KafkaQueryChangeLog(queryProducer, queryConsumer, changeLogTopic);
    queryRepo = new InMemoryQueryRepository(changeLog, Scheduler.newFixedRateSchedule(0L, 5, TimeUnit.SECONDS));

    // Initialize the Statements Producer and the Results Consumer.
    stmtProducer = KafkaTestUtil.makeProducer(kafka, StringSerializer.class, VisibilityStatementSerializer.class);
    resultConsumer = KafkaTestUtil.fromStartConsumer(kafka, StringDeserializer.class, VisibilityBindingSetDeserializer.class);
}
 

private static int[] getPartitionsByTopic(String topic, Producer<byte[], byte[]> producer) {
	// the fetched list is immutable, so we're creating a mutable copy in order to sort it
	List<PartitionInfo> partitionsList = new ArrayList<>(producer.partitionsFor(topic));

	// sort the partitions by partition id to make sure the fetched partition list is the same across subtasks
	Collections.sort(partitionsList, new Comparator<PartitionInfo>() {
		@Override
		public int compare(PartitionInfo o1, PartitionInfo o2) {
			return Integer.compare(o1.partition(), o2.partition());
		}
	});

	int[] partitions = new int[partitionsList.size()];
	for (int i = 0; i < partitions.length; i++) {
		partitions[i] = partitionsList.get(i).partition();
	}

	return partitions;
}
 

public void start() throws InterruptedException {
  RandomGenerator random = RandomManager.getRandom();

  Properties props = ConfigUtils.keyValueToProperties(
      "bootstrap.servers", "localhost:" + kafkaPort,
      "key.serializer", "org.apache.kafka.common.serialization.StringSerializer",
      "value.serializer", "org.apache.kafka.common.serialization.StringSerializer",
      "compression.type", "gzip",
      "linger.ms", 0,
      "batch.size", 0,
      "acks", 1,
      "max.request.size", 1 << 26 // TODO
  );
  try (Producer<String,String> producer = new KafkaProducer<>(props)) {
    for (int i = 0; i < howMany; i++) {
      Pair<String,String> datum = datumGenerator.generate(i, random);
      ProducerRecord<String,String> record =
          new ProducerRecord<>(topic, datum.getFirst(), datum.getSecond());
      producer.send(record);
      log.debug("Sent datum {} = {}", record.key(), record.value());
      if (intervalMsec > 0) {
        Thread.sleep(intervalMsec);
      }
    }
  }
}
 

@Test
@SuppressWarnings("unchecked")
public void shouldThrowExceptionIfProducerFails() throws NotificationException,
        ExecutionException, InterruptedException {
    Properties configProperties = mock(Properties.class);
    KafkaNotification kafkaNotification = new KafkaNotification(configProperties);

    Producer producer = mock(Producer.class);
    String topicName = kafkaNotification.getProducerTopicName(NotificationInterface.NotificationType.HOOK);
    String message = "This is a test message";
    Future returnValue = mock(Future.class);
    when(returnValue.get()).thenThrow(new RuntimeException("Simulating exception"));
    ProducerRecord expectedRecord = new ProducerRecord(topicName, message);
    when(producer.send(expectedRecord)).thenReturn(returnValue);

    try {
        kafkaNotification.sendInternalToProducer(producer,
            NotificationInterface.NotificationType.HOOK, Arrays.asList(new String[]{message}));
        fail("Should have thrown NotificationException");
    } catch (NotificationException e) {
        assertEquals(e.getFailedMessages().size(), 1);
        assertEquals(e.getFailedMessages().get(0), "This is a test message");
    }
}
 

/**
 * Create a {@link Producer} that is able to write to a topic in Kafka.
 *
 * @param kafkaHostname - The Kafka broker hostname. (not null)
 * @param kafkaPort - The Kafka broker port.
 * @param keySerializerClass - Serializes the keys. (not null)
 * @param valueSerializerClass - Serializes the values. (not null)
 * @return A {@link Producer} that can be used to write records to a topic.
 */
private static <K, V> Producer<K, V> makeProducer(
        final String kafkaHostname,
        final int kakfaPort,
        final Class<? extends Serializer<K>> keySerializerClass,
        final Class<? extends Serializer<V>> valueSerializerClass) {
    requireNonNull(kafkaHostname);
    requireNonNull(keySerializerClass);
    requireNonNull(valueSerializerClass);

    final Properties producerProps = new Properties();
    producerProps.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, kafkaHostname + ":" + kakfaPort);
    producerProps.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, keySerializerClass.getName());
    producerProps.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, valueSerializerClass.getName());
    return new KafkaProducer<>(producerProps);
}
 

public Producer Create() {
    Properties props = new Properties();
    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, this.kafkaAddress);
    props.put(ProducerConfig.CLIENT_ID_CONFIG, IKafkaConfiguration.PRODUCER_CLIENT_ID);

    // Key is set as long and Value is given by concrete implementation
    props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, LongSerializer.class.getName());

    Serializer serializer = getSerializer();
    props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, serializer.getClass().getName());

    //TODO: Optimizing partition
    // props.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, CustomPartitioner.class.getName());

    return new KafkaProducer<>(props);
}
 

public static void writeEmailMessage(String subject, String contents, String dataSchema, Producer<String, String> producer) {
    try {
        // 发邮件
        ControlMessage gm = new ControlMessage(System.currentTimeMillis(), ControlType.COMMON_EMAIL_MESSAGE.toString(), BoltCommandHandlerHelper.class.getName());

        gm.addPayload("subject", subject);
        gm.addPayload("contents", contents);
        gm.addPayload("datasource_schema", Utils.getDatasource().getDsName() + "/" + dataSchema);

        String topic = PropertiesHolder.getProperties(Constants.Properties.CONFIGURE, Constants.ConfigureKey.GLOBAL_EVENT_TOPIC);
        ProducerRecord<String, String> record = new ProducerRecord<>(topic, gm.getType(), gm.toJSONString());
        producer.send(record, (metadata, exception) -> {
            if (exception != null) {
                logger.error("Send global event error.{}", exception.getMessage());
            }
        });
    } catch (Exception e) {
        logger.error("send email error. schema:{}, subject:{}, content:{}", dataSchema, subject, contents, e);
    } finally {
        if (producer != null) producer.close();
    }
}
 

/**
 * Send mock message to kafka topic .
 */
public boolean mockMessage(String clusterAlias, String topic, String message) {
	Properties props = new Properties();
	props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias));
	props.put(Kafka.KEY_SERIALIZER, StringSerializer.class.getCanonicalName());
	props.put(Kafka.VALUE_SERIALIZER, StringSerializer.class.getCanonicalName());
	props.put(Kafka.PARTITION_CLASS, KafkaPartitioner.class.getName());

	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		sasl(props, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		ssl(props, clusterAlias);
	}
	Producer<String, String> producer = new KafkaProducer<>(props);
	producer.send(new ProducerRecord<String, String>(topic, new Date().getTime() + "", message));
	producer.close();

	return true;
}
 

/**
 * Send a message to the {@link KafkaCluster} on the given topic.
 */
public void sendMessage(String topic, String message) {
    Optional<Entity> anyBrokerNodeInCluster = Iterables.tryFind(cluster.getCluster().getChildren(), Predicates.and(
            Predicates.instanceOf(KafkaBroker.class),
            EntityPredicates.attributeEqualTo(KafkaBroker.SERVICE_UP, true)));
    if (anyBrokerNodeInCluster.isPresent()) {
        KafkaBroker broker = (KafkaBroker)anyBrokerNodeInCluster.get();

        Properties props = new Properties();

        props.put("metadata.broker.list", format("%s:%d", broker.getAttribute(KafkaBroker.HOSTNAME), broker.getKafkaPort()));
        props.put("bootstrap.servers", format("%s:%d", broker.getAttribute(KafkaBroker.HOSTNAME), broker.getKafkaPort()));
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");

        Producer<String, String> producer = new KafkaProducer<>(props);
        ((KafkaZooKeeper)cluster.getZooKeeper()).createTopic(topic);

        ProducerRecord<String, String> data = new ProducerRecord<>(topic, message);
        producer.send(data);
        producer.close();
    } else {
        throw new InvalidParameterException("No kafka broker node found");
    }
}
 

@Override
public Producer<byte[], byte[]> createProducer(Properties transportProps) {
  VerifiableProperties transportProviderProperties = new VerifiableProperties(transportProps);
  String clientId = transportProviderProperties.getString(ProducerConfig.CLIENT_ID_CONFIG);
  String bootstrapServers = transportProviderProperties.getString(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG);
  Properties producerConfig = transportProviderProperties.getDomainProperties(DOMAIN_PRODUCER);

  Validate.notEmpty(clientId, "clientId cannot be empty.");
  Validate.notEmpty(bootstrapServers, "bootstrapServers cannot be empty.");

  producerConfig = buildProducerProperties(producerConfig, clientId, bootstrapServers, DEFAULT_ENABLE_LARGE_MESSAGE);

  // Default DeSerializer for Key and Payload
  producerConfig.putIfAbsent(LiKafkaProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
      ByteArraySerializer.class.getCanonicalName());
  producerConfig.putIfAbsent(LiKafkaProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
      ByteArraySerializer.class.getCanonicalName());

  return new LiKafkaProducerImpl<>(producerConfig);
}
 

public static void main(String[] args) throws ExecutionException, InterruptedException{
    Properties properties = new Properties();
    properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList);
    properties.put(ProducerConfig.CLIENT_ID_CONFIG, "spark-producer-demo-client");
    properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
    properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

    Producer<String,String> producer = new KafkaProducer<>(properties);

    Random random = new Random();
    while (true) {
        int value = random.nextInt(10);
        ProducerRecord<String, String> message =
                new ProducerRecord<>(topic, value+"");
        producer.send(message, (recordMetadata, e) -> {
            if (recordMetadata != null) {
                System.out.println(recordMetadata.topic() + "-" + recordMetadata.partition() + ":" +
                        recordMetadata.offset());
            }
        });
        TimeUnit.SECONDS.sleep(1);
    }
}
 

@Override
public <K, V, E> boolean send(Producer<K, V> producer, ProducerRecord<K, V> record, final E event,
                              final FailedDeliveryCallback<E> failedDeliveryCallback) {
    try {
        producer.send(record, new Callback() {
            @Override
            public void onCompletion(RecordMetadata metadata, Exception exception) {
                if (exception != null) {
                    failedDeliveryCallback.onFailedDelivery(event, exception);
                }
            }
        });
        return true;
    } catch (BufferExhaustedException | TimeoutException e) {
        failedDeliveryCallback.onFailedDelivery(event, e);
        return false;
    }
}
 

@Nullable
private Producer<String, String> takeUnderLock(final boolean canCreate) {
    final Lock lock = canCreate ? rwLock.writeLock() : rwLock.readLock();
    lock.lock();
    try {
        if (null != activeProducer) {
            useCount.get(activeProducer).incrementAndGet();
            return activeProducer;
        } else if (canCreate) {
            activeProducer = createProducerInstance();
            useCount.put(activeProducer, new AtomicInteger(1));
            LOG.info("New producer instance created: " + activeProducer);
            return activeProducer;
        } else {
            return null;
        }
    } finally {
        lock.unlock();
    }
}
 

private String produceMessage(String topicName, Object msg, Boolean storeSchemaInHeader) {
    String bootstrapServers = CLUSTER.bootstrapServers();
    Map<String, Object> config = new HashMap<>();
    config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
    config.putAll(SCHEMA_REGISTRY_TEST_SERVER_CLIENT_WRAPPER.exportClientConf(true));
    config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName());
    config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName());
    config.put(KafkaAvroSerializer.STORE_SCHEMA_VERSION_ID_IN_HEADER, storeSchemaInHeader.toString());

    final Producer<String, Object> producer = new KafkaProducer<>(config);
    final Callback callback = new ProducerCallback();
    LOG.info("Sending message: [{}] to topic: [{}]", msg, topicName);
    ProducerRecord<String, Object> producerRecord = new ProducerRecord<>(topicName, getKey(msg), msg);
    producer.send(producerRecord, callback);
    producer.flush();
    LOG.info("Message successfully sent to topic: [{}]", topicName);
    producer.close(5, TimeUnit.SECONDS);

    return bootstrapServers;
}
 

private Producer<String, String> createProducer() throws Exception {
    Properties props = PropertiesHolder.getProperties(Constants.Properties.PRODUCER_CONFIG);
    props.setProperty("client.id", this.topologyId + "_writer_" + context.getThisTaskId());

    Producer<String, String> producer = new KafkaProducer<>(props);
    return producer;
}
 

private void testTeamBClientPart1() throws Exception {

        Producer<String, String> teamBProducer = getProducer(TEAM_B_CLIENT);

        //
        // team-b-client should fail to produce to a_* topic
        //
        produceFail(teamBProducer, TOPIC_A);

        // Re-init producer because message to topicA is stuck in the queue, and any subsequent message to another queue
        // won't be handled until first message makes it through.
        teamBProducer = newProducer(TEAM_B_CLIENT);

        //
        // team-b-client should succeed producing to b_* topic
        //
        produce(teamBProducer, TOPIC_B);

        //
        // team-b-client should fail to produce to x_* topic
        //
        produceFail(teamBProducer, TOPIC_X);


        Consumer<String, String> teamBConsumer = newConsumer(TEAM_B_CLIENT, TOPIC_A);

        //
        // team-b-client should fail consuming from a_* topic
        //
        consumeFail(teamBConsumer, TOPIC_A);

        // Close and re-init consumer
        teamBConsumer = newConsumer(TEAM_B_CLIENT, TOPIC_B);

        //
        // team-b-client should succeed consuming from b_* topic
        //
        consume(teamBConsumer, TOPIC_B);
    }
 

@Test
public void testConsumerBuilderWithCustomSpanNameProvider() throws InterruptedException {
  Producer<Integer, String> producer = createTracingProducer();
  producer.send(new ProducerRecord<>("messages", 1, "test"));
  producer.close();

  assertEquals(1, mockTracer.finishedSpans().size());

  ExecutorService executorService = Executors.newSingleThreadExecutor();
  final CountDownLatch latch = new CountDownLatch(1);

  executorService.execute(() -> {
    BiFunction<String, ConsumerRecord, String> operationNameProvider =
        (operationName, consumerRecord) -> createSpanNameProvider();
    Consumer<Integer, String> consumer = createConsumerWithSpanNameProvider(
        operationNameProvider);

    while (latch.getCount() > 0) {
      ConsumerRecords<Integer, String> records = consumer.poll(Duration.ofMillis(100));
      for (ConsumerRecord<Integer, String> record : records) {
        SpanContext spanContext = TracingKafkaUtils
            .extractSpanContext(record.headers(), mockTracer);
        assertNotNull(spanContext);
        assertEquals("test", record.value());
        assertEquals((Integer) 1, record.key());

        consumer.commitSync();
        latch.countDown();
      }
    }
    consumer.close();
  });

  assertTrue(latch.await(30, TimeUnit.SECONDS));

  assertEquals("Test_SpanNameProvider", mockTracer.finishedSpans().get(1).operationName());
}
 

public static Producer<String, Person> createPersonProducer() {
    Properties props = new Properties();
    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:19092");
    props.put(ProducerConfig.CLIENT_ID_CONFIG, "person");
    props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
    props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonbSerializer.class.getName());
    return new KafkaProducer<>(props);
}
 

@Test
public void testAuthorizedWrite() throws Exception {
    // Create the Producer
    Properties producerProps = new Properties();
    producerProps.put("bootstrap.servers", "localhost:" + port);
    producerProps.put("acks", "all");
    producerProps.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    producerProps.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    producerProps.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_SSL");
    producerProps.put("sasl.mechanism", "PLAIN");
    
    producerProps.put(SslConfigs.SSL_KEYSTORE_TYPE_CONFIG, "JKS");
    producerProps.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, serviceKeystorePath);
    producerProps.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, "sspass");
    producerProps.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, "skpass");
    producerProps.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, truststorePath);
    producerProps.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, "security");
    
    final Producer<String, String> producer = new KafkaProducer<>(producerProps);
    
    // Send a message
    Future<RecordMetadata> record = 
        producer.send(new ProducerRecord<String, String>("dev", "somekey", "somevalue"));
    producer.flush();
    record.get();

    producer.close();
}
 

private static Producer<String, byte[]> createProducer() {
  Properties config = new Properties();
  config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS);
  config.put(ProducerConfig.CLIENT_ID_CONFIG, "CouchbaseJsonProducerExample");
  config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
  config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
  return new KafkaProducer<String, byte[]>(config);
}
 

/**
 * Notifies producer cache, that this producer should be marked as obsolete. All methods, that are using this
 * producer instance right now can continue using it, but new calls to {@link #takeProducer()} will use some other
 * producers.
 * It is allowed to call this method only between {@link #takeProducer()} and {@link #releaseProducer(Producer)}
 * method calls. (You can not terminate something that you do not own)
 *
 * @param producer Producer instance to terminate.
 */
public void terminateProducer(final Producer<String, String> producer) {
    LOG.info("Received signal to terminate producer " + producer);
    rwLock.writeLock().lock();
    try {
        if (producer == this.activeProducer) {
            producerTerminations.inc();
            this.activeProducer = null;
        } else {
            LOG.info("Signal for producer termination already received: " + producer);
        }
    } finally {
        rwLock.writeLock().unlock();
    }
}
 

public TracingKafkaProducer(Producer<K, V> producer, Tracer tracer,
    BiFunction<String, ProducerRecord, String> producerSpanNameProvider) {
  this.producer = producer;
  this.tracer = tracer;
  this.spanDecorators = Collections.singletonList(STANDARD_TAGS);
  this.producerSpanNameProvider = (producerSpanNameProvider == null)
      ? ClientSpanNameProvider.PRODUCER_OPERATION_NAME
      : producerSpanNameProvider;
}
 

@Override
public void executeApp() throws Exception {
    Producer<Long, String> producer = SendRecord.createProducer();
    ProducerRecord<Long, String> record =
            new ProducerRecord<Long, String>("demo", "message");
    producer.send(record).get();
    producer.close();

    consumer = createConsumer();
    transactionMarker();
    consumer.close();
}
 

public KafkaByteBufferCommandTransport(Capability remoteCapability, String topic, String key
        , Producer<String, ByteBuffer> producer, Consumer<String, ByteBuffer> consumer) {
    this.remoteCapability = remoteCapability;
    this.producer = producer;
    this.consumer = consumer;
    this.topic = topic;
    this.key = key;
}
 

private  Producer<String, String> getKafkaProducer() {
    Properties props = new Properties();
    props.put("bootstrap.servers", "localhost:9092");
    props.put("acks", "all");
    props.put("retries", 0);
    props.put("batch.size", 16384);
    props.put("linger.ms", 1);
    props.put("buffer.memory", 33554432);
    props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");

    return new KafkaProducer<>(props);
}
 

@Test
public void produceWithCallback() throws InterruptedException {
    Properties properties = new Properties();
    properties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    try (Producer<String, String> producer = getClientFactory().newProducer(properties)) {
        ExecutorService executorService = Executors.newFixedThreadPool(topics.size());
        AtomicInteger failedCount = new AtomicInteger();
        CountDownLatch latch = new CountDownLatch(topics.size());
        try {
            topics.forEach(t -> {
                executorService.submit(() -> {
                    try {
                        produceMessagesWithCallback(producer, t);
                    } catch (Throwable e) {
                        failedCount.incrementAndGet();
                        System.out.println("produce messages failed for " + t);
                        e.printStackTrace();
                    } finally {
                        latch.countDown();
                    }
                });
            });
        } finally {
            executorService.shutdown();
        }

        if (!latch.await(_waitTimeout, TimeUnit.MILLISECONDS))
            Assert.fail("produce message timeout: " + _waitTimeout);

        Assert.assertEquals(0, failedCount.get());

        checkOtherApis(producer);
    }
}
 

protected void produceMessages() throws InterruptedException {
    Properties properties = new Properties();
    properties.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    properties.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    try (Producer<String, String> producer = getClientFactory().newProducer(properties)) {
        ExecutorService executorService = Executors.newFixedThreadPool(_topics.size());
        AtomicInteger failedCount = new AtomicInteger();
        CountDownLatch latch = new CountDownLatch(_topics.size());
        try {
            _topics.forEach(t -> {
                executorService.submit(() -> {
                    try {
                        produceMessages(producer, t);
                    } catch (Throwable e) {
                        failedCount.incrementAndGet();
                        System.out.println("produce messages failed for " + t);
                        e.printStackTrace();
                    } finally {
                        latch.countDown();
                    }
                });
            });
        } finally {
            executorService.shutdown();
        }

        if (!latch.await(_waitTimeout, TimeUnit.MILLISECONDS))
            Assert.fail("produce message timeout: " + _waitTimeout);

        Assert.assertEquals(0, failedCount.get());
    }
}