Java Code Examples for kafka.javaapi.producer.Producer#close()

public static void main(String[] args) {
	String topic= "test";
	long events = 100;
       Random rand = new Random();

       Properties props = new Properties();
       props.put("metadata.broker.list", "localhost:9092");
       props.put("serializer.class", "kafka.serializer.StringEncoder");
       props.put("request.required.acks", "1");

       ProducerConfig config = new ProducerConfig(props);

       Producer<String, String> producer = new Producer<String, String>(config);

       for (long nEvents = 0; nEvents < events; nEvents++) {                
              String msg = "NativeMessage-" + rand.nextInt() ; 
              KeyedMessage<String, String> data = new KeyedMessage<String, String>(topic, nEvents + "", msg);
              producer.send(data);
       }
       producer.close();

}
 

public static void publish(String url, String topic, String testCaseFolderName, String dataFileName) {
    log.info("Starting Kafka EventPublisher Client");

    Properties props = new Properties();
    props.put("metadata.broker.list", url);
    props.put("producer.type", "sync");
    props.put("serializer.class", "kafka.serializer.StringEncoder");

    ProducerConfig config = new ProducerConfig(props);
    Producer<String, Object> producer = new Producer<String, Object>(config);

    try {
        List<String> messagesList = readMsg(getTestDataFileLocation(testCaseFolderName, dataFileName));
        for (String message : messagesList) {
            log.info(String.format("Sending message: %s", message));
            KeyedMessage<String, Object> data = new KeyedMessage<String, Object>(topic, message);
            producer.send(data);
            Thread.sleep(100);
        }
        Thread.sleep(1000);
    } catch (Throwable t) {
        log.error("Error when sending the messages", t);
    } finally {
        producer.close();
    }
}
 

@Override
public void changed(BrokerService brokerService) {
  if (listenerCancelled.get()) {
    return;
  }

  String newBrokerList = brokerService.getBrokerList();

  // If there is no change, whether it is empty or not, just return
  if (Objects.equal(brokerList, newBrokerList)) {
    return;
  }

  Producer<Integer, ByteBuffer> newProducer = null;
  if (!newBrokerList.isEmpty()) {
    Properties props = new Properties();
    props.put("metadata.broker.list", newBrokerList);
    props.put("serializer.class", ByteBufferEncoder.class.getName());
    props.put("key.serializer.class", IntegerEncoder.class.getName());
    props.put("request.required.acks", Integer.toString(ack.getAck()));
    props.put("compression.codec", compression.getCodec());
    props.put("message.max.bytes", Integer.toString(MAX_MESSAGE_BYTES));

    ProducerConfig config = new ProducerConfig(props);
    newProducer = new Producer<>(config);
  }

  // If the broker list is empty, the producer will be set to null
  Producer<Integer, ByteBuffer> oldProducer = producer.getAndSet(newProducer);
  if (oldProducer != null) {
    oldProducer.close();
  }

  if (newBrokerList.isEmpty()) {
    LOG.warn("Empty Kafka producer broker list, publish will fail.");
  } else {
    LOG.info("Updated Kafka producer broker list: {}", newBrokerList);
  }
  brokerList = newBrokerList;
}
 

@Override
public void run() {
  // Call from cancel() through executor only.
  cancelChangeListener.cancel();
  Producer<Integer, ByteBuffer> kafkaProducer = producer.get();
  kafkaProducer.close();
  executor.shutdownNow();
}
 

public static void sendOne(Producer<String, String> producer, String topic) throws InterruptedException {
boolean sleepFlag = false;
   KeyedMessage<String, String> message1 = new KeyedMessage<String, String>(topic, "0", "test 0");
   producer.send(message1);
   if(sleepFlag) Thread.sleep(5000);
   KeyedMessage<String, String> message2 = new KeyedMessage<String, String>(topic, "1", "test 1");
   producer.send(message2);
   if(sleepFlag) Thread.sleep(5000);
   KeyedMessage<String, String> message3 = new KeyedMessage<String, String>(topic, "2", "test 2");
   producer.send(message3);
   if(sleepFlag) Thread.sleep(5000);
   KeyedMessage<String, String> message4 = new KeyedMessage<String, String>(topic, "3", "test 3");
   producer.send(message4);
   if(sleepFlag) Thread.sleep(5000);
   KeyedMessage<String, String> message5 = new KeyedMessage<String, String>(topic, "4", "test 4");
   producer.send(message5);
   if(sleepFlag) Thread.sleep(5000);
   KeyedMessage<String, String> message6 = new KeyedMessage<String, String>(topic, "5", "test 5");
   producer.send(message6);
   if(sleepFlag) Thread.sleep(5000);
   KeyedMessage<String, String> message7 = new KeyedMessage<String, String>(topic, "6", "test 6");
   producer.send(message7);
   if(sleepFlag) Thread.sleep(5000);
   KeyedMessage<String, String> message8 = new KeyedMessage<String, String>(topic, "7", "test 7");
   producer.send(message8);
   if(sleepFlag) Thread.sleep(5000);
   KeyedMessage<String, String> message9 = new KeyedMessage<String, String>(topic, "8", "test 8");
   producer.send(message9);
   if(sleepFlag) Thread.sleep(5000);
   producer.close();
 }
 

public static void main(String[] args) {
    Properties props = new Properties();
    props.put("zookeeper.connect",
            "wxb-1:2181,wxb-1:2181,wxb-12181");
    props.put("serializer.class", "kafka.serializer.StringEncoder");
    props.put("producer.type", "async");
    props.put("compression.codec", "1");
    props.put(
            "metadata.broker.list",
            "wxb-1:6667,wxb-2:6667,wxb-3:6667");

    ProducerConfig config = new ProducerConfig(props);
    Producer<String, String> producer = new Producer<String, String>(config);

    DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    Random r = new Random();
    for (int i = 0; i < 1000; i++) {
        int id = r.nextInt(10000000);
        int memberid = r.nextInt(100000);
        int totalprice = r.nextInt(1000) + 100;
        int preferential = r.nextInt(100);
        int sendpay = r.nextInt(3);

        StringBuffer data = new StringBuffer();
        data.append(String.valueOf(id)).append("\t")
                .append(String.valueOf(memberid)).append("\t")
                .append(String.valueOf(totalprice)).append("\t")
                .append(String.valueOf(preferential)).append("\t")
                .append(String.valueOf(sendpay)).append("\t")
                .append(df.format(new Date()));
        System.out.println(data.toString());
        producer.send(new KeyedMessage<String, String>("order", data
                .toString()));
    }
    producer.close();
    System.out.println("send over ------------------");
}
 

private static void publishMessage(Arguments arguments) {
    // (2) Create producer
    Properties properties2 = new Properties();
    properties2.put(BROKER_URL, arguments.serviceUrl);
    properties2.put(PRODUCER_TYPE, "sync");
    properties2.put(SERIALIZER_CLASS, TestEncoder.class.getName());
    properties2.put(KEY_SERIALIZER_CLASS, StringEncoder.class.getName());
    properties2.put(PARTITIONER_CLASS, TestPartitioner.class.getName());
    properties2.put(COMPRESSION_CODEC, "gzip"); // compression: ZLIB
    properties2.put(QUEUE_ENQUEUE_TIMEOUT_MS, "-1"); // block queue if full => -1 = true
    properties2.put(QUEUE_BUFFERING_MAX_MESSAGES, "6000"); // queue max message
    properties2.put(QUEUE_BUFFERING_MAX_MS, "100"); // batch delay
    properties2.put(BATCH_NUM_MESSAGES, "500"); // batch msg
    properties2.put(CLIENT_ID, "test");
    ProducerConfig config = new ProducerConfig(properties2);
    Producer<String, Tweet> producer = new Producer<>(config);

    String name = "user";
    String msg = arguments.messageValue;
    for (int i = 0; i < arguments.totalMessages; i++) {
        String sendMessage = msg + i;
        Tweet tweet = new Tweet(name, sendMessage);
        KeyedMessage<String, Tweet> message = new KeyedMessage<>(arguments.topicName, name, tweet);
        producer.send(message);
    }

    producer.close();
    log.info("Successfully published messages {}", arguments.totalMessages);

}
 

public static void main(String args[]) {
    log.info("Command line arguments passed: " + Arrays.deepToString(args));
    log.info("Starting Kafka Client");

    String url = args[0];
    String topic = args[1];
    String filePath = args[2];
    String sampleNumber = args[3];

    Properties props = new Properties();
    props.put("metadata.broker.list", url);
    props.put("serializer.class", "kafka.serializer.StringEncoder");

    ProducerConfig config = new ProducerConfig(props);
    Producer<String, Object> producer = new Producer<String, Object>(config);

    try {

        filePath = KafkaUtil.getEventFilePath(sampleNumber, topic, filePath);
        readMsg(filePath);

        for (String message : messagesList) {
            System.out.println("Sending message:");
            System.out.println(message);
            KeyedMessage<String, Object> data = new KeyedMessage<String, Object>(topic, message);
            producer.send(data);
        }
        Thread.sleep(500);

    } catch (Throwable t) {
        log.error("Error when sending the messages", t);
    } finally {
        producer.close();
    }
}
 

@Override
public void shutdown() {
    try {
        super.shutdown();
    } finally {
        for (Producer producer : producerMap.values()) {
            producer.close();
        }
    }
}
 

/**
 * The MapReduce driver - setup and launch the job.
 *
 * @param args the command-line arguments
 * @return the process exit code
 * @throws Exception if something goes wrong
 */
public int run(final String[] args) throws Exception {

  Cli cli = Cli.builder().setArgs(args).addOptions(Options.values()).build();
  int result = cli.runCmd();

  if (result != 0) {
    return result;
  }

  File inputFile = new File(cli.getArgValueAsString(Options.STOCKSFILE));
  String brokerList = cli.getArgValueAsString(Options.BROKER_LIST);
  String kTopic = cli.getArgValueAsString(Options.TOPIC);

  Properties props = new Properties();
  props.put("metadata.broker.list", brokerList);
  props.put("serializer.class", kafka.serializer.DefaultEncoder.class.getName());

  ProducerConfig config = new ProducerConfig(props);

  Producer<Integer, byte[]> producer = new Producer<Integer, byte[]>(config);

  for (Stock stock : AvroStockUtils.fromCsvFile(inputFile)) {
    KeyedMessage<Integer, byte[]> msg = new KeyedMessage<Integer, byte[]>(kTopic, toBytes(stock));
    System.out.println("Sending " + msg + " to kafka @ topic " + kTopic);
    producer.send(msg);

  }
  producer.close();
  System.out.println("done!");
  return 0;
}
 

public synchronized void close() {
  for (final Producer<String, T> producer : cachedProducers.values()) {
    try {
      producer.close();
    } catch (final Exception e) {
      LOGGER.warn("Unable to close kafka producer", e);
    }
  }
  cachedProducers.clear();
}
 

public void createKafkaData(int startNum) {
  //Write numOfMessages to Kafka
  Properties producerProps = new Properties();
  producerProps.put("metadata.broker.list", "localhost:9092");
  producerProps.put("serializer.class", "kafka.serializer.StringEncoder");
  producerProps.put("request.required.acks", "1");
  
  Producer<String, String> producer = new Producer<String, String>(new ProducerConfig(producerProps));
  for(int i =startNum ; i < startNum+numOfMessages; i++) {
    KeyedMessage<String, String> data = new KeyedMessage<String, String>(TOPIC,"Neverwinter"+Integer.toString(i));
    producer.send(data);
  }
  producer.close();
}