kafka.javaapi.producer.Producer Java Examples

private static Producer<String, String> initProducer() {
    Properties props = new Properties();
    props.put("metadata.broker.list", BROKER_LIST);
    // props.put("serializer.class", "kafka.serializer.StringEncoder");
    props.put("serializer.class", StringEncoder.class.getName());
    props.put("partitioner.class", HashPartitioner.class.getName());
//    props.put("compression.codec", "0");
    props.put("producer.type", "sync");
    props.put("batch.num.messages", "1");
    props.put("queue.buffering.max.messages", "1000000");
    props.put("queue.enqueue.timeout.ms", "20000000");

    
    ProducerConfig config = new ProducerConfig(props);
    Producer<String, String> producer = new Producer<String, String>(config);
    return producer;
  }
 

/**
 * 读取配置文件，创建线程池，运行线程
 */
public void go() {
    Constant constant = new Constant();
    kafkaProperties kafkaProperties = new kafkaProperties();
    ProducerConfig config = new ProducerConfig(kafkaProperties.properties());

    ExecutorService executorService = Executors.newFixedThreadPool(Integer.parseInt(constant.THREAD_POOL_SIZE));

    String topic = constant.TOPIC_NAME;
    Task[] tasks = new Task[Integer.parseInt(constant.THREAD_NUM)];
    String[] folders = constant.FILE_FOLDERS.split(";");
    int batchSize = Integer.parseInt(constant.BATCH_SIZE);
    CopyOnWriteArrayList<String> fileList = addFiles(folders);

    for (int i = 0; i < tasks.length; ++i) {
        tasks[i] = new Task(i, topic, new Producer<String, String>(config), fileList, batchSize);
    }

    for (Task task : tasks) {
        executorService.execute(task);
    }
    executorService.shutdown();
}
 

public static void main(String[] args) throws Exception {
	//read config file
	Properties prop = PropertyFileReader.readPropertyFile();		
	String zookeeper = prop.getProperty("com.iot.app.kafka.zookeeper");
	String brokerList = prop.getProperty("com.iot.app.kafka.brokerlist");
	String topic = prop.getProperty("com.iot.app.kafka.topic");
	logger.info("Using Zookeeper=" + zookeeper + " ,Broker-list=" + brokerList + " and topic " + topic);

	// set producer properties
	Properties properties = new Properties();
	properties.put("zookeeper.connect", zookeeper);
	properties.put("metadata.broker.list", brokerList);
	properties.put("request.required.acks", "1");
	properties.put("serializer.class", "com.iot.app.kafka.util.IoTDataEncoder");
	//generate event
	Producer<String, IoTData> producer = new Producer<String, IoTData>(new ProducerConfig(properties));
	IoTDataProducer iotProducer = new IoTDataProducer();
	iotProducer.generateIoTEvent(producer,topic);		
}
 

static Producer<String, byte[]> createProducer() {
        Properties props = new Properties();
        try {
//            props.put("zk.connect", p.getProperty("kafka.zk.connect"));   // not need zk in new version
            props.put("key.serializer.class", p.getProperty("kafka.key.serializer.class"));
            props.put("serializer.class", p.getProperty("kafka.serializer.class"));
            props.put("metadata.broker.list", p.getProperty("kafka.metadata.broker.list"));
            props.put("request.required.acks", p.getProperty("kafka.request.required.acks"));
            props.put("producer.type", p.getProperty("kafka.async"));
            props.put("partitioner.class", PARTITIONER_CLASS_NAME);

            props.put("queue.buffering.max.ms", p.getProperty("kafka.queue.buffering.max.ms"));
            props.put("queue.buffering.max.messages", p.getProperty("kafka.queue.buffering.max.messages"));
            props.put("queue.enqueue.timeout.ms", p.getProperty("kafka.queue.enqueue.timeout.ms"));
            // 41,0000,0000 / 24 / 60 / 60 = 47454 / 24 = 1977
            props.put("batch.num.messages", p.getProperty("kafka.batch.num.messages"));
            props.put("send.buffer.bytes", p.getProperty("kafka.send.buffer.bytes"));
//            props.put("compression.type", "lz4");
        } catch (Exception e) {
            _log.error("Connect with kafka failed {}!", e.getMessage());
            throw new RuntimeException(e);
        }
        _log.info("Connect with kafka successfully!");
        return new Producer<>(new ProducerConfig(props));
    }
 

/**
 * Creates kafka producers based on given message serializer type.
 *
 * @param serializerType    message serializer type
 * @param properties        properties to be used to send a message
 * @return                  created kafka producer
 */
public Producer createProducer(MessageSerializerType serializerType, Properties properties) {
    Producer producer;

    switch (serializerType) {
            case BYTE_SERIALIZER:
                producer = new Producer<String, byte[]>(new ProducerConfig(properties));
                break;
            case STRING_SERIALIZER:
                producer = new Producer<String, String>(new ProducerConfig(properties));
                break;
            default:
                throw new IllegalArgumentException("Incorrect serialazier class specified...");
    }
    return producer;
}
 

@Override
  public synchronized void start() {
      super.start();

Properties props = new Properties();
  	props.put("metadata.broker.list", brokerList);
  	props.put("request.required.acks", String.valueOf(requestRequiredAcks));
  	props.put("request.timeout.ms", String.valueOf(requestTimeoutms));
props.put("serializer.class", serializerClass);
props.put("partitioner.class", partitionerClass);
props.put("producer.type", producerType);
props.put("batch.num.messages", String.valueOf(batchNumMessages));
props.put("queue.buffering.max.messages", String.valueOf(queueBufferingMaxMessages));
props.put("topic.metadata.refresh.interval.ms", "30000");

producer = new Producer<String, String>(new ProducerConfig(props));
  }
 

public KafkaStreamProxyProducerImpl(String streamId, StreamSinkConfig streamConfig) {
    Preconditions.checkNotNull(streamConfig, "Stream sink config for " + streamId + " is null");
    this.streamId = streamId;
    this.config = (KafkaStreamSinkConfig) streamConfig;
    Properties properties = new Properties();
    Preconditions.checkNotNull(config.getBrokerList(), "brokerList is null");
    properties.put("metadata.broker.list", config.getBrokerList());
    properties.put("serializer.class", config.getSerializerClass());
    properties.put("key.serializer.class", config.getKeySerializerClass());
    // new added properties for async producer
    properties.put("producer.type", config.getProducerType());
    properties.put("batch.num.messages", config.getNumBatchMessages());
    properties.put("request.required.acks", config.getRequestRequiredAcks());
    properties.put("queue.buffering.max.ms", config.getMaxQueueBufferMs());
    ProducerConfig producerConfig = new ProducerConfig(properties);
    this.producer = new Producer(producerConfig);
}
 

@Override
public void start() {
    try {
        LOGGER.info("Starting Kafka Channel: " + getName());
        producer = new Producer<String, byte[]>(new ProducerConfig(kafkaConf));
        // We always have just one topic being read by one thread
        LOGGER.info("Topic = " + topic.get());
        topicCountMap.put(topic.get(), 1);
        counter.start();
        super.start();
    } catch (Exception e) {
        LOGGER.error("Could not start producer");
        throw new FlumeException("Unable to create Kafka Connections. " +
                "Check whether Kafka Brokers are up and that the " +
                "Flume agent can connect to it.", e);
    }
}
 

/**
 * Creates a new KafkaSink.
 * This method initializes and starts a new Kafka producer that will be
 * used to produce messages to kafka topics.
 * @param properties KafkaSink propertiers. You should provide a kafka_broker property
 *                   set to the Kafka host address. If none is provided localhost will
 *                   be used
 */

public KafkaSink(Map<String, Object> properties) {
    super(properties);
    // The producer config attributes
    Properties props = new Properties();
    if (properties != null && properties.get("kafka_brokers") != null) {
        props.put("metadata.broker.list", properties.get("kafka_brokers"));
    } else {
        props.put("metadata.broker.list", "127.0.0.1:9092");
    }
    props.put("serializer.class", "kafka.serializer.StringEncoder");
    props.put("request.required.acks", "1");
    props.put("message.send.max.retries", "60");
    props.put("retry.backoff.ms", "1000");
    props.put("producer.type", "async");
    props.put("queue.buffering.max.messages", "10000");
    props.put("queue.buffering.max.ms", "500");
    props.put("partitioner.class", "net.redborder.cep.sinks.kafka.SimplePartitioner");

    // Initialize the producer
    ProducerConfig config = new ProducerConfig(props);
    producer = new Producer<>(config);
}
 

public void execute(JavaPairRDD<String, byte[]> inputMessage) {
    JavaPairRDD<String, byte[]> partitionedRDD;
    if (config.getLocalMode())
        partitionedRDD = inputMessage;
    else {
        // Helps scale beyond number of input partitions in kafka
        partitionedRDD = inputMessage.repartition(config.getRepartitionCount());

    }

    partitionedRDD.foreachPartition(prdd -> {
        // You can choose binary or string encoder
        Producer validProducer = ConnectionManager.getKafkaSingletonConnectionWithBinaryEncoder(config);
        prdd.forEachRemaining(records -> {
            byte[] msg = records._2();
            try {
                // TODO: Add your logic here to process data
                // As default we are just publishing back to another kafka topic
                logger.info("Processing event=" + new String(msg));
                publishMessagesToKafka(validProducer, msg);
            } catch (Exception e){
                logger.error("Error processing message:" + msg);
            }
        });
    });
}
 

public static void main(String[] args) throws Exception {
	//read config file
	Properties prop = PropertyFileReader.readPropertyFile();		
	String zookeeper = prop.getProperty("com.iot.app.kafka.zookeeper");
	String brokerList = prop.getProperty("com.iot.app.kafka.brokerlist");
	String topic = prop.getProperty("com.iot.app.kafka.topic");
	logger.info("Using Zookeeper=" + zookeeper + " ,Broker-list=" + brokerList + " and topic " + topic);

	// set producer properties
	Properties properties = new Properties();
	properties.put("zookeeper.connect", zookeeper);
	properties.put("metadata.broker.list", brokerList);
	properties.put("request.required.acks", "1");
	properties.put("serializer.class", "com.iot.app.kafka.util.IoTDataEncoder");
	//generate event
	Producer<String, IoTData> producer = new Producer<String, IoTData>(new ProducerConfig(properties));
	IoTDataProducer iotProducer = new IoTDataProducer();
	iotProducer.generateIoTEvent(producer,topic);		
}
 

@Override
public void configure(Context context) {
    topic = context.getString(CONF_TOPIC);
    if (topic == null) {
        throw new ConfigurationException("Kafka topic must be specified.");
    }

    writeBody = context.getBoolean(CONF_WRITE_BODY, DEFAULT_WRITE_BODY);

    ImmutableMap<String, String> subProperties = context.getSubProperties(CONF_KAFKA);
    Properties properties = new Properties();
    properties.putAll(subProperties);

    producer = new Producer<String, String>(new ProducerConfig(properties));

    mapper = new ObjectMapper();
}
 

private org.apache.pulsar.client.api.Producer<byte[]> createNewProducer(String topic) {
    try {
        pulsarProducerBuilder.messageRoutingMode(MessageRoutingMode.CustomPartition);
        pulsarProducerBuilder.messageRouter(new MessageRouter() {
            private static final long serialVersionUID = 1L;

            @Override
            public int choosePartition(Message<?> msg, TopicMetadata metadata) {
                // https://kafka.apache.org/08/documentation.html#producerapi
                // The default partitioner is based on the hash of the key.
                return partitioner.partition(msg.getKey(), metadata.numPartitions());
            }
        });
        log.info("Creating producer for topic {} with config {}", topic, pulsarProducerBuilder.toString());
        return pulsarProducerBuilder.clone().topic(topic).create();
    } catch (PulsarClientException e) {
        throw new RuntimeException(e);
    }
}
 

private KafkaReporter(MetricRegistry registry, String name,
		TimeUnit rateUnit, TimeUnit durationUnit, boolean showSamples, MetricFilter filter,
		String topic, ProducerConfig config, String prefix,
		String hostName, String ip) {
	super(registry, name, filter, rateUnit, durationUnit);
	this.topic = topic;
	this.config = config;
	this.prefix = prefix;
	this.hostName = hostName;
	this.ip = ip;
	
	this.mapper = new ObjectMapper().registerModule(new MetricsModule(rateUnit,
               durationUnit,
               showSamples));

	producer = new Producer<String, String>(config);

	kafkaExecutor = Executors
			.newSingleThreadExecutor(new ThreadFactoryBuilder()
					.setNameFormat("kafka-producer-%d").build());
}
 

@Override
protected void processFile(
    final URL file,
    final String typeName,
    final GeoWaveAvroFormatPlugin<?, ?> plugin,
    final StageKafkaData<?> runData) {

  try {
    final Producer<String, Object> producer =
        (Producer<String, Object>) runData.getProducer(typeName, plugin);
    try (final CloseableIterator<?> avroRecords = plugin.toAvroObjects(file)) {
      while (avroRecords.hasNext()) {
        final Object avroRecord = avroRecords.next();
        final KeyedMessage<String, Object> data = new KeyedMessage<>(typeName, avroRecord);
        producer.send(data);
      }
    }
  } catch (final Exception e) {
    LOGGER.info(
        "Unable to send file [" + file.getPath() + "] to Kafka topic: " + e.getMessage(),
        e);
  }
}
 

@Override
public ListenableFuture<Integer> send() {
  try {
    int size = messages.size();
    Producer<Integer, ByteBuffer> kafkaProducer = producer.get();
    if (kafkaProducer == null) {
      return Futures.immediateFailedFuture(new IllegalStateException("No kafka producer available."));
    }
    kafkaProducer.send(messages);
    return Futures.immediateFuture(size);
  } catch (Exception e) {
    return Futures.immediateFailedFuture(e);
  } finally {
    messages.clear();
  }
}
 

public void createKafkaData(int startNum) {
  //Write numOfMessages to Kafka
  Properties producerProps = new Properties();
  producerProps.put("metadata.broker.list", "localhost:9092");
  producerProps.put("serializer.class", "kafka.serializer.StringEncoder");
  producerProps.put("request.required.acks", "1");
  
  Producer<String, String> producer = new Producer<String, String>(new ProducerConfig(producerProps));
  for(int i =startNum ; i < startNum+numOfMessages; i++) {
    KeyedMessage<String, String> data = new KeyedMessage<String, String>(TOPIC,"Neverwinter"+Integer.toString(i));
    producer.send(data);
  }
  producer.close();
}
 

public void processCounter(MetricName name, Counter counter, Context context) {
    final String header = "# time,count";
    final Producer producer = context.getProducer();
    final String topic = "%s-metrics-counter".format(prefix);
    final String message = valueOf(counter.count());
    send(producer, header, topic, message);
}
 

public StreamToActionBusCallback(Set<StreamAction> activeActions, String streamName,
        Producer<String, byte[]> avroProducer,
        Serializer<StratioStreamingMessage, Event> javaToSiddhiSerializer,
        Serializer<StratioStreamingMessage, byte[]> javaToAvroSerializer,
        String groupId) {

    this(activeActions, streamName, avroProducer, javaToSiddhiSerializer,
            javaToAvroSerializer);
    this.groupId = groupId;
}
 

@Override
    public void run()
    {
      logger.info("Start produce data .... ");
      Properties props = new Properties();
      props.setProperty("serializer.class", "kafka.serializer.StringEncoder");
      props.setProperty("key.serializer.class", "kafka.serializer.StringEncoder");
      props.put("metadata.broker.list", brokerList);
//      props.put("metadata.broker.list", "localhost:9092");
      props.setProperty("partitioner.class", KafkaTestPartitioner.class.getCanonicalName());
      props.setProperty("producer.type", "async");
//      props.setProperty("send.buffer.bytes", "1048576");
      props.setProperty("topic.metadata.refresh.interval.ms", "10000");

      if (producer == null) {
        producer = new Producer<String, String>(new ProducerConfig(props));
      }
      long k = 0;

      while (k < msgsSecThread || !controlThroughput) {
        long key = (stickyKey >= 0 ? stickyKey : k);
        k++;
        producer.send(new KeyedMessage<String, String>(topic, "" + key, new String(constantMsg)));
        if (k == Long.MAX_VALUE) {
          k = 0;
        }
      }
    }
 

/**
 * Initialize a Kafka producer using configs in LinkConfiguration
 * @param linkConfiguration
 * @return
 */
Producer getProducer(LinkConfiguration linkConfiguration) {
  Properties kafkaProps =  generateDefaultKafkaProps();
  kafkaProps.put(KafkaConstants.BROKER_LIST_KEY, linkConfiguration.linkConfig.brokerList);
  ProducerConfig config = new ProducerConfig(kafkaProps);
  return new Producer<String, String>(config);
}
 

public static void main(String args[]) {
    log.info("Command line arguments passed: " + Arrays.deepToString(args));
    log.info("Starting Kafka Client");

    String url = args[0];
    String topic = args[1];
    String filePath = args[2];
    String sampleNumber = args[3];

    Properties props = new Properties();
    props.put("metadata.broker.list", url);
    props.put("serializer.class", "kafka.serializer.StringEncoder");

    ProducerConfig config = new ProducerConfig(props);
    Producer<String, Object> producer = new Producer<String, Object>(config);

    try {

        filePath = KafkaUtil.getEventFilePath(sampleNumber, topic, filePath);
        readMsg(filePath);

        for (String message : messagesList) {
            System.out.println("Sending message:");
            System.out.println(message);
            KeyedMessage<String, Object> data = new KeyedMessage<String, Object>(topic, message);
            producer.send(data);
        }
        Thread.sleep(500);

    } catch (Throwable t) {
        log.error("Error when sending the messages", t);
    } finally {
        producer.close();
    }
}
 

private static void publishMessage(Arguments arguments) {
    // (2) Create producer
    Properties properties2 = new Properties();
    properties2.put(BROKER_URL, arguments.serviceUrl);
    properties2.put(PRODUCER_TYPE, "sync");
    properties2.put(SERIALIZER_CLASS, TestEncoder.class.getName());
    properties2.put(KEY_SERIALIZER_CLASS, StringEncoder.class.getName());
    properties2.put(PARTITIONER_CLASS, TestPartitioner.class.getName());
    properties2.put(COMPRESSION_CODEC, "gzip"); // compression: ZLIB
    properties2.put(QUEUE_ENQUEUE_TIMEOUT_MS, "-1"); // block queue if full => -1 = true
    properties2.put(QUEUE_BUFFERING_MAX_MESSAGES, "6000"); // queue max message
    properties2.put(QUEUE_BUFFERING_MAX_MS, "100"); // batch delay
    properties2.put(BATCH_NUM_MESSAGES, "500"); // batch msg
    properties2.put(CLIENT_ID, "test");
    ProducerConfig config = new ProducerConfig(properties2);
    Producer<String, Tweet> producer = new Producer<>(config);

    String name = "user";
    String msg = arguments.messageValue;
    for (int i = 0; i < arguments.totalMessages; i++) {
        String sendMessage = msg + i;
        Tweet tweet = new Tweet(name, sendMessage);
        KeyedMessage<String, Tweet> message = new KeyedMessage<>(arguments.topicName, name, tweet);
        producer.send(message);
    }

    producer.close();
    log.info("Successfully published messages {}", arguments.totalMessages);

}
 

@Override
public synchronized void start() {
    // instantiate the producer
    ProducerConfig config = new ProducerConfig(producerProps);
    producer = new Producer<String, String>(config);
    super.start();
}
 

public RemoteKafkaPoster(MetricRegistry metricRegistry, String kafkaHost, int kafkaPort) {
    this.metricRegistry = metricRegistry;
    String broker = kafkaHost + ":" + kafkaPort;
    Properties props = new Properties();
    props.put("metadata.broker.list", broker);
    props.put("serializer.class", "kafka.serializer.StringEncoder");
    ProducerConfig config = new ProducerConfig(props);
    producer = new Producer<String, String>(config);
}
 

public StreamToActionBusCallback(Set<StreamAction> activeActions, String streamName,
        Producer<String, byte[]> avroProducer,
        Serializer<StratioStreamingMessage, Event> javaToSiddhiSerializer,
        Serializer<StratioStreamingMessage, byte[]> javaToAvroSerializer) {
    super(activeActions);
    this.streamName = streamName;
    this.avroProducer = avroProducer;
    this.javaToSiddhiSerializer = javaToSiddhiSerializer;
    this.javaToAvroSerializer = javaToAvroSerializer;
}
 

private KafkaReporter(MetricRegistry registry,
                      String name,
                      MetricFilter filter,
                      TimeUnit rateUnit,
                      TimeUnit durationUnit,
                      String kafkaTopic,
                      Properties kafkaProperties) {
    super(registry, name, filter, rateUnit, durationUnit);
    this.registry = registry;
    mapper = new ObjectMapper().registerModule(new MetricsModule(rateUnit,
                                                                 durationUnit,
                                                                 false));
    this.kafkaTopic = kafkaTopic;
    kafkaProducer = new Producer<String, String>(new ProducerConfig(kafkaProperties));
}
 

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    Map configMap = (Map) stormConf.get(KAFKA_BROKER_PROPERTIES);
    Properties properties = new Properties();
    properties.putAll(configMap);
    ProducerConfig config = new ProducerConfig(properties);
    producer = new Producer<K, V>(config);
    this.topic = (String) stormConf.get(TOPIC);
    this.collector = collector;
}
 

public void processMeter(MetricName name, Metered meter, Context context) {
    final String header = "# time,count,1 min rate,mean rate,5 min rate,15 min rate";
    final Producer producer = context.getProducer();
    final String topic = "%s-metrics-meter".format(prefix);
    final String message = valueOf(meter.count()) + ',' + meter.oneMinuteRate() + ',' + meter.meanRate() + ','
            + meter.fiveMinuteRate() + ',' + meter.fifteenMinuteRate();
    send(producer, header, topic, message);
}
 

public void processHistogram(MetricName name, Histogram histogram, Context context) {
    final String header = "# time,min,max,mean,median,stddev,95%,99%,99.9%";
    final Producer producer = context.getProducer();
    final Snapshot snapshot = histogram.getSnapshot();
    final String topic="%s-metrics-histogram".format(prefix);
    final String message = valueOf(histogram.min()) + ',' + histogram.max() + ',' + histogram.mean() + ','
            + snapshot.getMedian() + ',' + histogram.stdDev() + ',' + snapshot.get95thPercentile() + ',' + snapshot.get99thPercentile() + ','
            + snapshot.get999thPercentile();
    send(producer, header, topic, message);
}