Python confluent_kafka.Producer() Examples

def _get_kafka_producer(self):

        try:

            if self.kafka_endpoint.startswith('@'):
                try:
                    _k_endpoint = get_endpoint_from_consul(self.consul_endpoint,
                                                           self.kafka_endpoint[1:])
                    log.debug('found-kafka-service', endpoint=_k_endpoint)

                except Exception as e:
                    log.exception('no-kafka-service-in-consul', e=e)

                    self.kproducer = None
                    self.kclient = None
                    return
            else:
                _k_endpoint = self.kafka_endpoint
            self.kproducer = _kafkaProducer(
                {'bootstrap.servers' :_k_endpoint}
                )
            pass
        except Exception, e:
            log.exception('failed-get-kafka-producer', e=e)
            return 

def test_conf_none():
    """ Issue #133
    Test that None can be passed for NULL by setting bootstrap.servers
    to None. If None would be converted to a string then a broker would
    show up in statistics. Verify that it doesnt. """

    def stats_cb_check_no_brokers(stats_json_str):
        """ Make sure no brokers are reported in stats """
        global seen_stats_cb_check_no_brokers
        stats = json.loads(stats_json_str)
        assert len(stats['brokers']) == 0, "expected no brokers in stats: %s" % stats_json_str
        seen_stats_cb_check_no_brokers = True

    conf = {'bootstrap.servers': None,  # overwrites previous value
            'statistics.interval.ms': 10,
            'stats_cb': stats_cb_check_no_brokers}

    p = confluent_kafka.Producer(conf)
    p.poll(timeout=1)

    global seen_stats_cb_check_no_brokers
    assert seen_stats_cb_check_no_brokers 

def _initialize_members(self, topic, server, port, zk_server, zk_port, partitions):

        # get logger isinstance
        self._logger = logging.getLogger("SPOT.INGEST.KafkaProducer")

        # kafka requirements
        self._server = server
        self._port = port
        self._zk_server = zk_server
        self._zk_port = zk_port
        self._topic = topic
        self._num_of_partitions = partitions
        self._partitions = []
        self._partitioner = None
        self._kafka_brokers = '{0}:{1}'.format(self._server, self._port)

        # create topic with partitions
        self._create_topic()

        self._kafka_conf = self._producer_config(self._kafka_brokers)

        self._p = Producer(**self._kafka_conf) 

def test_ksql_create_stream_w_properties(self):
        """ Test GET requests """
        topic = self.exist_topic
        stream_name = self.test_prefix + "test_ksql_create_stream"
        stream_name = "test_ksql_create_stream"
        ksql_string = "CREATE STREAM {} (ORDER_ID INT, TOTAL_AMOUNT DOUBLE, CUSTOMER_NAME VARCHAR) \
                       WITH (kafka_topic='{}', value_format='JSON');".format(stream_name, topic)
        streamProperties = {"ksql.streams.auto.offset.reset": "earliest"}
        if 'TEST_KSQL_CREATE_STREAM' not in utils.get_all_streams(self.api_client):
            r = self.api_client.ksql(ksql_string, stream_properties=streamProperties)
            self.assertEqual(r[0]['commandStatus']['status'], 'SUCCESS')
        producer = Producer({'bootstrap.servers': self.bootstrap_servers})
        producer.produce(self.exist_topic, '''{"order_id":3,"total_amount":43,"customer_name":"Palo Alto"}''')
        producer.flush()
        print()
        chunks = self.api_client.query("select * from {}".format(stream_name), stream_properties=streamProperties, idle_timeout=10)
        for chunk in chunks:
            pass
            assert json.loads(chunk)['row']['columns'][-1]=='Palo Alto' 

def prepareProducer(self,groupID = "pythonproducers"):
        options ={
                'bootstrap.servers':  self.kafka_brokers,
                'group.id': groupID
        }
        # We need this test as local kafka does not expect SSL protocol.
        if (self.kafka_env != 'LOCAL'):
            options['security.protocol'] = 'SASL_SSL'
            options['sasl.mechanisms'] = 'PLAIN'
            options['sasl.username'] = 'token'
            options['sasl.password'] = self.kafka_apikey
        if (self.kafka_env == 'OCP'):
            options['ssl.ca.location'] = os.environ['PEM_CERT']
        print("[KafkaProducer] - This is the configuration for the producer:")
        print('[KafkaProducer] - {}'.format(options))
        self.producer = Producer(options) 

def create_idempotent_producer(self, retry_limit, buffering_max):
        console_out_many(["Creating idempotent producer with:", 
                f"       bootstrap.servers={self.broker_manager.get_bootstrap_servers()}",
                f"       acks={self.acks_mode}",
                f"       retries={retry_limit}",
                "        metadata.max.age.ms: 60000",
                f"       buffering={buffering_max}"], self.get_actor())

        self.producer = Producer({'bootstrap.servers': self.broker_manager.get_bootstrap_servers(),
                            'message.send.max.retries': retry_limit,
                            'enable.idempotence': True,
                            'queue.buffering.max.ms': buffering_max,
                            #'batch.num.messages': 1000,
                            #'stats_cb': my_stats_callback,
                            #'statistics.interval.ms': 100,
                            'metadata.max.age.ms': 60000,
                            'default.topic.config': { 'request.required.acks': self.acks_mode }
                        }) 

def create_producer(self, retry_limit, buffering_max):
        console_out_many(["Creating producer with:", 
                f"       bootstrap.servers={self.broker_manager.get_bootstrap_servers()}",
                f"       acks={self.acks_mode}",
                f"       retries={retry_limit}",
                f"       buffering={buffering_max}"], self.get_actor())

        self.producer = Producer({'bootstrap.servers': self.broker_manager.get_bootstrap_servers(),
                            'message.send.max.retries': retry_limit,
                            'queue.buffering.max.ms': buffering_max,
                            #'queue.buffering.max.ms': 100,
                            #'batch.num.messages': 1000,
                            #'stats_cb': my_stats_callback,
                            #'statistics.interval.ms': 100,
                            'metadata.max.age.ms': 60000,
                            'default.topic.config': { 'request.required.acks': self.acks_mode }}) 

def init():

        global log
        global kafka_producer

        if not log:
            log = create_logger(Config().get("logging"))

        if kafka_producer:
            raise Exception("XOSKafkaProducer already initialized")

        else:
            log.info(
                "Connecting to Kafka with bootstrap servers: %s"
                % Config.get("kafka_bootstrap_servers")
            )

            try:
                producer_config = {
                    "bootstrap.servers": ",".join(Config.get("kafka_bootstrap_servers"))
                }

                kafka_producer = confluent_kafka.Producer(**producer_config)

                log.info("Connected to Kafka: %s" % kafka_producer)

            except confluent_kafka.KafkaError as e:
                log.exception("Kafka Error: %s" % e) 

def test_error_cb():
    """ Test the error callback. """

    global seen_all_brokers_down

    # Configure an invalid broker and make sure the ALL_BROKERS_DOWN
    # error is seen in the error callback.
    p = Producer({'bootstrap.servers': '127.0.0.1:1', 'socket.timeout.ms': 10,
                  'error_cb': error_cb})

    t_end = time.time() + 5

    while not seen_all_brokers_down and time.time() < t_end:
        p.poll(1)

    assert seen_all_brokers_down 

def test_fatal():
    """ Test fatal exceptions """

    # Configure an invalid broker and make sure the ALL_BROKERS_DOWN
    # error is seen in the error callback.
    p = Producer({'error_cb': error_cb})

    with pytest.raises(KafkaException) as exc:
        raise KafkaException(KafkaError(KafkaError.MEMBER_ID_REQUIRED,
                                        fatal=True))
    err = exc.value.args[0]
    assert isinstance(err, KafkaError)
    assert err.fatal()
    assert not err.retriable()
    assert not err.txn_requires_abort()

    p.poll(0)  # Need some p use to avoid flake8 unused warning 

def test_logging_producer():
    """ Tests that logging works """

    logger = logging.getLogger('producer')
    logger.setLevel(logging.DEBUG)
    f = CountingFilter('producer')
    logger.addFilter(f)

    p = confluent_kafka.Producer({'debug': 'all'}, logger=logger)

    while f.cnt == 0:
        p.poll(timeout=0.5)

    print('%s: %d log messages seen' % (f.name, f.cnt)) 

def test_thread_safety():
    """ Basic thread safety tests. """

    q = Queue()
    p = Producer({'socket.timeout.ms': 10,
                  'message.timeout.ms': 10})

    threads = list()
    for i in range(1, 5):
        thr = threading.Thread(target=thread_run, name=str(i), args=[i, p, q])
        thr.start()
        threads.append(thr)

    for thr in threads:
        thr.join()

    # Count the number of threads that exited cleanly
    cnt = 0
    try:
        for x in iter(q.get_nowait, None):
            cnt += 1
    except Empty:
        pass

    if cnt != len(threads):
        raise Exception('Only %d/%d threads succeeded' % (cnt, len(threads)))

    print('Done') 

def terminate(self):
        """ Terminate Producer and Consumer """
        soak.logger.info("Terminating (ran for {}s)".format(time.time() - self.start_time))
        self.run = False
        self.producer_thread.join()
        self.consumer_thread.join() 

def __init__(self, topic, rate, conf):
        """ SoakClient constructor. conf is the client configuration """
        self.topic = topic
        self.rate = rate
        self.disprate = int(rate * 10)
        self.run = True
        self.stats_cnt = {'producer': 0, 'consumer': 0}
        self.start_time = time.time()

        self.logger = logging.getLogger('soakclient')
        self.logger.setLevel(logging.DEBUG)
        handler = logging.StreamHandler()
        handler.setFormatter(logging.Formatter('%(asctime)-15s %(levelname)-8s %(message)s'))
        self.logger.addHandler(handler)

        # Create topic (might already exist)
        self.create_topic(self.topic, conf)

        #
        # Create Producer and Consumer, each running in its own thread.
        #
        conf['stats_cb'] = self.stats_cb
        conf['statistics.interval.ms'] = 10000

        # Producer
        conf['error_cb'] = self.producer_error_cb
        self.producer = Producer(conf)

        # Consumer
        conf['error_cb'] = self.consumer_error_cb
        conf['on_commit'] = self.consumer_commit_cb
        self.logger.info("consumer: using group.id {}".format(conf['group.id']))
        self.consumer = Consumer(conf)

        self.producer_thread = threading.Thread(target=self.producer_thread_main)
        self.producer_thread.start()

        self.consumer_thread = threading.Thread(target=self.consumer_thread_main)
        self.consumer_thread.start() 

def producer_error_cb(self, err):
        """ Producer error callback """
        self.logger.error("producer: error_cb: {}".format(err))
        self.producer_error_cb_cnt += 1 

def producer_run(self):
        """ Producer main loop """
        sleep_intvl = 1.0 / self.rate

        self.producer_msgid = 0
        self.dr_cnt = 0
        self.dr_err_cnt = 0
        self.producer_error_cb_cnt = 0

        next_stats = time.time() + 10

        while self.run:

            self.produce_record()

            now = time.time()
            t_end = now + sleep_intvl
            while True:
                if now > next_stats:
                    self.producer_stats()
                    next_stats = now + 10

                remaining_time = t_end - now
                if remaining_time < 0:
                    remaining_time = 0
                self.producer.poll(remaining_time)
                if remaining_time <= 0:
                    break
                now = time.time()

        remaining = self.producer.flush(30)
        self.logger.warning("producer: {} message(s) remaining in queue after flush()".format(remaining))
        self.producer_stats() 

def __init__(self, conf):
        """
        conf is a config dict passed to confluent_kafka.Producer()
        """
        super(VerifiableProducer, self).__init__(conf)
        self.conf['on_delivery'] = self.dr_cb
        self.producer = Producer(**self.conf)
        self.num_acked = 0
        self.num_sent = 0
        self.num_err = 0 

def dr_cb(self, err, msg):
        """ Producer delivery report callback """
        if err is not None:
            self.logger.warning("producer: delivery failed: {} [{}]: {}".format(msg.topic(), msg.partition(), err))
            self.dr_err_cnt += 1
        else:
            self.dr_cnt += 1
            if (self.dr_cnt % self.disprate) == 0:
                self.logger.debug("producer: delivered message to {} [{}] at offset {}".format(
                    msg.topic(), msg.partition(), msg.offset())) 

def produce(self, **kwargs):
        """
            Asynchronously sends message to Kafka by encoding with specified or default avro schema.

            :param str topic: topic name
            :param object value: An object to serialize
            :param str value_schema: Avro schema for value
            :param object key: An object to serialize
            :param str key_schema: Avro schema for key

            Plus any other parameters accepted by confluent_kafka.Producer.produce

            :raises SerializerError: On serialization failure
            :raises BufferError: If producer queue is full.
            :raises KafkaException: For other produce failures.
        """
        # get schemas from  kwargs if defined
        key_schema = kwargs.pop('key_schema', self._key_schema)
        value_schema = kwargs.pop('value_schema', self._value_schema)
        topic = kwargs.pop('topic', None)
        if not topic:
            raise ClientError("Topic name not specified.")
        value = kwargs.pop('value', None)
        key = kwargs.pop('key', None)

        if value is not None:
            if value_schema:
                value = self._serializer.encode_record_with_schema(topic, value_schema, value)
            else:
                raise ValueSerializerError("Avro schema required for values")

        if key is not None:
            if key_schema:
                key = self._serializer.encode_record_with_schema(topic, key_schema, key, True)
            else:
                raise KeySerializerError("Avro schema required for key")

        super(AvroProducer, self).produce(topic, value, key, **kwargs) 

def test_topic_config_update():
    # *NOTE* default.topic.config has been deprecated.
    # This example remains to ensure backward-compatibility until its removal.
    confs = [{"message.timeout.ms": 600000, "default.topic.config": {"message.timeout.ms": 1000}},
             {"message.timeout.ms": 1000},
             {"default.topic.config": {"message.timeout.ms": 1000}}]

    def on_delivery(err, msg):
        # Since there is no broker, produced messages should time out.
        global seen_delivery_cb
        seen_delivery_cb = True
        assert err.code() == confluent_kafka.KafkaError._MSG_TIMED_OUT

    for conf in confs:
        p = confluent_kafka.Producer(conf)

        start = time.time()

        timeout = start + 10.0

        p.produce('mytopic', value='somedata', key='a key', on_delivery=on_delivery)
        while time.time() < timeout:
            if seen_delivery_cb:
                return
            p.poll(1.0)

        if "CI" in os.environ:
            pytest.xfail("Timeout exceeded")
        pytest.fail("Timeout exceeded") 

def __init__(self, configs):
        self._producer = confluent_kafka.Producer(configs)
        self._cancelled = False
        self._poll_thread = Thread(target=self._poll_loop)
        self._poll_thread.start() 

def throttle_cb_instantiate_fail():
    """ Ensure noncallables raise TypeError"""
    with pytest.raises(ValueError):
        confluent_kafka.Producer({'throttle_cb': 1}) 

def startup_event():
    global producer, aio_producer
    aio_producer = AIOProducer(config)
    producer = Producer(config) 

def __init__(self, configs, loop=None):
        self._loop = loop or asyncio.get_event_loop()
        self._producer = confluent_kafka.Producer(configs)
        self._cancelled = False
        self._poll_thread = Thread(target=self._poll_loop)
        self._poll_thread.start() 

def setUp(self):
        self.url = "http://localhost:8088"
        self.api_client = KSQLAPI(url=self.url, check_version=False)
        self.test_prefix = "ksql_python_test"
        self.exist_topic = 'exist_topic'
        self.bootstrap_servers = 'localhost:29092'
        if utils.check_kafka_available(self.bootstrap_servers):
            producer = Producer({'bootstrap.servers': self.bootstrap_servers})
            producer.produce(self.exist_topic, "test_message")
            producer.flush() 

def setUp(self):
        self.url = "http://localhost:8088"
        self.api_client = KSQLAPI(url=self.url, check_version=False)
        self.test_prefix = "ksql_python_test"

        self.exist_topic = self.test_prefix + '_exist_topic'
        self.bootstrap_servers = 'localhost:29092'
        if utils.check_kafka_available(self.bootstrap_servers):
            producer = Producer({'bootstrap.servers': self.bootstrap_servers})
            producer.produce(self.exist_topic, "test_message")
            producer.flush() 

def __init__(self):
        super(TwitterStreamListener, self).__init__()
        self.producer = Producer(**KAFKA_CONF)
        self.count = 0
        self.tweets = [] 

def init():

        global log
        global kafka_producer

        if not log:
            log = create_logger(Config().get("logging"))

        if kafka_producer:
            raise Exception("XOSKafkaProducer already initialized")

        else:
            log.info(
                "Connecting to Kafka with bootstrap servers: %s"
                % Config.get("kafka_bootstrap_servers")
            )

            try:
                producer_config = {
                    "bootstrap.servers": ",".join(Config.get("kafka_bootstrap_servers"))
                }

                kafka_producer = confluent_kafka.Producer(**producer_config)

                log.info("Connected to Kafka: %s" % kafka_producer)

            except confluent_kafka.KafkaError as e:
                log.exception("Kafka Error: %s" % e) 

def __init__(self):
        self.LOG_CLASS = "StoreLog"
        self.method = settings.LOG_TYPE
        if self.method == settings.STORE:
             self.conf = settings.STORE_CONF
        elif self.method == settings.TOSERVER:
            self.conf = settings.SERVER_CONF
        elif self.method == settings.TOKAFKA:
            self.producer = Producer({'bootstrap.servers':'localhost:9092'}) 

def producer(self):
        # TODO: Must set all config values applicable to a producer
        return kafka.Producer({'bootstrap.servers': self.config.BOOTSTRAP_SERVERS})