Python kafka.KafkaProducer() Examples

def __init__(self, env, is_external_queue: bool):
        super().__init__(env, is_external_queue, queue_type='kafka', logger=logger)

        eq_host = env.config.get(ConfigKeys.HOST, domain=self.domain_key, default=None)
        eq_queue = env.config.get(ConfigKeys.QUEUE, domain=self.domain_key, default=None)

        if eq_host is None or len(eq_host) == 0 or (type(eq_host) == str and len(eq_host.strip()) == 0):
            logging.warning('blank external host specified, not setting up external publishing')
            return

        if eq_queue is None or len(eq_queue.strip()) == 0:
            logging.warning('blank external queue specified, not setting up external publishing')
            return

        if type(eq_host) == str:
            eq_host = [eq_host]

        from kafka import KafkaProducer
        import json

        self.queue = eq_queue
        self.queue_connection = KafkaProducer(
            bootstrap_servers=eq_host,
            value_serializer=lambda v: json.dumps(v).encode('utf-8'))
        logger.info('setting up pubsub for type "{}: and host(s) "{}"'.format(self.queue_type, ','.join(eq_host))) 

def try_publish(self, message):
        if self.env.enrichment_manager is not None:
            message = self.env.enrichment_manager.handle(message)

        topic_key = None

        # try to get some consistency
        try:
            target = message.get('target', dict())
            topic_key = target.get('id', None)

            if topic_key is None:
                actor = message.get('actor', dict())
                topic_key = actor.get('id', None)

            # kafka publisher can't handle string keys
            topic_key = bytes(str(topic_key), encoding='utf-8')

        except Exception as partition_e:
            logger.exception(traceback.format_exc())
            environ.env.capture_exception(partition_e)

        # for kafka, the queue_connection is the KafkaProducer and queue is the topic name
        self.queue_connection.send(
            topic=self.queue, value=message, key=topic_key) 

def _create_producer(self):
        import kafka

        if self.security_protocol == 'SASL_PLAINTEXT':
            self.producer = kafka.KafkaProducer(
                bootstrap_servers=[self.host],
                value_serializer=lambda v: json.dumps(v).encode('utf-8'),
                sasl_plain_username=self.sasl_username,
                sasl_plain_password=self.sasl_password,
                sasl_mechanism='PLAIN',
                security_protocol=self.security_protocol)
        elif self.security_protocol == 'SSL':
            self.producer = kafka.KafkaProducer(
                bootstrap_servers=[self.host],
                value_serializer=lambda v: json.dumps(v).encode('utf-8'),
                ssl_cafile=self.ssl_cafile,
                ssl_certfile=self.ssl_certfile,
                ssl_keyfile=self.ssl_keyfile,
                ssl_check_hostname=False,
                security_protocol=self.security_protocol) 

def kafka(TOPIC=None):
    # Lazy init of the Kafka producer
    #
    global PRODUCER
    if PRODUCER is None:
        PRODUCER = KafkaProducer(
            bootstrap_servers=KAFKA_BOOSTRAP_SERVERS,
            sasl_mechanism=KAFKA_SASL_MECHANISM,
            sasl_plain_username=KAFKA_USER,
            sasl_plain_password=KAFKA_PASSWORD)
    try:
        future = PRODUCER.send(TOPIC, request.get_data())
        future.get(timeout=60)
        return "OK", 200, None
    except KafkaTimeoutError:
        return "Internal Server Error", 500, None 

def _create_producer(self):
        while True:
            try:
                return KafkaProducer(
                    bootstrap_servers=self.config["bootstrap_uri"],
                    security_protocol=self.config["security_protocol"],
                    ssl_cafile=self.config["ssl_cafile"],
                    ssl_certfile=self.config["ssl_certfile"],
                    ssl_keyfile=self.config["ssl_keyfile"],
                    api_version=(1, 0, 0),
                    metadata_max_age_ms=self.config["metadata_max_age_ms"],
                    max_block_ms=2000  # missing topics will block unless we cache cluster metadata and pre-check
                )
            except:  # pylint: disable=bare-except
                self.log.exception("Unable to create producer, retrying")
                time.sleep(1) 

def init_producer(self):
    """Initialize KafkaProducer
    """
    print("[{}: log] Initializing producer...".format(self.pp))
    # Gather optional parameters
    dict_args = dict()
    #dict_args = self.get_servers(dict_args, 'producer_servers')
    #dict_args = self.get_security(dict_args, 'producer_security')
    dict_args = self.get_servers(dict_args, 'servers')
    dict_args = self.get_security(dict_args, 'security')
    # Instantiate producer
    try:
      self.producer = KafkaProducer(**dict_args)
    except Exception as inst:
      msg = "[{}: ERROR] Could not initialize producer with arguments {}. Error was: {}"
      raise RuntimeError(msg.format(self.pp, dict_args, inst))
    self.topic_name = self.get_required_param("topic_name") 

def _create_producer(self) -> None:
        import kafka

        if self.security_protocol == "SASL_PLAINTEXT":
            self.producer = kafka.KafkaProducer(
                bootstrap_servers=[self.host],
                value_serializer=lambda v: json.dumps(v).encode(DEFAULT_ENCODING),
                sasl_plain_username=self.sasl_username,
                sasl_plain_password=self.sasl_password,
                sasl_mechanism="PLAIN",
                security_protocol=self.security_protocol,
            )
        elif self.security_protocol == "SSL":
            self.producer = kafka.KafkaProducer(
                bootstrap_servers=[self.host],
                value_serializer=lambda v: json.dumps(v).encode(DEFAULT_ENCODING),
                ssl_cafile=self.ssl_cafile,
                ssl_certfile=self.ssl_certfile,
                ssl_keyfile=self.ssl_keyfile,
                ssl_check_hostname=False,
                security_protocol=self.security_protocol,
            ) 

def main(n):
    """Stream the video into a Kafka producer in an infinite loop"""
    
    topic = choose_channel(n)
    video_reader = imageio.get_reader(DATA + topic + '.mp4', 'ffmpeg')
    metadata = video_reader.get_meta_data()
    fps = metadata['fps']

    producer = KafkaProducer(bootstrap_servers='localhost:9092',
                             batch_size=15728640,
                             linger_ms=1000,
                             max_request_size=15728640,
                             value_serializer=lambda v: json.dumps(v.tolist()))
    
    while True:
        video_loop(video_reader, producer, topic, fps) 

def __init__(self, bootstrap_servers, kafkatopic):
        self.kafkatopic = kafkatopic
        self.producer = KafkaProducer(bootstrap_servers=bootstrap_servers) 

def kafka_connect(self):
        self.producer = KafkaProducer(
            bootstrap_servers=KAFKA_URL,
            value_serializer=lambda v: json.dumps(v).encode('utf-8')) 

def _create_producer(self, settings):
        """Tries to establish a Kafka consumer connection"""
        try:
            brokers = settings['KAFKA_HOSTS']
            self.logger.debug("Creating new kafka producer using brokers: " +
                               str(brokers))

            return KafkaProducer(bootstrap_servers=brokers,
                                 value_serializer=lambda m: json.dumps(m),
                                 retries=3,
                                 linger_ms=settings['KAFKA_PRODUCER_BATCH_LINGER_MS'],
                                 buffer_memory=settings['KAFKA_PRODUCER_BUFFER_BYTES'])
        except KeyError as e:
            self.logger.error('Missing setting named ' + str(e),
                               {'ex': traceback.format_exc()})
        except:
            self.logger.error("Couldn't initialize kafka producer in plugin.",
                               {'ex': traceback.format_exc()})
            raise 

def _create_producer(self):
        """Tries to establish a Kafka consumer connection"""
        if not self.closed:
            try:
                self.logger.debug("Creating new kafka producer using brokers: " +
                                   str(self.settings['KAFKA_HOSTS']))

                return KafkaProducer(bootstrap_servers=self.settings['KAFKA_HOSTS'],
                                     value_serializer=lambda v: json.dumps(v).encode('utf-8'),
                                     retries=3,
                                     linger_ms=self.settings['KAFKA_PRODUCER_BATCH_LINGER_MS'],
                                     buffer_memory=self.settings['KAFKA_PRODUCER_BUFFER_BYTES'])
            except KeyError as e:
                self.logger.error('Missing setting named ' + str(e),
                                   {'ex': traceback.format_exc()})
            except:
                self.logger.error("Couldn't initialize kafka producer.",
                                   {'ex': traceback.format_exc()})
                raise 

def _create_producer(self):
        """Tries to establish a Kafka consumer connection"""
        try:
            brokers = self.settings['KAFKA_HOSTS']
            self.logger.debug("Creating new kafka producer using brokers: " +
                               str(brokers))

            return KafkaProducer(bootstrap_servers=brokers,
                                 value_serializer=lambda m: json.dumps(m),
                                 retries=3,
                                 linger_ms=self.settings['KAFKA_PRODUCER_BATCH_LINGER_MS'],
                                 buffer_memory=self.settings['KAFKA_PRODUCER_BUFFER_BYTES'])
        except KeyError as e:
            self.logger.error('Missing setting named ' + str(e),
                               {'ex': traceback.format_exc()})
        except:
            self.logger.error("Couldn't initialize kafka producer.",
                               {'ex': traceback.format_exc()})
            raise 

def producer(hosts):
    return KafkaProducer(bootstrap_servers=hosts)


# noinspection PyShadowingNames 

def from_settings(cls, settings):
        my_level = settings.get('SC_LOG_LEVEL', 'INFO')
        my_name = settings.get('SC_LOGGER_NAME', 'sc-logger')
        my_output = settings.get('SC_LOG_STDOUT', True)
        my_json = settings.get('SC_LOG_JSON', False)
        my_dir = settings.get('SC_LOG_DIR', 'logs')
        my_bytes = settings.get('SC_LOG_MAX_BYTES', '10MB')
        my_file = settings.get('SC_LOG_FILE', 'main.log')
        my_backups = settings.get('SC_LOG_BACKUPS', 5)
        my_appids = settings.get('KAFKA_APPID_TOPICS', False)

        logger = LogFactory.get_instance(json=my_json,
                                         name=my_name,
                                         stdout=my_output,
                                         level=my_level,
                                         dir=my_dir,
                                         file=my_file,
                                         bytes=my_bytes,
                                         backups=my_backups)

        try:
            producer = KafkaProducer(bootstrap_servers=settings['KAFKA_HOSTS'],
                                 retries=3,
                                 linger_ms=settings['KAFKA_PRODUCER_BATCH_LINGER_MS'],
                                 buffer_memory=settings['KAFKA_PRODUCER_BUFFER_BYTES'])
        except Exception as e:
                logger.error("Unable to connect to Kafka in Pipeline"\
                    ", raising exit flag.")
                # this is critical so we choose to exit.
                # exiting because this is a different thread from the crawlers
                # and we want to ensure we can connect to Kafka when we boot
                sys.exit(1)
        topic_prefix = settings['KAFKA_TOPIC_PREFIX']
        use_base64 = settings['KAFKA_BASE_64_ENCODE']

        return cls(producer, topic_prefix, logger, appids=my_appids,
                   use_base64=use_base64) 

def __init__(
        self,
        host,
        sasl_username=None,
        sasl_password=None,
        ssl_cafile=None,
        ssl_certfile=None,
        ssl_keyfile=None,
        ssl_check_hostname=False,
        topic="rasa_core_events",
        security_protocol="SASL_PLAINTEXT",
        loglevel=logging.ERROR,
    ) -> None:
        raise_warning(
            "The `KafkaProducer` class is deprecated, please inherit "
            "from `KafkaEventBroker` instead. `KafkaProducer` will be "
            "removed in future Rasa versions.",
            FutureWarning,
            docs=DOCS_URL_EVENT_BROKERS,
        )

        super(KafkaProducer, self).__init__(
            host,
            sasl_username,
            sasl_password,
            ssl_cafile,
            ssl_certfile,
            ssl_keyfile,
            ssl_check_hostname,
            topic,
            security_protocol,
            loglevel,
        ) 

def initialize_output(self):
    """ Use information contained in `self.global_conf` to initialize Kafka output
    """
    self.out_servers = self.get_param('out_servers')
    self.out_topic = self.get_required_param('out_topic')
    if self.out_servers:
      self.producer = KafkaProducer(bootstrap_servers=self.out_servers)
    else:
      self.producer = KafkaProducer()
    init_out_msg = "[{}: log] CDRIngester output initialized with values:\n- out_servers: {};\n- out_topic: {}."
    print init_out_msg.format(self.pp, self.out_servers, self.out_topic) 

def init_producer(self):
    """Initialize ``self.producer``
    """
    print("[{}: log] Initializing producer...".format(self.pp))
    # Gather optional parameters
    dict_args = dict()
    dict_args = self.get_servers(dict_args, 'producer_servers')
    dict_args = self.get_security(dict_args, 'producer_security')
    # Instantiate producer
    try:
      self.producer = KafkaProducer(**dict_args)
    except Exception as inst:
      # Would be OK for ingester that do not output to kafka...
      print("[{}: warning] Could not initialize producer with arguments {}. Error was: {}".format(self.pp, dict_args, inst)) 

def start_bundle(self):
        self._producer = KafkaProducer(bootstrap_servers=self.attributes["servers"]) 

def __init__(self, url='kafka://localhost:9092', channel='socketio',
                 write_only=False):
        if kafka is None:
            raise RuntimeError('kafka-python package is not installed '
                               '(Run "pip install kafka-python" in your '
                               'virtualenv).')

        super(KafkaManager, self).__init__(channel=channel,
                                           write_only=write_only)

        self.kafka_url = url[8:] if url != 'kafka://' else 'localhost:9092'
        self.producer = kafka.KafkaProducer(bootstrap_servers=self.kafka_url)
        self.consumer = kafka.KafkaConsumer(self.channel,
                                            bootstrap_servers=self.kafka_url) 

def getProducer(validatedParams, timeout_ms):
    connectionHash = getConnectionHash(validatedParams)

    if globals().get("cached_producers") is None:
        logging.info("dictionary was None")
        globals()["cached_producers"] = dict()

    # remove arbitrary connection to make room for new one
    if len(globals()["cached_producers"]) == max_cached_producers:
        poppedProducer = globals()["cached_producers"].popitem()[1]
        poppedProducer.close(timeout=1)
        logging.info("Removed cached producer")

    if connectionHash not in globals()["cached_producers"]:
        logging.info("cache miss")
        # create a new connection

        producer = KafkaProducer(
            api_version_auto_timeout_ms=15000,
            batch_size=0,
            bootstrap_servers=validatedParams['brokers'],
            max_block_ms=timeout_ms,
            request_timeout_ms=timeout_ms,
        )

        logging.info("Created producer")

        # store the producer globally for subsequent invocations
        globals()["cached_producers"][connectionHash] = producer

        # return it
        return producer
    else:
        logging.info("Reusing existing producer")
        return globals()["cached_producers"][connectionHash] 

def __init__(self, bootstrap_servers, security_protocol="PLAINTEXT", sasl_mechanism=None, sasl_plain_username=None,
                 sasl_plain_password=None, ssl_keyfile=None, ssl_password=None, ssl_certfile=None, ssl_cafile=None,
                 ssl_crlfile=None, sasl_kerberos_service_name="kafka", sasl_kerberos_domain_name="kafka",
                 topic="nebula-reports"):
        self.topic = topic
        self.producer = KafkaProducer(value_serializer=lambda m: json.dumps(m).encode('ascii'),
                                      bootstrap_servers=bootstrap_servers, security_protocol=security_protocol,
                                      sasl_mechanism=sasl_mechanism, sasl_plain_username=sasl_plain_username,
                                      sasl_plain_password=sasl_plain_password, ssl_keyfile=ssl_keyfile,
                                      ssl_password=ssl_password, ssl_certfile=ssl_certfile, ssl_cafile=ssl_cafile,
                                      ssl_crlfile=ssl_crlfile, sasl_kerberos_service_name=sasl_kerberos_service_name,
                                      sasl_kerberos_domain_name=sasl_kerberos_domain_name) 

def test_queue_properties(queue, hosts, topic):
    assert hosts in repr(queue)
    assert topic in repr(queue)
    assert queue.producer.config['bootstrap_servers'] == hosts
    assert isinstance(queue.hosts, str) and queue.hosts == hosts
    assert isinstance(queue.topic, str) and queue.topic == topic
    assert isinstance(queue.producer, KafkaProducer)
    assert isinstance(queue.timeout, (int, float))
    assert callable(queue.serializer) or queue.serializer is None


# noinspection PyTypeChecker 

def connect_kafka_producer():
    _producer = None
    try:
        _producer = KafkaProducer(bootstrap_servers=['localhost:9092'], api_version=(0, 10))
    except Exception as ex:
        print('Exception while connecting Kafka')
        print(str(ex))
    finally:
        return _producer 

def connect_kafka_producer():
    _producer = None
    try:
        _producer = KafkaProducer(bootstrap_servers=['localhost:9092'], api_version=(0, 10))
    except Exception as ex:
        print('Exception while connecting Kafka')
        print(str(ex))
    finally:
        return _producer 

def client(self):
        if not self._client:
            kwargs = self._get_client_config()
            self._client = kafka.KafkaProducer(**kwargs)
        return self._client 

def init(self):
        self.producer = KafkaProducer(bootstrap_servers=self.bootstrap, key_serializer=str.encode,
                                      value_serializer=lambda v: json.dumps(v).encode('utf-8')) 

def send(self,topic,message):
        if not self._producer:
            try:
                self._producer=KafkaProducer(bootstrap_servers=kafka_hosts,api_version=(0,10),acks=0)
            except:
                print(traceback.format_exc(), flush=True)
                self._producer=None

        try:
            self._producer.send(topic,message.encode('utf-8'))
        except:
            print(traceback.format_exc(), flush=True) 

def init_producer(self):
        self.producer = KafkaProducer(
            bootstrap_servers=self.config["bootstrap_uri"],
            security_protocol=self.config["security_protocol"],
            ssl_cafile=self.config["ssl_cafile"],
            ssl_certfile=self.config["ssl_certfile"],
            ssl_keyfile=self.config["ssl_keyfile"],
            api_version=(1, 0, 0),
        ) 

def fixture_producer(kafka_server):
    if REST_URI in os.environ or REGISTRY_URI in os.environ:
        kafka_uri = f"{get_broker_ip()}:9093"
    else:
        kafka_uri = "127.0.0.1:{}".format(kafka_server["kafka_port"])
    prod = KafkaProducer(bootstrap_servers=kafka_uri)
    try:
        yield prod
    finally:
        prod.close()