Python kafka.SimpleProducer() Examples

The following are 4 code examples of kafka.SimpleProducer(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module kafka , or try the search function .
Example #1
Source File: tweet_sampler.py    From straw with MIT License 5 votes vote down vote up 
def __init__(self, kafka, chunk_size):
        self.producer = SimpleProducer(kafka)
        self.queues = {}
        self.chunk_size = chunk_size
Example #2
Source File: util.py    From kafka-utils with Apache License 2.0 5 votes vote down vote up 
def produce_example_msg(topic, num_messages=1):
    kafka = KafkaToolClient(KAFKA_URL)
    producer = SimpleProducer(kafka)
    for i in range(num_messages):
        try:
            producer.send_messages(topic, b'some message')
        except LeaderNotAvailableError:
            # Sometimes kafka takes a bit longer to assign a leader to a new
            # topic
            time.sleep(10)
            producer.send_messages(topic, b'some message')
Example #3
Source File: redis-monitor.py    From openslack-crawler with Apache License 2.0 5 votes vote down vote up 
def setup(self):
        '''
        Connection stuff here so we can mock it
        '''
        self.redis_conn = redis.Redis(host=REDIS_HOST, port=REDIS_PORT)

        # set up kafka
        self.kafka_conn = KafkaClient(KAFKA_HOSTS)
        self.producer = SimpleProducer(self.kafka_conn)
        self.topic_prefix = KAFKA_TOPIC_PREFIX
Example #4
Source File: straw_app.py    From straw with MIT License 4 votes vote down vote up 
def __init__(self, config):
    
        app = Flask(__name__)
        app.secret_key = 'i love to search full text in real time'

        # attach a redis connection pool
        app.pool = redis.ConnectionPool(host="localhost", port=6379)

        # user -> channels mapping
        app.user_channels = {}

        # how to handle messages that enter the stream from redis pub sub
        def redis_message_handler(msg):
            redis_connection = redis.Redis(connection_pool=app.pool)
            # get channel and content of incoming message
            channel = msg['channel']
            data = msg['data']

            # word highlighting -- TODO: this would be better to do in the search engine!
            query = redis_connection.get(channel)
            words = list(set(query.split(" ")))
            for w in words:
                data=data.lower().replace(w.lower(), highlight(w.lower()))

            # find users subscribed to this channel
            if app.user_channels.get(channel) is not None:
                for user in app.user_channels.get(channel):
                    redis_connection.lpush(user, data)
            else:
                # no more users for this channel, unsubscribe from it
                redis_connection.unsubscribe(channel)            
            
        # Add Redis query subscriber to app
        app.disp = []
        app.subscriber = QuerySubscriber("localhost", 6379, redis_message_handler)

        # setup kafka producer in the app
        kafka = KafkaClient("{0}:{1}".format(config["zookeeper_host"], 9092))
        app.producer = SimpleProducer(kafka)

        # add the app
        self.app = app