Python kombu.Queue() Examples

def connect(self, exchange, channel):  # pragma: no cover
        """
        Readies the StorageNotify for publishing notification messages by
        setting up a kombu.Producer.

        :param exchange: The exchange for publishing notifications.
        :type exchange: kombu.Exchange
        :param channel: The channel to bind to.
        :type channel: kombu.transport.base.StdChannel
        """
        name = self.__class__.__name__
        self.logger.debug('Connecting {}'.format(name))

        self._queue = kombu.Queue(exchange=exchange, channel=channel)
        self._queue.declare()

        self._producer = kombu.Producer(channel, exchange) 

def __init__(self, logs, tracker, start_at=None, transient=True):
        self._logs = logs
        super(_AMQPProcessor, self).__init__()
        self.handle_begin()
        self.transient = transient
        self.__tracker = tracker
        self.__inbound_queue = gevent.queue.Queue()
        self.__run_till = None
        self.__tail_timeout = None
        self.__in_finish_mode = False
        self.__ignore_misses = False
        # THIS is a hack to allow raw access to underlying tracker-records until we get a common
        # validation phase. See get_raw_tracker_events() below for details
        self.__matches_in_order = []
        self.__started_at = tracker.add_processor(self, start_at=start_at)
        self.__match_greenlet = gevent.spawn(self.__match_greenlet_run)
        self.__match_greenlet.greenlet_name = 'processor-match-loop-gl' 

def __init__(self, logs, connection, name, exchange, routing_key, queue_name):
        self.__logs = logs
        self.__ignore_some_stuff = False
        self.name = name
        self.__event_callbacks = []
        if queue_name is None:
            queue_name = ''
            exclusive = True
        else:
            exclusive = False
        chan = connection.channel()
        ex = Exchange(exchange, 'topic', channel=chan)
        queue = Queue(exchange=ex, routing_key=routing_key, exclusive=exclusive)
        consumer = Consumer(chan, queues=[queue], callbacks=[self.__message_cb])
        consumer.consume()
        self.exchange = ex 

def setup_rabbitmq_client(options):
    global RABBITMQ_CLIENT
    try:
        RABBITMQ_CLIENT
    except NameError:
        mqConnString = 'amqp://{0}:{1}@{2}:{3}//'.format(
            options.mquser,
            options.mqpassword,
            options.mqalertserver,
            options.mqport
        )
        mqAlertConn = Connection(mqConnString)
        alertExchange = Exchange(name=options.alertExchange, type='topic', durable=True, delivery_mode=1)
        alertExchange(mqAlertConn).declare()

        alertQueue = Queue(options.queueName,
                           exchange=alertExchange,
                           routing_key=options.alerttopic,
                           durable=False,
                           no_ack=(not options.mqack))
        alertQueue(mqAlertConn).declare()

        RABBITMQ_CLIENT = mqAlertConn.Consumer(alertQueue, accept=['json'])
    return RABBITMQ_CLIENT 

def test(url):
    from kombu import Exchange, Queue, Connection, Consumer, Producer
    task_queue = Queue('tasks', exchange=Exchange('celery', type='direct'), routing_key='tasks')
    # 生产者
    with Connection(url) as conn:
        with conn.channel() as channel:
            producer = Producer(channel)
            producer.publish({'hello': 'world'},
                             retry=True,
                             exchange=task_queue.exchange,
                             routing_key=task_queue.routing_key,
                             declare=[task_queue])

    def get_message(body, message):
        print("receive message: %s" % body)
        # message.ack()

    # 消费者
    with Connection(url) as conn:
        with conn.channel() as channel:
            consumer = Consumer(channel, queues=task_queue, callbacks=[get_message, ], prefetch_count=10)
            consumer.consume(no_ack=True) 

def start_listener(runtime_context):
    # Need to keep the amqp logger level at least as high as INFO,
    # or else it send heartbeat check messages every second
    logging.getLogger('amqp').setLevel(max(logger.level, getattr(logging, 'INFO')))
    logger.info('Starting pipeline listener')

    fits_exchange = Exchange(runtime_context.FITS_EXCHANGE, type='fanout')
    listener = RealtimeModeListener(runtime_context)

    with Connection(runtime_context.broker_url) as connection:
        listener.connection = connection.clone()
        listener.queue = Queue(runtime_context.queue_name, fits_exchange)
        try:
            listener.run()
        except listener.connection.connection_errors:
            listener.connection = connection.clone()
            listener.ensure_connection(max_retries=10)
        except KeyboardInterrupt:
            logger.info('Shutting down pipeline listener.') 

def __init__(self, crawl_name, num_urls=DEFAULT_NUM_URLS):
        """
        Create a NutchUrlTrails instance for visualizing a running Nutch crawl in real-time using Bokeh
        :param name: The name of the crawl (as identified by the queue)
        :param num_urls: The number of URLs to display in the visualization
        :return: A NutchUrLTrails instance
        """
        self.crawl_name = crawl_name
        self.num_urls = num_urls
        self.open_urls = {}
        self.closed_urls = {}
        self.old_segments = None
        self.old_circles = None
        
        self.session = Session()
        self.session.use_doc(self.crawl_name)
        self.document = Document()

        con = Connection()

        exchange = Exchange(EXCHANGE_NAME, 'direct', durable=False)
        queue = Queue(crawl_name, exchange=exchange, routing_key=crawl_name)
        self.queue = con.SimpleQueue(name=queue) 

def main():
    # connect and declare the message queue/kombu objects.
    # Event server/exchange/queue
    mqConnString = 'amqp://{0}:{1}@{2}:{3}//'.format(
        options.mquser,
        options.mqpassword,
        options.mqalertserver,
        options.mqport
    )
    mqAlertConn = Connection(mqConnString)

    # Exchange for alerts we pass to actions
    alertExchange = Exchange(name=options.alertExchange,
                             type='topic',
                             durable=True,
                             delivery_mode=1)

    alertExchange(mqAlertConn).declare()

    # Queue for the exchange
    alertQueue = Queue(options.queueName,
                       exchange=alertExchange,
                       routing_key=options.alerttopic,
                       durable=False,
                       no_ack=(not options.mqack))
    alertQueue(mqAlertConn).declare()

    # consume our alerts.
    alertConsumer(mqAlertConn, alertQueue, alertExchange).run() 

def get_trigger_cud_queue(name, routing_key, exclusive=False):
    return Queue(name, TRIGGER_CUD_XCHG, routing_key=routing_key, exclusive=exclusive) 

def get_trigger_instances_queue(name, routing_key):
    return Queue(name, TRIGGER_INSTANCE_XCHG, routing_key=routing_key) 

def get_sensor_cud_queue(name, routing_key):
    return Queue(name, SENSOR_CUD_XCHG, routing_key=routing_key) 

def get_queue(name, routing_key):
    return kombu.Queue(name, WORKFLOW_EXECUTION_XCHG, routing_key=routing_key) 

def get_status_management_queue(name, routing_key):
    return kombu.Queue(name, WORKFLOW_EXECUTION_STATUS_MGMT_XCHG, routing_key=routing_key) 

def _make_queue(self, routing_key, exchange, channel=None):
        """Make a named queue for the given exchange."""
        queue_name = "%s_%s" % (self._exchange_name, routing_key)
        return kombu.Queue(name=queue_name,
                           routing_key=routing_key, durable=False,
                           exchange=exchange, auto_delete=True,
                           channel=channel) 

def _queue(self):
        queue_name = 'flask-socketio.' + str(uuid.uuid4())
        options = {'durable': False, 'queue_arguments': {'x-expires': 300000}}
        options.update(self.queue_options)
        return kombu.Queue(queue_name, self._exchange(), **options) 

def get_queue(name=None, routing_key=None, exclusive=False, auto_delete=False):
    return Queue(name, EXECUTION_XCHG, routing_key=routing_key, exclusive=exclusive,
                 auto_delete=auto_delete) 

def _configureKombu(self):
        """
        Configure kombu for amqp or sqs
        """
        try:
            connString = self.__build_conn_string()
            self.mqConn = kombu.Connection(connString)
            if connString.find('sqs') == 0:
                self.mqConn.transport_options['region'] = os.getenv('DEFAULT_AWS_REGION', 'us-west-2')
                self.mqConn.transport_options['is_secure'] = True
                self.alertExchange = kombu.Exchange(
                    name=RABBITMQ["alertexchange"], type="topic", durable=True
                )
                self.alertExchange(self.mqConn).declare()
                alertQueue = kombu.Queue(
                    os.getenv('OPTIONS_ALERTSQSQUEUEURL').split('/')[4], exchange=self.alertExchange
                )
            else:
                self.alertExchange = kombu.Exchange(
                    name=RABBITMQ["alertexchange"], type="topic", durable=True
                )
                self.alertExchange(self.mqConn).declare()
                alertQueue = kombu.Queue(
                    RABBITMQ["alertqueue"], exchange=self.alertExchange
                )
            alertQueue(self.mqConn).declare()
            self.mqproducer = self.mqConn.Producer(serializer="json")
            self.log.debug("Kombu configured")
        except Exception as e:
            self.log.error(
                "Exception while configuring kombu for alerts: {0}".format(e)
            ) 

def consume_alerts(bot):
    # connect and declare the message queue/kombu objects.
    # server/exchange/queue
    mq_conn_str = 'amqp://{0}:{1}@{2}:{3}//'.format(
        options.mq_user,
        options.mq_password,
        options.mq_alert_server,
        options.mq_port
    )
    mq_alert_conn = Connection(mq_conn_str)

    # Exchange for alerts we pass to plugins
    alert_exchange = Exchange(
        name=options.alert_exchange,
        type='topic',
        durable=True,
        delivery_mode=1
    )

    alert_exchange(mq_alert_conn).declare()

    # Queue for the exchange
    alert_queue = Queue(
        options.queue_name,
        exchange=alert_exchange,
        routing_key=options.alerttopic,
        durable=False,
        no_ack=(not options.mq_ack)
    )
    alert_queue(mq_alert_conn).declare()

    # consume our alerts.
    AlertConsumer(mq_alert_conn, alert_queue, alert_exchange, bot).run() 

def test_helper_sync_send_msg(self, exchange, ex_rk, send_rk, payload):
        ex = Exchange(exchange, 'topic')
        queue = Queue(exchange=ex, routing_key=ex_rk + '.*', exclusive=True, channel=self.__connection)
        queue.declare()
        prod = Producer(self.__connection, exchange=ex, routing_key=send_rk)
        prod.publish(payload)
        return queue 

def __assure_named_queue(self, connection, exchange, queue_name, routing_key_extension='*'):
        routing_key = '{}.{}'.format(queue_name, routing_key_extension)
        queue = Queue(queue_name, exchange, routing_key, connection)
        queue.declare() 

def make_queue_obj(exchange, queue, routing_key, type='topic'):
    return Queue(queue, \
           Exchange(exchange, type=type), \
           routing_key=routing_key) 

def get_consumers(self, Consumer, channel):
        """
        Returns a list of kombu.Consumer instances to service all registered
        notification callbacks.

        If using the kombu.mixin.ConsumerMixin mixin class, these instances
        should be included in its get_consumers() method.

        :param Consumer: Message consumer class.
        :type Consumer: class
        :param channel: An open channel.
        :type channel: kombu.transport.*.Channel
        :returns: A list of consumer instances
        :rtype: [kombu.Consumer, ....]
        """
        consumer_list = []
        exchange = self.bus_mixin.producer.exchange
        for routing_key, callbacks in self.notify_callbacks.items():
            queue = kombu.Queue(
                exchange=exchange, routing_key=routing_key)
            consumer = Consumer(
                queues=queue, callbacks=callbacks)
            consumer_list.append(consumer)
            self.bus_mixin.logger.info(
                'Listening for "%s" notifications', routing_key)
        return consumer_list 

def send(self):

        try:
            # Connection
            conn = Connection(self.broker)

            # Channel
            channel = conn.channel()

            # Exchange
            task_exchange = Exchange(self._exchange_name,
                                     type=self._queue_type)

            # Queues
            if self._queue_name:
                queue = Queue(name=self._queue_name, channel=channel,
                              exchange=task_exchange,
                              routing_key=self._routing_key)
                queue.declare()

            # Producer
            producer = Producer(exchange=task_exchange, channel=channel,
                                routing_key=self._routing_key)

            # Send message
            for message in self._msgs:
                serialized_message = json.dumps(message, ensure_ascii=False)
                producer.publish(serialized_message)

            conn.close()

        except Exception, e:

            self.log.error(
                u'QueueManagerError - Error on sending objects from queue.')
            self.log.debug(e)
            raise Exception(
                'QueueManagerError - Error on sending objects to queue.') 

def __init__(self, env, is_external_queue: bool):
        super().__init__(env, is_external_queue, queue_type='redis', logger=logger)

        conf = env.config

        bind_port = self.get_port()
        if bind_port is None:
            logger.info('skipping pubsub setup, no port specified')
            return

        queue_host = conf.get(ConfigKeys.HOST, domain=self.domain_key, default=None)
        exchange = conf.get(ConfigKeys.EXCHANGE, domain=self.domain_key, default='node_exchange')
        queue_db = conf.get(ConfigKeys.DB, domain=self.domain_key, default=0)
        queue_name = conf.get(ConfigKeys.QUEUE, domain=self.domain_key, default=None)

        if queue_name is None or len(queue_name.strip()) == 0:
            queue_name = 'node_queue_%s_%s_%s' % (
                conf.get(ConfigKeys.ENVIRONMENT),
                self.get_host(),
                bind_port
            )

        if self.is_external_queue:
            self.exchange = Exchange(exchange, type='direct')
        else:
            self.exchange = Exchange(exchange, type='fanout')

        self.queue_connection = Connection(queue_host, transport_options={'db': queue_db})
        logger.info('queue connection: {}'.format(str(self.queue_connection)))
        self.queue_name = queue_name
        self.queue = Queue(self.queue_name, self.exchange) 

def __init__(self, _conf):
        amqp_conf = conf.get(ConfigKeys.AMQP)
        queue_host = amqp_conf.get(ConfigKeys.HOST)
        if queue_host is None or len(queue_host.strip()) == 0:
            return

        queue_port = amqp_conf.get(ConfigKeys.PORT)
        queue_vhost = amqp_conf.get(ConfigKeys.VHOST)
        queue_user = amqp_conf.get(ConfigKeys.USER)
        queue_pass = amqp_conf.get(ConfigKeys.PASSWORD)

        queue_host = ';'.join(['amqp://%s' % host for host in queue_host.split(';')])
        queue_exchange = '%s_%s' % (
            amqp_conf.get(ConfigKeys.EXCHANGE),
            amqp_conf.get(ConfigKeys.ENVIRONMENT)
        )

        queue_name = amqp_conf.get(ConfigKeys.QUEUE)
        self.exchange = Exchange(queue_exchange, type='direct')

        self.queue_connection = Connection(
            hostname=queue_host,
            port=queue_port,
            virtual_host=queue_vhost,
            userid=queue_user,
            password=queue_pass
        )
        self.queue = Queue(queue_name, self.exchange)
        logger.info('setting up pubsub for host(s) "{}"'.format(queue_host)) 

def start_sync(self):
        exchange = Exchange(self.exchange_name, self.exchange_type, durable=self.durable)
        queue = Queue(self.queue_name, exchange=exchange, routing_key=self.routing_key)
        with Connection(self.amqp_url) as conn:
            # producer = conn.Producer(serializer='json')
            # producer.publish({'name': '/tmp/lolcat1.avi', 'size': 1301013},
            #                  exchange=exchange, routing_key=self.routing_key,
            #                  declare=[queue])
            # producer.publish({'name': '/tmp/lolcat1.avi', 'size': 1301013},
            #                  exchange=exchange, routing_key=self.routing_key,
            #                  declare=[queue])
            with conn.Consumer(queue, callbacks=[self.rabbitmq_callback]) as consumer:
                # Process messages and handle events on all channels
                while True:
                    conn.drain_events() 

def _init_task_queues(self):
        """
        Assuming we init with parameter `trackers = ['mousekey.com', 'digidog.com',
        'futuredigi.com']`
        then this should set self.task_queues as below:

        self.task_queues = [
           Queue('mousekey.com', task_exchange, routing_key='mousekey.com'),
           Queue('digidog.com', task_exchange, routing_key='digidog.com'),
           Queue('futuredigi.com', task_exchange, routing_key='futuredigi.com')]
        """
        for tracker_name in self.trackers:
            queue = Queue(tracker_name, self.task_exchange, routing_key=tracker_name)
            self.task_queues.append(queue) 

def __queue(self, name):
        queue_name = self.__queue_name(name)
        queue = kombu.Queue(queue_name, self.__exchange, routing_key=queue_name)
        return queue 

def handle(self, queue, *args, **options):
        fmt = '%(asctime)s %(name)-12s: %(levelname)-8s %(message)s'
        log_level = 40 - (options['verbosity'] * 10)
        logging.basicConfig(level=log_level, format=fmt)

        # TODO: Ensure that failed processing does not requeue task into
        # work queue
        set_event_loop(Hub())
        kwargs = {
            'transport_options': conf.broker_transport_options,
        }
        with app.connection(**kwargs) as conn:
            logger.info("Launching worker")
            worker = Worker(conn, queues=[Queue(queue)])
            worker.connect_max_retries = 1
            while True:
                try:
                    breakers.celery.call(worker.run)
                except KeyboardInterrupt:
                    logger.info("KeyboardInterrupt, exiting. Bye!")
                    break
                except breakers.celery.expected_errors:
                    rest_val = 5
                    logger.warning(
                        "Open circuit detected. "
                        "Sleeping for %s seconds and then will try again.",
                        rest_val
                    )
                    time.sleep(rest_val) 

def get_consumers(self, _, default_channel):
        queues = [
            Queue(preprocessor.routing_key, exchange=raw_events_exchange,
                  routing_key=preprocessor.routing_key, durable=True)
            for routing_key, preprocessor in self.preprocessors.items()
        ]
        return [Consumer(default_channel,
                         queues=queues,
                         accept=['json'],
                         callbacks=[self.do_preprocess_raw_event])]