Python pika.BlockingConnection() Examples

def __init__(self, jobs, responses, host='localhost', port=5672,
                 user='guest', password='guest', rabbit_queue='rpc_queue'):
        # Set connection and channel
        self.credentials = pika.PlainCredentials(user, password)
        self.parameters = pika.ConnectionParameters(host, port, '/', self.credentials)
        self.connection = pika.BlockingConnection(self.parameters)
        self.channel = self.connection.channel()
        # Set queue for jobs and callback queue for responses
        result = self.channel.queue_declare(exclusive=True)
        self.callback_queue = result.method.queue
        self.channel.basic_consume(self.on_response, no_ack=True, queue=self.callback_queue)
        self.rabbit_queue = rabbit_queue
        self.channel.queue_declare(queue=self.rabbit_queue)
        self.response = None
        self.id = None
        # Local queues shared between threads
        self.jobs = jobs
        self.responses = responses
        # Report to the RabbitMQ server
        heartbeat_thread = threading.Thread(target=self.heartbeat)
        heartbeat_thread.daemon = True
        heartbeat_thread.start() 

def oCreateConnection(self):
        import pika
        global oCONNECTION
        if not self.oConnection:
            try:
                oConnection = pika.BlockingConnection(self.oParameters)
                assert oConnection, "oCreateConnection: no oConnection created"
                self.oConnection = oConnection
                oCONNECTION = oConnection
                vDebug("Created connection " +str(id(oConnection)))
            except Exception as e:
                #     raise exceptions.ProbableAuthenticationError
                oLOG.exception("Error in oCreateConnection " + str(e))
                raise

        return self.oConnection 

def _listen_thread(self, thread_idx):
        conn = pika.BlockingConnection(pika.ConnectionParameters(host=self._rmq_server_addr,port=self._port,heartbeat=self.heartbeat,blocked_connection_timeout=None,virtual_host='/',credentials=pika.PlainCredentials(self._username,self._username)))

        channel_request = conn.channel()
        channel_request.queue_declare(queue=self._request_pipe_name)

        channel_response = conn.channel()
        channel_response.queue_declare(queue=self._response_pipe_name)

        def fail(*args,**kwargs):
            print('args:',args)
            print('kwargs:',kwargs)
            raise NotImplementedError

        channel_response.add_on_cancel_callback(fail)

        channel_request.basic_qos(prefetch_count=1) 
        channel_request.basic_consume(partial(self._request_callback, channel_response=channel_response), queue=self._request_pipe_name)

        printgreen(info_prefix(),'Listening ({})'.format(thread_idx))
        print()
        channel_request.start_consuming() 

def get_connection_amqp():
    try:
        port = int(config.get('ckan.harvest.mq.port', PORT))
    except ValueError:
        port = PORT
    userid = config.get('ckan.harvest.mq.user_id', USERID)
    password = config.get('ckan.harvest.mq.password', PASSWORD)
    hostname = config.get('ckan.harvest.mq.hostname', HOSTNAME)
    virtual_host = config.get('ckan.harvest.mq.virtual_host', VIRTUAL_HOST)

    credentials = pika.PlainCredentials(userid, password)
    parameters = pika.ConnectionParameters(host=hostname,
                                           port=port,
                                           virtual_host=virtual_host,
                                           credentials=credentials,
                                           frame_max=10000)
    log.debug("pika connection using %s" % parameters.__dict__)

    return pika.BlockingConnection(parameters) 

def open_connection(self):
        """
        Connect to RabbitMQ.
        """
        # Set logger for pika.
        # See if something went wrong connecting to RabbitMQ.
        handler = logging.StreamHandler()
        handler.setFormatter(self.formatter)
        rabbitmq_logger = logging.getLogger('pika')
        rabbitmq_logger.addHandler(handler)
        rabbitmq_logger.propagate = False
        rabbitmq_logger.setLevel(logging.WARNING)

        if not self.connection or self.connection.is_closed:
            self.connection = pika.BlockingConnection(pika.ConnectionParameters(**self.connection_params))

        if not self.channel or self.channel.is_closed:
            self.channel = self.connection.channel()

        if self.exchange_declared is False:
            self.channel.exchange_declare(exchange=self.exchange, exchange_type='topic', durable=True, auto_delete=False)
            self.exchange_declared = True

        # Manually remove logger to avoid shutdown message.
        rabbitmq_logger.removeHandler(handler) 

def publish():
    connection = pika.BlockingConnection(parameters)
    channel = connection.channel()  

    channel.exchange_declare(exchange=args.rabbit_exchange,
                             durable=str2bool(args.exchange_durable),
                             auto_delete=str2bool(args.exchange_auto_delete),
                             type="topic")
    channel.queue_declare(queue=args.rabbit_queue,
                          durable=str2bool(args.queue_durable),
                          auto_delete=str2bool(args.queue_auto_delete))
    channel.queue_bind(args.rabbit_queue, args.rabbit_exchange,
                       args.routing_key)

    message = 'Gremlin Coming!'
    count = 0
    while count < args.msg_per_thread:
        channel.basic_publish(exchange=args.rabbit_exchange,
                              routing_key=args.routing_key,
                              body=message)
        count = count + 1
    connection.close() 

def nova_amq(self):
        """
        Method used to listen to nova events

        """

        connection = pika.BlockingConnection(pika.ConnectionParameters(host=self.rabbit_host,
                                                                       credentials=pika.PlainCredentials(
                                                                           username=self.rabbit_user,
                                                                           password=self.rabbit_pass)))
        channel = connection.channel()
        result = channel.queue_declare(exclusive=True)
        queue_name = result.method.queue
        channel.exchange_declare(exchange='nova', type='topic')
        channel.queue_bind(exchange='nova', queue=queue_name, routing_key='notifications.#')
        channel.queue_bind(exchange='nova', queue=queue_name, routing_key='compute.#')
        channel.basic_consume(self.nova_callback, queue=queue_name, no_ack=True)
        channel.start_consuming() 

def keystone_amq(self):
        """
        Method used to listen to keystone events
        """

        connection = pika.BlockingConnection(pika.ConnectionParameters(host=self.rabbit_host,
                                                                       credentials=pika.PlainCredentials(
                                                                           username=self.rabbit_user,
                                                                           password=self.rabbit_pass)))
        channel = connection.channel()
        result = channel.queue_declare(exclusive=True)
        queue_name = result.method.queue
        channel.exchange_declare(exchange='keystone', type='topic')
        channel.queue_bind(exchange='openstack', queue=queue_name, routing_key='notifications.#')
        channel.queue_bind(exchange='keystone', queue=queue_name, routing_key='keystone.#')
        channel.basic_consume(self.keystone_callback, queue=queue_name, no_ack=True)
        channel.start_consuming() 

def connect(self):
        """
            主进程的消费队列连接
        """
        self.connection = pika.BlockingConnection(pika.ConnectionParameters(
            host=MqConfig.host,
            port=MqConfig.port,
            credentials=self.auth,
            heartbeat=0
        ))
        self.channel = self.connection.channel()
        self.channel.basic_qos(prefetch_count=1)
        self.channel.exchange_declare(exchange=MqConfig.exchange, exchange_type=MqConfig.exchange_type, durable=True)
        self.channel.queue_declare(queue=MqConfig.main_queue, durable=True)
        self.channel.queue_bind(exchange=MqConfig.exchange, queue=MqConfig.main_queue)
        self.channel.basic_consume(queue=MqConfig.main_queue, on_message_callback=self.callback, auto_ack=True) 

def _job_monitor_thread(job_key, total_calls, rabbit_amqp_url, job_monitor_q):
    executor_id, job_id = job_key.rsplit('-', 1)
    exchange = 'pywren-{}-{}'.format(executor_id, job_id)
    queue_0 = '{}-0'.format(exchange)
    total_calls_rcvd = 0

    def callback(ch, method, properties, body):
        nonlocal total_calls_rcvd
        call_status = json.loads(body.decode("utf-8"))
        job_monitor_q.put(call_status)
        if call_status['type'] == '__end__':
            total_calls_rcvd += 1
        if total_calls_rcvd == total_calls:
            ch.stop_consuming()

    logger.debug('ExecutorID {} | JobID {} - Consuming from RabbitMQ '
                 'queue'.format(executor_id, job_id))
    params = pika.URLParameters(rabbit_amqp_url)
    connection = pika.BlockingConnection(params)
    channel = connection.channel()
    channel.basic_consume(callback, queue=queue_0, no_ack=True)
    channel.start_consuming() 

def delete_rabbitmq_resources(rabbit_amqp_url, executor_id, job_id):
    """
    Deletes RabbitMQ queues and exchanges of a given job.
    Only called when an exception is produced, otherwise resources are
    automatically deleted.
    """
    exchange = 'pywren-{}-{}'.format(executor_id, job_id)
    queue_0 = '{}-0'.format(exchange)  # For waiting
    queue_1 = '{}-1'.format(exchange)  # For invoker

    params = pika.URLParameters(rabbit_amqp_url)
    connection = pika.BlockingConnection(params)
    channel = connection.channel()
    channel.queue_delete(queue=queue_0)
    channel.queue_delete(queue=queue_1)
    channel.exchange_delete(exchange=exchange)
    connection.close() 

def create_rabbitmq_resources(rabbit_amqp_url, executor_id, job_id):
    """
    Creates RabbitMQ queues and exchanges of a given job in a thread.
    Called when a job is created.
    """
    logger.debug('ExecutorID {} | JobID {} - Creating RabbitMQ resources'.format(executor_id, job_id))

    def create_resources(rabbit_amqp_url, executor_id, job_id):
        exchange = 'pywren-{}-{}'.format(executor_id, job_id)
        queue_0 = '{}-0'.format(exchange)  # For waiting
        queue_1 = '{}-1'.format(exchange)  # For invoker

        params = pika.URLParameters(rabbit_amqp_url)
        connection = pika.BlockingConnection(params)
        channel = connection.channel()
        channel.exchange_declare(exchange=exchange, exchange_type='fanout', auto_delete=True)
        channel.queue_declare(queue=queue_0, auto_delete=True)
        channel.queue_bind(exchange=exchange, queue=queue_0)
        channel.queue_declare(queue=queue_1, auto_delete=True)
        channel.queue_bind(exchange=exchange, queue=queue_1)
        connection.close()

    th = threading.Thread(target=create_resources, args=(rabbit_amqp_url, executor_id, job_id))
    th.daemon = True
    th.start() 

def send(message_json):
    """
    :param message:
    :return:
    """
    global logger
    connection = pika.BlockingConnection(pika.ConnectionParameters(rabbit_host))
    channel = connection.channel()
    queue = channel.queue_declare(queue=queue_name, durable=True, auto_delete=True)
    logger.debug("send> number of messages in the queue is: "+str(queue.method.message_count))
    message = json.dumps(message_json)
    logger.debug("send> sending message")
    logger.debug(message)
    channel.basic_publish(exchange='',
                          routing_key=queue_name,
                          body=message,
                          properties=pika.BasicProperties(
                              delivery_mode=2,  # make message persistent
                          ))
    connection.close() 

def single_worker(worker_id):
    """
    :param worker_id:
    :return:
    """
    global lock
    global logger
    logger.debug('worker_id: '+str(worker_id))
    connection = pika.BlockingConnection(pika.ConnectionParameters(rabbit_host))
    channel = connection.channel()
    queue = channel.queue_declare(queue=queue_name, durable=True, auto_delete=True)
    channel.basic_qos(prefetch_count=1)
    channel.basic_consume(queue=queue_name, on_message_callback=callback)
    print("Rabbit consuming is started ... "+str(worker_id))
    logger.debug("Setting the logger ..."+str(worker_id))
    logger.debug("test connection ..."+str(channel.is_open))
    logger.debug("single_worker> number of messages in the queue is: " + str(queue.method.message_count))
    channel.start_consuming() 

def get_pending_messages():
    try:
        connection = pika.BlockingConnection(pika.ConnectionParameters(rabbit_host))
    except:
        print("exception 1 in connecting")
        sleep(3)
        connection = pika.BlockingConnection(pika.ConnectionParameters(rabbit_host))


    channel = connection.channel()
    queue = channel.queue_declare(queue=queue_name, durable=True, auto_delete=True)
    num = queue.method.message_count
    connection.close()
    sleep(0.1)
    return num


# This it to force the tests to run sequentially (while the default is to run them in parallel)
# This is needed as the tests are executed using the same repo and user 

def connect(self):
        try:
            self.connected_node = self.broker_manager.get_current_node(self.consumer_id)
            ip = self.broker_manager.get_node_ip(self.connected_node)
            console_out(f"Connecting to {self.connected_node}", self.get_actor())
            credentials = pika.PlainCredentials('jack', 'jack')
            parameters = pika.ConnectionParameters(ip,
                                                self.broker_manager.get_consumer_port(self.connected_node, self.consumer_id),
                                                '/',
                                                credentials)
            self.connection = pika.BlockingConnection(parameters)
            self.channel = self.connection.channel()

            if self.prefetch > 0:
                self.channel.basic_qos(prefetch_count=self.prefetch)

            return True
        except Exception as e:
            console_out_exception("Failed trying to connect.", e, self.get_actor())
            return False 

def sendData(self, server, queue, ci, data):
        self.logger.debug('Sending [{}] to [{}] Start'.format(ci, server))
        ## conn = pika.BlockingConnection(
                    ## pika.ConnectionParameters(server)
                ## )

        with pika.BlockingConnection(pika.ConnectionParameters(host = server,
                                        port = self.MQ_PORT)) as conn:
            for line in data:
                chan = conn.channel()
                chan.queue_declare(queue = queue, durable = True)
                chan.basic_publish(exchange = '', routing_key = queue, body = line,
                                    properties = pika.BasicProperties(delivery_mode = 2, ))
        self.logger.debug('Sending [{}] to [{}] Done'.format(ci, server))
        return(True)

    ## run func 

def create_channel(self):
        credentials = pika.PlainCredentials(self.user, self.pwd)
        self.__connection = pika.BlockingConnection(
            pika.ConnectionParameters(self.addr, self.port, self.vhost, credentials=credentials))
        channel = self.__connection.channel()
        return channel 

def RmqConsumer(host):
    global LOGGER
    global RABBITMQ_USER
    global RABBITMQ_PASSWORD
    while True:
        try:
            credentials = pika.PlainCredentials(RABBITMQ_USER, RABBITMQ_PASSWORD)
            connection = pika.BlockingConnection(
                    pika.ConnectionParameters(
                            host=host,
                            credentials=credentials
                        )
                )
            channel = connection.channel()
            channel.basic_qos(
                    prefetch_size=0,
                    prefetch_count=0,
                    global_qos=True
                )
            # Consume pool block records
            channel.basic_consume(
                    queue='poolblocks',
                    on_message_callback=poolblock_handler,
                    auto_ack=False
                )
            # Consume worker shares records
            channel.basic_consume(
                    queue='shares',
                    on_message_callback=share_handler,
                    auto_ack=False
                )
            channel.start_consuming()
        except Exception as e:
            LOGGER.exception("Something went wrong: {}".format(traceback.format_exc()))
            sys.stdout.flush()
            time.sleep(10) 

def run():

    try:
        parameters = AMQP_CONNECTION_PARAMETERS
        connection = pika.BlockingConnection(parameters)
        channel = connection.channel()
        channel.queue_declare(queue='security-scan-responses')
        channel.basic_qos(prefetch_count=1)
        channel.basic_consume(handle_message, queue='security-scan-responses')
        channel.start_consuming()

    except AMQPError as error:
        print('AMQP Error {}'.format(error))
    except Exception as error:
        print('Exception: {}'.format(error)) 

def __init__(self):

        # 数据库连接初始化
        self.sql_operator = SqlOperator()
        # 订阅频道
        connection = pika.BlockingConnection(pika.ConnectionParameters(host=RABBIT_MQ_HOST))
        channel = connection.channel()
        # 如果确定已经声明了，可以不声明。但是你不知道消费者生产者哪个先运行，所以要声明两次。
        channel.queue_declare(queue='jd_crawl')
        # prefetch_count 如果消费者中有一条消息没有处理完，就不会继续给这个消费者发送新消息
        channel.basic_qos(prefetch_count=SELENIUM_THREAD)
        channel.basic_consume('jd_crawl', self.callback)
        logging.info('开始监听Queue：jd_crawl')
        channel.start_consuming() 

def check_connection(self) -> bool:
        """
            检查连接
        """
        try:
            pika.BlockingConnection(pika.ConnectionParameters(
                host=MqConfig.host,
                port=MqConfig.port,
                credentials=self.auth,
                heartbeat=600
            ))
            return True
        except Exception as e:
            return False 

def connection_channel(self) -> Tuple[pika.BlockingConnection, pika.channel.Channel]:
        """
        Gets or creates the queue, and returns a tuple containing the object's VirtualHost's blocking connection,
        and its channel
        """
        connection = self.virtual_host.blocking_connection
        channel = connection.channel()

        return connection, channel 

def publish(self, connection: pika.BlockingConnection, channel: pika.channel.Channel) -> None:
        """
        Publishes a message to the channel
        """
        kwargs = self.publish_kwargs()
        kwargs['properties'] = pika.BasicProperties(**self.properties())
        channel.basic_publish(**kwargs)
        connection.close() 

def blocking_connection(self) -> pika.BlockingConnection:
        """
        Connect to the VHOST
        """
        credentials = pika.PlainCredentials(username=self.username, password=self.password)
        if self.name == '%2f':
            vhost = '/'
        else:
            vhost = self.name

        params = pika.ConnectionParameters(host=self.host, port=self.port, virtual_host=vhost,
                                           credentials=credentials, connection_attempts=10, ssl=self.secure,
                                           heartbeat=1200)
        return pika.BlockingConnection(parameters=params) 

def connect():
    curr_node = get_node_index(sys.argv[1])
    while True:
        try:
            credentials = pika.credentials.PlainCredentials('jack', 'jack')
            connection = pika.BlockingConnection(pika.ConnectionParameters(host=nodes[curr_node], port=5672, credentials=credentials))
            channel = connection.channel()
            print("Connected to " + nodes[curr_node])
            return channel
        except:
            curr_node += 1
            if curr_node > 2:
                print("Could not connect. Trying again in 5 seconds")
                time.sleep(5)
                curr_node  = 0 

def rabbitmq_connect(ip=None):
    """Connects to ip using standard port and credentials."""
    credentials = pika.credentials.PlainCredentials('testguest', 'secrete')
    parameters = pika.ConnectionParameters(
        host=ip, virtual_host='/testvhost', credentials=credentials)
    try:
        connection = pika.BlockingConnection(parameters)
        connection.channel()
    except Exception:
        sys.exit("Can't connect to %s" % ip)
    else:
        print("Connected.") 

def connect():
    curr_node = 0
    while True:
        try:
            credentials = pika.credentials.PlainCredentials('jack', 'jack')
            connection = pika.BlockingConnection(pika.ConnectionParameters(host=nodes[curr_node], port=5672, credentials=credentials))
            channel = connection.channel()
            return channel
        except:
            curr_node += 1
            if curr_node > 2:
                print("Could not connect. Trying again in 5 seconds")
                time.sleep(5)
                curr_node  = 0 

def connect():
    curr_node = get_node_index(sys.argv[1])
    while True:
        try:
            credentials = pika.credentials.PlainCredentials('jack', 'jack')
            connection = pika.BlockingConnection(pika.ConnectionParameters(host=nodes[curr_node], port=5672, credentials=credentials))
            channel = connection.channel()
            print("Connected to " + nodes[curr_node])
            return channel
        except:
            curr_node += 1
            if curr_node > 2:
                print("Could not connect. Trying again in 5 seconds")
                time.sleep(5)
                curr_node  = 0 

def get_queue_client() -> SimpleQueueClient:
    global queue_client
    if queue_client is None:
        if settings.RUNNING_INSIDE_TORNADO and settings.USING_RABBITMQ:
            queue_client = TornadoQueueClient()
        elif settings.USING_RABBITMQ:
            queue_client = SimpleQueueClient()

    return queue_client

# We using a simple lock to prevent multiple RabbitMQ messages being
# sent to the SimpleQueueClient at the same time; this is a workaround
# for an issue with the pika BlockingConnection where using
# BlockingConnection for multiple queues causes the channel to
# randomly close.