Python pika.BasicProperties() Examples

def push_to_result_queue(self, messages):
        current_app.logger.debug("Pushing result to RabbitMQ...")
        for message in messages:
            while message is not None:
                try:
                    self.result_channel.basic_publish(
                        exchange=current_app.config['SPARK_RESULT_EXCHANGE'],
                        routing_key='',
                        body=json.dumps(message),
                        properties=pika.BasicProperties(delivery_mode=2,),
                    )
                    break
                except (pika.exceptions.ConnectionClosed, pika.exceptions.ChannelClosed) as e:
                    current_app.logger.error('RabbitMQ Connection error while publishing results: %s', str(e), exc_info=True)
                    time.sleep(1)
                    self.rabbitmq.close()
                    self.connect_to_rabbitmq()
                    self.init_rabbitmq_channels()
        current_app.logger.debug("Done!") 

def on_mutation_request(self, ch, method, props, body):
        """Callback for messages in the 'rpc_mutations_queue'

        They say: "Hey, do you have a mutation for me?"
        """

        # This is the "remote procedure"
        # being called and returning a value
        mutation_obj = self.get_mutation()

        ch.basic_publish(exchange = '',
                         routing_key = props.reply_to,
                         properties = pika.BasicProperties(
                                    correlation_id = props.correlation_id),
                         body = mutation_obj.serialize_me())

        ch.basic_ack(delivery_tag = method.delivery_tag) 

def send_json_msg(self, rk, body, **kwargs):
		if self.connection.is_open:
			try:
				logging.debug("Sending message to manager")
				properties = pika.BasicProperties(content_type='application/json')
				self.channel.basic_publish(exchange=utils.EXCHANGE, routing_key=rk, body=json.dumps(body), properties=properties, **kwargs)
				return True
			except Exception as e:
				logging.exception("Error while sending data to queue:\n%s" % e)
				return False
		else:
			logging.error("Can't send message to manager")
			message = {"rk":rk, "body": body, "kwargs": kwargs, "json": True}
			if message not in self.message_queue: # could happen if we have another disconnect when we try to clear the message queue
				self.message_queue.append(message)
				logging.info("Added message to message queue")
			else:
				logging.debug("Message already in queue")

			return False
	
	# Try to resend the messages which couldn't be sent before 

def poll_mutation_queue(self):
        """
        In this paradigm calling means pushing our message
        to the queue (the callback will take care of it)
        and wait for the response and process it.
        @returns: string, serialized MutationObject (only attributes)
        """
        self.response = None
        self.corr_id = str(uuid.uuid4())
        self.channel.basic_publish(exchange = '',   # default exchange
                                   routing_key = 'rpc_mutations_queue',
                                   properties = pika.BasicProperties(
                                         reply_to = self.callback_queue,
                                         correlation_id = self.corr_id),
                                   body = 'POLL MUTATION QUEUE')

        self.ae.m_info("[x] Sent mutation queue poll")

        while self.response is None:
            # Waiting for a response
            self.connection.process_data_events()

        return self.response 

def send_evaluation(self, mutation_object):
        """
        In this paradigm calling means pushing our message
        to the queue (the callback will take care of it)
        and wait for the response and process it.
        @returns: string, serialized MutationObject (only attributes)
        """
        self.response = None
        self.corr_id = str(uuid.uuid4())
        self.channel.basic_publish(exchange = '',   # default exchange
                                   routing_key = 'rpc_evaluations_queue',
                                   properties = pika.BasicProperties(
                                         reply_to = self.callback_queue,
                                         correlation_id = self.corr_id),
                                   # This should be a serialized
                                   # evaluation object
                                   body = mutation_object.serialize_me())

        self.ae.m_info("[x] Sent evaluation")

        while self.response is None:
            # Waiting for a response
            self.connection.process_data_events()

        return self.response 

def whois_push(**whois_recv_info):
    global channel_whois
    # print 'whois push:', whois_recv_info
    result = ''
    try:
        result = json.dumps(whois_recv_info)
    except UnicodeDecodeError:
        for key in whois_recv_info.keys():
            if type(whois_recv_info[key]) == str:
                whois_recv_info[key] = whois_recv_info[key].decode('latin-1').encode("utf-8")
        result = json.dumps(whois_recv_info)
    if result != '':
        channel_whois.basic_publish(
            exchange='',
            routing_key='whois_queue',
            body=json.dumps(result),
            properties=pika.BasicProperties(
                delivery_mode=2)
        )


# 用来判断com_manage函数中，得到的whois信息是否包含xxx标志，若包括则需要重新发送 

def properties(self) -> dict:
        """
        Returns a dict from which a :class:`pika.BasicProperties` object can be created

        In this implementation, the following is returned:
        - headers
        - content message_type
        - priority
        - message id
        - message message_type

        """
        headers = {self.type_header: self.message.task}
        content_type = self.content_type
        priority = self.message.priority
        message_id = str(self.message.uuid)
        message_type = self.message_type

        return dict(headers=headers, content_type=content_type, priority=priority, message_id=message_id,
                    type=message_type) 

def message_worker(self):
        while 1:
            try:
                record, routing_key = self.queue.get()

                if not self.connection or self.connection.is_closed or not self.channel or self.channel.is_closed:
                    self.open_connection()

                self.channel.basic_publish(
                    exchange=self.exchange,
                    routing_key=routing_key,
                    body=record,
                    properties=pika.BasicProperties(
                        delivery_mode=2,
                        headers=self.message_headers
                    )
                )
            except Exception:
                self.channel, self.connection = None, None
                self.handleError(record)
            finally:
                self.queue.task_done()
                if self.close_after_emit:
                    self.close_connection() 

def publish_messages():
    global connection, channel, published, count, published, curr_pos, last_acked, curr_del_tag, exit_triggered
    for iteration in xrange(curr_pos, count):
        if channel.is_open:
            curr_del_tag += 1
            channel.basic_publish('', queue,
                                'Hello World!',
                                pika.BasicProperties(content_type='text/plain',
                                                        delivery_mode=2))
            published += 1
            curr_pos += 1
            
            # don't let the publishing advance further than 10000 messages past acks
            if curr_del_tag - last_acked > 9999:
                if channel.is_open:
                    connection.add_timeout(5, publish_messages)
                break
        else:
            print("Channel closed, ceasing publishing")
            break

    if curr_pos == count and exit_triggered == False:
        exit_triggered = True
        exit_program() 

def _make_amqp_properties_and_routing_keys(self):
        input_properties = pika.BasicProperties(**{
            'message_id': self.MESSAGE_ID,
            'type': 'rather-not-used-by-parsers :-)',
            'timestamp': self.MESSAGE_TIMESTAMP,
            'headers': dict(
                **self.MESSAGE_EXTRA_HEADERS),
        })
        if issubclass(self.PARSER_CLASS, BlackListParser):
            assert not issubclass(self.PARSER_CLASS, AggregatedEventParser)
            event_type = 'bl'
        elif issubclass(self.PARSER_CLASS, AggregatedEventParser):
            event_type = 'hifreq'
        else:
            event_type = 'event'
        input_rk = self.PARSER_SOURCE
        if self.PARSER_RAW_FORMAT_VERSION_TAG is not sentinel.not_set:
            input_rk = input_rk + '.' + self.PARSER_RAW_FORMAT_VERSION_TAG
        expected_output_rk = (event_type + '.parsed.' + self.PARSER_SOURCE)
        return input_properties, input_rk, expected_output_rk 

def register_consumer(self, queue_name: str, consumer: Consumer) -> None:
        def wrapped_consumer(ch: BlockingChannel,
                             method: Basic.Deliver,
                             properties: pika.BasicProperties,
                             body: str) -> None:
            try:
                consumer(ch, method, properties, body)
                ch.basic_ack(delivery_tag=method.delivery_tag)
            except Exception as e:
                ch.basic_nack(delivery_tag=method.delivery_tag)
                raise e

        self.consumers[queue_name].add(wrapped_consumer)
        self.ensure_queue(queue_name,
                          lambda: self.channel.basic_consume(queue_name, wrapped_consumer,
                                                             consumer_tag=self._generate_ctag(queue_name))) 

def register_consumer(self, queue_name: str, consumer: Consumer) -> None:
        def wrapped_consumer(ch: BlockingChannel,
                             method: Basic.Deliver,
                             properties: pika.BasicProperties,
                             body: str) -> None:
            consumer(ch, method, properties, body)
            ch.basic_ack(delivery_tag=method.delivery_tag)

        if not self.ready():
            self.consumers[queue_name].add(wrapped_consumer)
            return

        self.consumers[queue_name].add(wrapped_consumer)
        self.ensure_queue(queue_name,
                          lambda: self.channel.basic_consume(queue_name, wrapped_consumer,
                                                             consumer_tag=self._generate_ctag(queue_name))) 

def poll_mutation_queue(self):
        """
        In this paradigm calling means pushing our message
        to the queue (the callback will take care of it)
        and wait for the response and process it.
        @returns: string, serialized MutationObject (only attributes)
        """
        self.response = None
        self.corr_id = str(uuid.uuid4())
        self.channel.basic_publish(exchange = '',   # default exchange
                                   routing_key = 'rpc_mutations_queue',
                                   properties = pika.BasicProperties(
                                         reply_to = self.callback_queue,
                                         correlation_id = self.corr_id),
                                   body = 'POLL MUTATION QUEUE')

        self.ae.m_info("[x] Sent mutation queue poll")

        while self.response is None:
            # Waiting for a response
            self.connection.process_data_events()

        return self.response 

def send_evaluation(self, mutation_object):
        """
        In this paradigm calling means pushing our message
        to the queue (the callback will take care of it)
        and wait for the response and process it.
        @returns: string, serialized MutationObject (only attributes)
        """
        self.response = None
        self.corr_id = str(uuid.uuid4())
        self.channel.basic_publish(exchange = '',   # default exchange
                                   routing_key = 'rpc_evaluations_queue',
                                   properties = pika.BasicProperties(
                                         reply_to = self.callback_queue,
                                         correlation_id = self.corr_id),
                                   # This should be a serialized
                                   # evaluation object
                                   body = mutation_object.serialize_me())

        self.ae.m_info("[x] Sent evaluation")

        while self.response is None:
            # Waiting for a response
            self.connection.process_data_events()

        return self.response 

def publish_message(self, body, durable=True):
        self.__channel.exchange_declare(exchange=self.__exchange, exchange_type=self.__exchange_type)
        if self.__queue_name:
            result = self.__channel.queue_declare(queue=self.__queue_name)
        else:
            result = self.__channel.queue_declare(exclusive=True)

        self.__channel.queue_bind(exchange=self.__exchange, queue=result.method.queue)

        if durable:
            properties = pika.BasicProperties(delivery_mode=2)
            self.__channel.basic_publish(exchange=self.__exchange, routing_key=self.__routing_key, body=body,
                                         properties=properties)
        else:
            self.__channel.basic_publish(exchange=self.__exchange, routing_key=self.__routing_key, body=body)
        ins_log.read_log('info', 'Publish message %s sucessfuled.' % body) 

def call(self, parameters):
        assert type(parameters) == str
        self.id = str(uuid.uuid4())
        properties = pika.BasicProperties(reply_to=self.callback_queue, correlation_id=self.id)
        self.channel.basic_publish(
            exchange='', routing_key=self.rabbit_queue, properties=properties, body=parameters
        )
        while self.response is None:
            time.sleep(3)
        print(" [*] Got fitness for individual {}".format(json.loads(parameters)[0]))
        self.responses.put(self.response)
        # Job is completed, remove job order from queue
        self.jobs.get()
        self.jobs.task_done()
        # Close RabbitMQ connection to prevent file descriptors limit from blocking server
        self.connection.close() 

def on_request(self, channel, method, properties, body):
        i, genes, additional_parameters = json.loads(body)
        # If an additional parameter is received as a list, convert to tuple
        for param in additional_parameters.keys():
            if isinstance(additional_parameters[param], list):
                additional_parameters[param] = tuple(additional_parameters[param])
        print(" [.] Evaluating individual {}".format(i))
        # print("     ... Genes: {}".format(str(genes)))
        # print("     ... Other: {}".format(str(additional_parameters)))
        # Run model and return fitness metric
        individual = self.individual(self.x_train, self.y_train, genes=genes, **additional_parameters)
        fitness = individual.get_fitness()
        # Prepare response for master and send it
        response = json.dumps([i, fitness])
        channel.basic_publish(
            exchange='', routing_key=properties.reply_to,
            properties=pika.BasicProperties(correlation_id=properties.correlation_id), body=response
        )
        channel.basic_ack(delivery_tag=method.delivery_tag) 

def send(message_json):
    """
    :param message:
    :return:
    """
    global logger
    connection = pika.BlockingConnection(pika.ConnectionParameters(rabbit_host))
    channel = connection.channel()
    queue = channel.queue_declare(queue=queue_name, durable=True, auto_delete=True)
    logger.debug("send> number of messages in the queue is: "+str(queue.method.message_count))
    message = json.dumps(message_json)
    logger.debug("send> sending message")
    logger.debug(message)
    channel.basic_publish(exchange='',
                          routing_key=queue_name,
                          body=message,
                          properties=pika.BasicProperties(
                              delivery_mode=2,  # make message persistent
                          ))
    connection.close() 

def sendData(self, server, queue, ci, data):
        self.logger.debug('Sending [{}] to [{}] Start'.format(ci, server))
        ## conn = pika.BlockingConnection(
                    ## pika.ConnectionParameters(server)
                ## )

        with pika.BlockingConnection(pika.ConnectionParameters(host = server,
                                        port = self.MQ_PORT)) as conn:
            for line in data:
                chan = conn.channel()
                chan.queue_declare(queue = queue, durable = True)
                chan.basic_publish(exchange = '', routing_key = queue, body = line,
                                    properties = pika.BasicProperties(delivery_mode = 2, ))
        self.logger.debug('Sending [{}] to [{}] Done'.format(ci, server))
        return(True)

    ## run func 

def send_request_to_spark_cluster(message):
    with create_app().app_context():
        rabbitmq_connection = utils.connect_to_rabbitmq(
            username=current_app.config['RABBITMQ_USERNAME'],
            password=current_app.config['RABBITMQ_PASSWORD'],
            host=current_app.config['RABBITMQ_HOST'],
            port=current_app.config['RABBITMQ_PORT'],
            virtual_host=current_app.config['RABBITMQ_VHOST'],
            error_logger=current_app.logger,
        )
        try:
            channel = rabbitmq_connection.channel()
            channel.exchange_declare(exchange=current_app.config['SPARK_REQUEST_EXCHANGE'], exchange_type='fanout')
            channel.basic_publish(
                exchange=current_app.config['SPARK_REQUEST_EXCHANGE'],
                routing_key='',
                body=message,
                properties=pika.BasicProperties(delivery_mode=2,),
            )
        except Exception:
            # this is a relatively non critical part of LB for now, so just log the error and
            # move ahead
            current_app.logger.error('Could not send message to spark cluster: %s', ujson.dumps(message), exc_info=True) 

def broadcastmsgproducer(self,queuename,message, emulatername, action):
    # Get the service resource
        try:
            routingkey = queuename+".*"
            exchange = queuename+".exchange"

            self.response = None
            self.corr_id = str(uuid.uuid4())

            res = self.channel.basic_publish(exchange=exchange,
                      routing_key=routingkey,
                      body=message,properties = pika.BasicProperties(reply_to = self.callback_queue,
                                             correlation_id = self.corr_id,headers={'emulator_name':emulatername, 'action': action}))

            if res:
                self.connection.close()
                return self.corr_id

            else:
                self.connection.close()
                return False
        except Exception as e:
            print e
            return False 

def msgproducer(self,queuename,message, emulatername, action):
    # Get the service resource
        try:
            routingkey = queuename+".*"
            exchange = queuename+".exchange"

            self.response = None
            self.corr_id = str(uuid.uuid4())

            res = self.channel.basic_publish(exchange=exchange,
                      routing_key=routingkey,
                      body=message,properties = pika.BasicProperties(reply_to = self.callback_queue,
                                             correlation_id = self.corr_id,headers={'emulator_name':emulatername, 'action': action}))

            if res:
                self.connection.close()
                return self.corr_id

            else:
                self.connection.close()
                return False
        except Exception as e:
            print e
            return False 

def msgproducer(self,queuename,message, emulatername):
    # Get the service resource
        try:
            routingkey = queuename+".*"
            exchange = queuename+".exchange"

            self.response = None
            self.corr_id = str(uuid.uuid4())

            res = self.channel.basic_publish(exchange=exchange,
                      routing_key=routingkey,
                      body=message,properties = pika.BasicProperties(reply_to = self.callback_queue,
                                             correlation_id = self.corr_id,headers={'emulator_name':emulatername}))

            if res:
                self.connection.close()
                return self.corr_id

            else:
                self.connection.close()
                return False
        except Exception as e:
            print e
            return False 

def msgproducer(self,queuename):
    # Get the service resource
        try:
            routingkey = queuename+".*"
            exchange = queuename+".exchange"

            self.response = None
            self.corr_id = str(uuid.uuid4())

            res = self.channel.basic_publish(exchange=exchange,
                      routing_key=routingkey,
                      body=self.corr_id,properties = pika.BasicProperties(reply_to = self.callback_queue,
                                             correlation_id = self.corr_id))

            if res:
                self.connection.close()
                return self.corr_id

            else:
                self.connection.close()
                return False
        except Exception as e:
            print e
            return False 

def whois_push(**whois_recv_info):
    global channel_whois
    # print 'whois push:', whois_recv_info
    result = ''
    try:
        result = json.dumps(whois_recv_info)
    except UnicodeDecodeError:
        for key in whois_recv_info.keys():
            if type(whois_recv_info[key]) == str:
                whois_recv_info[key] = whois_recv_info[key].decode('latin-1').encode("utf-8")
        result = json.dumps(whois_recv_info)
    if result != '':
        channel_whois.basic_publish(
            exchange='',
            routing_key='whois_queue',
            body=json.dumps(result),
            properties=pika.BasicProperties(
                delivery_mode=2)
        )


# 用来判断com_manage函数中，得到的whois信息是否包含xxx标志，若包括则需要重新发送 

def got_config_request(self, ch, method, properties, body):
		ip_addresses = json.loads(body)
		logging.info("Got config request with following IP addresses: %s" % ip_addresses)

		pi_id = None
		worker = db.session.query(db.objects.Worker).filter(db.objects.Worker.address.in_(ip_addresses)).first()
		if worker:
			pi_id = worker.id
			logging.debug("Found worker id %s for IP address %s" % (pi_id, worker.address))
		else: # wasn't able to find worker with given ip address(es)
			logging.error("Wasn't able to find worker for given IP adress(es)")
			reply_properties = pika.BasicProperties(correlation_id=properties.correlation_id)
			self.channel.basic_publish(exchange=utils.EXCHANGE, properties=reply_properties, routing_key=properties.reply_to, body="")
			return
		
		config = self.prepare_config(pi_id)
		logging.info("Sending intial config to worker with id %s" % pi_id)
		reply_properties = pika.BasicProperties(correlation_id=properties.correlation_id, content_type='application/json')
		self.channel.basic_publish(exchange=utils.EXCHANGE, properties=reply_properties, routing_key=properties.reply_to, body=json.dumps(config))

	# callback method for when the manager recieves data after a worker executed its actions 

def main(self):
        """Make connections to sock and rabbit and receive messages from sock
        to sent to rabbit
        """
        # ensure connections to the socket and rabbit before getting messages
        if self.rabbit_conn() and self.socket_conn():
            socket_ok = True
            events = []
            while socket_ok:
                if not events:
                    socket_ok, events = self.poll_events()
                try:
                    for event in events:
                        self.channel.basic_publish(
                            exchange=self.exchange,
                            routing_key=self.routing_key,
                            body=event,
                            properties=pika.BasicProperties(delivery_mode=2,))
                    events = []
                except pika.exceptions.AMQPError as err:
                    print("Unable to send events %s to RabbitMQ: %s, retrying" % (  # pylint: disable=print-statement
                        events, err))
                    time.sleep(1)
                    self.rabbit_conn()
                    sys.stdout.flush()
            self.sock.close() 

def send(self, body="null", priority=1):
        if not self.connection.is_open:
            self.connection = pika.BlockingConnection(self.parameters)
            self.channel = self.connection.channel()
            self.priority_queue = {"x-max-priority": 20, "x-message-ttl": 60000}
            self.channel.queue_declare(queue=self.queue, arguments=self.priority_queue)
        self.channel.basic_publish(
            exchange="",
            routing_key=self.queue,
            body=body,
            properties=pika.BasicProperties(
                content_type="text/plain", priority=priority
            ),
        )
        # self.connection.close() 

def publish_message(self, body, durable=True):
        self.__channel.exchange_declare(exchange=self.__exchange, exchange_type=self.__exchange_type)
        if self.__queue_name:
            result = self.__channel.queue_declare(queue=self.__queue_name)
        else:
            result = self.__channel.queue_declare(exclusive=True)
        self.__channel.queue_bind(exchange=self.__exchange, queue=result.method.queue)
        if durable:
            properties = pika.BasicProperties(delivery_mode=2)
            self.__channel.basic_publish(exchange=self.__exchange, routing_key=self.__routing_key, body=body,
                                         properties=properties)
        else:
            self.__channel.basic_publish(exchange=self.__exchange, routing_key=self.__routing_key, body=body)
        ins_log.read_log('info', 'Publish message %s sucessfuled.' % body) 

def _publish(self, data, routing_key, custom_prop_kwargs):
        if data is DoNotPublish:
            return
        prop_kwargs = self._prop_kwargs
        if custom_prop_kwargs:
            prop_kwargs = dict(prop_kwargs, **custom_prop_kwargs)
        properties = pika.BasicProperties(**prop_kwargs)
        with self._connection_lock_nonblocking:
            self._channel.basic_publish(exchange=self._exchange_name,
                                        routing_key=routing_key,
                                        body=data,
                                        properties=properties,
                                        mandatory=self._mandatory)