Python zmq.NOBLOCK Examples

def receive(self, timeout=30):
        """
        Try to send the message until it either has been received or
        hits timeout.

        :param timeout: Timeout to retry receiving the message
        :type timeout: ``int``

        :return: The received message
        :rtype: ``object`` or ``str`` or ``zmq.sugar.frame.Frame``
        """
        return retry_until_timeout(
            exception=zmq.ZMQError,
            item=self._socket.recv,
            kwargs={"flags": zmq.NOBLOCK},
            timeout=timeout,
            raise_on_timeout=True,
        ) 

def shutdown(self):
        """Shut down paired listener with <END> signal."""
        if hasattr(self, 'socket'):
            try:
                self.socket.send_unicode('<END>', zmq.NOBLOCK)
            except zmq.error.ZMQError:
                # may need to listen first
                try:
                    self.socket.recv_unicode(zmq.NOBLOCK)
                    self.socket.send_unicode('<END>', zmq.NOBLOCK)
                except zmq.error.ZMQError:
                    # paired process is probably dead already
                    pass
        if hasattr(self, 'process'):
            # try to let the subprocess clean up, but don't wait too long
            try:
                self.process.communicate(timeout=1)
            except subprocess.TimeoutExpired:
                self.process.kill() 

def test_root_topic(self):
        logger, handler, sub = self.connect_handler()
        handler.socket.bind(self.iface)
        sub2 = sub.context.socket(zmq.SUB)
        self.sockets.append(sub2)
        sub2.connect(self.iface)
        sub2.setsockopt(zmq.SUBSCRIBE, b'')
        handler.root_topic = b'twoonly'
        msg1 = 'ignored'
        logger.info(msg1)
        self.assertRaisesErrno(zmq.EAGAIN, sub.recv, zmq.NOBLOCK)
        topic,msg2 = sub2.recv_multipart()
        self.assertEqual(topic, b'twoonly.INFO')
        self.assertEqual(msg2, b(msg1)+b'\n')
        
        logger.removeHandler(handler) 

def senddat(self, data):
        data['stationid'] = self.stationid
        data['dt'] = int(time.time())
        data.update(self.loc)
        data['rand'] = str(uuid4())[:8]
        m = md5()
        m.update((self.station_pass + data['rand'] + str(data['dt']))
                .encode('utf-8'))
        data['sign'] = m.hexdigest()[:12]

        self.datq.put(data)

        if not self.connected:
            return

        while not self.datq.empty():
            try:
                self.datsock.send_string(json.dumps(self.datq.get()),
                        zmq.NOBLOCK)
            except Exception as e:
                return 

def _handle_recv(self):
        """Handle a recv event."""
        if self._flushed:
            return
        try:
            msg = self.socket.recv_multipart(zmq.NOBLOCK, copy=self._recv_copy)
        except zmq.ZMQError as e:
            if e.errno == zmq.EAGAIN:
                # state changed since poll event
                pass
            else:
                raise
        else:
            if self._recv_callback:
                callback = self._recv_callback
                self._run_callback(callback, msg) 

def build_device(self, mon_sub=b"", in_prefix=b'in', out_prefix=b'out'):
        self.device = devices.ThreadMonitoredQueue(zmq.PAIR, zmq.PAIR, zmq.PUB,
                                            in_prefix, out_prefix)
        alice = self.context.socket(zmq.PAIR)
        bob = self.context.socket(zmq.PAIR)
        mon = self.context.socket(zmq.SUB)
        
        aport = alice.bind_to_random_port('tcp://127.0.0.1')
        bport = bob.bind_to_random_port('tcp://127.0.0.1')
        mport = mon.bind_to_random_port('tcp://127.0.0.1')
        mon.setsockopt(zmq.SUBSCRIBE, mon_sub)
        
        self.device.connect_in("tcp://127.0.0.1:%i"%aport)
        self.device.connect_out("tcp://127.0.0.1:%i"%bport)
        self.device.connect_mon("tcp://127.0.0.1:%i"%mport)
        self.device.start()
        time.sleep(.2)
        try:
            # this is currenlty necessary to ensure no dropped monitor messages
            # see LIBZMQ-248 for more info
            mon.recv_multipart(zmq.NOBLOCK)
        except zmq.ZMQError:
            pass
        self.sockets.extend([alice, bob, mon])
        return alice, bob, mon 

def test_root_topic(self):
        logger, handler, sub = self.connect_handler()
        handler.socket.bind(self.iface)
        sub2 = sub.context.socket(zmq.SUB)
        self.sockets.append(sub2)
        sub2.connect(self.iface)
        sub2.setsockopt(zmq.SUBSCRIBE, b'')
        handler.root_topic = b'twoonly'
        msg1 = 'ignored'
        logger.info(msg1)
        self.assertRaisesErrno(zmq.EAGAIN, sub.recv, zmq.NOBLOCK)
        topic,msg2 = sub2.recv_multipart()
        self.assertEqual(topic, b'twoonly.INFO')
        self.assertEqual(msg2, b(msg1)+b'\n')
        
        logger.removeHandler(handler) 

def _read_next_input_message():
  """Reads the next message from stdin_socket.

  Returns:
    _NOT_READY if input is not available.
  """
  kernel = ipython.get_kernel()
  stdin_socket = kernel.stdin_socket

  reply = None
  try:
    _, reply = kernel.session.recv(stdin_socket, zmq.NOBLOCK)
  except Exception:  # pylint: disable=broad-except
    # We treat invalid messages as empty replies.
    pass
  if reply is None:
    return _NOT_READY

  # We want to return '' even if reply is malformed.
  return reply.get('content', {}).get('value', '') 

def build_device(self, mon_sub=b"", in_prefix=b'in', out_prefix=b'out'):
        self.device = devices.ThreadMonitoredQueue(zmq.PAIR, zmq.PAIR, zmq.PUB,
                                            in_prefix, out_prefix)
        alice = self.context.socket(zmq.PAIR)
        bob = self.context.socket(zmq.PAIR)
        mon = self.context.socket(zmq.SUB)
        
        aport = alice.bind_to_random_port('tcp://127.0.0.1')
        bport = bob.bind_to_random_port('tcp://127.0.0.1')
        mport = mon.bind_to_random_port('tcp://127.0.0.1')
        mon.setsockopt(zmq.SUBSCRIBE, mon_sub)
        
        self.device.connect_in("tcp://127.0.0.1:%i"%aport)
        self.device.connect_out("tcp://127.0.0.1:%i"%bport)
        self.device.connect_mon("tcp://127.0.0.1:%i"%mport)
        self.device.start()
        time.sleep(.2)
        try:
            # this is currenlty necessary to ensure no dropped monitor messages
            # see LIBZMQ-248 for more info
            mon.recv_multipart(zmq.NOBLOCK)
        except zmq.ZMQError:
            pass
        self.sockets.extend([alice, bob, mon])
        return alice, bob, mon 

def _handle_recv(self):
        """Handle a recv event."""
        if self._flushed:
            return
        try:
            msg = self.socket.recv_multipart(zmq.NOBLOCK, copy=self._recv_copy)
        except zmq.ZMQError as e:
            if e.errno == zmq.EAGAIN:
                # state changed since poll event
                pass
            else:
                raise
        else:
            if self._recv_callback:
                callback = self._recv_callback
                self._run_callback(callback, msg) 

def _transmit_one_msg_throughlistener(self, msg, ident) -> Tuple[bool, Optional[str], bool]:

        def prepare_error_msg(ex):
            err_str = '{} got error {} while sending through listener to {}' \
                .format(self, ex, ident)
            print(err_str)
            return err_str

        need_to_resend = False
        if isinstance(ident, str):
            ident = ident.encode()
        try:
            msg = self._prepare_to_send(msg)
            self.listener.send_multipart([ident, msg], flags=zmq.NOBLOCK)
        except zmq.Again as ex:
            need_to_resend = True
            return False, prepare_error_msg(ex), need_to_resend
        except zmq.ZMQError as ex:
            need_to_resend = (ex.errno == 113)
            return False, prepare_error_msg(ex), need_to_resend
        except Exception as ex:
            return False, prepare_error_msg(ex), need_to_resend
        return True, None, need_to_resend 

def poll_command_request(self):
        """If the command RPC socket has an incoming request,
        separate it into its action and its params and put it
        on the command request queue.
        """
        try:
            message = self.rpc.recv(zmq.NOBLOCK)
        except zmq.ZMQError as exc:
            if exc.errno == zmq.EAGAIN:
                return
            else:
                raise

        _logger.debug("Received command %s", message)
        segments = _unpack(message)
        action, params = segments[0], segments[1:]
        _logger.debug("Adding %s, %s to the request queue", action, params)
        self._command = _Command(action, params) 

def get_command(self):
        """Attempt to return a unicode object from the command socket
        
        If no message is available without blocking (as opposed to a blank 
        message), return None
        """
        try:
            message_bytes = self.socket.recv(zmq.NOBLOCK)
            log.debug("Received message: %r", message_bytes)
        except zmq.ZMQError as exc:
            if exc.errno == zmq.EAGAIN:
                return None
            else:
                raise
        else:
            return message_bytes.decode(config.CODEC) 

def generator_from_zmq_pull(context, host):
    socket = context.socket(zmq.PULL)
    # TODO: Configure socket with clean properties to avoid message overload.
    if host.endswith('/'):
        host = host[:-1]
    print_item("+", "Binding ZMQ pull socket : " + colorama.Fore.CYAN + "{0}".format(host) + colorama.Style.RESET_ALL)
    socket.bind(host)

    while True:
        try:
            message = socket.recv(flags=zmq.NOBLOCK)
        except zmq.Again as e:
            message = None
        if message is None:
            yield None # NOTE: We have to make the generator non blocking.
        else:
            task = json.loads(message)
            yield task 

def _handle_recv(self):
        """Handle a recv event."""
        if self._flushed:
            return
        try:
            msg = self.socket.recv_multipart(zmq.NOBLOCK, copy=self._recv_copy)
        except zmq.ZMQError as e:
            if e.errno == zmq.EAGAIN:
                # state changed since poll event
                pass
            else:
                gen_log.error("RECV Error: %s"%zmq.strerror(e.errno))
        else:
            if self._recv_callback:
                callback = self._recv_callback
                # self._recv_callback = None
                self._run_callback(callback, msg)
                
        # self.update_state() 

def _abort_queue(self, stream):
        poller = zmq.Poller()
        poller.register(stream.socket, zmq.POLLIN)
        while True:
            idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True)
            if msg is None:
                return

            self.log.info("Aborting:")
            self.log.info("%s", msg)
            msg_type = msg['header']['msg_type']
            reply_type = msg_type.split('_')[0] + '_reply'

            status = {'status' : 'aborted'}
            md = {'engine' : self.ident}
            md.update(status)
            reply_msg = self.session.send(stream, reply_type, metadata=md,
                        content=status, parent=msg, ident=idents)
            self.log.debug("%s", reply_msg)
            # We need to wait a bit for requests to come in. This can probably
            # be set shorter for true asynchronous clients.
            poller.poll(50) 

def receive(self, timeout=30):
        """
        Try to receive the message until it has either been received or
        hits timeout.

        :param timeout: Timeout to retry receiving the message.
        :type timeout: ``int``

        :return: The received message.
        :rtype: ``bytes`` or ``zmq.sugar.frame.Frame`` or ``memoryview``
        """
        return retry_until_timeout(
            exception=zmq.ZMQError,
            item=self._socket.recv,
            kwargs={"flags": zmq.NOBLOCK},
            timeout=timeout,
            raise_on_timeout=True,
        ) 

def send(self, data, timeout=30):
        """
        Try to send the message until it either sends or hits timeout.

        :param data: The content of the message.
        :type data: ``bytes`` or ``zmq.sugar.frame.Frame`` or ``memoryview``
        :param timeout: Timeout to retry sending the message.
        :type timeout: ``int``

        :return: ``None``
        :rtype: ``NoneType``
        """
        return retry_until_timeout(
            exception=zmq.ZMQError,
            item=self._socket.send,
            kwargs={"data": data, "flags": zmq.NOBLOCK},
            timeout=timeout,
            raise_on_timeout=True,
        ) 

def recv_array(socket, flags=0, copy=False, track=False, block=True):
    """recv a numpy array"""
    if block:
        md = socket.recv_json(flags=flags)
        msg = socket.recv(flags=flags, copy=copy, track=track)
        buf = bytearray(msg)
        A = np.frombuffer(buf, dtype=md['dtype'])
        return A.reshape(md['shape'])
    else:
        try:
            md = socket.recv_json(flags=flags | zmq.NOBLOCK)
            msg = socket.recv(flags=flags | zmq.NOBLOCK, copy=copy, track=track)
            buf = bytearray(msg)
            A = np.frombuffer(buf, dtype=md['dtype'])
            return A.reshape(md['shape'])
        except zmq.Again:
            return False 

def transmit(self, msg, uid, timeout=None, serialized=False, is_batch=False):
        remote = self.remotes.get(uid)
        err_str = None
        if not remote:
            return False, err_str
        socket = remote.socket
        if not socket:
            return False, err_str
        try:
            if not serialized:
                msg = self.prepare_to_send(msg)

            print('{} transmitting message {} to {} by socket {} {}'
                  .format(self, msg, uid, socket.FD, socket.underlying))
            socket.send(msg, flags=zmq.NOBLOCK)

            return True, err_str
        except zmq.Again:
            print('{} could not transmit message to {}'.format(self, uid))
        return False, err_str 

def _receiveFromListener(self, quota: Quota) -> int:
        """
        Receives messages from listener
        :param quota: number of messages to receive
        :return: number of received messages
        """
        i = 0
        incoming_size = 0
        try:
            ident, msg = self.listener.recv_multipart(flags=zmq.NOBLOCK)
            if msg:
                # Router probing sends empty message on connection
                incoming_size += len(msg)
                i += 1
                self._verifyAndAppend(msg, ident)
        except zmq.Again as e:
            return i
        except zmq.ZMQError as e:
            print("Strange ZMQ behaviour during node-to-node message receiving, experienced {}".format(e))
        if i > 0:
            print('{} got {} messages through listener'.
                  format(self, i))
        return i 

def _sendReply(self, destination, fid, *args):
        """Send a REPLY directly to its destination. If it doesn't work, launch
        it back to the broker."""
        # Try to send the result directly to its parent
        self.addPeer(destination)

        try:
            self.direct_socket.send_multipart([
                destination,
                REPLY,
            ] + list(args),
                flags=zmq.NOBLOCK)
        except zmq.error.ZMQError as e:
            # Fallback on Broker routing if no direct connection possible
            scoop.logger.debug(
                "{0}: Could not send result directly to peer {1}, routing through "
                "broker.".format(scoop.worker, destination)
            )
            self.socket.send_multipart([
                REPLY, 
                ] + list(args) + [
                destination,
            ])

        self.socket.send_multipart([
            STATUS_DONE,
            fid,
        ]) 

def step_async(self, actions):
        action_dicts = dlist_to_listd(actions)
        # zmq send
        for socket, action in zip(self._zmq_sockets, action_dicts):
            msg = json.dumps({k: v.item() for k, v in action.items()})
            socket.send(msg.encode(), zmq.NOBLOCK, copy=False, track=False)

        self.waiting = True 

def receive_loop(self):
        """
        This is the receive loop for Banyan messages.

        This method may be overwritten to meet the needs
        of the application before handling received messages.

        """
        while True:
            try:
                data = self.subscriber.recv_multipart(zmq.NOBLOCK)
                if self.numpy:
                    payload2 = {}
                    payload = msgpack.unpackb(data[1], object_hook=m.decode)
                    # convert keys to strings
                    # this compensates for the breaking change in msgpack-numpy 0.4.1 to 0.4.2
                    for key, value in payload.items():
                        if not type(key) == str:
                            key = key.decode('utf-8')
                            payload2[key] = value

                    if payload2:
                        payload = payload2
                    self.incoming_message_processing(data[0].decode(), payload)
                else:
                    self.incoming_message_processing(data[0].decode(),
                                                     msgpack.unpackb(data[1], raw=False))
            # if no messages are available, zmq throws this exception
            except zmq.error.Again:
                try:
                    if self.receive_loop_idle_addition:
                        self.receive_loop_idle_addition()
                    time.sleep(self.loop_time)
                except KeyboardInterrupt:
                    self.clean_up()
                    raise KeyboardInterrupt 

def receive(self):
        """ Reception and pyobj de-serialization of one message. """
        return self.socket.recv_pyobj(zmq.NOBLOCK) 

def send_check_address(self, address_name):
        """ Send request to check address. """
        self.logger.trace('send CHECK_ADDRESS {}'.format(address_name))
        try:
            self.socket.send_pyobj((DeferredRequestHeaders.CHECK_ADDRESS,
                                    (address_name, )),
                                   zmq.NOBLOCK)
        except zmq.error.Again:
            self.logger.error('CHECK_ADDRESS not sent') 

def run(self):
        while True:
            try:
                message = self.pull.recv(flags=zmq.NOBLOCK)
            except zmq.Again as e:
                message = None
            if message is not None:
                task = json.loads(message)
                self.redis.setex(
                    task['transaction'],
                    self.result_expiration,
                    json.dumps(task['data'])
                ) 

def send_isolate_addresses(self, address_names):
        """ Send request to isolate address. """
        self.logger.trace('send ISOLATE_ADDRESSES {}'.format(address_names))
        try:
            self.socket.send_pyobj((DeferredRequestHeaders.ISOLATE_ADDRESSES,
                                    address_names),
                                   zmq.NOBLOCK)
        except zmq.error.Again:
            self.logger.error('ISOLATE_ADDRESSES not sent') 

def receive(self):
        """
        Worker tries to receive the response to the message sent until timeout.

        :return: Response to the message sent.
        :type: :py:class:`~testplan.runners.pools.communication.Message`
        """
        start_time = time.time()

        while self.active:
            try:
                received = self._sock.recv(flags=zmq.NOBLOCK)
                try:
                    loaded = pickle.loads(received)
                except Exception as exc:
                    print("Deserialization error. - {}".format(exc))
                    raise
                else:
                    return loaded
            except zmq.Again:
                if time.time() - start_time > self._recv_timeout:
                    print(
                        "Transport receive timeout {}s reached!".format(
                            self._recv_timeout
                        )
                    )
                    return None
                time.sleep(self._recv_sleep)
        return None 

def send_start_process(self, address_name, namespec, extra_args):
        """ Send request to start process. """
        self.logger.trace('send START_PROCESS {} to {} with {}'.format(
            namespec, address_name, extra_args))
        try:
            self.socket.send_pyobj((DeferredRequestHeaders.START_PROCESS,
                                    (address_name, namespec, extra_args)),
                                   zmq.NOBLOCK)
        except zmq.error.Again:
            self.logger.error('START_PROCESS not sent')