Python zmq.eventloop.zmqstream.ZMQStream() Examples

def  __init__(self, zmq_context, trade_in_socket, trade_pub = None, reopen=True):
    self.zmq_context      = zmq_context
    self.connection_id    = None
    self.trade_in_socket  = trade_in_socket
    self.is_logged        = False
    self.user_id          = None
    self.reopen           = reopen
    self.trade_pub        = trade_pub

    self.trade_pub_socket = None
    self.trade_pub_socket_stream = None

    if self.trade_pub:
      self.trade_pub_socket = self.zmq_context.socket(zmq.SUB)
      self.trade_pub_socket.connect(self.trade_pub)
      self.trade_pub_socket_stream = ZMQStream(self.trade_pub_socket)
      self.trade_pub_socket_stream.on_recv(self._on_trade_publish) 

def connect(self):
    if not self.trade_pub_socket and self.trade_pub:
      self.trade_pub_socket = self.zmq_context.socket(zmq.SUB)
      self.trade_pub_socket.connect(self.trade_pub)
      self.trade_pub_socket_stream = ZMQStream(self.trade_pub_socket)
      self.trade_pub_socket_stream.on_recv(self._on_trade_publish)

    self.trade_in_socket.send( "OPN," + base64.b32encode(os.urandom(10)))
    response_message = self.trade_in_socket.recv()
    opt_code    = response_message[:3]
    raw_message = response_message[4:]

    if opt_code != 'OPN':
      if opt_code == 'ERR':
        raise TradeClientException( error_message = raw_message )

      raise TradeClientException( error_message = 'Protocol Error: Unknown message opt_code received' )

    self.connection_id = raw_message 

def register(self):
        """send the registration_request"""

        self.log.info("Registering with controller at %s"%self.url)
        ctx = self.context
        connect,maybe_tunnel = self.init_connector()
        reg = ctx.socket(zmq.DEALER)
        reg.setsockopt(zmq.IDENTITY, self.bident)
        connect(reg, self.url)
        self.registrar = zmqstream.ZMQStream(reg, self.loop)


        content = dict(uuid=self.ident)
        self.registrar.on_recv(lambda msg: self.complete_registration(msg, connect, maybe_tunnel))
        # print (self.session.key)
        self.session.send(self.registrar, "registration_request", content=content) 

def init_networking(self):
        """
        Creates socket, connects to specified port on localhost,
        and starts the :meth:`~Net_Node.threaded_loop` as a daemon thread.
        """
        self.sock = self.context.socket(zmq.DEALER)
        self.sock.identity = self.id
        #self.sock.probe_router = 1

        # net nodes are local only
        self.sock.connect('tcp://localhost:{}'.format(self.port))

        # wrap in zmqstreams and start loop thread
        self.sock = ZMQStream(self.sock, self.loop)
        self.sock.on_recv(self.handle_listen)

        self.loop_thread = threading.Thread(target=self.threaded_loop)
        self.loop_thread.daemon = True
        self.loop_thread.start()

        self.repeat_thread = threading.Thread(target=self.repeat)
        self.repeat_thread.daemon = True
        self.repeat_thread.start()

        #self.connected = True 

def __init__(self, queue):
        self.queue = queue
        self.io_loop = IOLoop(make_current=False)
        context = zmq.Context.instance()
        self.pause_socket = context.socket(zmq.REP)
        self.pause_socket_port = self.pause_socket.bind_to_random_port(
            "tcp://127.0.0.1")
        self.status_socket = context.socket(zmq.PULL)
        self.status_socket_port = self.status_socket.bind_to_random_port(
            "tcp://127.0.0.1")

        self.pause_stream = ZMQStream(self.pause_socket, self.io_loop)
        self.status_stream = ZMQStream(self.status_socket, self.io_loop)

        self.pause_stream.on_recv(self.recv_pause)
        self.status_stream.on_recv(self.recv_status)
        self.thread = threading.Thread(target=self._thread_main)
        self.thread.daemon = True 

def run(self):
        """The thread's main activity.  Call start() instead."""
        self.socket = self.context.socket(zmq.DEALER)
        self.socket.setsockopt(zmq.IDENTITY, self.session.bsession)
        self.socket.connect(self.address)
        self.stream = zmqstream.ZMQStream(self.socket, self.ioloop)
        self.stream.on_recv(self._handle_recv)
        self._run_loop()
        try:
            self.socket.close()
        except:
            pass 

def start(self):
        """Start ZAP authentication"""
        super(IOLoopAuthenticator, self).start()
        self.zap_stream = zmqstream.ZMQStream(self.zap_socket, self.io_loop)
        self.zap_stream.on_recv(self.handle_zap_message) 

def setup_zmq(self, url):
        zmq_socket = self.context.socket(zmq.REP)
        zmq_socket.bind(url)
        zmq_stream = ZMQStream(zmq_socket)
        zmq_stream.on_recv(self.handle_zmq)
        return zmq_socket, zmq_stream, url 

def __init__(self, name, end_point, context=None):
        super(CommandHandlerConnector, self).__init__(name, end_point, context)
        self._sockt = self._context.socket(zmq.REQ)
        self._stream = zmqstream.ZMQStream(self._sockt)
        self._stream.on_recv_stream(self._on_recv) 

def __init__(self, registry, name, end_point, sockt):
        self._name = name
        self._end_point = end_point
        self._stream = zmqstream.ZMQStream(sockt)
        self._registry = registry
        logger.info('command handler initialized') 

def register_socket(self, socket, loop, on_recv):
        stream = ZMQStream(socket, loop)
        stream.on_recv(on_recv)
        self._streams.append(stream) 

def run(self):
        """
        A :class:`zmq.Context` and :class:`tornado.IOLoop` are spawned,
        the listener and optionally the pusher are instantiated and
        connected to :meth:`~.Station.handle_listen` using
        :meth:`~zmq.eventloop.zmqstream.ZMQStream.on_recv` .

        The process is kept open by the :class:`tornado.IOLoop` .
        """
        # init zmq objects
        self.context = zmq.Context()
        self.loop = IOLoop()

        # Our networking topology is treelike:
        # each Station object binds one Router to
        # send and receive messages from its descendants
        # each Station object may have one Dealer that
        # connects it with its antecedents.
        self.listener  = self.context.socket(zmq.ROUTER)
        self.listener.identity = self.id.encode('utf-8')
        self.listener.bind('tcp://*:{}'.format(self.listen_port))
        self.listener = ZMQStream(self.listener, self.loop)
        self.listener.on_recv(self.handle_listen)

        if self.pusher is True:
            self.pusher = self.context.socket(zmq.DEALER)
            self.pusher.identity = self.id.encode('utf-8')
            self.pusher.connect('tcp://{}:{}'.format(self.push_ip, self.push_port))
            self.pusher = ZMQStream(self.pusher, self.loop)
            self.pusher.on_recv(self.handle_listen)
            # TODO: Make sure handle_listen knows how to handle ID-less messages

        self.logger.info('Starting IOLoop')
        self.loop.start() 

def setUp(self):
        if tornado is None:
            pytest.skip()
        if asyncio:
            asyncio.set_event_loop(asyncio.new_event_loop())
        self.context = zmq.Context()
        self.loop = ioloop.IOLoop.instance()
        self.push = zmqstream.ZMQStream(self.context.socket(zmq.PUSH))
        self.pull = zmqstream.ZMQStream(self.context.socket(zmq.PULL))
        port = self.push.bind_to_random_port('tcp://127.0.0.1')
        self.pull.connect('tcp://127.0.0.1:%i' % port)
        self.stream = self.push 

def setUp(self):
        try:
            from tornado import ioloop
        except ImportError:
            pytest.skip("Requires tornado")
        from zmq.eventloop import zmqstream
        self.fail_msg = None
        self.io_loop = ioloop.IOLoop()
        super(TestIOLoopAuthentication, self).setUp()
        self.server = self.socket(zmq.PUSH)
        self.client = self.socket(zmq.PULL)
        self.pushstream = zmqstream.ZMQStream(self.server, self.io_loop)
        self.pullstream = zmqstream.ZMQStream(self.client, self.io_loop) 

def start(self):
        """Start ZAP authentication"""
        super(IOLoopAuthenticator, self).start()
        self.zap_stream = zmqstream.ZMQStream(self.zap_socket, self.io_loop)
        self.zap_stream.on_recv(self.handle_zap_message) 

def run(self):
        """The thread's main activity.  Call start() instead."""
        self.socket = self.context.socket(zmq.SUB)
        self.socket.setsockopt(zmq.SUBSCRIBE,b'')
        self.socket.setsockopt(zmq.IDENTITY, self.session.bsession)
        self.socket.connect(self.address)
        self.stream = zmqstream.ZMQStream(self.socket, self.ioloop)
        self.stream.on_recv(self._handle_recv)
        self._run_loop()
        try:
            self.socket.close()
        except:
            pass 

def start(self):
        """Start ZAP authentication"""
        super(IOLoopAuthenticator, self).start()
        self.zap_stream = zmqstream.ZMQStream(self.zap_socket, self.io_loop)
        self.zap_stream.on_recv(self.handle_zap_message) 

def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True):
        """Receive and unpack a message.

        Parameters
        ----------
        socket : ZMQStream or Socket
            The socket or stream to use in receiving.

        Returns
        -------
        [idents], msg
            [idents] is a list of idents and msg is a nested message dict of
            same format as self.msg returns.
        """
        if isinstance(socket, ZMQStream):
            socket = socket.socket
        try:
            msg_list = socket.recv_multipart(mode, copy=copy)
        except zmq.ZMQError as e:
            if e.errno == zmq.EAGAIN:
                # We can convert EAGAIN to None as we know in this case
                # recv_multipart won't return None.
                return None,None
            else:
                raise
        # split multipart message into identity list and message dict
        # invalid large messages can cause very expensive string comparisons
        idents, msg_list = self.feed_identities(msg_list, copy)
        try:
            return idents, self.unserialize(msg_list, content=content, copy=copy)
        except Exception as e:
            # TODO: handle it
            raise e 

def send_raw(self, stream, msg_list, flags=0, copy=True, ident=None):
        """Send a raw message via ident path.

        This method is used to send a already serialized message.

        Parameters
        ----------
        stream : ZMQStream or Socket
            The ZMQ stream or socket to use for sending the message.
        msg_list : list
            The serialized list of messages to send. This only includes the
            [p_header,p_parent,p_metadata,p_content,buffer1,buffer2,...] portion of
            the message.
        ident : ident or list
            A single ident or a list of idents to use in sending.
        """
        to_send = []
        if isinstance(ident, bytes):
            ident = [ident]
        if ident is not None:
            to_send.extend(ident)

        to_send.append(DELIM)
        to_send.append(self.sign(msg_list))
        to_send.extend(msg_list)
        stream.send_multipart(to_send, flags, copy=copy) 

def test_tracking(self):
        """test tracking messages"""
        a,b = self.create_bound_pair(zmq.PAIR, zmq.PAIR)
        s = self.session
        s.copy_threshold = 1
        stream = ZMQStream(a)
        msg = s.send(a, 'hello', track=False)
        self.assertTrue(msg['tracker'] is ss.DONE)
        msg = s.send(a, 'hello', track=True)
        self.assertTrue(isinstance(msg['tracker'], zmq.MessageTracker))
        M = zmq.Message(b'hi there', track=True)
        msg = s.send(a, 'hello', buffers=[M], track=True)
        t = msg['tracker']
        self.assertTrue(isinstance(t, zmq.MessageTracker))
        self.assertRaises(zmq.NotDone, t.wait, .1)
        del M
        t.wait(1) # this will raise


    # def test_rekey(self):
    #     """rekeying dict around json str keys"""
    #     d = {'0': uuid.uuid4(), 0:uuid.uuid4()}
    #     self.assertRaises(KeyError, ss.rekey, d)
    #
    #     d = {'0': uuid.uuid4(), 1:uuid.uuid4(), 'asdf':uuid.uuid4()}
    #     d2 = {0:d['0'],1:d[1],'asdf':d['asdf']}
    #     rd = ss.rekey(d)
    #     self.assertEqual(d2,rd)
    #
    #     d = {'1.5':uuid.uuid4(),'1':uuid.uuid4()}
    #     d2 = {1.5:d['1.5'],1:d['1']}
    #     rd = ss.rekey(d)
    #     self.assertEqual(d2,rd)
    #
    #     d = {'1.0':uuid.uuid4(),'1':uuid.uuid4()}
    #     self.assertRaises(KeyError, ss.rekey, d)
    # 

def as_zmqstream(f):
    def wrapped(self, *args, **kwargs):
        socket = f(self, *args, **kwargs)
        return ZMQStream(socket, self.loop)
    return wrapped 

def __init__(self, **kwargs):
        super(LogWatcher, self).__init__(**kwargs)
        s = self.context.socket(zmq.SUB)
        s.bind(self.url)
        self.stream = zmqstream.ZMQStream(s, self.loop)
        self.subscribe()
        self.on_trait_change(self.subscribe, 'topics') 

def subscribe(self,zmq_context,trade_pub_connection_string,trade_client):

        """" subscribe. """
        self.md_pub_socket = zmq_context.socket(zmq.SUB)
        self.md_pub_socket.connect(trade_pub_connection_string)
        self.md_pub_socket.setsockopt(zmq.SUBSCRIBE,"^MD_FULL_REFRESH_" +self.symbol + '$')
        self.md_pub_socket.setsockopt(zmq.SUBSCRIBE,"^MD_TRADE_" + self.symbol + '$')
        self.md_pub_socket.setsockopt(zmq.SUBSCRIBE,"^MD_INCREMENTAL_" +self.symbol +".0$")
        self.md_pub_socket.setsockopt(zmq.SUBSCRIBE,"^MD_INCREMENTAL_" +self.symbol +".1$")

        self.md_pub_socket_stream = ZMQStream(self.md_pub_socket)
        self.md_pub_socket_stream.on_recv(self.on_md_publish)

        md_subscription_msg = {
            'MsgType': 'V',
            'MDReqID': '0',  # not important.
            'SubscriptionRequestType': '0',
            'MarketDepth': 0,
            'TradeDate': time.strftime("%Y%m%d", time.localtime()),
            'MDUpdateType': '0',
            'MDEntryTypes': ['0', '1', '2'],
            'Instruments': [self.symbol]
        }

        self.application.log('DEBUG', 'MARKET_DATA_SUBSCRIBER', 'SUBSCRIBE' )

        return trade_client.sendJSON(md_subscription_msg) 

def setUp(self):
        if tornado is None:
            pytest.skip()
        self.context = zmq.Context()
        self.loop = ioloop.IOLoop.instance()
        self.push = zmqstream.ZMQStream(self.context.socket(zmq.PUSH))
        self.pull = zmqstream.ZMQStream(self.context.socket(zmq.PULL))
        port = self.push.bind_to_random_port('tcp://127.0.0.1')
        self.pull.connect('tcp://127.0.0.1:%i' % port)
        self.stream = self.push 

def setUp(self):
        try:
            from tornado import ioloop
        except ImportError:
            pytest.skip("Requires tornado")
        from zmq.eventloop import zmqstream
        self.fail_msg = None
        self.io_loop = ioloop.IOLoop()
        super(TestIOLoopAuthentication, self).setUp()
        self.server = self.socket(zmq.PUSH)
        self.client = self.socket(zmq.PULL)
        self.pushstream = zmqstream.ZMQStream(self.server, self.io_loop)
        self.pullstream = zmqstream.ZMQStream(self.client, self.io_loop) 

def __init__(self, **kwargs):
        """
        # universal:
        loop: IOLoop for creating future connections
        session: streamsession for sending serialized data
        # engine:
        queue: ZMQStream for monitoring queue messages
        query: ZMQStream for engine+client registration and client requests
        heartbeat: HeartMonitor object for tracking engines
        # extra:
        db: ZMQStream for db connection (NotImplemented)
        engine_info: zmq address/protocol dict for engine connections
        client_info: zmq address/protocol dict for client connections
        """

        super(Hub, self).__init__(**kwargs)
        self.registration_timeout = max(10000, 5*self.heartmonitor.period)

        # register our callbacks
        self.query.on_recv(self.dispatch_query)
        self.monitor.on_recv(self.dispatch_monitor_traffic)

        self.heartmonitor.add_heart_failure_handler(self.handle_heart_failure)
        self.heartmonitor.add_new_heart_handler(self.handle_new_heart)

        self.monitor_handlers = {b'in' : self.save_queue_request,
                                b'out': self.save_queue_result,
                                b'intask': self.save_task_request,
                                b'outtask': self.save_task_result,
                                b'tracktask': self.save_task_destination,
                                b'incontrol': _passer,
                                b'outcontrol': _passer,
                                b'iopub': self.save_iopub_message,
        }

        self.query_handlers = {'queue_request': self.queue_status,
                                'result_request': self.get_results,
                                'history_request': self.get_history,
                                'db_request': self.db_query,
                                'purge_request': self.purge_results,
                                'load_request': self.check_load,
                                'resubmit_request': self.resubmit_task,
                                'shutdown_request': self.shutdown_request,
                                'registration_request' : self.register_engine,
                                'unregistration_request' : self.unregister_engine,
                                'connection_request': self.connection_request,
        }

        # ignore resubmit replies
        self.resubmit.on_recv(lambda msg: None, copy=False)

        self.log.info("hub::created hub") 

def launch_scheduler(in_addr, out_addr, mon_addr, not_addr, reg_addr, config=None,
                        logname='root', log_url=None, loglevel=logging.DEBUG,
                        identity=b'task', in_thread=False):

    ZMQStream = zmqstream.ZMQStream

    if config:
        # unwrap dict back into Config
        config = Config(config)

    if in_thread:
        # use instance() to get the same Context/Loop as our parent
        ctx = zmq.Context.instance()
        loop = ioloop.IOLoop.instance()
    else:
        # in a process, don't use instance()
        # for safety with multiprocessing
        ctx = zmq.Context()
        loop = ioloop.IOLoop()
    ins = ZMQStream(ctx.socket(zmq.ROUTER),loop)
    util.set_hwm(ins, 0)
    ins.setsockopt(zmq.IDENTITY, identity + b'_in')
    ins.bind(in_addr)

    outs = ZMQStream(ctx.socket(zmq.ROUTER),loop)
    util.set_hwm(outs, 0)
    outs.setsockopt(zmq.IDENTITY, identity + b'_out')
    outs.bind(out_addr)
    mons = zmqstream.ZMQStream(ctx.socket(zmq.PUB),loop)
    util.set_hwm(mons, 0)
    mons.connect(mon_addr)
    nots = zmqstream.ZMQStream(ctx.socket(zmq.SUB),loop)
    nots.setsockopt(zmq.SUBSCRIBE, b'')
    nots.connect(not_addr)
    
    querys = ZMQStream(ctx.socket(zmq.DEALER),loop)
    querys.connect(reg_addr)
    
    # setup logging.
    if in_thread:
        log = Application.instance().log
    else:
        if log_url:
            log = connect_logger(logname, ctx, log_url, root="scheduler", loglevel=loglevel)
        else:
            log = local_logger(logname, loglevel)

    scheduler = TaskScheduler(client_stream=ins, engine_stream=outs,
                            mon_stream=mons, notifier_stream=nots,
                            query_stream=querys,
                            loop=loop, log=log,
                            config=config)
    scheduler.start()
    if not in_thread:
        try:
            loop.start()
        except KeyboardInterrupt:
            scheduler.log.critical("Interrupted, exiting...") 

def __init__(self, config, driver_info, driver):
    # Clone config so we can update it.
    config = json.loads(json.dumps(config))

    self._config = config
    self._exiting = False
    self._engine_id = str(uuid.uuid4())
    self._wire = WireProtocol(self._engine_id, config["key"], config["signature_scheme"])

    connection = config["transport"] + "://" + config["ip"]

    def bind(socket, port):
      if port <= 0:
        return socket.bind_to_random_port(connection)
      else:
        socket.bind("%s:%s" % (connection, port))
      return port

    def wrap_with_deserialization(fn):
      def accept(wire_msg):
        return fn(*self._wire.deserialize_wire_msg(wire_msg))

      return accept

    ## Initialize:
    ioloop.install()

    ctx = zmq.Context()
    self._heartbeat_socket = ctx.socket(zmq.REP)
    config["hb_port"] = bind(self._heartbeat_socket, config["hb_port"])

    # IOPub/Sub: also called SubSocketChannel in IPython sources
    self._iopub_socket = ctx.socket(zmq.PUB)
    config["iopub_port"] = bind(self._iopub_socket, config["iopub_port"])
    iopub_stream = zmqstream.ZMQStream(self._iopub_socket)
    iopub_stream.on_recv(wrap_with_deserialization(self._iopub_handler))
    iopub = OutgoingStream(self._wire, iopub_stream)

    self._control_socket = ctx.socket(zmq.ROUTER)
    config["control_port"] = bind(self._control_socket, config["control_port"])
    control_stream = zmqstream.ZMQStream(self._control_socket)
    control_stream.on_recv(wrap_with_deserialization(self._control_handler))

    self._stdin_socket = ctx.socket(zmq.ROUTER)
    config["stdin_port"] = bind(self._stdin_socket, config["stdin_port"])
    stdin_stream = zmqstream.ZMQStream(self._stdin_socket)
    stdin_stream.on_recv(wrap_with_deserialization(self._stdin_handler))

    self._shell_socket = ctx.socket(zmq.ROUTER)
    config["shell_port"] = bind(self._shell_socket, config["shell_port"])
    shell_stream = zmqstream.ZMQStream(self._shell_socket)
    shell = OutgoingStream(self._wire, shell_stream)
    shell_stream.on_recv(wrap_with_deserialization(self._shell_handler))

    self._shell_handler_impl = ShellHandler(self._engine_id, iopub, shell, driver_info, driver)