Python zmq.IDENTITY Examples
The following are 30
code examples of zmq.IDENTITY().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
zmq
, or try the search function
.
.
Example #1
| Source File: parallel_map.py From dataflow with Apache License 2.0 | 8 votes |
|
def run(self):
enable_death_signal(_warn=self.identity == b'0')
ctx = zmq.Context()
# recv jobs
socket = ctx.socket(zmq.PULL)
socket.setsockopt(zmq.IDENTITY, self.identity)
socket.set_hwm(self.hwm * self.batch_size)
socket.connect(self.input_pipe)
# send results
out_socket = ctx.socket(zmq.PUSH)
out_socket.set_hwm(max(self.hwm, 5))
out_socket.connect(self.result_pipe)
batch = []
while True:
dp = loads(socket.recv(copy=False))
dp = self.map_func(dp)
if dp is not None:
batch.append(dp)
if len(batch) == self.batch_size:
dp = BatchData.aggregate_batch(batch)
out_socket.send(dumps(dp), copy=False)
del batch[:]
Example #2
| Source File: parallel_map.py From tensorpack with Apache License 2.0 | 6 votes |
|
def run(self):
enable_death_signal(_warn=self.identity == b'0')
ctx = zmq.Context()
# recv jobs
socket = ctx.socket(zmq.PULL)
socket.setsockopt(zmq.IDENTITY, self.identity)
socket.set_hwm(self.hwm * self.batch_size)
socket.connect(self.input_pipe)
# send results
out_socket = ctx.socket(zmq.PUSH)
out_socket.set_hwm(max(self.hwm, 5))
out_socket.connect(self.result_pipe)
batch = []
while True:
dp = loads(socket.recv(copy=False))
dp = self.map_func(dp)
if dp is not None:
batch.append(dp)
if len(batch) == self.batch_size:
dp = BatchData.aggregate_batch(batch)
out_socket.send(dumps(dp), copy=False)
del batch[:]
Example #3
| Source File: proxydevice.py From Carnets with BSD 3-Clause "New" or "Revised" License | 6 votes |
|
def _setup_sockets(self):
ins,outs = Device._setup_sockets(self)
ctx = self._context
mons = ctx.socket(self.mon_type)
# set sockopts (must be done first, in case of zmq.IDENTITY)
for opt,value in self._mon_sockopts:
mons.setsockopt(opt, value)
for iface in self._mon_binds:
mons.bind(iface)
for iface in self._mon_connects:
mons.connect(iface)
return ins,outs,mons
Example #4
| Source File: socket.py From Carnets with BSD 3-Clause "New" or "Revised" License | 6 votes |
|
def get(self, option):
c_data = new_pointer_from_opt(option, length=255)
c_value_pointer = c_data[0]
c_sizet_pointer = c_data[1]
_retry_sys_call(C.zmq_getsockopt,
self._zmq_socket,
option,
c_value_pointer,
c_sizet_pointer)
sz = c_sizet_pointer[0]
v = value_from_opt_pointer(option, c_value_pointer, sz)
if option != zmq.IDENTITY and option in zmq.constants.bytes_sockopts and v.endswith(b'\0'):
v = v[:-1]
return v
Example #5
| Source File: simulator.py From ternarynet with Apache License 2.0 | 6 votes |
|
def run(self):
player = self._build_player()
context = zmq.Context()
c2s_socket = context.socket(zmq.PUSH)
c2s_socket.setsockopt(zmq.IDENTITY, self.identity)
c2s_socket.set_hwm(2)
c2s_socket.connect(self.c2s)
s2c_socket = context.socket(zmq.DEALER)
s2c_socket.setsockopt(zmq.IDENTITY, self.identity)
#s2c_socket.set_hwm(5)
s2c_socket.connect(self.s2c)
state = player.current_state()
reward, isOver = 0, False
while True:
c2s_socket.send(dumps(
(self.identity, state, reward, isOver)),
copy=False)
action = loads(s2c_socket.recv(copy=False).bytes)
reward, isOver = player.action(action)
state = player.current_state()
# compatibility
Example #6
| Source File: parallel_map.py From ADL with MIT License | 6 votes |
|
def run(self):
enable_death_signal(_warn=self.identity == b'0')
ctx = zmq.Context()
socket = ctx.socket(zmq.PULL)
socket.setsockopt(zmq.IDENTITY, self.identity)
socket.set_hwm(self.hwm)
socket.connect(self.input_pipe)
out_socket = ctx.socket(zmq.PUSH)
out_socket.set_hwm(max(self.hwm // self.batch_size, 5))
out_socket.connect(self.result_pipe)
batch = []
while True:
dp = loads(socket.recv(copy=False))
dp = self.map_func(dp)
if dp is not None:
batch.append(dp)
if len(batch) == self.batch_size:
dp = BatchData.aggregate_batch(batch)
out_socket.send(dumps(dp), copy=False)
del batch[:]
Example #7
| Source File: simulator.py From VDAIC2017 with MIT License | 6 votes |
|
def run(self):
player = self._build_player()
context = zmq.Context()
c2s_socket = context.socket(zmq.PUSH)
c2s_socket.setsockopt(zmq.IDENTITY, self.identity)
c2s_socket.set_hwm(2)
c2s_socket.connect(self.c2s)
s2c_socket = context.socket(zmq.DEALER)
s2c_socket.setsockopt(zmq.IDENTITY, self.identity)
#s2c_socket.set_hwm(5)
s2c_socket.connect(self.s2c)
state = player.current_state()
reward, isOver = 0, False
while True:
c2s_socket.send(dumps(
(self.identity, state, reward, isOver)),
copy=False)
action = loads(s2c_socket.recv(copy=False).bytes)
reward, isOver = player.action(action)
state = player.current_state()
# compatibility
Example #8
| Source File: proxydevice.py From pySINDy with MIT License | 6 votes |
|
def _setup_sockets(self):
ins,outs = Device._setup_sockets(self)
ctx = self._context
mons = ctx.socket(self.mon_type)
# set sockopts (must be done first, in case of zmq.IDENTITY)
for opt,value in self._mon_sockopts:
mons.setsockopt(opt, value)
for iface in self._mon_binds:
mons.bind(iface)
for iface in self._mon_connects:
mons.connect(iface)
return ins,outs,mons
Example #9
| Source File: socket.py From pySINDy with MIT License | 6 votes |
|
def get(self, option):
c_data = new_pointer_from_opt(option, length=255)
c_value_pointer = c_data[0]
c_sizet_pointer = c_data[1]
_retry_sys_call(C.zmq_getsockopt,
self._zmq_socket,
option,
c_value_pointer,
c_sizet_pointer)
sz = c_sizet_pointer[0]
v = value_from_opt_pointer(option, c_value_pointer, sz)
if option != zmq.IDENTITY and option in zmq.constants.bytes_sockopts and v.endswith(b'\0'):
v = v[:-1]
return v
Example #10
| Source File: test_socket.py From pySINDy with MIT License | 6 votes |
|
def test_unicode_sockopts(self):
"""test setting/getting sockopts with unicode strings"""
topic = "tést"
if str is not unicode:
topic = topic.decode('utf8')
p,s = self.create_bound_pair(zmq.PUB, zmq.SUB)
self.assertEqual(s.send_unicode, s.send_unicode)
self.assertEqual(p.recv_unicode, p.recv_unicode)
self.assertRaises(TypeError, s.setsockopt, zmq.SUBSCRIBE, topic)
self.assertRaises(TypeError, s.setsockopt, zmq.IDENTITY, topic)
s.setsockopt_unicode(zmq.IDENTITY, topic, 'utf16')
self.assertRaises(TypeError, s.setsockopt, zmq.AFFINITY, topic)
s.setsockopt_unicode(zmq.SUBSCRIBE, topic)
self.assertRaises(TypeError, s.getsockopt_unicode, zmq.AFFINITY)
self.assertRaisesErrno(zmq.EINVAL, s.getsockopt_unicode, zmq.SUBSCRIBE)
identb = s.getsockopt(zmq.IDENTITY)
identu = identb.decode('utf16')
identu2 = s.getsockopt_unicode(zmq.IDENTITY, 'utf16')
self.assertEqual(identu, identu2)
time.sleep(0.1) # wait for connection/subscription
p.send_unicode(topic,zmq.SNDMORE)
p.send_unicode(topic*2, encoding='latin-1')
self.assertEqual(topic, s.recv_unicode())
self.assertEqual(topic*2, s.recv_unicode(encoding='latin-1'))
Example #11
| Source File: jrpc_py.py From TradeSim with Apache License 2.0 | 6 votes |
|
def _do_connect(self):
client_id = str(random.randint(1000000, 100000000))
socket = self._ctx.socket(zmq.DEALER)
identity = (client_id) + '$' + str(random.randint(1000000, 1000000000))
identity = identity.encode('utf-8')
socket.setsockopt(zmq.IDENTITY, identity)
socket.setsockopt(zmq.RCVTIMEO, 500)
socket.setsockopt(zmq.SNDTIMEO, 500)
socket.setsockopt(zmq.LINGER, 0)
socket.connect(self._addr)
return socket
Example #12
| Source File: brokerLaunch.py From scoop with GNU Lesser General Public License v3.0 | 6 votes |
|
def sendConnect(self, data):
"""Send a CONNECT command to the broker
:param data: List of other broker main socket URL"""
# Imported dynamically - Not used if only one broker
if self.backend == 'ZMQ':
import zmq
self.context = zmq.Context()
self.socket = self.context.socket(zmq.DEALER)
if sys.version_info < (3,):
self.socket.setsockopt_string(zmq.IDENTITY, unicode('launcher'))
else:
self.socket.setsockopt_string(zmq.IDENTITY, 'launcher')
self.socket.connect(
"tcp://{hostname}:{port}".format(
port=self.brokerPort,
hostname = self.hostname
)
)
self.socket.send_multipart([b"CONNECT",
pickle.dumps(data,
pickle.HIGHEST_PROTOCOL)])
else:
# TODO
pass
Example #13
| Source File: brokerLaunch.py From scoop with GNU Lesser General Public License v3.0 | 6 votes |
|
def sendConnect(self, data):
"""Send a CONNECT command to the broker
:param data: List of other broker main socket URL"""
# Imported dynamically - Not used if only one broker
if self.backend == 'ZMQ':
import zmq
self.context = zmq.Context()
self.socket = self.context.socket(zmq.DEALER)
self.socket.setsockopt(zmq.IDENTITY, b'launcher')
self.socket.connect(
"tcp://127.0.0.1:{port}".format(
port=self.brokerPort,
)
)
self.socket.send_multipart([b"CONNECT",
pickle.dumps(data,
pickle.HIGHEST_PROTOCOL)])
else:
# TODO
pass
Example #14
| Source File: job_engine.py From ibeis with Apache License 2.0 | 6 votes |
|
def initialize_client_thread(jobiface):
"""
Creates a ZMQ object in this thread. This talks to background processes.
"""
print = partial(ut.colorprint, color='blue')
if jobiface.verbose:
print('Initializing JobInterface')
jobiface.engine_deal_sock = ctx.socket(zmq.DEALER)
jobiface.engine_deal_sock.setsockopt_string(zmq.IDENTITY,
'client%s.engine.DEALER' %
(jobiface.id_,))
jobiface.engine_deal_sock.connect(jobiface.port_dict['engine_url1'])
if jobiface.verbose:
print('connect engine_url1 = %r' % (jobiface.port_dict['engine_url1'],))
jobiface.collect_deal_sock = ctx.socket(zmq.DEALER)
jobiface.collect_deal_sock.setsockopt_string(zmq.IDENTITY,
'client%s.collect.DEALER'
% (jobiface.id_,))
jobiface.collect_deal_sock.connect(jobiface.port_dict['collect_url1'])
if jobiface.verbose:
print('connect collect_url1 = %r' % (jobiface.port_dict['collect_url1'],))
Example #15
| Source File: heartmonitor.py From Computable with MIT License | 6 votes |
|
def __init__(self, in_addr, out_addr, mon_addr=None, in_type=zmq.SUB, out_type=zmq.DEALER, mon_type=zmq.PUB, heart_id=None):
if mon_addr is None:
self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
else:
self.device = ThreadMonitoredQueue(in_type, out_type, mon_type, in_prefix=b"", out_prefix=b"")
# do not allow the device to share global Context.instance,
# which is the default behavior in pyzmq > 2.1.10
self.device.context_factory = zmq.Context
self.device.daemon=True
self.device.connect_in(in_addr)
self.device.connect_out(out_addr)
if mon_addr is not None:
self.device.connect_mon(mon_addr)
if in_type == zmq.SUB:
self.device.setsockopt_in(zmq.SUBSCRIBE, b"")
if heart_id is None:
heart_id = uuid.uuid4().bytes
self.device.setsockopt_out(zmq.IDENTITY, heart_id)
self.id = heart_id
Example #16
| Source File: engine.py From Computable with MIT License | 6 votes |
|
def register(self):
"""send the registration_request"""
self.log.info("Registering with controller at %s"%self.url)
ctx = self.context
connect,maybe_tunnel = self.init_connector()
reg = ctx.socket(zmq.DEALER)
reg.setsockopt(zmq.IDENTITY, self.bident)
connect(reg, self.url)
self.registrar = zmqstream.ZMQStream(reg, self.loop)
content = dict(uuid=self.ident)
self.registrar.on_recv(lambda msg: self.complete_registration(msg, connect, maybe_tunnel))
# print (self.session.key)
self.session.send(self.registrar, "registration_request", content=content)
Example #17
| Source File: test_socket.py From vnpy_crypto with MIT License | 6 votes |
|
def test_unicode_sockopts(self):
"""test setting/getting sockopts with unicode strings"""
topic = "tést"
if str is not unicode:
topic = topic.decode('utf8')
p,s = self.create_bound_pair(zmq.PUB, zmq.SUB)
self.assertEqual(s.send_unicode, s.send_unicode)
self.assertEqual(p.recv_unicode, p.recv_unicode)
self.assertRaises(TypeError, s.setsockopt, zmq.SUBSCRIBE, topic)
self.assertRaises(TypeError, s.setsockopt, zmq.IDENTITY, topic)
s.setsockopt_unicode(zmq.IDENTITY, topic, 'utf16')
self.assertRaises(TypeError, s.setsockopt, zmq.AFFINITY, topic)
s.setsockopt_unicode(zmq.SUBSCRIBE, topic)
self.assertRaises(TypeError, s.getsockopt_unicode, zmq.AFFINITY)
self.assertRaisesErrno(zmq.EINVAL, s.getsockopt_unicode, zmq.SUBSCRIBE)
identb = s.getsockopt(zmq.IDENTITY)
identu = identb.decode('utf16')
identu2 = s.getsockopt_unicode(zmq.IDENTITY, 'utf16')
self.assertEqual(identu, identu2)
time.sleep(0.1) # wait for connection/subscription
p.send_unicode(topic,zmq.SNDMORE)
p.send_unicode(topic*2, encoding='latin-1')
self.assertEqual(topic, s.recv_unicode())
self.assertEqual(topic*2, s.recv_unicode(encoding='latin-1'))
Example #18
| Source File: proxydevice.py From Computable with MIT License | 6 votes |
|
def _setup_sockets(self):
ins,outs = Device._setup_sockets(self)
ctx = self._context
mons = ctx.socket(self.mon_type)
# set sockopts (must be done first, in case of zmq.IDENTITY)
for opt,value in self._mon_sockopts:
mons.setsockopt(opt, value)
for iface in self._mon_binds:
mons.bind(iface)
for iface in self._mon_connects:
mons.connect(iface)
return ins,outs,mons
Example #19
| Source File: socket.py From Computable with MIT License | 6 votes |
|
def get(self, option):
low_level_data = new_pointer_from_opt(option, length=255)
low_level_value_pointer = low_level_data[0]
low_level_sizet_pointer = low_level_data[1]
rc = C.zmq_getsockopt(self._zmq_socket,
option,
low_level_value_pointer,
low_level_sizet_pointer)
_check_rc(rc)
sz = low_level_sizet_pointer[0]
v = value_from_opt_pointer(option, low_level_value_pointer, sz)
if option != zmq.IDENTITY and option in zmq.constants.bytes_sockopts and v.endswith(b'\0'):
v = v[:-1]
return v
Example #20
| Source File: socket.py From vnpy_crypto with MIT License | 6 votes |
|
def get(self, option):
c_data = new_pointer_from_opt(option, length=255)
c_value_pointer = c_data[0]
c_sizet_pointer = c_data[1]
_retry_sys_call(C.zmq_getsockopt,
self._zmq_socket,
option,
c_value_pointer,
c_sizet_pointer)
sz = c_sizet_pointer[0]
v = value_from_opt_pointer(option, c_value_pointer, sz)
if option != zmq.IDENTITY and option in zmq.constants.bytes_sockopts and v.endswith(b'\0'):
v = v[:-1]
return v
Example #21
| Source File: proxydevice.py From vnpy_crypto with MIT License | 6 votes |
|
def _setup_sockets(self):
ins,outs = Device._setup_sockets(self)
ctx = self._context
mons = ctx.socket(self.mon_type)
# set sockopts (must be done first, in case of zmq.IDENTITY)
for opt,value in self._mon_sockopts:
mons.setsockopt(opt, value)
for iface in self._mon_binds:
mons.bind(iface)
for iface in self._mon_connects:
mons.connect(iface)
return ins,outs,mons
Example #22
| Source File: probe.py From parsl with Apache License 2.0 | 5 votes |
|
def __init__(self, addresses, port):
uid = str(uuid.uuid4())
self.context = zmq.Context()
self.task_incoming = self.context.socket(zmq.DEALER)
self.task_incoming.setsockopt(zmq.IDENTITY, uid.encode('utf-8'))
# Linger is set to 0, so that the manager can exit even when there might be
# messages in the pipe
self.task_incoming.setsockopt(zmq.LINGER, 0)
address = probe_addresses(addresses, port)
print("Viable address :", address)
self.task_incoming.connect("tcp://{}:{}".format(address, port))
print("Here")
Example #23
| Source File: client.py From bluesky with GNU General Public License v3.0 | 5 votes |
|
def connect(self, hostname='localhost', event_port=0, stream_port=0, protocol='tcp'):
conbase = '{}://{}'.format(protocol, hostname)
econ = conbase + (':{}'.format(event_port) if event_port else '')
scon = conbase + (':{}'.format(stream_port) if stream_port else '')
self.event_io.setsockopt(zmq.IDENTITY, self.client_id)
self.event_io.connect(econ)
self.send_event(b'REGISTER')
self.host_id = self.event_io.recv_multipart()[0]
print('Client {} connected to host {}'.format(self.client_id, self.host_id))
self.stream_in.connect(scon)
self.poller.register(self.event_io, zmq.POLLIN)
self.poller.register(self.stream_in, zmq.POLLIN)
Example #24
| Source File: node.py From bluesky with GNU General Public License v3.0 | 5 votes |
|
def connect(self):
''' Connect node to the BlueSky server. '''
# Initialization of sockets.
self.event_io.setsockopt(zmq.IDENTITY, self.node_id)
self.event_io.connect('tcp://localhost:{}'.format(self.event_port))
self.stream_out.connect('tcp://localhost:{}'.format(self.stream_port))
# Start communication, and receive this node's ID
self.send_event(b'REGISTER')
self.host_id = self.event_io.recv_multipart()[0]
# print('Node connected, id={}'.format(self.node_id))
Example #25
| Source File: simulator.py From Distributed-BA3C with Apache License 2.0 | 5 votes |
|
def run(self):
player = self._build_player()
context = zmq.Context()
c2s_socket = context.socket(zmq.PUSH)
c2s_socket.setsockopt(zmq.IDENTITY, self.identity)
c2s_socket.set_hwm(2)
c2s_socket.connect(self.c2s)
s2c_socket = context.socket(zmq.DEALER)
s2c_socket.setsockopt(zmq.IDENTITY, self.identity)
#s2c_socket.set_hwm(5)
s2c_socket.connect(self.s2c)
state = player.current_state()
reward, isOver = 0, False
ts = 0
while True:
c2s_socket.send(dumps(
(self.identity, state, reward, isOver, ts, True)),
copy=False)
#t.grel here we get the action
(action, ts, isAlive) = loads(s2c_socket.recv(copy=False).bytes)
if not isAlive:
c2s_socket.send(dumps(
(self.identity, 0, 0, 0, 0, False)),
copy=False)
print("closing thread : {}".format(self.identity))
break
reward, isOver = player.action(action)
state = player.current_state()
# compatibility
Example #26
| Source File: simulator.py From Distributed-BA3C with Apache License 2.0 | 5 votes |
|
def run(self):
self.player = self._build_player()
self.ctx = zmq.Context()
self.c2s_socket = self.ctx.socket(zmq.PUSH)
self.c2s_socket.setsockopt(zmq.IDENTITY, self.identity)
self.c2s_socket.set_hwm(5)
self.c2s_socket.connect(self.pipe_c2s)
self._prepare()
for dp in self.get_data():
self.c2s_socket.send(dumps(dp), copy=False)
Example #27
| Source File: jrpc_py.py From TradeApi with Apache License 2.0 | 5 votes |
|
def _do_connect(self):
client_id = str(random.randint(1000000, 100000000))
socket = self._ctx.socket(zmq.DEALER)
identity = (client_id) + '$' + str(random.randint(1000000, 1000000000))
identity = identity.encode('utf-8')
socket.setsockopt(zmq.IDENTITY, identity)
socket.setsockopt(zmq.RCVTIMEO, 500)
socket.setsockopt(zmq.SNDTIMEO, 500)
socket.setsockopt(zmq.LINGER, 0)
socket.connect(self._addr)
return socket
Example #28
| Source File: test_socket.py From vnpy_crypto with MIT License | 5 votes |
|
def test_dir(self):
ctx = self.Context()
s = ctx.socket(zmq.PUB)
self.assertTrue('send' in dir(s))
self.assertTrue('IDENTITY' in dir(s))
self.assertTrue('AFFINITY' in dir(s))
self.assertTrue('FD' in dir(s))
s.close()
ctx.term()
Example #29
| Source File: simulator.py From ternarynet with Apache License 2.0 | 5 votes |
|
def run(self):
self.player = self._build_player()
self.ctx = zmq.Context()
self.c2s_socket = self.ctx.socket(zmq.PUSH)
self.c2s_socket.setsockopt(zmq.IDENTITY, self.identity)
self.c2s_socket.set_hwm(5)
self.c2s_socket.connect(self.pipe_c2s)
self._prepare()
for dp in self.get_data():
self.c2s_socket.send(dumps(dp), copy=False)
Example #30
| Source File: data_provider.py From PoseFix_RELEASE with MIT License | 5 votes |
|
def run(self):
print('Start data provider {}-{}'.format(self.pipename, self.identity))
setproctitle('data provider {}-{}'.format(self.pipename, self.identity))
ctx = zmq.Context()
socket = ctx.socket(zmq.DEALER)
socket.setsockopt(zmq.IDENTITY, self.identity)
socket.set_hwm(self.hwm)
socket.connect(self.pipename)
while True:
dp = loads(socket.recv(copy=False).bytes)
dp = self.map_func(dp)
socket.send(dumps(dp), copy=False)
