Python multiprocessing.Pipe() Examples

def test_ignore_listener(self):
        conn, child_conn = multiprocessing.Pipe()
        try:
            p = multiprocessing.Process(target=self._test_ignore_listener,
                                        args=(child_conn,))
            p.daemon = True
            p.start()
            child_conn.close()
            address = conn.recv()
            time.sleep(0.1)
            os.kill(p.pid, signal.SIGUSR1)
            time.sleep(0.1)
            client = multiprocessing.connection.Client(address)
            self.assertEqual(client.recv(), 'welcome')
            p.join()
        finally:
            conn.close()

#
#
# 

def __init__(self, env_fns, spaces=None):
        """
        Arguments:

        env_fns: iterable of callables -  functions that create environments to run in subprocesses. Need to be cloud-pickleable
        """
        self.waiting = False
        self.closed = False
        nenvs = len(env_fns)
        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])
        self.ps = [Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn)))
                   for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)]
        for p in self.ps:
            p.daemon = True  # if the main process crashes, we should not cause things to hang
            p.start()
        for remote in self.work_remotes:
            remote.close()

        self.remotes[0].send(('get_spaces', None))
        observation_space, action_space = self.remotes[0].recv()
        self.viewer = None
        VecEnv.__init__(self, len(env_fns), observation_space, action_space) 

def tcp_server_piped(port=19543, use_stderr_logger=False):
    """
    TCP server context-manager used for integration tests.

    It starts server on localhost/given-port during context-manager start()
    and performs server-cleanup during context-manager stop()

    :param port: where to start server at
    :param use_stderr_logger: configure logging to output into stderr?
    :return: pair (server, inter-process-pipe used to control server)
    """
    client_process_pipe_endpoint, server_pipe_endpoint = Pipe()
    tcp_server = TcpServerPiped(host='localhost', port=port,
                                pipe_in=server_pipe_endpoint, delay=0,
                                use_stderr_logger=use_stderr_logger)
    tcp_server.start()
    time.sleep(0.5)  # allow server to boot-up
    # returning tcp_server let's you call tcp_server.terminate() to kill server
    # for testing dropped TCP connection
    yield (tcp_server, client_process_pipe_endpoint)
    client_process_pipe_endpoint.send(("shutdown", {}))
    tcp_server.join()

# ------------------- TCP server running in separate process --------------------- 

def _pipe(self):
        """On Windows we use a pipe to emulate a Linux style character
        buffer."""
        if self._evdev:
            return None

        if not self.__pipe:
            target_function = self._get_target_function()
            if not target_function:
                return None

            self.__pipe, child_conn = Pipe(duplex=False)
            self._listener = Process(target=target_function,
                                     args=(child_conn,), daemon=True)
            self._listener.start()
        return self.__pipe 

def main():
    parent_conn, child_conn = Pipe()

    child = Process(target=work, args=(child_conn,))

    for item in (
        42,
        'some string',
        {'one': 1},
        CustomClass(),
        None,
    ):
        print(
            "PRNT: send: {}".format(item)
        )
        parent_conn.send(item)

    child.start()
    child.join() 

def __init__(self, env_fns, horizon, trajectories, seed=0):
        """Launch subprocess workers and store configuration parameters.
        :param env_fns (list<()->ResettableEnv>): list of thunks.
        :param horizon (int): length of trajectories to search over.
        :param trajectories (int): minimum number of trajectories to evaluate.
               It will be rounded up to the nearest multiple of len(make_env)."""
        super().__init__(horizon, trajectories)
        nremotes = len(env_fns)
        # Integer ceiling of self.trajectories / nworkers
        traj_per_worker = (self.trajectories - 1) // nremotes + 1

        pipes = [Pipe() for _ in range(nremotes)]
        self.remotes, self.work_remotes = zip(*pipes)
        worker_cfgs = zip(self.work_remotes, self.remotes, env_fns)
        self.ps = []
        for i, (work_remote, remote, dynamic_fn) in enumerate(worker_cfgs):
            args = (work_remote, remote, CloudpickleWrapper(dynamic_fn), horizon, traj_per_worker)
            process = Process(target=_worker, args=args)
            process.daemon = True
            # If the main process crashes, we should not cause things to hang
            process.start()
            self.ps.append(process)
        for remote in self.work_remotes:
            remote.close() 

def __init__(self, env_fns, render_interval):
        """ Minor addition to SubprocVecEnv, automatically renders environments

        envs: list of gym environments to run in subprocesses
        """
        self.closed = False
        nenvs = len(env_fns)
        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])
        self.ps = [Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn)))
            for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)]
        for p in self.ps:
            p.daemon = True # if the main process crashes, we should not cause things to hang
            p.start()
        for remote in self.work_remotes:
            remote.close()

        self.remotes[0].send(('get_spaces', None))
        self.action_space, self.observation_space = self.remotes[0].recv()

        self.render_interval = render_interval
        self.render_timer = 0 

def __init__(self, env_fns, spaces=None):
        """
        envs: list of gym environments to run in subprocesses
        """
        self.waiting = False
        self.closed = False
        nenvs = len(env_fns)
        self.nenvs = nenvs
        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])
        self.ps = [Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn)))
            for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)]
        for p in self.ps:
            p.daemon = True # if the main process crashes, we should not cause things to hang
            p.start()
        for remote in self.work_remotes:
            remote.close()

        self.remotes[0].send(('get_spaces', None))
        observation_space, action_space = self.remotes[0].recv()
        VecEnv.__init__(self, len(env_fns), observation_space, action_space) 

def __init__(self, env_fns, spaces=None):
        """
        envs: list of gym environments to run in subprocesses
        """
        self.waiting = False
        self.closed = False
        nenvs = len(env_fns)
        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])
        self.ps = [Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn)))
            for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)]
        for p in self.ps:
            p.daemon = True # if the main process crashes, we should not cause things to hang
            p.start()
        for remote in self.work_remotes:
            remote.close()

        self.remotes[0].send(('get_spaces', None))
        observation_space, action_space = self.remotes[0].recv()
        VecEnv.__init__(self, len(env_fns), observation_space, action_space) 

def run(engine_factory, source_name, input_queue_maxsize, port, num_tokens,
        message_max_size=None):
    engine_read, server_write = multiprocessing.Pipe(duplex=False)

    # We cannot read from multiprocessing.Pipe without blocking the event
    # loop
    server_read, engine_write = os.pipe()

    local_server = _LocalServer(
        num_tokens, input_queue_maxsize, server_write, server_read)
    local_server.add_source_consumed(source_name)

    engine_process = multiprocessing.Process(
        target=_run_engine,
        args=(engine_factory, engine_read, server_read, engine_write))
    try:
        engine_process.start()
        os.close(engine_write)
        local_server.launch(port, message_max_size)
    finally:
        local_server.cleanup()
        os.close(server_read)

    raise Exception('Server stopped') 

def test_recursion(self):
        rconn, wconn = self.Pipe(duplex=False)
        self._test_recursion(wconn, [])

        time.sleep(DELTA)
        result = []
        while rconn.poll():
            result.append(rconn.recv())

        expected = [
            [],
              [0],
                [0, 0],
                [0, 1],
              [1],
                [1, 0],
                [1, 1]
            ]
        self.assertEqual(result, expected) 

def test_spawn_close(self):
        # We test that a pipe connection can be closed by parent
        # process immediately after child is spawned.  On Windows this
        # would have sometimes failed on old versions because
        # child_conn would be closed before the child got a chance to
        # duplicate it.
        conn, child_conn = self.Pipe()

        p = self.Process(target=self._echo, args=(child_conn,))
        p.daemon = True
        p.start()
        child_conn.close()    # this might complete before child initializes

        msg = latin('hello')
        conn.send_bytes(msg)
        self.assertEqual(conn.recv_bytes(), msg)

        conn.send_bytes(SENTINEL)
        conn.close()
        p.join() 

def test_fd_transfer(self):
        if self.TYPE != 'processes':
            self.skipTest("only makes sense with processes")
        conn, child_conn = self.Pipe(duplex=True)

        p = self.Process(target=self._writefd, args=(child_conn, b"foo"))
        p.daemon = True
        p.start()
        self.addCleanup(support.unlink, support.TESTFN)
        with open(support.TESTFN, "wb") as f:
            fd = f.fileno()
            if msvcrt:
                fd = msvcrt.get_osfhandle(fd)
            reduction.send_handle(conn, fd, p.pid)
        p.join()
        with open(support.TESTFN, "rb") as f:
            self.assertEqual(f.read(), b"foo") 

def test_large_fd_transfer(self):
        # With fd > 256 (issue #11657)
        if self.TYPE != 'processes':
            self.skipTest("only makes sense with processes")
        conn, child_conn = self.Pipe(duplex=True)

        p = self.Process(target=self._writefd, args=(child_conn, b"bar", True))
        p.daemon = True
        p.start()
        self.addCleanup(support.unlink, support.TESTFN)
        with open(support.TESTFN, "wb") as f:
            fd = f.fileno()
            for newfd in range(256, MAXFD):
                if not self._is_fd_assigned(newfd):
                    break
            else:
                self.fail("could not find an unassigned large file descriptor")
            os.dup2(fd, newfd)
            try:
                reduction.send_handle(conn, newfd, p.pid)
            finally:
                os.close(newfd)
        p.join()
        with open(support.TESTFN, "rb") as f:
            self.assertEqual(f.read(), b"bar") 

def test_ignore(self):
        conn, child_conn = multiprocessing.Pipe()
        try:
            p = multiprocessing.Process(target=self._test_ignore,
                                        args=(child_conn,))
            p.daemon = True
            p.start()
            child_conn.close()
            self.assertEqual(conn.recv(), 'ready')
            time.sleep(0.1)
            os.kill(p.pid, signal.SIGUSR1)
            time.sleep(0.1)
            conn.send(1234)
            self.assertEqual(conn.recv(), 1234)
            time.sleep(0.1)
            os.kill(p.pid, signal.SIGUSR1)
            self.assertEqual(conn.recv_bytes(), b'x' * self.CONN_MAX_SIZE)
            time.sleep(0.1)
            p.join()
        finally:
            conn.close() 

def __init__(self, constructor):
    """Step environment in a separate process for lock free paralellism.

    The environment will be created in the external process by calling the
    specified callable. This can be an environment class, or a function
    creating the environment and potentially wrapping it. The returned
    environment should not access global variables.

    Args:
      constructor: Callable that creates and returns an OpenAI gym environment.

    Attributes:
      observation_space: The cached observation space of the environment.
      action_space: The cached action space of the environment.
    """
    self._conn, conn = multiprocessing.Pipe()
    self._process = multiprocessing.Process(
        target=self._worker, args=(constructor, conn))
    atexit.register(self.close)
    self._process.start()
    self._observ_space = None
    self._action_space = None 

def __init__(self, env_fns, spaces=None):
        """
        envs: list of gym environments to run in subprocesses
        """
        self.waiting = False
        self.closed = False
        nenvs = len(env_fns)
        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])
        self.ps = [Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn)))
                   for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)]
        for p in self.ps:
            p.daemon = True # if the main process crashes, we should not cause things to hang
            p.start()
        for remote in self.work_remotes:
            remote.close()

        self.remotes[0].send(('get_spaces', None))
        observation_space, action_space = self.remotes[0].recv()
        VecEnv.__init__(self, len(env_fns), observation_space, action_space) 

def _run_context_manager_generator_in_process(cm_gen):
    parent_conn, child_conn = multiprocessing.Pipe(duplex=False)
    target_fn = _wrap_gen_in_process(child_conn)(cm_gen)

    process = multiprocessing.Process(target=target_fn)
    process.start()
    try:
        wrapped_result = parent_conn.recv()
        if wrapped_result.exc_info is not None:
            raise wrapped_result.exc_info[0].with_traceback(*wrapped_result.exc_info[1:])
        else:
            yield wrapped_result.result
    finally:
        try:
            if parent_conn.poll():
                exc_info = parent_conn.recv().exc_info
                if exc_info is not None:
                    raise exc_info[0].with_traceback(*exc_info[1:])
        finally:
            process.terminate()
            process.join()
            parent_conn.close() 

def run_tests_in_workers(*, target, num_workers):
    parent_conn, child_conn = multiprocessing.Pipe(duplex=False)

    @_wrap_gen_in_process(child_conn)
    def target_fn():
        target()
        yield

    processes = [multiprocessing.Process(target=target_fn) for _ in range(num_workers)]
    for process in processes:
        process.start()

    try:
        for _ in range(num_workers):
            wrapped_result = parent_conn.recv()
            if wrapped_result.exc_info is not None:
                raise wrapped_result.exc_info[0].with_traceback(*wrapped_result.exc_info[1:])
    finally:
        parent_conn.close()
        for process in processes:
            process.join() 

def SendAndRecv(self, sendCmd, sendValue, checkRecvCmd, checkReceiveValue):
        # send message - must be a tuple type
        self.pipe.send((sendCmd, sendValue))

        # wait recv message - must be a tuple type
        recv, value = self.pipe.recv()
        if self.simpleLog:
            logger.log(MINOR_INFO, "Pipe : Send %s and Recv %s in %s" % (
                get_command_name(sendCmd), get_command_name(recv), getTraceCallStack()))
        else:
            logger.log(MINOR_INFO, "Pipe : Send %s, %s and Recv %s, %s in %s" % (
                get_command_name(sendCmd), str(sendValue), get_command_name(recv), str(value), getTraceCallStack()))

        # check receive correct command and value
        if recv != checkRecvCmd or (checkReceiveValue is not None and checkReceiveValue != value):
            if self.simpleLog:
                logger.log(MINOR_INFO, "Pipe : RecvFailed %s and Send %s in %s" % (get_command_name(recv),
                                                                                   COMMAND.FAIL, getTraceCallStack()))
            else:
                logger.log(MINOR_INFO, "Pipe : RecvFailed %s, %s and Send %s, %s in %s" % (
                    get_command_name(recv), str(value), COMMAND.FAIL, "None", getTraceCallStack()))
            logger.error("ERROR : Received %s not %s" % (recv, checkRecvCmd))
            raise BaseException("Pipe receive error.")
        return value 

def __init__(self, env_fns):
        """
        envs: list of gym environments to run in subprocesses
        """
        self.closed = False
        nenvs = len(env_fns)
        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])
        self.ps = [Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn)))
            for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)]
        for p in self.ps:
            #p.daemon = True # if the main process crashes, we should not cause things to hang
            p.start()
        for remote in self.work_remotes:
            remote.close()

        self.remotes[0].send(('get_spaces', None))
        self.action_space, self.observation_space = self.remotes[0].recv() 

def initialize_pyplot(self):
        """
        Call me before use!
        [Supposed to be done inside already running server process]
        """
        if not self.ready:
            from multiprocessing import Pipe
            self.out_pipe, self.in_pipe = Pipe()

            if self.plt is None:
                import matplotlib
                matplotlib.use(self.plt_backend, force=True)
                import matplotlib.pyplot as plt

            self.plt = plt
            self.ready = True 

def __init__(self, env_fns, spaces=None):
        """
        envs: list of gym environments to run in subprocesses
        """
        self.waiting = False
        self.closed = False
        nenvs = len(env_fns)
        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])
        self.ps = [Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn)))
            for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)]
        for p in self.ps:
            p.daemon = True # if the main process crashes, we should not cause things to hang
            p.start()
        for remote in self.work_remotes:
            remote.close()

        self.remotes[0].send(('get_spaces', None))
        observation_space, action_space = self.remotes[0].recv()
        VecEnv.__init__(self, len(env_fns), observation_space, action_space) 

def __init__(self, env_fns, spaces=None):
        """
        Arguments:

        env_fns: iterable of callables -  functions that create environments to run in subprocesses. Need to be cloud-pickleable
        """
        self.waiting = False
        self.closed = False
        nenvs = len(env_fns)
        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])
        self.ps = [Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn)))
                   for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)]
        for p in self.ps:
            p.daemon = True  # if the main process crashes, we should not cause things to hang
            p.start()
        for remote in self.work_remotes:
            remote.close()

        self.remotes[0].send(('get_spaces', None))
        observation_space, action_space = self.remotes[0].recv()
        self.viewer = None
        VecEnv.__init__(self, len(env_fns), observation_space, action_space) 

def __init__(self, env_fns, spaces=None):
        """
        Arguments:

        env_fns: iterable of callables -  functions that create environments to run in subprocesses. Need to be cloud-pickleable
        """
        self.waiting = False
        self.closed = False
        nenvs = len(env_fns)
        self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)])
        self.ps = [Process(target=worker, args=(work_remote, remote, CloudpickleWrapper(env_fn)))
                   for (work_remote, remote, env_fn) in zip(self.work_remotes, self.remotes, env_fns)]
        for p in self.ps:
            p.daemon = True  # if the main process crashes, we should not cause things to hang
            p.start()
        for remote in self.work_remotes:
            remote.close()

        self.remotes[0].send(('get_spaces', None))
        observation_space, action_space = self.remotes[0].recv()
        self.viewer = None
        VecEnv.__init__(self, len(env_fns), observation_space, action_space) 

def __init__(self, maxsize=0):
        if maxsize <= 0:
            maxsize = _multiprocessing.SemLock.SEM_VALUE_MAX
        self._maxsize = maxsize
        self._reader, self._writer = Pipe(duplex=False)
        self._rlock = Lock()
        self._opid = os.getpid()
        if sys.platform == 'win32':
            self._wlock = None
        else:
            self._wlock = Lock()
        self._sem = BoundedSemaphore(maxsize)

        self._after_fork()

        if sys.platform != 'win32':
            register_after_fork(self, Queue._after_fork) 

def __init__(self, maxsize=0):
        if maxsize <= 0:
            maxsize = _multiprocessing.SemLock.SEM_VALUE_MAX
        self._maxsize = maxsize
        self._reader, self._writer = Pipe(duplex=False)
        self._rlock = Lock()
        self._opid = os.getpid()
        if sys.platform == 'win32':
            self._wlock = None
        else:
            self._wlock = Lock()
        self._sem = BoundedSemaphore(maxsize)

        self._after_fork()

        if sys.platform != 'win32':
            register_after_fork(self, Queue._after_fork) 

def start(self):
        parent_conn, child_conn = multiprocessing.Pipe()
        event = multiprocessing.Event()

        self.__conn = parent_conn
        self.__run_process = multiprocessing.Process(target=self.run, args=(child_conn, event))
        self.__run_process.start()

        # To avoid defunct process
        self.__join_thread = threading.Thread(target=self.wait)
        self.__join_thread.start()

        # If no sleep then CommonProcess will be terminated with exitcode SIGSEGV.
        # It may be python bug/
        time.sleep(conf.SLEEP_SECONDS_FOR_INIT_COMMON_PROCESS)
        event.wait() 

def test_basic_usage(self):
        """"""
        pipp, pipc = Pipe()
        pips, pipr = Pipe()
        self.print_bar()
        print('Test Task Interface')
        ret_process = ProcessTask(id='test-1', target=testfun, args=(5,), 
                                  status_monitor_pipe=pipc,
                                  result_pipe=pips, 
                                  result_hook_function=result_callback)
        ret_process.start()
        print('Test get threads status')
        time.sleep(1)
        #print(ret_process.subthreads_count)
        
        
        threads_status = pipp.recv()
        self.assertIsInstance(threads_status, dict)
        #print pipr.recv()
        #print pipr.recv()
        #print pipr.recv()
        #print pipr.recv()
        self.print_end_bar() 

def test_basic_usage(self):
        """"""
        pipp, pipc = Pipe()
        pips, pipr = Pipe()
        self.print_bar()
        print('Test Task Interface')
        ret_process = ProcessTask(id='test-1', target=testfun, args=(5,), 
                                  status_monitor_pipe=pipc,
                                  result_pipe=pips, 
                                  result_hook_function=result_callback)
        ret_process.start()
        print('Test get threads status')
        time.sleep(1)
        #print(ret_process.subthreads_count)
        
        
        threads_status = pipp.recv()
        self.assertIsInstance(threads_status, dict)
        #print pipr.recv()
        #print pipr.recv()
        #print pipr.recv()
        #print pipr.recv()
        self.print_end_bar()