Python multiprocessing.connection.wait() Examples

def shutdown(self, wait=True):
        with self._shutdown_lock:
            self._shutdown_thread = True
        if self._queue_management_thread:
            # Wake up queue management thread
            self._queue_management_thread_wakeup.wakeup()
            if wait:
                self._queue_management_thread.join()
        # To reduce the risk of opening too many files, remove references to
        # objects that use file descriptors.
        self._queue_management_thread = None
        if self._call_queue is not None:
            self._call_queue.close()
            if wait:
                self._call_queue.join_thread()
            self._call_queue = None
        self._result_queue = None
        self._processes = None

        if self._queue_management_thread_wakeup:
            self._queue_management_thread_wakeup.close()
            self._queue_management_thread_wakeup = None 

def shutdown(self, wait=True):
        with self._shutdown_lock:
            self._shutdown_thread = True
        if self._queue_management_thread:
            # Wake up queue management thread
            self._queue_management_thread_wakeup.wakeup()
            if wait:
                self._queue_management_thread.join()
        # To reduce the risk of opening too many files, remove references to
        # objects that use file descriptors.
        self._queue_management_thread = None
        if self._call_queue is not None:
            self._call_queue.close()
            if wait:
                self._call_queue.join_thread()
            self._call_queue = None
        self._result_queue = None
        self._processes = None

        if self._queue_management_thread_wakeup:
            self._queue_management_thread_wakeup.close()
            self._queue_management_thread_wakeup = None 

def rollout(self, policy, train, epoch, discard_other_rollout_type=False):
        if not self.workers_alive():
            raise RuntimeError("Worker has died prematurely.")
        # prevent starvation if only one worker
        if discard_other_rollout_type:
            self._discard_rollout_promises(not train)

        self._alloc_rollout_promise(policy, train, epoch)
        # Check if remote path has been collected.
        ready_promises = wait(self.rollout_promise_list[train], timeout=0)
        for rollout_promise in ready_promises:
            rollout = rollout_promise.recv()
            path_epoch, _ = self.pipe_info[rollout_promise]
            self._free_rollout_promise(rollout_promise)
            # Throw away eval paths from previous epochs
            if path_epoch != epoch and train == False:
                continue
            self._alloc_rollout_promise(policy, train, epoch)
            return rollout
        return None 

def poll(self, flag=os.WNOHANG):
        if self.returncode is None:
            from multiprocessing.connection import wait
            timeout = 0 if flag == os.WNOHANG else None
            if not wait([self.sentinel], timeout):
                return None
            try:
                self.returncode = forkserver.read_signed(self.sentinel)
            except (OSError, EOFError):
                # This should not happen usually, but perhaps the forkserver
                # process itself got killed
                self.returncode = 255

        return self.returncode 

def map(self, fn, *iterables, timeout=None, chunksize=1):
        """Returns an iterator equivalent to map(fn, iter).

        Args:
            fn: A callable that will take as many arguments as there are
                passed iterables.
            timeout: The maximum number of seconds to wait. If None, then there
                is no limit on the wait time.
            chunksize: If greater than one, the iterables will be chopped into
                chunks of size chunksize and submitted to the process pool.
                If set to one, the items in the list will be sent one at a time.

        Returns:
            An iterator equivalent to: map(func, *iterables) but the calls may
            be evaluated out-of-order.

        Raises:
            TimeoutError: If the entire result iterator could not be generated
                before the given timeout.
            Exception: If fn(*args) raises for any values.
        """
        if chunksize < 1:
            raise ValueError("chunksize must be >= 1.")

        results = super().map(partial(_process_chunk, fn),
                              _get_chunks(*iterables, chunksize=chunksize),
                              timeout=timeout)
        return _chain_from_iterable_of_lists(results) 

def prediction_worker(self):
        logger.debug("prediction_worker started")
        average_prediction_size = []
        last_model_check_time = time()
        while True:
            if last_model_check_time+60 < time():
                self.try_reload_model()
                last_model_check_time = time()
                logger.debug(f"average_prediction_size={np.average(average_prediction_size)}")
                average_prediction_size = []
            ready_conns = connection.wait(self.connections, timeout=0.001)  # type: list[Connection]
            if not ready_conns:
                continue
            data = []
            size_list = []
            for conn in ready_conns:
                x = conn.recv()
                data.append(x)  # shape: (k, 2, 8, 8)
                size_list.append(x.shape[0])  # save k
            average_prediction_size.append(np.sum(size_list))
            array = np.concatenate(data, axis=0)
            policy_ary, value_ary = self.model.model.predict_on_batch(array)
            idx = 0
            for conn, s in zip(ready_conns, size_list):
                conn.send((policy_ary[idx:idx+s], value_ary[idx:idx+s]))
                idx += s 

def poll(self, flag=os.WNOHANG):
        if self.returncode is None:
            from multiprocessing.connection import wait
            timeout = 0 if flag == os.WNOHANG else None
            if not wait([self.sentinel], timeout):
                return None
            try:
                self.returncode = forkserver.read_unsigned(self.sentinel)
            except (OSError, EOFError):
                # The process ended abnormally perhaps because of a signal
                self.returncode = 255
        return self.returncode 

def terminate(self):
        if self.returncode is None:
            try:
                os.kill(self.pid, signal.SIGTERM)
            except ProcessLookupError:
                pass
            except OSError:
                if self.wait(timeout=0.1) is None:
                    raise 

def wait(self, timeout=None):
        if self.returncode is None:
            if timeout is not None:
                from multiprocessing.connection import wait
                if not wait([self.sentinel], timeout):
                    return None
            # This shouldn't block if wait() returned successfully.
            return self.poll(os.WNOHANG if timeout == 0.0 else 0)
        return self.returncode 

def shutdown(self, wait=True):
        with self._shutdown_lock:
            self._shutdown_thread = True
        if self._queue_management_thread:
            # Wake up queue management thread
            self._result_queue.put(None)
            if wait:
                self._queue_management_thread.join()
        # To reduce the risk of opening too many files, remove references to
        # objects that use file descriptors.
        self._queue_management_thread = None
        self._call_queue = None
        self._result_queue = None
        self._processes = None 

def map(self, fn, *iterables, timeout=None, chunksize=1):
        """Returns an iterator equivalent to map(fn, iter).

        Args:
            fn: A callable that will take as many arguments as there are
                passed iterables.
            timeout: The maximum number of seconds to wait. If None, then there
                is no limit on the wait time.
            chunksize: If greater than one, the iterables will be chopped into
                chunks of size chunksize and submitted to the process pool.
                If set to one, the items in the list will be sent one at a time.

        Returns:
            An iterator equivalent to: map(func, *iterables) but the calls may
            be evaluated out-of-order.

        Raises:
            TimeoutError: If the entire result iterator could not be generated
                before the given timeout.
            Exception: If fn(*args) raises for any values.
        """
        if chunksize < 1:
            raise ValueError("chunksize must be >= 1.")

        results = super().map(partial(_process_chunk, fn),
                              _get_chunks(*iterables, chunksize=chunksize),
                              timeout=timeout)
        return itertools.chain.from_iterable(results) 

def terminate(self):
            if self.returncode is None:
                try:
                    os.kill(self.pid, signal.SIGTERM)
                except ProcessLookupError:
                    pass
                except OSError:
                    if self.wait(timeout=0.1) is None:
                        raise 

def wait(self, timeout=None):
            if sys.version_info < (3, 3):
                import time
                if timeout is None:
                    return self.poll(0)
                deadline = time.time() + timeout
                delay = 0.0005
                while 1:
                    res = self.poll()
                    if res is not None:
                        break
                    remaining = deadline - time.time()
                    if remaining <= 0:
                        break
                    delay = min(delay * 2, remaining, 0.05)
                    time.sleep(delay)
                return res

            if self.returncode is None:
                if timeout is not None:
                    from multiprocessing.connection import wait
                    if not wait([self.sentinel], timeout):
                        return None
                # This shouldn't block if wait() returned successfully.
                return self.poll(os.WNOHANG if timeout == 0.0 else 0)
            return self.returncode 

def terminate(self):
        if self.returncode is None:
            try:
                os.kill(self.pid, signal.SIGTERM)
            except ProcessLookupError:
                pass
            except OSError:
                if self.wait(timeout=0.1) is None:
                    raise 

def wait(self, timeout=None):
        if self.returncode is None:
            if timeout is not None:
                from multiprocessing.connection import wait
                if not wait([self.sentinel], timeout):
                    return None
            # This shouldn't block if wait() returned successfully.
            return self.poll(os.WNOHANG if timeout == 0.0 else 0)
        return self.returncode 

def shutdown(self, wait=True):
        with self._shutdown_lock:
            self._shutdown_thread = True
        if self._queue_management_thread:
            # Wake up queue management thread
            self._result_queue.put(None)
            if wait:
                self._queue_management_thread.join()
        # To reduce the risk of opening too many files, remove references to
        # objects that use file descriptors.
        self._queue_management_thread = None
        self._call_queue = None
        self._result_queue = None
        self._processes = None 

def _send_signal(self, sig):
        if self.returncode is None:
            try:
                os.kill(self.pid, sig)
            except ProcessLookupError:
                pass
            except OSError:
                if self.wait(timeout=0.1) is None:
                    raise 

def wait(self, timeout=None):
        if self.returncode is None:
            if timeout is not None:
                from multiprocessing.connection import wait
                if not wait([self.sentinel], timeout):
                    return None
            # This shouldn't block if wait() returned successfully.
            return self.poll(os.WNOHANG if timeout == 0.0 else 0)
        return self.returncode 

def map(self, fn, *iterables, timeout=None, chunksize=1):
        """Returns an iterator equivalent to map(fn, iter).

        Args:
            fn: A callable that will take as many arguments as there are
                passed iterables.
            timeout: The maximum number of seconds to wait. If None, then there
                is no limit on the wait time.
            chunksize: If greater than one, the iterables will be chopped into
                chunks of size chunksize and submitted to the process pool.
                If set to one, the items in the list will be sent one at a time.

        Returns:
            An iterator equivalent to: map(func, *iterables) but the calls may
            be evaluated out-of-order.

        Raises:
            TimeoutError: If the entire result iterator could not be generated
                before the given timeout.
            Exception: If fn(*args) raises for any values.
        """
        if chunksize < 1:
            raise ValueError("chunksize must be >= 1.")

        results = super().map(partial(_process_chunk, fn),
                              _get_chunks(*iterables, chunksize=chunksize),
                              timeout=timeout)
        return _chain_from_iterable_of_lists(results) 

def poll(self, flag=os.WNOHANG):
        if self.returncode is None:
            from multiprocessing.connection import wait
            timeout = 0 if flag == os.WNOHANG else None
            if not wait([self.sentinel], timeout):
                return None
            try:
                self.returncode = forkserver.read_unsigned(self.sentinel)
            except (OSError, EOFError):
                # The process ended abnormally perhaps because of a signal
                self.returncode = 255
        return self.returncode 

def terminate(self):
        if self.returncode is None:
            try:
                os.kill(self.pid, signal.SIGTERM)
            except ProcessLookupError:
                pass
            except OSError:
                if self.wait(timeout=0.1) is None:
                    raise 

def wait(self, timeout=None):
        if self.returncode is None:
            if timeout is not None:
                from multiprocessing.connection import wait
                if not wait([self.sentinel], timeout):
                    return None
            # This shouldn't block if wait() returned successfully.
            return self.poll(os.WNOHANG if timeout == 0.0 else 0)
        return self.returncode 

def shutdown(self, wait=True):
        with self._shutdown_lock:
            self._shutdown_thread = True
        if self._queue_management_thread:
            # Wake up queue management thread
            self._result_queue.put(None)
            if wait:
                self._queue_management_thread.join()
        # To reduce the risk of opening too many files, remove references to
        # objects that use file descriptors.
        self._queue_management_thread = None
        self._call_queue = None
        self._result_queue = None
        self._processes = None 

def map(self, fn, *iterables, timeout=None, chunksize=1):
        """Returns an iterator equivalent to map(fn, iter).

        Args:
            fn: A callable that will take as many arguments as there are
                passed iterables.
            timeout: The maximum number of seconds to wait. If None, then there
                is no limit on the wait time.
            chunksize: If greater than one, the iterables will be chopped into
                chunks of size chunksize and submitted to the process pool.
                If set to one, the items in the list will be sent one at a time.

        Returns:
            An iterator equivalent to: map(func, *iterables) but the calls may
            be evaluated out-of-order.

        Raises:
            TimeoutError: If the entire result iterator could not be generated
                before the given timeout.
            Exception: If fn(*args) raises for any values.
        """
        if chunksize < 1:
            raise ValueError("chunksize must be >= 1.")

        results = super().map(partial(_process_chunk, fn),
                              _get_chunks(*iterables, chunksize=chunksize),
                              timeout=timeout)
        return itertools.chain.from_iterable(results) 

def shutdown(self, wait=True):
        with self._shutdown_lock:
            self._shutdown_thread = True
        if self._queue_management_thread:
            # Wake up queue management thread
            self._result_queue.put(None)
            if wait:
                self._queue_management_thread.join()
        # To reduce the risk of opening too many files, remove references to
        # objects that use file descriptors.
        self._queue_management_thread = None
        self._call_queue = None
        self._result_queue = None
        self._manager = None 

def __init__(self, _ActorClass, *args, **kwargs):
        process._check_system_limits()

        self._ActorClass = _ActorClass
        # todo: If we want to cancel futures we need to give the task_queue a
        # maximum size
        self._call_queue = multiprocessing.JoinableQueue()
        self._call_queue._ignore_epipe = True
        self._result_queue = multiprocessing.Queue()
        self._work_ids = queue.Queue()
        self._queue_management_thread = None

        # We only maintain one process for our actor
        self._manager = None

        # Shutdown is a two-step process.
        self._shutdown_thread = False
        self._shutdown_lock = threading.Lock()
        self._broken = False
        self._queue_count = 0
        self._pending_work_items = {}

        self._did_initialize = False

        if args or kwargs:
            # If given actor initialization args we must start the Actor
            # immediately. Otherwise just wait until we get a message
            print('Init with args')
            print('args = %r' % (args,))
            self._initialize_actor(*args, **kwargs) 

def _worker_loop(pipe, *worker_env_args, **worker_env_kwargs):
        env = RemoteRolloutEnv.WorkerEnv(*worker_env_args, **worker_env_kwargs)
        while True:
            wait([pipe])
            env_update, rollout_args = pipe.recv()
            if env_update is not None:
                env._env.update_env(**env_update)
            rollout = env.rollout(*rollout_args)
            pipe.send(rollout) 

def _discard_rollout_promises(self, train_type):
        ready_promises = wait(self.rollout_promise_list[train_type], timeout=0)
        for rollout_promise in ready_promises:
            self._free_rollout_promise(rollout_promise) 

def _free_rollout_promise(self, pipe):
        _, train = self.pipe_info[pipe]
        assert pipe not in self.free_pipes
        if wait([pipe], timeout=0):
            pipe.recv()
        self.free_pipes.add(pipe)
        del self.pipe_info[pipe]
        self.rollout_promise_list[train].remove(pipe) 

def poll(self, flag=os.WNOHANG):
        if self.returncode is None:
            from multiprocessing.connection import wait
            timeout = 0 if flag == os.WNOHANG else None
            if not wait([self.sentinel], timeout):
                return None
            try:
                self.returncode = forkserver.read_unsigned(self.sentinel)
            except (OSError, EOFError):
                # The process ended abnormally perhaps because of a signal
                self.returncode = 255
        return self.returncode