Python multiprocessing.TimeoutError() Examples

def abortable_worker(func, *args, **kwargs):
    # returns ("MULTIPROCESSING_TIMEOUT",) if timeout
    timeout = kwargs['timeout']
    p = multiprocessing.dummy.Pool(1)
    res = p.apply_async(func, args=args)
    # assumes timeout is an integer >= 1
    for i in range(timeout + 1):
        if i > 0:
            time.sleep(1)
        if res.ready():
            if res.successful():
                try:
                    return res.get(timeout=1)
                except multiprocessing.TimeoutError:
                    logger.info("Aborting due to timeout in get")
                    p.terminate()
                    return (TIMEOUT_ID,)
    logger.info("Aborting due to timeout")
    p.terminate()
    return (TIMEOUT_ID,) 

def wait_for_device():
    """
    Wait for device to boot. 1 minute timeout.
    """

    wait_for_device_cmd = 'wait-for-device shell getprop sys.boot_completed'
    p = ADB.exec_cmd(wait_for_device_cmd, stdout=subprocess.PIPE)
    boot_completed = p.stdout.readline().strip('\r\n')

    try:
        with Timeout(seconds=60):
            while boot_completed != '1':
                time.sleep(1)
                p = ADB.exec_cmd(wait_for_device_cmd, stdout=subprocess.PIPE)
                boot_completed = p.stdout.readline().strip('\r\n')
                Logger.debug('Waiting for device to finish booting (adb shell getprop sys.boot_completed)')
    except TimeoutError:
        Logger.debug('Timed out while waiting for sys.boot_completed, there might not be a default launcher set, trying to run anyway')
        pass 

def get(self, timeout=None):
        self.wait(timeout)
        if self._result_thread.is_alive():
            raise TimeoutError

        results = []
        for batch in self._result_thread.results():
            for result in batch:
                if isinstance(result, PoolTaskError):
                    raise result.underlying
                elif isinstance(result, Exception):
                    raise result
            results.extend(batch)

        if self._single_result:
            return results[0]

        return results 

def wait_for_image(self, ctx: commands.Context) -> discord.Message:
        await ctx.send(_("Upload an image for me to use! Type `exit` to cancel."))
        msg = None
        while msg is None:

            def check(m):
                return m.author == ctx.author and (m.attachments or "exit" in m.content)

            try:
                msg = await self.bot.wait_for("message", check=check, timeout=60)
            except asyncio.TimeoutError:
                await ctx.send(_("Image adding timed out."))
                break
            if "exit" in msg.content.lower():
                await ctx.send(_("Image adding cancelled."))
                break
        return msg 

def map(self, func, iterable, chunksize=None):
        """
        Equivalent of `map()` built-in, without swallowing
        `KeyboardInterrupt`.

        :param func:
            The function to apply to the items.

        :param iterable:
            An iterable of items that will have `func` applied to them.

        """
        # The key magic is that we must call r.get() with a timeout, because
        # a Condition.wait() without a timeout swallows KeyboardInterrupts.
        r = self.map_async(func, iterable, chunksize)

        while True:
            try:
                return r.get(self.wait_timeout)
            except multiprocessing.TimeoutError:
                pass
            except KeyboardInterrupt:
                self.terminate()
                self.join()
                raise 

def wait_for_multiple_responses(self, ctx: commands.Context) -> List[discord.Message]:
        msg_text = _(
            "Please enter your desired phrase to be used for this trigger."
            "Type `exit` to stop adding responses."
        )
        await ctx.send(msg_text)
        responses: list = []
        while True:

            def check(m):
                return m.author == ctx.author

            try:
                message = await self.bot.wait_for("message", check=check, timeout=60)
                await message.add_reaction("✅")
            except asyncio.TimeoutError:
                return responses
            if message.content == "exit":
                return responses
            else:
                responses.append(message.content) 

def get(self, timeout=None):
        """
        Return the result when it arrives. If timeout is not None and the
        result does not arrive within timeout seconds then
        multiprocessing.TimeoutError is raised. If the remote call raised an
        exception then that exception will be reraised by get().
        """

        try:
            res = self._q.get(timeout=timeout)
        except Queue.Empty:
            raise multiprocessing.TimeoutError("Timed out")

        if isinstance(res, Exception):
            raise res
        return res 

def kill(self, retries=3, pause=.1):
        try:
            try:
                self.task.get(1)
                self.__set_info('KILLED', "None")
            except (AttributeError, TimeoutError):
                self.__set_info('CANCELLED', "None")
            except UnicodeEncodeError:
                self.__set_info('CRASHED', "None")
            for pid in self.pids:
                try:
                    with open(pid) as f:
                        os.kill(int(f.read().strip()), signal.SIGTERM)
                    os.remove(pid)
                except (IOError, OSError):
                    pass  # simply fail silently when no PID or OS cannot kill it as it is already terminated
            if self.command.__name__.lstrip('_') == 'run' and retries > 0:
                time.sleep(pause)  # wait ... sec that the next call from the command starts
                                   # this is necessary e.g. with cooja command (when Cooja starts a first time for
                                   #  a simulation without a malicious mote then a second time with)
                self.kill(retries - 1, 2 * pause)  # then kill it
        except KeyboardInterrupt:
            self.kill(0, 0) 

def map(self, func, items, chunksize=None):
        """
        A replacement for :func:`map` that handles :exc:`KeyboardInterrupt`.

        :param func:
            Function to apply to the items.

        :param items:
            Iterable of items to have `func` applied to.
        """
        # Call r.get() with a timeout, since a Condition.wait() swallows
        # KeyboardInterrupts without a timeout
        r = self.map_async(func, items, chunksize)

        while True:
            try:
                return r.get(self._wait_timeout)
            except TimeoutError:
                pass
            except KeyboardInterrupt:
                self.terminate()
                self.join()
                raise 

def map(self, func, items, chunksize=None):
        """ Catch keyboard interuppts to allow the pool to exit cleanly.

        Parameters
        ----------
        func: function
            Function to call
        items: list of tuples
            Arguments to pass
        chunksize: int, Optional
            Number of calls for each process to handle at once
        """
        results = self.map_async(func, items, chunksize)
        while True:
            try:
                return results.get(1800)
            except TimeoutError:
                pass
            except KeyboardInterrupt:
                self.terminate()
                self.join()
                raise KeyboardInterrupt 

def get(self, timeout=None):
        """
        Return the result when it arrives. If timeout is not None and the
        result does not arrive within timeout seconds then
        multiprocessing.TimeoutError is raised. If the remote call raised an
        exception then that exception will be reraised by get().
        """

        try:
            res = self._q.get(timeout=timeout)
        except queue.Empty:
            raise multiprocessing.TimeoutError("Timed out")

        if isinstance(res, Exception):
            raise res
        return res 

def input_timeout(prompt, timeout):
    """
    """

    pool = ThreadPool(processes=1)
    answer = pool.apply_async(input, args=[prompt])

    try:
        return answer.get(timeout=timeout)
    except TimeoutError as e:
        sys.exit('Timed out.') 

def map(self, func, iterable, chunksize=None):
        '''
        Equivalent of ``map()`` built-in, without swallowing
        ``KeyboardInterrupt``.

        :param func:
            The function to apply to the items.

        :param iterable:
            An iterable of items that will have `func` applied to them.

        '''
        # The key magic is that we must call r.get() with a timeout, because
        # a Condition.wait() without a timeout swallows KeyboardInterrupts.
        r = self.map_async(func, iterable, chunksize)

        while True:
            try:
                return r.get(self.wait_timeout)
            except TimeoutError:
                pass
            except KeyboardInterrupt:
                self.terminate()
                self.join()
                raise
            # Other exceptions propagate up. 

def timeout(seconds, default="timeout"):
    def decorator(fn):
        @wraps(fn)
        def wrapper(*args, **kwargs):
            pool = ThreadPool(processes=1)
            async_result = pool.apply_async(fn, args=args, kwds=kwargs)
            try:
                return async_result.get(seconds)
            except TimeoutError:
                if callable(default):
                    return default()
                return default
        return wrapper
    return decorator 

def wait(self, timeout=None):
        """
        Returns once the result is ready or the timeout expires (does not
        raise TimeoutError).

        Args:
            timeout: timeout in milliseconds.
        """

        self._result_thread.join(timeout) 

def timeout(max_timeout):
    """Timeout decorator, parameter in seconds."""
    def timeout_decorator(f):
        """Wrap the original function."""
        @functools.wraps(f)
        def func_wrapper(self, *args, **kwargs):
            """Closure for function."""
            pool = multiprocessing.pool.ThreadPool(processes=1)
            async_result = pool.apply_async(f, (self,) + args, kwargs)
            timeout = kwargs.pop('timeout_max_timeout', max_timeout) or max_timeout
            # raises a TimeoutError if execution exceeds max_timeout
            return async_result.get(timeout)
        return func_wrapper
    return timeout_decorator 

def _get_resultl(self,resultl):
        retl=[]
        # get result from eflist
        for cur in resultl:
            try:
                retl.extend(cur.get(timeout=self._timeout))
            except TimeoutError as e:
                continue
        return retl 

def test_fetch_cvimage_from_url_timeout(monkeypatch):
    def long_func(*args, **kwargs):
        time.sleep(3)
    monkeypatch.setattr(requests, 'get', long_func)
    with pytest.raises(TimeoutError):
        classifiers.fetch_cvimage_from_url('this url is ignored', timeout_max_timeout=1) 

def test_map_chunksize(self):
        try:
            self.pool.map_async(sqr, [], chunksize=1).get(timeout=TIMEOUT1)
        except multiprocessing.TimeoutError:
            self.fail("pool.map_async with chunksize stalled on null list") 

def test_async_timeout(self):
        res = self.pool.apply_async(sqr, (6, TIMEOUT2 + 1.0))
        get = TimingWrapper(res.get)
        self.assertRaises(multiprocessing.TimeoutError, get, timeout=TIMEOUT2)
        self.assertTimingAlmostEqual(get.elapsed, TIMEOUT2) 

def parallel_execute(commands: List[List[str]], cpus: Optional[int] = None,
                     timeout: Optional[int] = None, verbose: bool = True) -> List[int]:
    """ Limited return vals, only returns return codes
    """
    if verbose:
        runner = verbose_child_process
    else:
        runner = child_process
    os.setpgid(0, 0)
    if not cpus:
        cpus = get_config().cpus
    assert isinstance(cpus, int)
    pool = multiprocessing.Pool(cpus)
    jobs = pool.map_async(runner, commands)

    try:
        errors = jobs.get(timeout=timeout)
    except multiprocessing.TimeoutError:
        pool.terminate()
        assert isinstance(timeout, int)
        raise RuntimeError("One of %d child processes timed out after %d seconds" % (
                cpus, timeout))

    except KeyboardInterrupt:
        logging.error("Interrupted by user")
        pool.terminate()
        raise

    pool.close()

    return errors 

def reset_wait(self, timeout=None):
        """
        Parameters
        ----------
        timeout : int or float, optional
            Number of seconds before the call to `reset_wait` times out. If
            `None`, the call to `reset_wait` never times out.

        Returns
        -------
        observations : sample from `observation_space`
            A batch of observations from the vectorized environment.
        """
        self._assert_is_running()
        if self._state != AsyncState.WAITING_RESET:
            raise NoAsyncCallError('Calling `reset_wait` without any prior '
                'call to `reset_async`.', AsyncState.WAITING_RESET.value)

        if not self._poll(timeout):
            self._state = AsyncState.DEFAULT
            raise mp.TimeoutError('The call to `reset_wait` has timed out after '
                '{0} second{1}.'.format(timeout, 's' if timeout > 1 else ''))

        results, successes = zip(*[pipe.recv() for pipe in self.parent_pipes])
        self._raise_if_errors(successes)
        self._state = AsyncState.DEFAULT

        if not self.shared_memory:
            concatenate(results, self.observations, self.single_observation_space)

        return deepcopy(self.observations) if self.copy else self.observations 

def close_extras(self, timeout=None, terminate=False):
        """
        Parameters
        ----------
        timeout : int or float, optional
            Number of seconds before the call to `close` times out. If `None`,
            the call to `close` never times out. If the call to `close` times
            out, then all processes are terminated.

        terminate : bool (default: `False`)
            If `True`, then the `close` operation is forced and all processes
            are terminated.
        """
        timeout = 0 if terminate else timeout
        try:
            if self._state != AsyncState.DEFAULT:
                logger.warn('Calling `close` while waiting for a pending '
                    'call to `{0}` to complete.'.format(self._state.value))
                function = getattr(self, '{0}_wait'.format(self._state.value))
                function(timeout)
        except mp.TimeoutError:
            terminate = True

        if terminate:
            for process in self.processes:
                if process.is_alive():
                    process.terminate()
        else:
            for pipe in self.parent_pipes:
                if (pipe is not None) and (not pipe.closed):
                    pipe.send(('close', None))
            for pipe in self.parent_pipes:
                if (pipe is not None) and (not pipe.closed):
                    pipe.recv()

        for pipe in self.parent_pipes:
            if pipe is not None:
                pipe.close()
        for process in self.processes:
            process.join() 

def test_reset_timeout_async_vector_env(shared_memory):
    env_fns = [make_slow_env(0.3, i) for i in range(4)]
    with pytest.raises(TimeoutError):
        try:
            env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
            env.reset_async()
            observations = env.reset_wait(timeout=0.1)
        finally:
            env.close(terminate=True) 

def test_step_timeout_async_vector_env(shared_memory):
    env_fns = [make_slow_env(0., i) for i in range(4)]
    with pytest.raises(TimeoutError):
        try:
            env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
            observations = env.reset()
            env.step_async([0.1, 0.1, 0.3, 0.1])
            observations, rewards, dones, _ = env.step_wait(timeout=0.1)
        finally:
            env.close(terminate=True) 

def wait_for_multiple_images(self, ctx: commands.Context) -> List[str]:
        await ctx.send(_("Upload an image for me to use! Type `exit` to cancel."))
        files: list = []
        while True:

            def check(m):
                return m.author == ctx.author

            try:
                msg = await self.bot.wait_for("message", check=check, timeout=60)
            except asyncio.TimeoutError:
                return files
            if "exit" in msg.content.lower():
                return files
            else:
                link = LINK_REGEX.search(msg.content)
                for a in msg.attachments:
                    if a.size > 8 * 1000 * 1000:
                        continue
                    try:
                        files.append(await self.save_image_location(a.url, ctx.guild))
                        await msg.add_reaction("✅")
                    except Exception:
                        pass
                if link:
                    try:
                        files.append(await self.save_image_location(link.group(0), ctx.guild))
                        await msg.add_reaction("✅")
                    except Exception:
                        pass
        return files 

def _query_num_gpus():
    """
    Returns the number of GPU devices on the host. Returns 0 if the host has no GPU devices.
    """

    global _num_gpus
    if _num_gpus is None:
        COMMAND = 'nvidia-smi -L 2>/dev/null | grep \'GPU [0-9]\' | wc -l'
        TIMEOUT_SECONDS = 75
        STATUS_POLL_INTERVAL_SECONDS = 0.025

        try:
            proc = subprocess.Popen(COMMAND, shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, bufsize=1)
        except (OSError, ValueError):
            logging.exception("Error launching /usr/bin/nvidia-smi.")
            return 0

        start_time = time.time()

        # Wait for the process to finish
        exitcode = None
        while exitcode is None and time.time() - start_time < TIMEOUT_SECONDS:
            time.sleep(STATUS_POLL_INTERVAL_SECONDS)
            exitcode = proc.poll()

        # Terminate the process if not finished
        if exitcode is None:
            logging.error("nvidia-smi timed out after %s secs", time.time() - start_time)
            proc.terminate()
            raise TimeoutError

        _num_gpus = int(proc.stdout.readline())
        logging.info("nvidia-smi took: %s secs to identify %d gpus", time.time() - start_time, _num_gpus)

    return _num_gpus 

def get_num_gpus(num_gpus=AUTODETECT_GPU_COUNT, **kwargs):
    """
    Returns the number of available GPUs based on configuration parameters and available hardware GPU devices.

    :param num_gpus: (int or "auto")
        If set to "auto", the function queries and returns the number of available GPUs.
        If set to an integer value, the function returns the value of min(num_gpus, auto_detected_gpu_count)
        Otherwise raises ValueError.
    :param kwargs: additional configuration parameters, not used
    :return: (int) number of GPUs
    """

    # Shortcut execution if what we want is 0 gpu, i.e. only cpu
    if num_gpus == 0:
        return 0

    try:
        num_available_gpus = _query_num_gpus()
    except TimeoutError:
        if num_gpus == AUTODETECT_GPU_COUNT:
            return 0
        else:
            return num_gpus

    if num_gpus == AUTODETECT_GPU_COUNT:
        return num_available_gpus
    else:
        try:
            num_requested_gpus = int(num_gpus)
        except ValueError:
            raise CustomerValueError(
                "Invalid value '{}' provided for hyperparameter '_num_gpus'. '_num_gpus' must be an integer or 'auto'. "
                "Please set the value of '_num_gpus' hyperparameter to 'auto' or an integer value and try again."
                .format(num_gpus))

        if num_requested_gpus > num_available_gpus:
            logging.warning("Request number of gpus: %d, Number of GPUs found: %d",
                            num_requested_gpus, num_available_gpus)
            return num_available_gpus
        else:
            return num_requested_gpus 

def call_gnugo(self, sgf_file_name, command):
        try:
            pool = multiprocessing.Pool(processes=1)
            result = pool.apply_async(run_gnugo, (sgf_file_name, command))
            output = result.get(timeout=10)
            pool.close()
            return output
        except multiprocessing.TimeoutError:
            pool.terminate()
            # if can't get answer from GnuGo, return no result
            return '' 

def test_timeout():
    @timeout(0.1)
    def timeout_function(secs: float):
        sleep(secs)
        return True

    with pytest.raises(MultiprocessingTimeoutError):
        timeout_function(1)
    assert timeout_function(0.01)