Python concurrent.futures.ALL_COMPLETED Examples

def test_all_completed(self):
        future1 = self.executor.submit(divmod, 2, 0)
        future2 = self.executor.submit(mul, 2, 21)

        finished, pending = futures.wait(
                [SUCCESSFUL_FUTURE,
                 CANCELLED_AND_NOTIFIED_FUTURE,
                 EXCEPTION_FUTURE,
                 future1,
                 future2],
                return_when=futures.ALL_COMPLETED)

        self.assertEqual(set([SUCCESSFUL_FUTURE,
                              CANCELLED_AND_NOTIFIED_FUTURE,
                              EXCEPTION_FUTURE,
                              future1,
                              future2]), finished)
        self.assertEqual(set(), pending) 

def test_timeout(self):
        future1 = self.executor.submit(mul, 6, 7)
        future2 = self.executor.submit(time.sleep, 6)

        finished, pending = futures.wait(
                [CANCELLED_AND_NOTIFIED_FUTURE,
                 EXCEPTION_FUTURE,
                 SUCCESSFUL_FUTURE,
                 future1, future2],
                timeout=5,
                return_when=futures.ALL_COMPLETED)

        self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE,
                              EXCEPTION_FUTURE,
                              SUCCESSFUL_FUTURE,
                              future1]), finished)
        self.assertEqual(set([future2]), pending) 

def _create_and_install_waiters(fs, return_when):
        if return_when == _AS_COMPLETED:
            waiter = _AsCompletedWaiter()
        elif return_when == FIRST_COMPLETED:
            waiter = _FirstCompletedWaiter()
        else:
            pending_count = sum(
                    f._state not in [CANCELLED_AND_NOTIFIED, FINISHED]
                    for f in fs)

            if return_when == FIRST_EXCEPTION:
                waiter = _AllCompletedWaiter(pending_count,
                                             stop_on_exception=True)
            elif return_when == ALL_COMPLETED:
                waiter = _AllCompletedWaiter(pending_count,
                                             stop_on_exception=False)
            else:
                raise ValueError("Invalid return condition: %r" % return_when)

        for f in fs:
            f._waiters.append(waiter)

        return waiter 

def test_timeout(self):
        # Make sure the executor has already started to avoid timeout happening
        # before future1 returns
        assert self.executor.submit(id_sleep, 42).result() == 42

        future1 = self.executor.submit(mul, 6, 7)
        future2 = self.executor.submit(self.wait_and_return, 5)

        assert future1.result() == 42

        finished, pending = futures.wait([CANCELLED_AND_NOTIFIED_FUTURE,
                                          EXCEPTION_FUTURE, SUCCESSFUL_FUTURE,
                                          future1, future2],
                                         timeout=.1,
                                         return_when=futures.ALL_COMPLETED)

        assert set([CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE,
                    SUCCESSFUL_FUTURE, future1]) == finished
        assert set([future2]) == pending

        _executor_mixin._test_event.set()
        assert future2.result(timeout=10)
        _executor_mixin._test_event.clear() 

def test_timeout(self):
        future1 = self.executor.submit(mul, 6, 7)
        future2 = self.executor.submit(time.sleep, 6)

        finished, pending = futures.wait(
                [CANCELLED_AND_NOTIFIED_FUTURE,
                 EXCEPTION_FUTURE,
                 SUCCESSFUL_FUTURE,
                 future1, future2],
                timeout=5,
                return_when=futures.ALL_COMPLETED)

        self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE,
                              EXCEPTION_FUTURE,
                              SUCCESSFUL_FUTURE,
                              future1]), finished)
        self.assertEqual(set([future2]), pending) 

def test_all_completed(self):
        future1 = self.executor.submit(divmod, 2, 0)
        future2 = self.executor.submit(mul, 2, 21)

        finished, pending = futures.wait(
                [SUCCESSFUL_FUTURE,
                 CANCELLED_AND_NOTIFIED_FUTURE,
                 EXCEPTION_FUTURE,
                 future1,
                 future2],
                return_when=futures.ALL_COMPLETED)

        self.assertEqual(set([SUCCESSFUL_FUTURE,
                              CANCELLED_AND_NOTIFIED_FUTURE,
                              EXCEPTION_FUTURE,
                              future1,
                              future2]), finished)
        self.assertEqual(set(), pending) 

def sed_msg(mobile, set_of_times, interval_time, remainder):
    init_mobile(mobile, set_of_times)
    available_msg = []
    while len(available_msg) < remainder:
        m = random.choice(all_message_info)
        if check_available(m):
            available_msg.append(m)
    pool = ThreadPoolExecutor(500)
    SHARE_Q = queue.Queue(remainder)
    while SHARE_Q.qsize() < remainder:
        SHARE_Q.put(random.choice(available_msg))
    all_task = []
    for t in range(SHARE_Q.qsize()):
        mg = SHARE_Q.get()
        try:
            all_task.append(pool.submit(eval(mg), mg, mobile))
        except KeyboardInterrupt:
            r.delete('%s_%d' % (mobile, set_of_times))
            sys.exit(0)
        if interval_time:
            time.sleep(interval_time)
    wait(all_task, return_when=ALL_COMPLETED) 

def assemble_python_lambdas(project_path, bundles_dir):
    from syndicate.core import CONFIG
    project_base_folder = os.path.basename(os.path.normpath(project_path))
    project_abs_path = build_path(CONFIG.project_path, project_path)
    _LOG.info('Going to process python project by path: {0}'.format(
        project_abs_path))
    executor = ThreadPoolExecutor(max_workers=5)
    futures = []
    for root, sub_dirs, files in os.walk(project_abs_path):
        for item in files:
            if item.endswith(LAMBDA_CONFIG_FILE_NAME):
                _LOG.info('Going to build artifact in: {0}'.format(root))
                arg = {
                    'item': item,
                    'project_base_folder': project_base_folder,
                    'project_path': project_path,
                    'root': root,
                    'target_folder': bundles_dir
                }
                futures.append(executor.submit(_build_python_artifact, arg))
    concurrent.futures.wait(futures, return_when=ALL_COMPLETED)
    executor.shutdown()
    _LOG.info('Python project was processed successfully') 

def test_all_completed(self):
        future1 = self.executor.submit(divmod, 2, 0)
        future2 = self.executor.submit(mul, 2, 21)

        finished, pending = futures.wait(
                [SUCCESSFUL_FUTURE,
                 CANCELLED_AND_NOTIFIED_FUTURE,
                 EXCEPTION_FUTURE,
                 future1,
                 future2],
                return_when=futures.ALL_COMPLETED)

        self.assertEqual(set([SUCCESSFUL_FUTURE,
                              CANCELLED_AND_NOTIFIED_FUTURE,
                              EXCEPTION_FUTURE,
                              future1,
                              future2]), finished)
        self.assertEqual(set(), pending) 

def create_pool(self, job, parameters, workers=None):
        """ Create resources in pool in sub processes.

        :param workers:
        :type parameters: iterable
        :type job: func
        """
        executor = ThreadPoolExecutor(
            workers) if workers else ThreadPoolExecutor()
        try:
            # futures = [executor.submit(func, i, kwargs) for i in args]
            futures = []
            for param_chunk in parameters:
                param_chunk['self'] = self
                futures.append(executor.submit(job, param_chunk))
            concurrent.futures.wait(futures, return_when=ALL_COMPLETED)
            responses = {}
            for future in futures:
                result = future.result()
                if result:
                    responses.update(result)
            return responses
        finally:
            executor.shutdown(wait=True) 

def test_timeout(self):
        future1 = self.executor.submit(mul, 6, 7)
        future2 = self.executor.submit(time.sleep, 6)

        finished, pending = futures.wait(
                [CANCELLED_AND_NOTIFIED_FUTURE,
                 EXCEPTION_FUTURE,
                 SUCCESSFUL_FUTURE,
                 future1, future2],
                timeout=5,
                return_when=futures.ALL_COMPLETED)

        self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE,
                              EXCEPTION_FUTURE,
                              SUCCESSFUL_FUTURE,
                              future1]), finished)
        self.assertEqual(set([future2]), pending) 

def _stop(self):
        """
        Stop this AnimationLoop

        Shuts down the loop and triggers cleanup tasks.
        """
        if not self.running:
            return False

        self.running = False

        for layer in self.layers[::-1]:
            await self.remove_layer(layer)

        if self._anim_task is not None and not self._anim_task.done():
            self._anim_task.cancel()
            await asyncio.wait([self._anim_task], return_when=futures.ALL_COMPLETED)

        self._logger.info("AnimationLoop stopped") 

def stop(self):
        if self.renderer.running:

            tasks = []
            if self.task is not None and not self.task.done():
                self.task.cancel()
                tasks.append(self.task)

            if self.waiter is not None and not self.waiter.done():
                self.waiter.cancel()
                tasks.append(self.waiter)

            await self.renderer._stop()

            if tasks:
                await asyncio.wait(tasks, return_when=futures.ALL_COMPLETED)

            self.renderer.finish(self._frame) 

def _close_input_devices(self):
        if not hasattr(self, '_opened') or not self._opened:
            return

        self._opened = False

        for event_device in self._event_devices:
            asyncio.get_event_loop().remove_reader(event_device.fileno())
            event_device.close()

        tasks = []
        for task in self._tasks:
            if not task.done():
                task.cancel()
                tasks.append(task)

        await asyncio.wait(tasks, return_when=futures.ALL_COMPLETED)
        self._event_devices.clear() 

def test_all_completed(self):
        future1 = self.executor.submit(divmod, 2, 0)
        future2 = self.executor.submit(mul, 2, 21)

        finished, pending = futures.wait(
                [SUCCESSFUL_FUTURE,
                 CANCELLED_AND_NOTIFIED_FUTURE,
                 EXCEPTION_FUTURE,
                 future1,
                 future2],
                return_when=futures.ALL_COMPLETED)

        self.assertEqual(set([SUCCESSFUL_FUTURE,
                              CANCELLED_AND_NOTIFIED_FUTURE,
                              EXCEPTION_FUTURE,
                              future1,
                              future2]), finished)
        self.assertEqual(set(), pending) 

def test_timeout(self):
        future1 = self.executor.submit(mul, 6, 7)
        future2 = self.executor.submit(time.sleep, 6)

        finished, pending = futures.wait(
                [CANCELLED_AND_NOTIFIED_FUTURE,
                 EXCEPTION_FUTURE,
                 SUCCESSFUL_FUTURE,
                 future1, future2],
                timeout=5,
                return_when=futures.ALL_COMPLETED)

        self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE,
                              EXCEPTION_FUTURE,
                              SUCCESSFUL_FUTURE,
                              future1]), finished)
        self.assertEqual(set([future2]), pending) 

def test_pending_calls_race(self):
        # Issue #14406: multi-threaded race condition when waiting on all
        # futures.
        event = threading.Event()
        def future_func():
            event.wait()
        oldswitchinterval = sys.getswitchinterval()
        sys.setswitchinterval(1e-6)
        try:
            fs = {self.executor.submit(future_func) for i in range(100)}
            event.set()
            futures.wait(fs, return_when=futures.ALL_COMPLETED)
        finally:
            sys.setswitchinterval(oldswitchinterval) 

def test_pending_calls_race(self):
        # Issue #14406: multi-threaded race condition when waiting on all
        # futures.
        event = threading.Event()
        def future_func():
            event.wait()
        oldswitchinterval = sys.getswitchinterval()
        sys.setswitchinterval(1e-6)
        try:
            fs = {self.executor.submit(future_func) for i in range(100)}
            event.set()
            futures.wait(fs, return_when=futures.ALL_COMPLETED)
        finally:
            sys.setswitchinterval(oldswitchinterval) 

def wait(fs, timeout=None, return_when=ALL_COMPLETED):
    raise NotImplementedError() 

def test_all_completed(self):
        future1 = self.executor.submit(divmod, 2, 0)
        future2 = self.executor.submit(mul, 2, 21)

        finished, pending = futures.wait([SUCCESSFUL_FUTURE, EXCEPTION_FUTURE,
                                          CANCELLED_AND_NOTIFIED_FUTURE,
                                          future1, future2],
                                         return_when=futures.ALL_COMPLETED)

        assert set([SUCCESSFUL_FUTURE, CANCELLED_AND_NOTIFIED_FUTURE,
                    EXCEPTION_FUTURE, future1, future2]) == finished
        assert set() == pending 

def discover(self, timeout=None):    # TODO: better name?
        """Discover sentinels and all monitored services within given timeout.

        If no sentinels discovered within timeout: TimeoutError is raised.
        If some sentinels were discovered but not all — it is ok.
        If not all monitored services (masters/slaves) discovered
        (or connections established) — it is ok.
        TBD: what if some sentinels/services unreachable;
        """
        # TODO: check not closed
        # TODO: discovery must be done with some customizable timeout.
        if timeout is None:
            timeout = self.discover_timeout
        tasks = []
        pools = []
        for addr in self._sentinels:    # iterate over unordered set
            tasks.append(self._connect_sentinel(addr, timeout, pools))
        done, pending = await asyncio.wait(tasks,
                                           return_when=ALL_COMPLETED)
        assert not pending, ("Expected all tasks to complete", done, pending)

        for task in done:
            result = task.result()
            if isinstance(result, Exception):
                continue    # FIXME
        if not pools:
            raise Exception("Could not connect to any sentinel")
        pools, self._pools[:] = self._pools[:], pools
        # TODO: close current connections
        for pool in pools:
            pool.close()
            await pool.wait_closed()

        # TODO: discover peer sentinels
        for pool in self._pools:
            await pool.execute_pubsub(
                b'psubscribe', self._monitor.pattern('*')) 

def wait_for_all(fs, timeout=None):
    """Wait for all of the futures to complete.

    Works correctly with both green and non-green futures (but not both
    together, since this can't be guaranteed to avoid dead-lock due to how
    the waiting implementations are different when green threads are being
    used).

    Returns pair (done futures, not done futures).
    """
    return _wait_for(fs, futures.ALL_COMPLETED, _wait_for_all_green,
                     'wait_for_all', timeout=timeout) 

def test_pending_calls_race(self):
        # Issue #14406: multi-threaded race condition when waiting on all
        # futures.
        event = threading.Event()
        def future_func():
            event.wait()
        oldswitchinterval = sys.getswitchinterval()
        sys.setswitchinterval(1e-6)
        try:
            fs = {self.executor.submit(future_func) for i in range(100)}
            event.set()
            futures.wait(fs, return_when=futures.ALL_COMPLETED)
        finally:
            sys.setswitchinterval(oldswitchinterval) 

def _apply_dynamic_changes(resources, output):
    from syndicate.core import PROCESSOR_FACADE
    pool = ThreadPoolExecutor(max_workers=5)
    futures = []
    for name, meta in resources.items():
        resource_type = meta['resource_type']
        apply_changes = meta.get('apply_changes')
        if apply_changes:
            for apply_item in apply_changes:
                change_type = apply_item['apply_type']
                dependency_name = apply_item['dependency_name']
                res_config = resources.get(dependency_name)
                if not res_config:
                    _LOG.debug('Dependency resource {0} is not found, '
                               'skipping the apply'.format(dependency_name))
                else:
                    dependency_type = res_config['resource_type']
                    func = PROCESSOR_FACADE.resource_identifier() \
                        .get(resource_type)
                    if func:
                        resource_output = __find_output_by_resource_name(
                            output, name)
                        identifier = func(name, resource_output)
                        apply_func = PROCESSOR_FACADE.mapping_applier() \
                            .get(change_type)
                        if apply_func:
                            alias = '#{' + name + '}'
                            f = pool.submit(apply_func, alias, identifier,
                                            apply_item)
                            futures.append(f)
                        else:
                            _LOG.warn('Dynamic apply is not defined '
                                      'for {0} type'.format(change_type))
                    else:
                        _LOG.warn('Resource identifier is not defined '
                                  'for {0} type'.format(dependency_type))
            _LOG.info('Dynamic changes were applied to {0}'.format(name))
    concurrent.futures.wait(futures, timeout=None, return_when=ALL_COMPLETED) 

def test_pending_calls_race(self):
        # Issue #14406: multi-threaded race condition when waiting on all
        # futures.
        event = threading.Event()
        def future_func():
            event.wait()
        oldswitchinterval = sys.getswitchinterval()
        sys.setswitchinterval(1e-6)
        try:
            fs = {self.executor.submit(future_func) for i in range(100)}
            event.set()
            futures.wait(fs, return_when=futures.ALL_COMPLETED)
        finally:
            sys.setswitchinterval(oldswitchinterval) 

def exit(loop):
        try:
            loop.run_until_complete(asyncio.wait( \
                    list(asyncio.Task.all_tasks()),
                    return_when=futures.ALL_COMPLETED))
            loop.close()

        except KeyboardInterrupt:
            pass 

def _run(self):
        dm = UChromaDeviceManager()

        atexit.register(UChromaServer.exit, self._loop)

        dbus = DeviceManagerAPI(dm, self._logger)
        power = PowerMonitor()

        for sig in (signal.SIGINT, signal.SIGTERM):
            self._loop.add_signal_handler(sig, self._shutdown_callback)

        try:
            dbus.run()
            power.start()

            ensure_future(dm.monitor_start(), loop=self._loop)

            self._loop.run_forever()

        except KeyboardInterrupt:
            pass

        finally:
            for sig in (signal.SIGTERM, signal.SIGINT):
                self._loop.remove_signal_handler(sig)

            power.stop()

            self._loop.run_until_complete(asyncio.wait( \
                    [dm.close_devices(), dm.monitor_stop()],
                    return_when=futures.ALL_COMPLETED)) 

def await_termination(self, timeout=None):
        with type(self)._POOL_LOCK:
            if not self.is_shutdown:
                raise AsyncArcticException("The workers pool has not been shutdown, please call shutdown() first.")
        LazySingletonTasksCoordinator.wait_tasks(
            [v[0] for v in itervalues(self.alive_tasks)],
            timeout=timeout, return_when=ALL_COMPLETED, raise_exceptions=False)
        with type(self)._POOL_LOCK:
            self.alive_tasks = {} 

def wait_tasks_or_abort(futures, timeout=60, kill_switch_ev=None):
        try:
            LazySingletonTasksCoordinator.wait_tasks(futures, return_when=FIRST_EXCEPTION, raise_exceptions=True)
        except Exception as e:
            if kill_switch_ev is not None:
                # Used when we want to keep both raise the exception and wait for all tasks to finish
                kill_switch_ev.set()
                LazySingletonTasksCoordinator.wait_tasks(futures, return_when=ALL_COMPLETED,
                                                         raise_exceptions=False, timeout=timeout)
            raise e 

def wait_tasks(futures, timeout=None, return_when=ALL_COMPLETED, raise_exceptions=True):
        running_futures = [fut for fut in futures if not fut.done()]
        done, _ = wait(running_futures, timeout=timeout, return_when=return_when)
        if raise_exceptions:
            [f.result() for f in done if not f.cancelled() and f.exception() is not None]  # raises the exception