Python concurrent.futures() Examples

def pool(self):
        result = self.parse_html(self.host)
        try:
            with concurrent.futures.ThreadPoolExecutor(max_workers=30) as executor:
                futures = [executor.submit(self.parse_html, i) for i in result]
                for future in concurrent.futures.as_completed(futures, timeout=3):
                    future.result()
        except (EOFError, concurrent.futures._base.TimeoutError):
            pass
        except Exception as e:
            logging.exception(e)
        
        jslink = JsLeaks().pool(self.js)
        
        self.result.extend(jslink)
        self.result = list(set(self.result))
        
        for i in self.result:
            console('Crawl', self.host, i + '\n')
        
        Sqldb(self.dbname).get_crawl(self.domain, self.result) 

def download_all_images(self, photo_list: [OSCPhoto],
                            track_path: str,
                            override=False,
                            workers: int = 10):
        """This method will download all images to a path overriding or not the files at
        that path. By default this method uses 10 parallel workers."""
        with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
            loop = asyncio.new_event_loop()
            futures = [
                loop.run_in_executor(executor,
                                     self.get_image, photo, track_path, override)
                for photo in photo_list
            ]
            if not futures:
                loop.close()
                return

            loop.run_until_complete(asyncio.gather(*futures))
            loop.close() 

def wait_task(self, return_when=concurrent.futures.FIRST_COMPLETED):
        """Waits for one or more running tasks to complete.

        Results of all completed tasks are copied into ``self.ready_task_results``.

        When no task is running, does nothing.

        """
        if len(self.running_tasks) > 0:
            self.config.log("Waiting for tasks to complete...")
            ready_tasks, self.running_tasks = concurrent.futures.wait(
                self.running_tasks, return_when=return_when
            )
            for task in ready_tasks:
                self.ready_task_results.append(task.result())

    # Overridden such that instances of search job can be pickled to workers 

def test_future_traceback(self):
        @return_future
        @gen.engine
        def f(callback):
            yield gen.Task(self.io_loop.add_callback)
            try:
                1 / 0
            except ZeroDivisionError:
                self.expected_frame = traceback.extract_tb(
                    sys.exc_info()[2], limit=1)[0]
                raise
        try:
            yield f()
            self.fail("didn't get expected exception")
        except ZeroDivisionError:
            tb = traceback.extract_tb(sys.exc_info()[2])
            self.assertIn(self.expected_frame, tb)

# The following series of classes demonstrate and test various styles
# of use, with and without generators and futures. 

def test_future_traceback(self):
        @return_future
        @gen.engine
        def f(callback):
            yield gen.Task(self.io_loop.add_callback)
            try:
                1 / 0
            except ZeroDivisionError:
                self.expected_frame = traceback.extract_tb(
                    sys.exc_info()[2], limit=1)[0]
                raise
        try:
            yield f()
            self.fail("didn't get expected exception")
        except ZeroDivisionError:
            tb = traceback.extract_tb(sys.exc_info()[2])
            self.assertIn(self.expected_frame, tb)

# The following series of classes demonstrate and test various styles
# of use, with and without generators and futures. 

def then(self, fn, deferred=False):
        result = Future(self._loop)
        result._register()

        def callback(_):
            try:
                if deferred:
                    temp = self._loop.run_later(fn, self.result())
                    temp.chain(result)
                elif not threading.current_thread() is self._loop:
                    temp = self._loop.run_async(fn, self.result())
                    temp.chain(result)
                else:
                    result.set_result(fn(self.result()))
            except Exception as e:
                self.logger.exception(
                    "Unhandled exception while chaining futures"
                )
                result.set_exception(e)
            except:
                result.cancel()

        self.add_done_callback(callback)
        return result 

def __init__(self, *args: Future, **kwargs: Future) -> None:
        if args and kwargs:
            raise ValueError("You must provide args or kwargs, not both")

        if kwargs:
            self._unfinished = dict((f, k) for (k, f) in kwargs.items())
            futures = list(kwargs.values())  # type: Sequence[Future]
        else:
            self._unfinished = dict((f, i) for (i, f) in enumerate(args))
            futures = args

        self._finished = collections.deque()  # type: Deque[Future]
        self.current_index = None  # type: Optional[Union[str, int]]
        self.current_future = None  # type: Optional[Future]
        self._running_future = None  # type: Optional[Future]

        for future in futures:
            future_add_done_callback(future, self._done_callback) 

def run_async(self, func, *args, **kwds):
        """
        Fills in a list with the function to be executed in the pomp thread
        and wakes up the pomp thread.
        """
        future = Future(self)
        future._register()

        if threading.current_thread() is not self:
            self.async_pomp_task.append((future, func, args, kwds))
            self._wake_up()
        else:
            try:
                ret = func(*args, **kwds)
            except Exception as e:
                self.logger.exception(
                    "Unhandled exception in async task function"
                )
                future.set_exception(e)
            else:
                if not isinstance(ret, concurrent.futures.Future):
                    future.set_result(ret)
                else:
                    ret.chain(future)
        return future 

def _run_task_list(self, task_list):
        """
        execute all pending functions located in the task list
        this is done in the order the list has been filled in
        """
        while len(task_list):
            future, f, args, kwds = task_list.pop(0)
            try:
                ret = f(*args, **kwds)
            except Exception as e:
                self.logger.exception(
                    "Unhandled exception in async task function"
                )
                self._unregister_future(future, ignore_error=True)
                future.set_exception(e)
                continue
            if not isinstance(ret, concurrent.futures.Future):
                future.set_result(ret)
            else:
                ret.chain(future) 

def __init__(self, *args: Future, **kwargs: Future) -> None:
        if args and kwargs:
            raise ValueError("You must provide args or kwargs, not both")

        if kwargs:
            self._unfinished = dict((f, k) for (k, f) in kwargs.items())
            futures = list(kwargs.values())  # type: Sequence[Future]
        else:
            self._unfinished = dict((f, i) for (i, f) in enumerate(args))
            futures = args

        self._finished = collections.deque()  # type: Deque[Future]
        self.current_index = None  # type: Optional[Union[str, int]]
        self.current_future = None  # type: Optional[Future]
        self._running_future = None  # type: Optional[Future]

        for future in futures:
            future_add_done_callback(future, self._done_callback) 

def test_async_await(self):
        class Object(object):
            def __init__(self):
                self.executor = futures.thread.ThreadPoolExecutor(1)

            @run_on_executor()
            def f(self):
                return 42

        o = Object()
        namespace = exec_test(globals(), locals(), """
        async def f():
            answer = await o.f()
            return answer
        """)
        result = yield namespace['f']()
        self.assertEqual(result, 42) 

def __init__(self, *args: Future, **kwargs: Future) -> None:
        if args and kwargs:
            raise ValueError("You must provide args or kwargs, not both")

        if kwargs:
            self._unfinished = dict((f, k) for (k, f) in kwargs.items())
            futures = list(kwargs.values())  # type: Sequence[Future]
        else:
            self._unfinished = dict((f, i) for (i, f) in enumerate(args))
            futures = args

        self._finished = collections.deque()  # type: Deque[Future]
        self.current_index = None  # type: Optional[Union[str, int]]
        self.current_future = None  # type: Optional[Future]
        self._running_future = None  # type: Optional[Future]

        for future in futures:
            future_add_done_callback(future, self._done_callback) 

def download_file(self, bucket, key, filename, object_size,
                      extra_args, callback=None):
        with self._executor_cls(max_workers=2) as controller:
            # 1 thread for the future that manages the uploading of files
            # 1 thread for the future that manages IO writes.
            download_parts_handler = functools.partial(
                self._download_file_as_future,
                bucket, key, filename, object_size, callback)
            parts_future = controller.submit(download_parts_handler)

            io_writes_handler = functools.partial(
                self._perform_io_writes, filename)
            io_future = controller.submit(io_writes_handler)
            results = concurrent.futures.wait(
                [parts_future, io_future],
                return_when=concurrent.futures.FIRST_EXCEPTION)
            self._process_future_results(results) 

def download_file(self, bucket, key, filename, object_size,
                      extra_args, callback=None):
        with self._executor_cls(max_workers=2) as controller:
            # 1 thread for the future that manages the uploading of files
            # 1 thread for the future that manages IO writes.
            download_parts_handler = functools.partial(
                self._download_file_as_future,
                bucket, key, filename, object_size, callback)
            parts_future = controller.submit(download_parts_handler)

            io_writes_handler = functools.partial(
                self._perform_io_writes, filename)
            io_future = controller.submit(io_writes_handler)
            results = concurrent.futures.wait(
                [parts_future, io_future],
                return_when=concurrent.futures.FIRST_EXCEPTION)
            self._process_future_results(results) 

def test_temp_table_concurrency(con, test_data_dir):
    # we don't install futures on windows in CI and we can't run this test
    # there anyway so we import here
    import concurrent.futures
    from concurrent.futures import as_completed

    def limit_10(i, hdfs_path):
        t = con.parquet_file(hdfs_path)
        return t.sort_by(t.r_regionkey).limit(1, offset=i).execute()

    nthreads = 4
    hdfs_path = pjoin(test_data_dir, 'parquet/tpch_region')

    with concurrent.futures.ThreadPoolExecutor(max_workers=nthreads) as e:
        futures = [e.submit(limit_10, i, hdfs_path) for i in range(nthreads)]
    assert all(map(len, (future.result() for future in as_completed(futures)))) 

def execute_runners(self, parallel=False, num_threads=1):
        '''
        Run each of the algorithms
        '''

        base_output_dir = self.output_settings.base_dir

        batches =  self.runners.keys()

        for batch in batches:
            if parallel==True:
                executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
                futures = [executor.submit(runner.run, base_output_dir)
                    for runner in self.runners[batch]]
                
                # https://stackoverflow.com/questions/35711160/detect-failed-tasks-in-concurrent-futures
                # Re-raise exception if produced
                for future in concurrent.futures.as_completed(futures):
                    future.result()
                executor.shutdown(wait=True)
            else:
                for runner in self.runners[batch]:
                    runner.run(output_dir=base_output_dir) 

def test_uncaught_exception_log(self):
        if IOLoop.configured_class().__name__.endswith('AsyncIOLoop'):
            # Install an exception handler that mirrors our
            # non-asyncio logging behavior.
            def exc_handler(loop, context):
                app_log.error('%s: %s', context['message'],
                              type(context.get('exception')))
            self.io_loop.asyncio_loop.set_exception_handler(exc_handler)

        @gen.coroutine
        def f():
            yield gen.moment
            1 / 0

        g = f()

        with ExpectLog(app_log,
                       "(?s)Future.* exception was never retrieved:"
                       ".*ZeroDivisionError"):
            yield gen.moment
            yield gen.moment
            # For some reason, TwistedIOLoop and pypy3 need a third iteration
            # in order to drain references to the future
            yield gen.moment
            del g
            gc.collect()  # for PyPy


# The following series of classes demonstrate and test various styles
# of use, with and without generators and futures. 

def attack(self, function, links, thread_count):
		links = list(links)
		threadpool = concurrent.futures.ThreadPoolExecutor(
				max_workers=thread_count)
		futures = (threadpool.submit(function, link) for link in links if link not in self.passed)
		for i, _ in enumerate(concurrent.futures.as_completed(futures)):
			if i + 1 == len(links) or (i + 1) % thread_count == 0:
				print(f'Progress: {i+1}/{len(links)}',
						end='\r')
		print('') 

def thread(self, function, thread_count, engines, domain, q, limit, count, key):
		threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=thread_count)
		futures = (threadpool.submit(function, name, domain, q, limit, count, key) for name in engines if name in self.meta['sources'])
		for _ in concurrent.futures.as_completed(futures):
			pass 

def _create_threadpool(
        cls, num_threads: int
    ) -> concurrent.futures.ThreadPoolExecutor:
        pid = os.getpid()
        if cls._threadpool_pid != pid:
            # Threads cannot survive after a fork, so if our pid isn't what it
            # was when we created the pool then delete it.
            cls._threadpool = None
        if cls._threadpool is None:
            cls._threadpool = concurrent.futures.ThreadPoolExecutor(num_threads)
            cls._threadpool_pid = pid
        return cls._threadpool 

def thread(self, function, thread_count, engines, q, limit, count):
		threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=thread_count)
		futures = (threadpool.submit(function, name, q, limit, count) for name in engines if name in self.meta['sources'])
		for _ in concurrent.futures.as_completed(futures):
			pass 

def thread(self, function, thread_count, engines, q, limit, count):
		threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=thread_count)
		futures = (threadpool.submit(function, name, q, limit, count) for name in engines if name in self.meta['sources'])
		for _ in concurrent.futures.as_completed(futures):
			pass 

def __init__(self, config, dataset, parent_job=None):
        super().__init__(config, dataset, parent_job)

        # create data structures for parallel job submission
        self.num_workers = self.config.get("search.num_workers")
        self.device_pool = self.config.get("search.device_pool")
        if len(self.device_pool) == 0:
            self.device_pool = [self.config.get("job.device")]
        if len(self.device_pool) < self.num_workers:
            self.device_pool = self.device_pool * self.num_workers
        self.device_pool = self.device_pool[: self.num_workers]
        self.config.log("Using device pool: {}".format(self.device_pool))
        self.free_devices = copy.deepcopy(self.device_pool)
        self.on_error = self.config.check("search.on_error", ["abort", "continue"])

        self.running_tasks = set()  #: set of futures currently runnning
        self.ready_task_results = list()  #: set of results
        if self.num_workers > 1:
            self.process_pool = concurrent.futures.ProcessPoolExecutor(
                max_workers=self.num_workers,
                mp_context=torch.multiprocessing.get_context("spawn"),
            )
        else:
            self.process_pool = None  # marks that we run in single process

        self.config.check_range("valid.every", 1, config.get("train.max_epochs"))

        if self.__class__ == SearchJob:
            for f in Job.job_created_hooks:
                f(self) 

def thread(self, function, hostname, wordlist, thread_count):
		threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=thread_count)
		futures = (threadpool.submit(function, hostname, word) for word in wordlist if not word.startswith('#'))
		counter = 1
		for _ in concurrent.futures.as_completed(futures):
			print(f"Checking payload {counter}, hits: {len(self.hostnames)}", end='\r')
			counter += 1

		print(os.linesep) 

def thread(self, function, hostname, wordlist, thread_count, method, header=(), content=[], status_codes=[], not_status_codes=[]):
		threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=thread_count)
		futures = (threadpool.submit(function, hostname, word, method, header, content, status_codes, not_status_codes) for word in wordlist if not '#' in word)
		counter = 1
		for _ in concurrent.futures.as_completed(futures):
			print(f"Checking payload {counter}, hits: {len(self.resp[method])}", end='\r')
			counter += 1
		print('') 

def thread(self, function, hostname, wordlist, thread_count):
		threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=thread_count)
		futures = (threadpool.submit(function, hostname, word) for word in wordlist if not word.startswith("#"))
		counter = 1
		for _ in concurrent.futures.as_completed(futures):
			print(f"Checking payload {counter}, hits: {len(self.hostnames)}", end='\r')
			counter += 1
		print(os.linesep) 

def thread(self, function, hostname, wordlist, thread_count, status_codes):
		threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=thread_count)
		futures = (threadpool.submit(function, hostname, word, status_codes) for word in wordlist if not word.startswith("#"))
		counter = 1
		for _ in concurrent.futures.as_completed(futures):
			print(f"Checking payload {counter}, hits: {len(self.hostnames)}", end='\r')
			counter += 1
		print(os.linesep) 

def set_center_freq(self, center_freq):
        """Set center frequency and clear averaged PSD data"""
        psd_state = {
            'repeats': 0,
            'freq_array': self._base_freq_array + self._lnb_lo + center_freq,
            'pwr_array': None,
            'update_lock': threading.Lock(),
            'futures': [],
        }
        return psd_state 

def wait_for_result(self, psd_state):
        """Wait for all PSD threads to finish and return result"""
        if len(psd_state['futures']) > 1:
            concurrent.futures.wait(psd_state['futures'])
        elif psd_state['futures']:
            psd_state['futures'][0].result()
        return self.result(psd_state) 

def update_async(self, psd_state, samples_array):
        """Compute PSD from samples and update average for given center frequency (asynchronously in another thread)"""
        future = self._executor.submit(self.update, psd_state, samples_array)
        future.add_done_callback(self._release_future_memory)
        psd_state['futures'].append(future)
        return future