Python queue.SimpleQueue() Examples

def download_file(url: str, out_path: str = None, buffer_size: int = 10*(1024**2)):
    data = Queue()
    def _download():
        b = data.get()
        with open(out_path or url.split('/')[-1], 'wb') as o:
            while b:
                o.write(b)
                b = data.get()
    
    s = ssl.SSLContext()
    f = req.urlopen(url, context=s)
    th = Thread(target=_download)
    th.start()
    b = f.read(buffer_size)
    while b:
        data.put(b)
        b = f.read(buffer_size)
    data.put([])
    th.join() 

def _setup_logging_queue(*handlers: Handler) -> QueueHandler:
    """Create a new LocalQueueHandler and start an associated QueueListener.
    """
    queue: Queue = Queue()
    queue_handler = LocalQueueHandler(queue)

    serving_listener = QueueListener(queue, *handlers, respect_handler_level=True)
    serving_listener.start()

    return queue_handler 

def __init__(self, bus, org, interface):
        self.bus = bus
        self.dbus_org = org
        self.dbus_interface = interface
        self._sent = False
        self._data = {'cmd': None}
        self.eventQueue = SimpleQueue()
        self.daemon = self._get_daemon(bus, org, interface)
        self.__async_thread = None

        logger.debug("%s daemon loaded - version :  %s" %
                     (interface, self.daemon.GetVersion())) 

def __init__(self, max_workers=None, thread_name_prefix='',
                 initializer=None, initargs=()):
        """Initializes a new ThreadPoolExecutor instance.

        Args:
            max_workers: The maximum number of threads that can be used to
                execute the given calls.
            thread_name_prefix: An optional name prefix to give our threads.
            initializer: An callable used to initialize worker threads.
            initargs: A tuple of arguments to pass to the initializer.
        """
        if max_workers is None:
            # Use this number because ThreadPoolExecutor is often
            # used to overlap I/O instead of CPU work.
            max_workers = (os.cpu_count() or 1) * 5
        if max_workers <= 0:
            raise ValueError("max_workers must be greater than 0")

        if initializer is not None and not callable(initializer):
            raise TypeError("initializer must be a callable")

        self._max_workers = max_workers
        self._work_queue = queue.SimpleQueue()
        self._threads = set()
        self._broken = False
        self._shutdown = False
        self._shutdown_lock = threading.Lock()
        self._thread_name_prefix = (thread_name_prefix or
                                    ("ThreadPoolExecutor-%d" % self._counter()))
        self._initializer = initializer
        self._initargs = initargs 

def _setup_queues(self):
        self._inqueue = self._ctx.SimpleQueue()
        self._outqueue = self._ctx.SimpleQueue()
        self._quick_put = self._inqueue._writer.send
        self._quick_get = self._outqueue._reader.recv 

def _setup_queues(self):
        self._inqueue = queue.SimpleQueue()
        self._outqueue = queue.SimpleQueue()
        self._quick_put = self._inqueue.put
        self._quick_get = self._outqueue.get 

def run_all(folder, s_target):
    sql_queue = SimpleQueue()
    size = 0
    for filename in glob.iglob(f'{folder}/**/*.sql', recursive=True):
        sql_queue.put(get_sql(filename))
        size += 1
    num_tries = 0
    max_tries = size * 2
    while not sql_queue.empty() and num_tries < max_tries:
        try:
            sql = sql_queue.get()
            s_target.execute(sql)
            s_target.commit()
            num_tries = 0
        except ProgrammingError as e:
            message = str(e.orig).strip()
            if 'relation' in message and 'does not exist' in message:
                s_target.rollback()
                print(f'Object does not exist yet: {message}. Re-queueing...')
                sql_queue.put(sql)
                num_tries += 1
            else:
                raise
    if num_tries >= max_tries:
        print(f'Number of attempts exceeded configured threshold of {max_tries}')
        sys.exit(1) 

def __init__(self, *, reader, loop: asyncio.AbstractEventLoop, executor):
        super().__init__(loop=loop, executor=executor)
        self.reader = reader
        self.queue = SimpleQueue()
        self.event = asyncio.Event()
        self.lock = asyncio.Lock()
        self.closed = False

        self._loop.create_task(self._exec(self._reader)) 

def __init__(self, rpc):
        super(GcRecorder, self).__init__()
        self.should_stop = False
        L.debug('session:{} initializing gc recorder'.format(rpc.session_id))
        self.record_rpc = rpc
        self.gc_recorder = dict()
        self.gc_queue = SimpleQueue()
        if "EGGROLL_GC_DISABLE" in os.environ and os.environ["EGGROLL_GC_DISABLE"] == '1':
            L.info("global gc is disable, "
                  "will not execute gc but only record temporary RollPair during the whole session")
        else:
            self.gc_thread = Thread(target=self.run, daemon=True)
            self.gc_thread.start()
            L.debug("starting gc_thread......") 

def __init__(self, max_workers=None, thread_name_prefix='',
                 initializer=None, initargs=()):
        """Initializes a new ThreadPoolExecutor instance.

        Args:
            max_workers: The maximum number of threads that can be used to
                execute the given calls.
            thread_name_prefix: An optional name prefix to give our threads.
            initializer: An callable used to initialize worker threads.
            initargs: A tuple of arguments to pass to the initializer.
        """
        if max_workers is None:
            # Use this number because ThreadPoolExecutor is often
            # used to overlap I/O instead of CPU work.
            max_workers = (os.cpu_count() or 1) * 5
        if max_workers <= 0:
            raise ValueError("max_workers must be greater than 0")

        if initializer is not None and not callable(initializer):
            raise TypeError("initializer must be a callable")

        self._max_workers = max_workers
        self._work_queue = queue.SimpleQueue()
        self._threads = set()
        self._broken = False
        self._shutdown = False
        self._shutdown_lock = threading.Lock()
        self._thread_name_prefix = (thread_name_prefix or
                                    ("ThreadPoolExecutor-%d" % self._counter()))
        self._initializer = initializer
        self._initargs = initargs 

def __init__(self, processes=None, initializer=None, initargs=(),
                 maxtasksperchild=None, context=None):
        self._ctx = context or get_context()
        self._setup_queues()
        self._taskqueue = queue.SimpleQueue()
        self._cache = {}
        self._state = RUN
        self._maxtasksperchild = maxtasksperchild
        self._initializer = initializer
        self._initargs = initargs

        if processes is None:
            processes = os.cpu_count() or 1
        if processes < 1:
            raise ValueError("Number of processes must be at least 1")

        if initializer is not None and not callable(initializer):
            raise TypeError('initializer must be a callable')

        self._processes = processes
        self._pool = []
        self._repopulate_pool()

        self._worker_handler = threading.Thread(
            target=Pool._handle_workers,
            args=(self._cache, self._taskqueue, self._ctx, self.Process,
                  self._processes, self._pool, self._inqueue, self._outqueue,
                  self._initializer, self._initargs, self._maxtasksperchild,
                  self._wrap_exception)
            )
        self._worker_handler.daemon = True
        self._worker_handler._state = RUN
        self._worker_handler.start()

        self._task_handler = threading.Thread(
            target=Pool._handle_tasks,
            args=(self._taskqueue, self._quick_put, self._outqueue,
                  self._pool, self._cache)
            )
        self._task_handler.daemon = True
        self._task_handler._state = RUN
        self._task_handler.start()

        self._result_handler = threading.Thread(
            target=Pool._handle_results,
            args=(self._outqueue, self._quick_get, self._cache)
            )
        self._result_handler.daemon = True
        self._result_handler._state = RUN
        self._result_handler.start()

        self._terminate = util.Finalize(
            self, self._terminate_pool,
            args=(self._taskqueue, self._inqueue, self._outqueue, self._pool,
                  self._worker_handler, self._task_handler,
                  self._result_handler, self._cache),
            exitpriority=15
            )