Python asyncio.Semaphore() Examples

def __init__(self, threads: int = None) -> None:
        try:  # get or create loop (threads don't have one)
            #: our asyncio loop
            self.loop = asyncio.get_event_loop()
        except RuntimeError:
            self.loop = asyncio.new_event_loop()
            asyncio.set_event_loop(self.loop)
        #: number of threads to use
        self.threads = threads or threads_to_use()
        #: semaphore to limit io parallelism
        self.io_sem: asyncio.Semaphore = asyncio.Semaphore(1)
        #: must never run more than one conda at the same time
        #: (used by PyPi when running skeleton)
        self.conda_sem: asyncio.Semaphore = asyncio.Semaphore(1)
        #: the filters successively applied to each item
        self.filters: List[AsyncFilter] = []
        #: executor running things in separate python processes
        self.proc_pool_executor = ProcessPoolExecutor(self.threads)

        self._shutting_down = False 

def test_acquire_cancel_before_awoken(self):
        sem = asyncio.Semaphore(value=0, loop=self.loop)

        t1 = asyncio.Task(sem.acquire(), loop=self.loop)
        t2 = asyncio.Task(sem.acquire(), loop=self.loop)
        t3 = asyncio.Task(sem.acquire(), loop=self.loop)
        t4 = asyncio.Task(sem.acquire(), loop=self.loop)

        test_utils.run_briefly(self.loop)

        sem.release()
        t1.cancel()
        t2.cancel()

        test_utils.run_briefly(self.loop)
        num_done = sum(t.done() for t in [t3, t4])
        self.assertEqual(num_done, 1)

        t3.cancel()
        t4.cancel()
        test_utils.run_briefly(self.loop) 

def test_acquire_cancel_before_awoken(self):
        sem = asyncio.Semaphore(value=0, loop=self.loop)

        t1 = asyncio.Task(sem.acquire(), loop=self.loop)
        t2 = asyncio.Task(sem.acquire(), loop=self.loop)
        t3 = asyncio.Task(sem.acquire(), loop=self.loop)
        t4 = asyncio.Task(sem.acquire(), loop=self.loop)

        test_utils.run_briefly(self.loop)

        sem.release()
        t1.cancel()
        t2.cancel()

        test_utils.run_briefly(self.loop)
        num_done = sum(t.done() for t in [t3, t4])
        self.assertEqual(num_done, 1)

        t3.cancel()
        t4.cancel()
        test_utils.run_briefly(self.loop) 

def test_semaphore(self):
        sem = asyncio.Semaphore(loop=self.loop)
        self.assertEqual(1, sem._value)

        @asyncio.coroutine
        def acquire_lock():
            return (yield from sem)

        res = self.loop.run_until_complete(acquire_lock())

        self.assertTrue(res)
        self.assertTrue(sem.locked())
        self.assertEqual(0, sem._value)

        sem.release()
        self.assertFalse(sem.locked())
        self.assertEqual(1, sem._value) 

def test_repr(self):
        sem = asyncio.Semaphore(loop=self.loop)
        self.assertTrue(repr(sem).endswith('[unlocked,value:1]>'))
        self.assertTrue(RGX_REPR.match(repr(sem)))

        self.loop.run_until_complete(sem.acquire())
        self.assertTrue(repr(sem).endswith('[locked]>'))
        self.assertTrue('waiters' not in repr(sem))
        self.assertTrue(RGX_REPR.match(repr(sem)))

        sem._waiters.append(mock.Mock())
        self.assertTrue('waiters:1' in repr(sem))
        self.assertTrue(RGX_REPR.match(repr(sem)))

        sem._waiters.append(mock.Mock())
        self.assertTrue('waiters:2' in repr(sem))
        self.assertTrue(RGX_REPR.match(repr(sem))) 

def interval_task(self):
        try:
            to_backup = self.bot.db.intervals.find({"next": {
                "$lt": datetime.utcnow()
            }})
            semaphore = Semaphore(10)
            async for interval in to_backup:
                async def run_interval():
                    try:
                        next = datetime.utcnow() + timedelta(minutes=interval["interval"])
                        await self.bot.db.intervals.update_one({"_id": interval["_id"]}, {"$set": {"next": next}})
                        await self.run_backup(interval["_id"])
                    finally:
                        semaphore.release()

                await semaphore.acquire()
                self.bot.loop.create_task(run_interval())
                await sleep(0)

        except Exception:
            pass 

def find_node_ports(address, ports):
    """
    Find all the open ports for a host.

    :param address: IP address of the host.
    :param ports: Port to check.
    :return: A list of the found open ports.
    """
    sem = asyncio.Semaphore(400)  # Change this value for concurrency limitation
    tasks = [asyncio.ensure_future(check_port_sem(sem, address, p)) for p in ports]
    found_ports = []
    for response in await asyncio.gather(*tasks):
        if response:
            found_ports.append(response)

    return found_ports 

def check_routers(self, addresses: List[str], ports: List[int]) -> List[BaseIndustrialRouter]:
        """
        Check for routers in a range of addressess and ports.

        :param addresses: List of addressess to be checked.
        :param ports: List of ports to be checked for each address.
        :return: A list of found routers.
        """
        async def check_routers_aio(addresses, ports):
            semaphore = Semaphore(50)
            futures = []
            routers = []
            for address in addresses:
                for port in ports:
                    futures.append(asyncio.ensure_future(self.check_is_router(address, port, semaphore=semaphore)))
            done, pending = await asyncio.wait(futures)
            for future in done:
                if future:
                    routers.append(future)
            return routers

        return asyncio.run(check_routers_aio(addresses, ports), debug=True) 

def check_router_credentials(self, routers: List[BaseIndustrialRouter]):
        """
        Check default credentials for a list of routers.

        :param routers: List of routers to be checked.
        """
        async def check_router_credentials_aio(routers):
            semaphore = Semaphore(100)
            futures = []
            for router in routers:
                if isinstance(router, self.__class__.router_cls):
                    futures.append(asyncio.ensure_future(self.check_default_password(router, semaphore=semaphore)))

            await asyncio.wait(futures, return_when=ALL_COMPLETED, )

        asyncio.run(check_router_credentials_aio(routers), debug=True) 

def main(global_delay, local_delay, concurrency):
    global global_sleep, local_sleep, semaphore, index
    global_sleep = global_delay
    local_sleep = local_delay
    semaphore = asyncio.Semaphore(concurrency)
    print('Global delay =', global_delay)
    print('Local delay =', local_delay)
    print('Max. concurrency =', concurrency)
    print('Building inverted index...')
    index = build_index()

    app = web.Application()
    app.router.add_get('/', usage)
    app.router.add_get('/index/{word}', index_for)
    app.router.add_get('/name/{char}', char_name)

    print('Listening on port', PORT)
    web.run_app(app, port=PORT) 

def scan_host(address, start_port, end_port, max_conns=400):
        """

        :param address: IPv4 address to scan
        :param start_port: First port value to scan
        :param end_port: Last port value to scan
        :param max_conns: Maximum simultaneous number of connections
        :return:
        """
        sem = asyncio.Semaphore(max_conns)
        ports = range(start_port, end_port)
        tasks = [asyncio.ensure_future(PortScanner.check_port_sem(sem, address, port)) for port in ports]
        responses = await asyncio.gather(*tasks)
        open_ports = list(filter(lambda x: x is not None, responses))

        return open_ports 

def test_context_manager_async_with(self):
        primitives = [
            asyncio.Lock(loop=self.loop),
            asyncio.Condition(loop=self.loop),
            asyncio.Semaphore(loop=self.loop),
            asyncio.BoundedSemaphore(loop=self.loop),
        ]

        async def test(lock):
            await asyncio.sleep(0.01, loop=self.loop)
            self.assertFalse(lock.locked())
            async with lock as _lock:
                self.assertIs(_lock, None)
                self.assertTrue(lock.locked())
                await asyncio.sleep(0.01, loop=self.loop)
                self.assertTrue(lock.locked())
            self.assertFalse(lock.locked())

        for primitive in primitives:
            self.loop.run_until_complete(test(primitive))
            self.assertFalse(primitive.locked()) 

def __init__(self, in_state, in_forward, search_threads):
        self.noise_eps = 0.25
        self.dirichlet_alpha = 0.3    #0.03
        self.p_ = (1 - self.noise_eps) * 1 + self.noise_eps * np.random.dirichlet([self.dirichlet_alpha])
        self.root = leaf_node(None, self.p_, in_state)
        self.c_puct = 5    #1.5
        # self.policy_network = in_policy_network
        self.forward = in_forward
        self.node_lock = defaultdict(Lock)

        self.virtual_loss = 3
        self.now_expanding = set()
        self.expanded = set()
        self.cut_off_depth = 30
        # self.QueueItem = namedtuple("QueueItem", "feature future")
        self.sem = asyncio.Semaphore(search_threads)
        self.queue = Queue(search_threads)
        self.loop = asyncio.get_event_loop()
        self.running_simulation_num = 0 

def _check_uris_async(
    links_to_check: Iterable[str], max_threads: int = 10, delay: float = 0
) -> Iterable[UrlResult]:
    tasks = []
    # create instance of Semaphore
    sem = asyncio.Semaphore(max_threads)

    # Create client session that will ensure we dont open new connection
    # per each request.
    async with ClientSession() as session:
        for uri in links_to_check:
            if delay:
                asyncio.sleep(delay)
            # pass Semaphore and session to every GET request
            task = asyncio.ensure_future(_check_uri_with_sem_async(sem, uri, session))
            tasks.append(task)

        results = await asyncio.gather(*tasks)
        return results 

def test_context_manager_with_await(self):
        primitives = [
            asyncio.Lock(loop=self.loop),
            asyncio.Condition(loop=self.loop),
            asyncio.Semaphore(loop=self.loop),
            asyncio.BoundedSemaphore(loop=self.loop),
        ]

        async def test(lock):
            await asyncio.sleep(0.01, loop=self.loop)
            self.assertFalse(lock.locked())
            with await lock as _lock:
                self.assertIs(_lock, None)
                self.assertTrue(lock.locked())
                await asyncio.sleep(0.01, loop=self.loop)
                self.assertTrue(lock.locked())
            self.assertFalse(lock.locked())

        for primitive in primitives:
            self.loop.run_until_complete(test(primitive))
            self.assertFalse(primitive.locked()) 

def __init__(self, config: Config, model, play_config=None):

        self.config = config
        self.model = model
        self.play_config = play_config or self.config.play
        self.api = Connect4ModelAPI(self.config, self.model)

        self.labels_n = config.n_labels
        self.var_n = defaultdict(lambda: np.zeros((self.labels_n,)))
        self.var_w = defaultdict(lambda: np.zeros((self.labels_n,)))
        self.var_q = defaultdict(lambda: np.zeros((self.labels_n,)))
        self.var_u = defaultdict(lambda: np.zeros((self.labels_n,)))
        self.var_p = defaultdict(lambda: np.zeros((self.labels_n,)))
        self.expanded = set()
        self.now_expanding = set()
        self.prediction_queue = Queue(self.play_config.prediction_queue_size)
        self.sem = asyncio.Semaphore(self.play_config.parallel_search_num)

        self.moves = []
        self.loop = asyncio.get_event_loop()
        self.running_simulation_num = 0

        self.thinking_history = {}  # for fun 

def __init__(
        self,
        context: InjectionContext,
        receive_inbound: Coroutine,
        return_inbound: Callable = None,
    ):
        """Initialize an `InboundTransportManager` instance."""
        self.context = context
        self.max_message_size = 0
        self.receive_inbound = receive_inbound
        self.return_inbound = return_inbound
        self.registered_transports = {}
        self.running_transports = {}
        self.sessions = OrderedDict()
        self.session_limit: asyncio.Semaphore = None
        self.task_queue = TaskQueue()
        self.undelivered_queue: DeliveryQueue = None 

def setup(self):
        """Perform setup operations."""
        # Load config settings
        if self.context.settings.get("transport.max_message_size"):
            self.max_message_size = self.context.settings["transport.max_message_size"]

        inbound_transports = (
            self.context.settings.get("transport.inbound_configs") or []
        )
        for transport in inbound_transports:
            module, host, port = transport
            self.register(
                InboundTransportConfiguration(module=module, host=host, port=port)
            )

        # Setup queue for undelivered messages
        if self.context.settings.get("transport.enable_undelivered_queue"):
            self.undelivered_queue = DeliveryQueue()

        # self.session_limit = asyncio.Semaphore(50) 

def __init__(self, in_state, in_forward, search_threads):
        self.noise_eps = 0.25
        self.dirichlet_alpha = 0.3    #0.03
        self.p_ = (1 - self.noise_eps) * 1 + self.noise_eps * np.random.dirichlet([self.dirichlet_alpha])
        self.root = leaf_node(None, self.p_, in_state)
        self.c_puct = 5    #1.5
        # self.policy_network = in_policy_network
        self.forward = in_forward
        self.node_lock = defaultdict(Lock)

        self.virtual_loss = 3
        self.now_expanding = set()
        self.expanded = set()
        self.cut_off_depth = 30
        # self.QueueItem = namedtuple("QueueItem", "feature future")
        self.sem = asyncio.Semaphore(search_threads)
        self.queue = Queue(search_threads)
        self.loop = asyncio.get_event_loop()
        self.running_simulation_num = 0 

def test_semaphore(self):
        sem = asyncio.Semaphore(loop=self.loop)
        self.assertEqual(1, sem._value)

        @asyncio.coroutine
        def acquire_lock():
            return (yield from sem)

        res = self.loop.run_until_complete(acquire_lock())

        self.assertTrue(res)
        self.assertTrue(sem.locked())
        self.assertEqual(0, sem._value)

        sem.release()
        self.assertFalse(sem.locked())
        self.assertEqual(1, sem._value) 

def __init__(self, daemon, coin, blocks_event):
        self.logger = class_logger(__name__, self.__class__.__name__)
        self.daemon = daemon
        self.coin = coin
        self.blocks_event = blocks_event
        self.blocks = []
        self.caught_up = False
        # Access to fetched_height should be protected by the semaphore
        self.fetched_height = None
        self.semaphore = asyncio.Semaphore()
        self.refill_event = asyncio.Event()
        # The prefetched block cache size.  The min cache size has
        # little effect on sync time.
        self.cache_size = 0
        self.min_cache_size = 10 * 1024 * 1024
        # This makes the first fetch be 10 blocks
        self.ave_size = self.min_cache_size // 10
        self.polling_delay = 5 

def print_tests():
    st = '''Available functions:
print_tests()  Print this list.
ack_test()  Test event acknowledge and Message class.
message_test() Test Message class.
event_test()  Test Event and Lock objects.
barrier_test()  Test the Barrier class.
semaphore_test(bounded=False)  Test Semaphore or BoundedSemaphore.
condition_test()  Test the Condition class.
queue_test()  Test the  Queue class

Recommended to issue ctrl-D after running each test.
'''
    print('\x1b[32m')
    print(st)
    print('\x1b[39m') 

def test_initial_value_zero(self):
        sem = asyncio.Semaphore(0, loop=self.loop)
        self.assertTrue(sem.locked()) 

def test_ctor_loop(self):
        loop = mock.Mock()
        sem = asyncio.Semaphore(loop=loop)
        self.assertIs(sem._loop, loop)

        sem = asyncio.Semaphore(loop=self.loop)
        self.assertIs(sem._loop, self.loop) 

def __init__ (self, host, port, ssl, nick, logger, channels=None,
            tempdir=None, destdir='.', processLimit=1,
            blacklist={}, needVoice=False, loop=None):
        self.needVoice = needVoice

        super().__init__ (host=host, port=port, ssl=ssl, nick=nick,
                logger=logger, channels=channels, loop=loop)

        self.jobs = {}
        self.tempdir = tempdir or tempfile.gettempdir()
        self.destdir = destdir
        self.processLimit = asyncio.Semaphore (processLimit)
        self.blacklist = blacklist 

def __init__(self, addr=None, port=None, horizontal_refresh_rate=None):

            if addr is None:
                addr = 0x27 # check with i2cdetect

            if port is None:
                port = 1    # depends on pin and rpi rev.
            
            if horizontal_refresh_rate is None:
                horizontal_refresh_rate = 20000 # per sec
            super().__init__(addr, port, 1 / horizontal_refresh_rate)
            self.loop = asyncio.get_event_loop()
            async def init():
                await self.write_cmd(0x03)
                await self.write_cmd(0x03)
                await self.write_cmd(0x03)
                await self.write_cmd(0x02)

                await self.write_cmd(LCD_FUNCTIONSET | LCD_2LINE | LCD_5x8DOTS | LCD_4BITMODE)
                await self.write_cmd(LCD_DISPLAYCONTROL | LCD_DISPLAYON)
                await self.write_cmd(LCD_CLEARDISPLAY)
                await self.write_cmd(LCD_ENTRYMODESET | LCD_ENTRYLEFT)
                await asyncio.sleep(0.2)
                await self.display("***", row=0, col=8)
                await self.display(" Total: - - -", row=1)
                await self.display("  Cash: - - -", row=2)
                await self.display("Change: - - -", row=3)

            self.sem = asyncio.Semaphore()
            self.column = 8
            self.loop.run_until_complete(init()) 

def rate(self, rate):
        self._rate = rate
        self.semaphore = asyncio.Semaphore(rate) 

def __init__(self, bot):
        self.bot = bot
        self.settings = dataIO.load_json(JSON)
        self._task = bot.loop.create_task(self.loop_task())
        self._semaphore = asyncio.Semaphore(PARALLEL_TASKS)

        try:
            self.analytics = CogAnalytics(self)
        except Exception as error:
            self.bot.logger.exception(error)
            self.analytics = None 

def test_semaphore_value(self):
        self.assertRaises(ValueError, asyncio.Semaphore, -1) 

def test_listen():
    player = PlayerNetworkChild(
        player_configuration=player_configuration,
        avatar=12,
        server_configuration=server_configuration,
        start_listening=False,
    )

    type(player).websocket_url = PropertyMock(return_value="ws://localhost:8899")

    player._handle_message = CoroutineMock()
    semaphore = asyncio.Semaphore()

    async def showdown_server_mock(websocket, path):
        semaphore.release()
        await websocket.ping()
        await websocket.send("error|test 1")
        await websocket.send("error|test 2")
        await websocket.send("error|test 3")

    await semaphore.acquire()

    gathered = asyncio.gather(websockets.serve(showdown_server_mock, "0.0.0.0", 8899))

    await player.listen()

    await gathered
    assert player._handle_message.await_count == 3
    player._handle_message.assert_awaited_with("error|test 3")