Python asyncio.BoundedSemaphore() Examples

def test_context_manager_with_await(self):
        primitives = [
            asyncio.Lock(loop=self.loop),
            asyncio.Condition(loop=self.loop),
            asyncio.Semaphore(loop=self.loop),
            asyncio.BoundedSemaphore(loop=self.loop),
        ]

        async def test(lock):
            await asyncio.sleep(0.01, loop=self.loop)
            self.assertFalse(lock.locked())
            with self.assertWarns(DeprecationWarning):
                with await lock as _lock:
                    self.assertIs(_lock, None)
                    self.assertTrue(lock.locked())
                    await asyncio.sleep(0.01, loop=self.loop)
                    self.assertTrue(lock.locked())
                self.assertFalse(lock.locked())

        for primitive in primitives:
            self.loop.run_until_complete(test(primitive))
            self.assertFalse(primitive.locked()) 

def test_context_manager_async_with(self):
        primitives = [
            asyncio.Lock(loop=self.loop),
            asyncio.Condition(loop=self.loop),
            asyncio.Semaphore(loop=self.loop),
            asyncio.BoundedSemaphore(loop=self.loop),
        ]

        async def test(lock):
            await asyncio.sleep(0.01, loop=self.loop)
            self.assertFalse(lock.locked())
            async with lock as _lock:
                self.assertIs(_lock, None)
                self.assertTrue(lock.locked())
                await asyncio.sleep(0.01, loop=self.loop)
                self.assertTrue(lock.locked())
            self.assertFalse(lock.locked())

        for primitive in primitives:
            self.loop.run_until_complete(test(primitive))
            self.assertFalse(primitive.locked()) 

def test_context_manager_async_with(self):
        primitives = [
            asyncio.Lock(loop=self.loop),
            asyncio.Condition(loop=self.loop),
            asyncio.Semaphore(loop=self.loop),
            asyncio.BoundedSemaphore(loop=self.loop),
        ]

        async def test(lock):
            await asyncio.sleep(0.01, loop=self.loop)
            self.assertFalse(lock.locked())
            async with lock as _lock:
                self.assertIs(_lock, None)
                self.assertTrue(lock.locked())
                await asyncio.sleep(0.01, loop=self.loop)
                self.assertTrue(lock.locked())
            self.assertFalse(lock.locked())

        for primitive in primitives:
            self.loop.run_until_complete(test(primitive))
            self.assertFalse(primitive.locked()) 

def test_context_manager_with_await(self):
        primitives = [
            asyncio.Lock(loop=self.loop),
            asyncio.Condition(loop=self.loop),
            asyncio.Semaphore(loop=self.loop),
            asyncio.BoundedSemaphore(loop=self.loop),
        ]

        async def test(lock):
            await asyncio.sleep(0.01, loop=self.loop)
            self.assertFalse(lock.locked())
            with await lock as _lock:
                self.assertIs(_lock, None)
                self.assertTrue(lock.locked())
                await asyncio.sleep(0.01, loop=self.loop)
                self.assertTrue(lock.locked())
            self.assertFalse(lock.locked())

        for primitive in primitives:
            self.loop.run_until_complete(test(primitive))
            self.assertFalse(primitive.locked()) 

def __init__(self, verbosity=0, max_tasks=512):
        """Constructor.

        Args:
            verbosity: set output verbosity: 0 (default) is none, 3 is debug
            max_tasks: the maximum number of tasks asyncio will queue (default 512)
        """
        self.tasks = []
        self.errors = []
        self.fqdn = []
        self.ignore_hosts = []
        asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
        self.loop = asyncio.get_event_loop()
        self.resolver = aiodns.DNSResolver(loop=self.loop, rotate=True)
        self.sem = asyncio.BoundedSemaphore(max_tasks)
        self.max_tasks = max_tasks
        self.verbosity = verbosity
        self.logger = ConsoleLogger(verbosity) 

def print_tests():
    st = '''Available functions:
print_tests()  Print this list.
ack_test()  Test event acknowledge and Message class.
message_test() Test Message class.
event_test()  Test Event and Lock objects.
barrier_test()  Test the Barrier class.
semaphore_test(bounded=False)  Test Semaphore or BoundedSemaphore.
condition_test()  Test the Condition class.
queue_test()  Test the  Queue class

Recommended to issue ctrl-D after running each test.
'''
    print('\x1b[32m')
    print(st)
    print('\x1b[39m') 

def test_context_manager_with_await(self):
        primitives = [
            asyncio.Lock(loop=self.loop),
            asyncio.Condition(loop=self.loop),
            asyncio.Semaphore(loop=self.loop),
            asyncio.BoundedSemaphore(loop=self.loop),
        ]

        async def test(lock):
            await asyncio.sleep(0.01, loop=self.loop)
            self.assertFalse(lock.locked())
            with await lock as _lock:
                self.assertIs(_lock, None)
                self.assertTrue(lock.locked())
                await asyncio.sleep(0.01, loop=self.loop)
                self.assertTrue(lock.locked())
            self.assertFalse(lock.locked())

        for primitive in primitives:
            self.loop.run_until_complete(test(primitive))
            self.assertFalse(primitive.locked()) 

def test_context_manager_async_with(self):
        primitives = [
            asyncio.Lock(loop=self.loop),
            asyncio.Condition(loop=self.loop),
            asyncio.Semaphore(loop=self.loop),
            asyncio.BoundedSemaphore(loop=self.loop),
        ]

        async def test(lock):
            await asyncio.sleep(0.01, loop=self.loop)
            self.assertFalse(lock.locked())
            async with lock as _lock:
                self.assertIs(_lock, None)
                self.assertTrue(lock.locked())
                await asyncio.sleep(0.01, loop=self.loop)
                self.assertTrue(lock.locked())
            self.assertFalse(lock.locked())

        for primitive in primitives:
            self.loop.run_until_complete(test(primitive))
            self.assertFalse(primitive.locked()) 

def __init__(self):
        self.logger = logging.getLogger("proxy_py/processor")
        self.logger.setLevel(logging.DEBUG if settings.DEBUG else logging.INFO)

        logger_handler = logging.StreamHandler(sys.stdout)
        logger_handler.setLevel(logging.DEBUG if settings.DEBUG else logging.INFO)
        logger_handler.setFormatter(logging.Formatter(settings.LOG_FORMAT_STRING))

        self.logger.addHandler(logger_handler)

        self.collectors_logger = logging.getLogger("proxy_py/collectors")
        self.collectors_logger.setLevel(logging.DEBUG if settings.DEBUG else logging.INFO)

        collectors_logger_handler = logging.StreamHandler(sys.stdout)
        collectors_logger_handler.setLevel(logging.DEBUG if settings.DEBUG else logging.INFO)
        collectors_logger_handler.setFormatter(logging.Formatter(settings.LOG_FORMAT_STRING))

        self.collectors_logger.addHandler(collectors_logger_handler)

        self.logger.debug("processor initialization...")

        self.proxies_semaphore = asyncio.BoundedSemaphore(settings.NUMBER_OF_CONCURRENT_TASKS)
        self.good_proxies_are_processed = False 

def __init__(self, storage_directory: Path, loglevel: int=logging.DEBUG) -> None:
        self.__init_logger(loglevel)
        self.ipv6_url = 'http://data.caida.org/datasets/routing/routeviews6-prefix2as/{}'
        self.ipv4_url = 'http://data.caida.org/datasets/routing/routeviews-prefix2as/{}'
        self.storage_root = storage_directory
        self.sema = asyncio.BoundedSemaphore(2) 

def __init__(self,
                 privkey: datatypes.PrivateKey,
                 context: BasePeerContext,
                 max_peers: int = DEFAULT_MAX_PEERS,
                 event_bus: EndpointAPI = None,
                 metrics_registry: MetricsRegistry = None,
                 ) -> None:
        self.logger = get_logger(self.__module__ + '.' + self.__class__.__name__)

        self.privkey = privkey
        self.max_peers = max_peers
        self.context = context

        self.connected_nodes: Dict[SessionAPI, BasePeer] = {}

        self._subscribers: List[PeerSubscriber] = []
        self._event_bus = event_bus

        if metrics_registry is None:
            # Initialize with a MetricsRegistry from pyformance as p2p can not depend on Trinity
            # This is so that we don't need to pass a MetricsRegistry in tests and mocked pools.
            metrics_registry = MetricsRegistry()
        self._active_peer_counter = metrics_registry.counter('trinity.p2p/peers.counter')
        self._peer_reporter_registry = self.get_peer_reporter_registry(metrics_registry)

        # Restricts the number of concurrent connection attempts can be made
        self._connection_attempt_lock = asyncio.BoundedSemaphore(MAX_CONCURRENT_CONNECTION_ATTEMPTS)

        # Ensure we can only have a single concurrent handshake in flight per remote
        self._handshake_locks = ResourceLock()

        self.peer_backends = self.setup_peer_backends()
        self.connection_tracker = self.setup_connection_tracker() 

def test_release_not_acquired(self):
        sem = asyncio.BoundedSemaphore(loop=self.loop)

        self.assertRaises(ValueError, sem.release) 

def test_release_not_acquired(self):
        sem = asyncio.BoundedSemaphore(loop=self.loop)

        self.assertRaises(ValueError, sem.release) 

def __init__(self, name, token):
        self.name = name
        self.token = token
        self.team = self.users = self.channels = self.directs = None
        # When we send messages asynchronously, we'll receive an RTM event before the HTTP request
        # returns. This lock will block event parsing whilst we're sending, to make sure the caller
        # can finish processing the new message (e.g. storing the ID) before receiving the event.
        self.lock = asyncio.BoundedSemaphore()
        self.callbacks = []
        # Internal tracking of the RTM task, used to cancel on plugin unload.
        self._task = None
        self._sess = aiohttp.ClientSession() 

def run_sema_test(bounded):
    num_coros = 5
    barrier = Barrier(num_coros + 1)
    if bounded:
        semaphore = asyncio.BoundedSemaphore(3)
    else:
        semaphore = asyncio.Semaphore(3)
    for n in range(num_coros):
        asyncio.create_task(run_sema(n, semaphore, barrier))
    await barrier  # Quit when all coros complete
    try:
        semaphore.release()
    except ValueError:
        print('Bounded semaphore exception test OK') 

def main():
    # pylint: disable=global-statement
    global CATEGORIES
    semaphore = asyncio.BoundedSemaphore(args.limit)
    async with aiohttp.ClientSession() as session:
        async with aiohttp.ClientSession(
                connector=aiohttp.TCPConnector(
                    ssl=args.do_not_verify_ssl)) as session:
            CATEGORIES = await get_existing_categories(
                session, args.url, args.username, args.password, semaphore)
            await upload_scripts(session, args.url, args.username,
                                 args.password, semaphore)
            await upload_extension_attributes(session, args.url, args.username,
                                              args.password, semaphore) 

def test_release_not_acquired(self):
        sem = asyncio.BoundedSemaphore(loop=self.loop)

        self.assertRaises(ValueError, sem.release) 

def test_release_not_acquired(self):
        sem = asyncio.BoundedSemaphore(loop=self.loop)

        self.assertRaises(ValueError, sem.release) 

def __init__(self, storage_directory: Path, collector: str='rrc00', hours: list=['0000'], loglevel: int=logging.DEBUG) -> None:
        self.__init_logger(loglevel)
        self.collector = collector
        self.hours = hours
        self.url = 'http://data.ris.ripe.net/{}'
        self.storage_root = storage_directory
        self.sema = asyncio.BoundedSemaphore(5) 

def setup_test_cases(cases, conn, num_jobs, verbose=False):
    setup = get_test_cases_setup(cases)

    async def _run():
        if num_jobs == 1:
            # Special case for --jobs=1
            for _case, dbname, setup_script in setup:
                await _setup_database(dbname, setup_script, conn)
                if verbose:
                    print(f' -> {dbname}: OK', flush=True)
        else:
            async with taskgroup.TaskGroup(name='setup test cases') as g:
                # Use a semaphore to limit the concurrency of bootstrap
                # tasks to the number of jobs (bootstrap is heavy, having
                # more tasks than `--jobs` won't necessarily make
                # things faster.)
                sem = asyncio.BoundedSemaphore(num_jobs)

                async def controller(coro, dbname, *args):
                    async with sem:
                        await coro(dbname, *args)
                        if verbose:
                            print(f' -> {dbname}: OK', flush=True)

                for _case, dbname, setup_script in setup:
                    g.create_task(controller(
                        _setup_database, dbname, setup_script, conn))

    return asyncio.run(_run()) 

def __init__(self, json_entry):
        """
        Init class structure. Each module takes a JSON entry object which
        can pass different values to the module with out changing up the API.
        adapted form  Empire Project:
        https://github.com/EmpireProject/Empire/blob/master/lib/modules/python_template.py

        :param json_entry: JSON data object passed to the module.
        """
        module_helpers.RequestsHelpers.__init__(self)
        self.json_entry = json_entry
        self.info = {
            # mod name
            'Module': 'subdomain_bruteforce.py',

            # long name of the module to be used
            'Name': 'Recursive Subdomain Bruteforce Using Wordlist',

            # version of the module to be used
            'Version': '1.0',

            # description
            'Description': ['Uses lists from dnspop',
                            'with high quality dns resolvers.'],

            # authors or sources to be quoted
            'Authors': ['@Killswitch-GUI', '@blark'],

            # list of resources or comments
            'comments': [
                'Searches and performs recursive dns-lookup.',
                ' adapted from https://github.com/blark/aiodnsbrute/blob/master/aiodnsbrute/cli.py'
            ],
            # priority of module (0) being first to execute
            'priority': 0
        }

        self.options = {
        }
        # ~ queue object
        self.word_count = int(self.json_entry['args'].wordlist_count)
        self.word_list_queue = queue.Queue(maxsize=0)
        self.tasks = []
        self.domain = ''
        self.errors = []
        self.fqdn = []
        self.runtime_queue = []
        # disable uvloop until supports windows
        # asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
        self.loop = asyncio.get_event_loop()
        self.resolver = aiodns.DNSResolver(loop=self.loop, rotate=True)
        # TODO: make max tasks defined in config.json
        self.max_tasks = 500
        # TODO: make total set from wordcount in config.json
        self.sem = asyncio.BoundedSemaphore(self.max_tasks)
        self.cs = core_scrub.Scrub()
        self.core_args = self.json_entry['args']
        self.silent = self.json_entry['silent'] 

def __init__(self, json_entry):
        """
        Init class structure. Each module takes a JSON entry object which
        can pass different values to the module with out changing up the API.
        adapted form  Empire Project:
        https://github.com/EmpireProject/Empire/blob/master/lib/modules/python_template.py

        :param json_entry: JSON data object passed to the module.
        """
        module_helpers.RequestsHelpers.__init__(self)
        self.json_entry = json_entry
        self.info = {
            # mod name
            'Module': 'subdomain_bruteforce.py',

            # long name of the module to be used
            'Name': 'Recursive Subdomain Bruteforce Using Wordlist',

            # version of the module to be used
            'Version': '1.0',

            # description
            'Description': ['Uses lists from dnspop',
                            'with high quality dns resolvers.'],

            # authors or sources to be quoted
            'Authors': ['@Killswitch-GUI', '@blark'],

            # list of resources or comments
            'comments': [
                'Searches and performs recursive dns-lookup.',
                ' adapted from https://github.com/blark/aiodnsbrute/blob/master/aiodnsbrute/cli.py'
            ],
            # priority of module (0) being first to execute
            'priority': 0
        }

        self.options = {
        }
        # ~ queue object
        self.word_count = int(self.json_entry['args'].raw_depth)
        self.word_list_queue = queue.Queue(maxsize=0)
        self.tasks = []
        self.domain = ''
        self.errors = []
        self.fqdn = []
        self.sub_gen_count = 0
        # disable uvloop until supports windows
        # asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
        self.loop = asyncio.get_event_loop()
        self.resolver = aiodns.DNSResolver(loop=self.loop, rotate=True)
        # TODO: make max tasks defined in config.json
        self.max_tasks = 1024
        # TODO: make total set from wordcount in config.json
        self.sem = asyncio.BoundedSemaphore(self.max_tasks)
        self.cs = core_scrub.Scrub()
        self.core_args = self.json_entry['args']
        self.core_resolvers = self.json_entry['resolvers']
        self.silent = self.json_entry['silent']