Python gevent.monkey.patch_all() Examples

def testGevent():

    # 创建一个协程对象，创建的时候就自动运行了，不需要手动启动
    g1 = gevent.spawn(work5, 10)
    g2 = gevent.spawn(work5, 10)
    g3 = gevent.spawn(work5, 10)

    # 阻塞等待协程执行完毕
    # 没使用monkey.patch_all()破解的时候不会自动切换，破解后就会随机协程
    # g1.join()
    # g2.join()
    # g3.join()

    # 程序从上到下执行，不管之前有没有异步代码，遇到join相当于一面墙，堵住下面的路
    gevent.joinall([g1,g2,g3])

    print("全部协程结束完毕.") 

def patch(self):
        from gevent import monkey
        monkey.noisy = False

        # if the new version is used make sure to patch subprocess
        if gevent.version_info[0] == 0:
            monkey.patch_all()
        else:
            monkey.patch_all(subprocess=True)

        # monkey patch sendfile to make it none blocking
        patch_sendfile()

        # patch sockets
        sockets = []
        for s in self.sockets:
            if sys.version_info[0] == 3:
                sockets.append(socket(s.FAMILY, _socket.SOCK_STREAM,
                    fileno=s.sock.fileno()))
            else:
                sockets.append(socket(s.FAMILY, _socket.SOCK_STREAM,
                    _sock=s))
        self.sockets = sockets 

def run():
    initEngine()
    if conf.ENGINE is ENGINE_MODE_STATUS.THREAD: #多线程模式conf.ENGINE=9
        for i in range(th.threads_num):
            t = threading.Thread(target=scan, name=str(i))
            setThreadDaemon(t)
            t.start()
        # It can quit with Ctrl-C
        while 1: #如果未扫描结束，主线程一直死循环等待，取决于线程数量和th.is_continue的值
            if th.thread_count > 0 and th.is_continue:
                time.sleep(0.01)
            else:
                break

    elif conf.ENGINE is ENGINE_MODE_STATUS.GEVENT: #协程模式conf.ENGINE=8
        from gevent import monkey
        monkey.patch_all()
        import gevent
        while th.queue.qsize() > 0 and th.is_continue:
            gevent.joinall([gevent.spawn(scan) for i in xrange(0, th.threads_num) if #生成10个concurrent
                            th.queue.qsize() > 0])

    if 'errmsg' in th:
        logger.error(th.errmsg) 

def run(servername, ip, port, softcron=True, logging=False, profiler=None,
        options=None):
    if servername == 'gevent':
        from gevent import monkey
        monkey.patch_all()
    elif servername == 'eventlet':
        import eventlet
        eventlet.monkey_patch()

    import gluon.main

    if logging:
        application = gluon.main.appfactory(wsgiapp=gluon.main.wsgibase,
                                            logfilename='httpserver.log',
                                            profiler_dir=profiler)
    else:
        application = gluon.main.wsgibase
    if softcron:
        from gluon.settings import global_settings
        global_settings.web2py_crontype = 'soft'
    getattr(Servers, servername)(application, (ip, int(port)), options=options) 

def verify_async(case_list,coroutine):
        """
        Verify used gevent lib
        :param case_list:
        :param coroutine:
        :return:
        """
        from gevent import monkey
        monkey.patch_all()
        result = []
        geventPool = pool.Pool(coroutine)
        tasks = [geventPool.spawn(Verify.request_and_verify, case) for case in case_list]
        gevent.joinall(tasks)
        for i in tasks:
            if i.value is not None:
                result.append(i.value)
        print_info('Total Verify-Case is: %s, %s error happened.' % (len(case_list), Verify.ERROR_COUNT))
        return result 

def patch(self):
        from gevent import monkey
        monkey.noisy = False

        # if the new version is used make sure to patch subprocess
        if gevent.version_info[0] == 0:
            monkey.patch_all()
        else:
            monkey.patch_all(subprocess=True)

        # monkey patch sendfile to make it none blocking
        patch_sendfile()

        # patch sockets
        sockets = []
        for s in self.sockets:
            sockets.append(socket(s.FAMILY, _socket.SOCK_STREAM,
                _sock=s))
        self.sockets = sockets 

def patch_os():
    """
    Replace :func:`os.fork` with :func:`gevent.fork`, and, on POSIX,
    :func:`os.waitpid` with :func:`gevent.os.waitpid` (if the
    environment variable ``GEVENT_NOWAITPID`` is not defined). Does
    nothing if fork is not available.

    .. caution:: This method must be used with :func:`patch_signal` to have proper SIGCHLD
         handling and thus correct results from ``waitpid``.
         :func:`patch_all` calls both by default.

    .. caution:: For SIGCHLD handling to work correctly, the event loop must run.
         The easiest way to help ensure this is to use :func:`patch_all`.
    """
    patch_module('os') 

def patch_signal():
    """
    Make the signal.signal function work with a monkey-patched os.

    This method must be used with :func:`patch_os` to have proper SIGCHLD
    handling. :func:`patch_all` calls both by default.

    .. seealso:: :mod:`gevent.signal`
    """
    patch_module("signal") 

def _get_script_help():
    from inspect import getargspec
    patch_all_args = getargspec(patch_all)[0]
    modules = [x for x in patch_all_args if 'patch_' + x in globals()]
    script_help = """gevent.monkey - monkey patch the standard modules to use gevent.

USAGE: python -m gevent.monkey [MONKEY OPTIONS] script [SCRIPT OPTIONS]

If no OPTIONS present, monkey patches all the modules it can patch.
You can exclude a module with --no-module, e.g. --no-thread. You can
specify a module to patch with --module, e.g. --socket. In the latter
case only the modules specified on the command line will be patched.

MONKEY OPTIONS: --verbose %s""" % ', '.join('--[no-]%s' % m for m in modules)
    return script_help, patch_all_args, modules 

def main():
    args = {}
    argv = sys.argv[1:]
    verbose = False
    script_help, patch_all_args, modules = _get_script_help()
    while argv and argv[0].startswith('--'):
        option = argv[0][2:]
        if option == 'verbose':
            verbose = True
        elif option.startswith('no-') and option.replace('no-', '') in patch_all_args:
            args[option[3:]] = False
        elif option in patch_all_args:
            args[option] = True
            if option in modules:
                for module in modules:
                    args.setdefault(module, False)
        else:
            sys.exit(script_help + '\n\n' + 'Cannot patch %r' % option)
        del argv[0]
        # TODO: break on --
    if verbose:
        import pprint
        import os
        print('gevent.monkey.patch_all(%s)' % ', '.join('%s=%s' % item for item in args.items()))
        print('sys.version=%s' % (sys.version.strip().replace('\n', ' '), ))
        print('sys.path=%s' % pprint.pformat(sys.path))
        print('sys.modules=%s' % pprint.pformat(sorted(sys.modules.keys())))
        print('cwd=%s' % os.getcwd())

    patch_all(**args)
    if argv:
        sys.argv = argv
        __package__ = None
        assert __package__ is None
        globals()['__file__'] = sys.argv[0]  # issue #302
        with open(sys.argv[0]) as f:
            exec(f.read())
    else:
        print(script_help) 

def patch_signal():
    """
    Make the signal.signal function work with a monkey-patched os.

    This method must be used with :func:`patch_os` to have proper SIGCHLD
    handling. :func:`patch_all` calls both by default.

    .. seealso:: :mod:`gevent.signal`
    """
    patch_module("signal") 

def patch_os():
    """
    Replace :func:`os.fork` with :func:`gevent.fork`, and, on POSIX,
    :func:`os.waitpid` with :func:`gevent.os.waitpid` (if the
    environment variable ``GEVENT_NOWAITPID`` is not defined). Does
    nothing if fork is not available.

    This method must be used with :func:`patch_signal` to have proper SIGCHLD
    handling. :func:`patch_all` calls both by default.
    """
    patch_module('os') 

def main():
    args = {}
    argv = sys.argv[1:]
    verbose = False
    script_help, patch_all_args, modules = _get_script_help()
    while argv and argv[0].startswith('--'):
        option = argv[0][2:]
        if option == 'verbose':
            verbose = True
        elif option.startswith('no-') and option.replace('no-', '') in patch_all_args:
            args[option[3:]] = False
        elif option in patch_all_args:
            args[option] = True
            if option in modules:
                for module in modules:
                    args.setdefault(module, False)
        else:
            sys.exit(script_help + '\n\n' + 'Cannot patch %r' % option)
        del argv[0]
        # TODO: break on --
    if verbose:
        import pprint
        import os
        print('gevent.monkey.patch_all(%s)' % ', '.join('%s=%s' % item for item in args.items()))
        print('sys.version=%s' % (sys.version.strip().replace('\n', ' '), ))
        print('sys.path=%s' % pprint.pformat(sys.path))
        print('sys.modules=%s' % pprint.pformat(sorted(sys.modules.keys())))
        print('cwd=%s' % os.getcwd())

    patch_all(**args)
    if argv:
        sys.argv = argv
        __package__ = None
        assert __package__ is None
        globals()['__file__'] = sys.argv[0]  # issue #302
        with open(sys.argv[0]) as f:
            exec(f.read())
    else:
        print(script_help) 

def basic(topic='topic', channel='channel', count=1e6, size=10, gevent=False,
    max_in_flight=2500, profile=False):
    '''Basic benchmark'''
    if gevent:
        from gevent import monkey
        monkey.patch_all()

    # Check the types of the arguments
    count = int(count)
    size = int(size)
    max_in_flight = int(max_in_flight)

    from nsq.http import nsqd
    from nsq.reader import Reader

    print('Publishing messages...')
    for batch in grouper(messages(count, size), 1000):
        nsqd.Client('http://localhost:4151').mpub(topic, batch)

    print('Consuming messages')
    client = Reader(topic, channel, nsqd_tcp_addresses=['localhost:4150'],
        max_in_flight=max_in_flight)
    with closing(client):
        start = -time.time()
        if profile:
            with profiler():
                for message in islice(client, count):
                    message.fin()
        else:
            for message in islice(client, count):
                message.fin()
        start += time.time()
    print('Finished %i messages in %fs (%5.2f messages / second)' % (
        count, start, count / start)) 

def _get_script_help():
    from inspect import getargspec
    patch_all_args = getargspec(patch_all)[0] # pylint:disable=deprecated-method
    modules = [x for x in patch_all_args if 'patch_' + x in globals()]
    script_help = """gevent.monkey - monkey patch the standard modules to use gevent.

USAGE: python -m gevent.monkey [MONKEY OPTIONS] script [SCRIPT OPTIONS]

If no OPTIONS present, monkey patches all the modules it can patch.
You can exclude a module with --no-module, e.g. --no-thread. You can
specify a module to patch with --module, e.g. --socket. In the latter
case only the modules specified on the command line will be patched.

MONKEY OPTIONS: --verbose %s""" % ', '.join('--[no-]%s' % m for m in modules)
    return script_help, patch_all_args, modules 

def main():
    args = {}
    argv = sys.argv[1:]
    verbose = False
    script_help, patch_all_args, modules = _get_script_help()
    while argv and argv[0].startswith('--'):
        option = argv[0][2:]
        if option == 'verbose':
            verbose = True
        elif option.startswith('no-') and option.replace('no-', '') in patch_all_args:
            args[option[3:]] = False
        elif option in patch_all_args:
            args[option] = True
            if option in modules:
                for module in modules:
                    args.setdefault(module, False)
        else:
            sys.exit(script_help + '\n\n' + 'Cannot patch %r' % option)
        del argv[0]
        # TODO: break on --
    if verbose:
        import pprint
        import os
        print('gevent.monkey.patch_all(%s)' % ', '.join('%s=%s' % item for item in args.items()))
        print('sys.version=%s' % (sys.version.strip().replace('\n', ' '), ))
        print('sys.path=%s' % pprint.pformat(sys.path))
        print('sys.modules=%s' % pprint.pformat(sorted(sys.modules.keys())))
        print('cwd=%s' % os.getcwd())

    patch_all(**args)
    if argv:
        sys.argv = argv
        __package__ = None
        assert __package__ is None
        globals()['__file__'] = sys.argv[0]  # issue #302
        with open(sys.argv[0]) as f:
            exec(f.read())
    else:
        print(script_help) 

def _get_script_help():
    from inspect import getargspec
    patch_all_args = getargspec(patch_all)[0]
    modules = [x for x in patch_all_args if 'patch_' + x in globals()]
    script_help = """gevent.monkey - monkey patch the standard modules to use gevent.

USAGE: python -m gevent.monkey [MONKEY OPTIONS] script [SCRIPT OPTIONS]

If no OPTIONS present, monkey patches all the modules it can patch.
You can exclude a module with --no-module, e.g. --no-thread. You can
specify a module to patch with --module, e.g. --socket. In the latter
case only the modules specified on the command line will be patched.

MONKEY OPTIONS: --verbose %s""" % ', '.join('--[no-]%s' % m for m in modules)
    return script_help, patch_all_args, modules 

def patch_signal():
    """
    Make the signal.signal function work with a monkey-patched os.

    .. caution:: This method must be used with :func:`patch_os` to have proper SIGCHLD
         handling. :func:`patch_all` calls both by default.

    .. caution:: For proper SIGCHLD handling, you must yield to the event loop.
         Using :func:`patch_all` is the easiest way to ensure this.

    .. seealso:: :mod:`gevent.signal`
    """
    patch_module("signal") 

def check_gevent_monkey_patch(raise_exc=True):
    if not monkey.is_module_patched('socket'):  # 随便选一个检测标志
        if raise_exc:
            warnings.warn(f'检测到 你还没有打gevent包的猴子补丁，请在所运行的起始脚本第一行写上  【import gevent.monkey;gevent.monkey.patch_all()】  这句话。')
            raise Exception(f'检测到 你还没有打gevent包的猴子补丁，请在所运行的起始脚本第一行写上  【import gevent.monkey;gevent.monkey.patch_all()】  这句话。')
    else:
        return 1 

def run(self):
        import gevent
        from gevent import monkey
        monkey.patch_all()
        from gevent import pool
        # default 200
        # g_pool = pool.Pool(200)
        g_pool = pool.Pool(self.coroutine)
        tasks = [g_pool.spawn(self.gen_traffic, url) for url in self.url_list]
        gevent.joinall(tasks)
        traffic_list = []
        for i in tasks:
            if i.value is not None:
                traffic_list.append(i.value)
        # save traffic for rescan
        Engine.save_traffic(traffic_list, self.id) 

def _get_script_help():
    from inspect import getargspec
    patch_all_args = getargspec(patch_all)[0]
    modules = [x for x in patch_all_args if 'patch_' + x in globals()]
    script_help = """gevent.monkey - monkey patch the standard modules to use gevent.

USAGE: python -m gevent.monkey [MONKEY OPTIONS] script [SCRIPT OPTIONS]

If no OPTIONS present, monkey patches all the modules it can patch.
You can exclude a module with --no-module, e.g. --no-thread. You can
specify a module to patch with --module, e.g. --socket. In the latter
case only the modules specified on the command line will be patched.

MONKEY OPTIONS: --verbose %s""" % ', '.join('--[no-]%s' % m for m in modules)
    return script_help, patch_all_args, modules 

def main():
    args = {}
    argv = sys.argv[1:]
    verbose = False
    script_help, patch_all_args, modules = _get_script_help()
    while argv and argv[0].startswith('--'):
        option = argv[0][2:]
        if option == 'verbose':
            verbose = True
        elif option.startswith('no-') and option.replace('no-', '') in patch_all_args:
            args[option[3:]] = False
        elif option in patch_all_args:
            args[option] = True
            if option in modules:
                for module in modules:
                    args.setdefault(module, False)
        else:
            sys.exit(script_help + '\n\n' + 'Cannot patch %r' % option)
        del argv[0]
        # TODO: break on --
    if verbose:
        import pprint
        import os
        print('gevent.monkey.patch_all(%s)' % ', '.join('%s=%s' % item for item in args.items()))
        print('sys.version=%s' % (sys.version.strip().replace('\n', ' '), ))
        print('sys.path=%s' % pprint.pformat(sys.path))
        print('sys.modules=%s' % pprint.pformat(sorted(sys.modules.keys())))
        print('cwd=%s' % os.getcwd())

    patch_all(**args)
    if argv:
        sys.argv = argv
        __package__ = None
        assert __package__ is None
        globals()['__file__'] = sys.argv[0]  # issue #302
        with open(sys.argv[0]) as f:
            exec(f.read())
    else:
        print(script_help) 

def patch_signal():
    """
    Make the signal.signal function work with a monkey-patched os.

    This method must be used with :func:`patch_os` to have proper SIGCHLD
    handling. :func:`patch_all` calls both by default.

    .. seealso:: :mod:`gevent.signal`
    """
    patch_module("signal") 

def patch_os():
    """
    Replace :func:`os.fork` with :func:`gevent.fork`, and, on POSIX,
    :func:`os.waitpid` with :func:`gevent.os.waitpid` (if the
    environment variable ``GEVENT_NOWAITPID`` is not defined). Does
    nothing if fork is not available.

    This method must be used with :func:`patch_signal` to have proper SIGCHLD
    handling. :func:`patch_all` calls both by default.
    """
    patch_module('os') 

def test_notify_gevent(case_setup, pyfile):

    def get_environ(writer):
        # I.e.: Make sure that gevent support is disabled
        env = os.environ.copy()
        env['GEVENT_SUPPORT'] = ''
        return env

    @pyfile
    def case_gevent():
        from gevent import monkey
        monkey.patch_all()
        print('TEST SUCEEDED')

    def additional_output_checks(writer, stdout, stderr):
        assert 'environment variable' in stderr
        assert 'GEVENT_SUPPORT=True' in stderr

    with case_setup.test_file(
            case_gevent,
            get_environ=get_environ,
            additional_output_checks=additional_output_checks,
            EXPECTED_RETURNCODE='any',
            FORCE_KILL_PROCESS_WHEN_FINISHED_OK=True
        ) as writer:
        json_facade = JsonFacade(writer)
        json_facade.write_launch()
        json_facade.write_make_initial_run()

        wait_for_condition(lambda: 'GEVENT_SUPPORT=True' in writer.get_stderr())

        writer.finished_ok = True 

def run(self, handler):
        from gevent import wsgi as wsgi_fast, pywsgi, monkey, local
        if self.options.get('monkey', True):
            if not threading.local is local.local: monkey.patch_all()
        wsgi = wsgi_fast if self.options.get('fast') else pywsgi
        wsgi.WSGIServer((self.host, self.port), handler).serve_forever() 

def start_server():
    logger.info(f"server port: {config.SERVER_PORT}")
    logger.info(f"pic root dir path: {config.PIC_DIR_PATH}")

    # check existed
    assert os.path.exists(
        config.PIC_DIR_PATH
    ), f"dir path not existed: {config.PIC_DIR_PATH}"

    from gevent import monkey, pywsgi

    monkey.patch_all()
    server = pywsgi.WSGIServer(("0.0.0.0", int(config.SERVER_PORT)), app)
    server.serve_forever() 

def global_server(spider_global_variable):
    # 初始化全局变量
    url_rule = UrlRuleClass(spider_global_variable)

    threads_list = []
    spider_threads = []

    threads_list.append(threading.Thread(target=spider_scheduling, args=(spider_global_variable, url_rule,)))
    threads_list.append(threading.Thread(target=global_scheduling, args=(spider_global_variable,)))

    for t in threads_list:
        t.setDaemon(True)
        t.start()

    if spider_global_variable.spider_use_gevent:
        import gevent
        from gevent import monkey
        monkey.patch_all(thread=False)
        for i in xrange(spider_global_variable.threads):
            spider_threads.append(gevent.spawn(spider, spider_global_variable))
        gevent.joinall(spider_threads)
    else:
        for i in xrange(spider_global_variable.threads):
            spider_threads.append(threading.Thread(target=spider, args=(spider_global_variable,)))
        for t in spider_threads:
            t.setDaemon(True)
            t.start()


    time.sleep(120)
    while True:
        if spider_global_variable.spider_urlnode_queue.qsize() == 0:
            spider_logger.critical('MSpider wait to exit!!')
            time.sleep(120)
            if spider_global_variable.spider_urlnode_queue.qsize() == 0:
                pass
            else:
                continue
            spider_global_variable.end_ctime = time.ctime()
            time.sleep(120)
            spider_logger.critical('MSpider exit!!')
            sys.exit(0)
        else:
            time.sleep(10) 

def patch_all(socket=True, dns=True, time=True, select=True, thread=True, os=True, ssl=True, httplib=False,
              subprocess=True, sys=False, aggressive=True, Event=False,
              builtins=True, signal=True):
    """
    Do all of the default monkey patching (calls every other applicable
    function in this module).

    .. versionchanged:: 1.1
       Issue a :mod:`warning <warnings>` if this function is called multiple times
       with different arguments. The second and subsequent calls will only add more
       patches, they can never remove existing patches by setting an argument to ``False``.
    .. versionchanged:: 1.1
       Issue a :mod:`warning <warnings>` if this function is called with ``os=False``
       and ``signal=True``. This will cause SIGCHLD handlers to not be called. This may
       be an error in the future.
    """
    # Check to see if they're changing the patched list
    _warnings, first_time = _check_repatching(**locals())
    if not _warnings and not first_time:
        # Nothing to do, identical args to what we just
        # did
        return

    # order is important
    if os:
        patch_os()
    if time:
        patch_time()
    if thread:
        patch_thread(Event=Event)
    # sys must be patched after thread. in other cases threading._shutdown will be
    # initiated to _MainThread with real thread ident
    if sys:
        patch_sys()
    if socket:
        patch_socket(dns=dns, aggressive=aggressive)
    if select:
        patch_select(aggressive=aggressive)
    if ssl:
        patch_ssl()
    if httplib:
        raise ValueError('gevent.httplib is no longer provided, httplib must be False')
    if subprocess:
        patch_subprocess()
    if builtins:
        patch_builtins()
    if signal:
        if not os:
            _queue_warning('Patching signal but not os will result in SIGCHLD handlers'
                           ' installed after this not being called and os.waitpid may not'
                           ' function correctly if gevent.subprocess is used. This may raise an'
                           ' error in the future.',
                           _warnings)
        patch_signal()

    _process_warnings(_warnings) 

def patch_all(socket=True, dns=True, time=True, select=True, thread=True, os=True, ssl=True, httplib=False,
              subprocess=True, sys=False, aggressive=True, Event=False,
              builtins=True, signal=True):
    """
    Do all of the default monkey patching (calls every other applicable
    function in this module).

    .. versionchanged:: 1.1
       Issue a :mod:`warning <warnings>` if this function is called multiple times
       with different arguments. The second and subsequent calls will only add more
       patches, they can never remove existing patches by setting an argument to ``False``.
    .. versionchanged:: 1.1
       Issue a :mod:`warning <warnings>` if this function is called with ``os=False``
       and ``signal=True``. This will cause SIGCHLD handlers to not be called. This may
       be an error in the future.
    """
    # Check to see if they're changing the patched list
    _warnings, first_time = _check_repatching(**locals())
    if not _warnings and not first_time:
        # Nothing to do, identical args to what we just
        # did
        return

    # order is important
    if os:
        patch_os()
    if time:
        patch_time()
    if thread:
        patch_thread(Event=Event)
    # sys must be patched after thread. in other cases threading._shutdown will be
    # initiated to _MainThread with real thread ident
    if sys:
        patch_sys()
    if socket:
        patch_socket(dns=dns, aggressive=aggressive)
    if select:
        patch_select(aggressive=aggressive)
    if ssl:
        patch_ssl()
    if httplib:
        raise ValueError('gevent.httplib is no longer provided, httplib must be False')
    if subprocess:
        patch_subprocess()
    if builtins:
        patch_builtins()
    if signal:
        if not os:
            _queue_warning('Patching signal but not os will result in SIGCHLD handlers'
                           ' installed after this not being called and os.waitpid may not'
                           ' function correctly if gevent.subprocess is used. This may raise an'
                           ' error in the future.',
                           _warnings)
        patch_signal()

    _process_warnings(_warnings)