Python gevent.sleep() Examples

def _publisher(self):
        while True:
            self._xmpp_client_ready.wait()

            try:
                while True:
                    cmd, indicator, value = self.q.peek()
                    if value is None:
                        value = {}
                    value['origins'] = [self.jid]
                    self._xmpp_publish(cmd, {
                        'indicator': indicator,
                        'value': value
                    })
                    _ = self.q.get()

            except gevent.GreenletExit:
                break

            except Exception as e:
                LOG.exception('%s - Exception in publishing message', self.name)
                gevent.sleep(30)
                self.statistics['xmpp.publish_error'] += 1 

def test_lock_out_of_context_pair_4(self):
        with raises(GIPCLocked):
            with pipe(True) as (h1, h2):
                # Write more to pipe than pipe buffer can hold
                # (makes `put` block when there is no reader).
                # Buffer is quite large on Windows.
                gw1 = gevent.spawn(lambda h: h.put(LONGERTHANBUFFER), h1)
                gw2 = gevent.spawn(lambda h: h.put(LONGERTHANBUFFER), h2)
                gevent.sleep(SHORTTIME)
                # Context fails closing h2 writer, succeeds upon closing h2
                # reader. Proceeds closing h1 writer, fails, closes h1
                # reader and succeeds.
        assert h2._reader._closed
        assert h1._reader._closed
        assert not h2._writer._closed
        assert not h1._writer._closed
        gw1.kill(block=False)
        gw2.kill(block=False)
        gevent.sleep(-1)
        h2.close()
        h1.close() 

def test_lock_out_of_context_pair_3(self):
        with raises(GIPCLocked):
            with pipe(True) as (h1, h2):
                gr1 = gevent.spawn(lambda h: h.get(), h1)
                gr2 = gevent.spawn(lambda h: h.get(), h2)
                gevent.sleep(SHORTTIME)
                # Context succeeds closing h2 writer, fails upon closing h2
                # reader. Proceeds closing h1 writer, succeeds, closes h1
                # reader and fails.
        assert not h2._reader._closed
        assert not h1._reader._closed
        assert h2._writer._closed
        assert h1._writer._closed
        gr1.kill(block=False)
        gr2.kill(block=False)
        gevent.sleep(-1)
        h2.close()
        h1.close() 

def _compact_loop(self):
        gevent.sleep(self.compact_delay)

        while True:
            try:
                gevent.idle()

                counter = 0
                for idx in self.indexes.keys():
                    for i in self.query(index=idx, include_value=False):
                        if counter % 512 == 0:
                            gevent.sleep(0.001)  # yield to other greenlets
                        counter += 1

            except gevent.GreenletExit:
                break
            except:
                LOG.exception('Exception in _compact_loop')

            try:
                gevent.sleep(self.compact_interval)

            except gevent.GreenletExit:
                break 

def test_lock_out_of_context_pair(self):
        with raises(GIPCLocked):
            with pipe(True) as (h1, h2):
                # Write more to pipe than pipe buffer can hold
                # (makes `put` block when there is no reader).
                # Buffer is quite large on Windows.
                gw = gevent.spawn(lambda h: h.put(LONGERTHANBUFFER), h1)
                gevent.sleep(SHORTTIME)
                # Context manager tries to close h2 reader, h2 writer, and
                # h1 writer first. Fails upon latter, must still close
                # h1 reader after that.
        assert not h1._writer._closed
        assert h1._reader._closed
        assert h2._writer._closed
        assert h2._reader._closed
        # Kill greenlet (free lock on h1 writer), close h1 writer.
        gw.kill(block=False)
        gevent.sleep(-1)
        h1.close()
        assert h1._writer._closed 

def _job_timeout_glet(self, job_group, jobid, timeout):
        gevent.sleep(timeout)

        prefix = REDIS_JOBS_GROUP_PREFIX.format(job_group)

        jobdata = self.SR.hget(prefix, jobid)
        if jobdata is None:
            return

        jobdata = json.loads(jobdata)
        status = jobdata.get('status', None)
        if status != 'RUNNING':
            LOG.info('Timeout for job {}-{} triggered but status not running'.format(prefix, jobid))
            return

        pid = jobdata.get('pid', None)
        if pid is None:
            LOG.error('Timeout for job {}-{} triggered but no pid available'.format(prefix, jobid))
            return

        LOG.error('Timeout for job {}-{} triggered, sending TERM signal'.format(prefix, jobid))
        os.kill(pid, signal.SIGTERM) 

def test_lock_out_of_context_single(self):
        r, w = pipe()
        g = gevent.spawn(lambda r: r.get(), r)
        gevent.sleep(SHORTTIME)
        with raises(GIPCLocked):
            with r:
                pass
                # The context manager can't close `r`, as it is locked in `g`.
        g.kill(block=False)
        # Ensure killing via 'context switch', i.e. yield control to other
        # coroutines (otherwise the subsequent close attempt will fail with
        # `GIPCLocked` error).
        gevent.sleep(-1)
        # Close writer first. otherwise, `os.close(r._fd)` would block on Win.
        w.close()
        r.close() 

def test_exitcode_previous_to_join(self):
        p = start_process(lambda: gevent.sleep(SHORTTIME))
        # Assume that the child process is still alive when the next
        # line is executed by the interpreter (there is no guarantee
        # for that, but it's rather likely).
        assert p.exitcode is None

        # Expect the child watcher mechanism to pick up
        # and process the child process termination event
        # (within at most two seconds). The `gevent.sleep()`
        # invocations allow for libev event loop iterations,
        # two of which are required after the OS delivers the
        # SIGCHLD signal to the parent process: one iteration
        # invokes the child reap loop, and the next invokes
        # the libev callback associated with the termination
        # event.
        deadline = time.time() + 2
        while time.time() < deadline:
            if p.exitcode is not None:
                assert p.exitcode == 0
                p.join()
                return
            gevent.sleep(ALMOSTZERO)
        raise Exception('Child termination not detected') 

def publish(self, method, params=None):
        if self.socket is None:
            raise RuntimeError('Not connected')

        if params is None:
            params = {}

        id_ = str(uuid.uuid1())

        body = {
            'method': method,
            'id': id_,
            'params': params
        }

        try:
            self.socket.send_json(
                obj=body,
                flags=zmq.NOBLOCK
            )
        except zmq.ZMQError:
            LOG.error('Topic {} queue full - dropping message'.format(self.topic))

        gevent.sleep(0) 

def watch(self):
        import gevent

        cur_endpoints = self.get()
        yield cur_endpoints

        while True:
            if len(cur_endpoints) == self._expected_instances:
                linger = 10
            else:
                linger = 1

            gevent.sleep(linger)
            mtime = self._get_mtime()
            if mtime == self._endpoint_last_modified:
                continue

            self._endpoint_last_modified = mtime
            cur_endpoints = self.get()
            yield cur_endpoints 

def daemon_thread(mars_app, app_client, key_prefix, endpoint_file):
    last_eps = set()
    while True:
        try:
            cid_to_endpoint = dict()
            for val in app_client.kv.get_prefix(key_prefix).values():
                ep, cid = to_str(val).split('@', 1)
                cid_to_endpoint[cid] = ep

            containers = app_client.get_containers([MarsSchedulerConfig.service_name], states=['RUNNING'])
            eps = set()
            for container in containers:
                if container.yarn_container_id not in cid_to_endpoint:
                    continue
                eps.add(cid_to_endpoint[container.yarn_container_id])

            if eps != last_eps:
                logger.info('New endpoints retrieved: %r', eps)
                with open(endpoint_file, 'w') as file_obj:
                    file_obj.write('\n'.join(eps))
                last_eps = eps
            time.sleep(1)
        except SkeinError:
            mars_app._running = False
            break 

def read(self, size):
        while len(self.buffer) < size:
            self.ensure_connection()

            try:
                packet = self.socket.recv(self.buffer_size)
            except socket.error as error:
                if error.errno in (EDEADLK, EAGAIN, EWOULDBLOCK):
                    gevent.sleep()
                    continue
                six.raise_from(NSQSocketError(*error.args), error)

            if not packet:
                self.close()

            self.buffer += packet

        data = self.buffer[:size]
        self.buffer = self.buffer[size:]

        return data 

def _age_out_loop(self):
        while True:
            with self.state_lock:
                if self.state != ft_states.STARTED:
                    break

            self._actor_queue.put(
                (utc_millisec(), 'age_out')
            )

            if self.age_out['interval'] is None:
                break

            try:
                gevent.sleep(self.age_out['interval'])
            except gevent.GreenletExit:
                break 

def wait_all_schedulers_ready(self):
        """
        Wait till all containers are ready, both in yarn and in ClusterInfoActor
        """
        from ...scheduler.utils import SchedulerClusterInfoActor

        # check if all schedulers are ready using Kubernetes API
        sleep_fun = (getattr(self, 'pool', None) or time).sleep
        while not self.scheduler_discoverer.is_all_ready():
            sleep_fun(1)
        yarn_schedulers = self.scheduler_discoverer.get()

        logger.debug('Schedulers all ready in yarn, waiting ClusterInfoActor to be ready')
        # check if all schedulers are registered in ClusterInfoActor
        actor_client = new_client()
        while True:
            cluster_info = actor_client.actor_ref(
                SchedulerClusterInfoActor.default_uid(), address=random.choice(yarn_schedulers))
            cluster_info_schedulers = cluster_info.get_schedulers()
            if set(cluster_info_schedulers) == set(yarn_schedulers):
                break
            sleep_fun(1)  # pragma: no cover 

def publish(self, method, params=None):
        high_bits = self.num_publish >> 12
        low_bits = self.num_publish & 0xfff

        if (low_bits % 128) == 127:
            lagger = self.lagger()
            LOG.debug('topic {} - sent {} lagger {}'.format(
                self.topic,
                self.num_publish,
                lagger
            ))

            while (self.num_publish - lagger) > 1024:
                LOG.debug('topic {} - waiting lagger delta: {}'.format(
                    self.topic,
                    self.num_publish - lagger
                ))
                gevent.sleep(0.1)
                lagger = self.lagger()

            if low_bits == 0xfff:
                # we are switching to a new list, gc
                self.gc(lagger)

        msg = {
            'method': method,
            'params': params
        }

        qname = '{}:queue:{:013X}'.format(
            self.prefix,
            high_bits
        )

        self.SR.rpush(qname, json.dumps(msg))
        self.num_publish += 1 

def _aggregate_iterator(self, iterator):
        self.agg_table = _bptable_factory(
            '{}.aggregate-temp'.format(self.name),
            truncate=True,
            type_in_key=True
        )

        for nitem, item in enumerate(iterator):
            if nitem != 0 and nitem % 1024 == 0:
                gevent.sleep(0.001)

            with self.state_lock:
                if self.state != ft_states.STARTED:
                    LOG.info(
                        '%s - state not STARTED, aggregation not performed',
                        self.name
                    )
                    self.agg_table.close()
                    return False

                try:
                    ipairs = self._process_item(item)

                except gevent.GreenletExit:
                    raise

                except:
                    self.statistics['error.parsing'] += 1
                    LOG.exception('%s - Exception parsing %s', self.name, item)
                    continue

                for indicator, attributes in ipairs:
                    self.agg_table.put(indicator, attributes)

        return True 

def _config_monitor(config_path):
    api_config_path = os.path.join(config_path, 'api')
    dirsnapshot = utils.DirSnapshot(api_config_path, CONFIG_FILES_RE)
    while True:
        try:
            with API_CONFIG_LOCK.acquire(timeout=600):
                new_snapshot = utils.DirSnapshot(api_config_path, CONFIG_FILES_RE)

                if new_snapshot != dirsnapshot:
                    try:
                        _load_config(config_path)
                        _load_auth_dbs(config_path)

                    except gevent.GreenletExit:
                        break

                    except:
                        LOG.exception('Error loading config')

                    dirsnapshot = new_snapshot

        except filelock.Timeout:
            LOG.error('Timeout locking config in config monitor')

        gevent.sleep(1)


# initialization 

def _restart_engine():
    LOG.info('Restarting minemeld-engine')

    supervisorurl = config.get('SUPERVISOR_URL',
                               'unix:///var/run/supervisor.sock')
    sserver = xmlrpclib.ServerProxy(
        'http://127.0.0.1',
        transport=supervisor.xmlrpc.SupervisorTransport(
            None,
            None,
            supervisorurl
        )
    )

    try:
        result = sserver.supervisor.stopProcess('minemeld-engine', False)
        if not result:
            LOG.error('Stop minemeld-engine returned False')
            return

    except xmlrpclib.Fault as e:
        LOG.error('Error stopping minemeld-engine: {!r}'.format(e))

    LOG.info('Stopped minemeld-engine for API request')

    now = time.time()
    info = None
    while (time.time()-now) < 60*10*1000:
        info = sserver.supervisor.getProcessInfo('minemeld-engine')
        if info['statename'] in ('FATAL', 'STOPPED', 'UNKNOWN', 'EXITED'):
            break
        gevent.sleep(5)
    else:
        LOG.error('Timeout during minemeld-engine restart')
        return

    sserver.supervisor.startProcess('minemeld-engine', False)
    LOG.info('Started minemeld-engine') 

def _controlled_emit_update(self, indicator, value):
        self._emit_counter += 1
        if self._emit_counter == 15937:
            gevent.sleep(0.001)
            self._emit_counter = 0
        self.emit_update(indicator, value) 

def sleep(self, t):
        value = self.wobject.wait(timeout=t)
        LOG.debug('value %s', value)
        if value is not None:
            raise CheckpointSet() 

def enable_prevent_write(locker, timeout=900):
    global PREVENT_WRITE

    def _cleanup_prevent_write():
        gevent.sleep(timeout)
        LOG.info('Checking if prevent write still enabled by locker {}'.format(locker))
        disable_prevent_write(locker)

    with PREVENT_WRITE_GUARD:
        if PREVENT_WRITE is None:
            PREVENT_WRITE = locker
            gevent.spawn(_cleanup_prevent_write)

    return False 

def send_message(self, address, message):
        context = zmq.Context()
        socket = context.socket(zmq.REQ)
        socket.connect(address)
        socket.send_json(message)

        import gevent
        gevent.sleep(0.05)

        return socket 

def process(n_clicks):
    print("Processing gevent sleep request #{}".format(n_clicks))
    gevent.sleep(10)
    return 'Done (request #{})'.format(n_clicks) 

def process(n_clicks):
    print("Processing cpu request #{}".format(n_clicks))
    time.sleep(10)
    return 'Done (request #{})'.format(n_clicks) 

def auto_push(self):
        while True:
            gevent.sleep(1 + random.randint(0, 2))
            try:
                self.process_cached_log()
            except Exception:
                logger.exception('process cached log fail.') 

def __refresh_menu_maps(self):
        while True:
            gevent.sleep(self.refresh_interval + random.randint(1, 60))
            logger.info('start refresh %s cache', self.scan_key)
            try:
                tmp_maps = self.__build_menu_maps()
                if tmp_maps:
                    self.__menu_maps = tmp_maps
            except Exception as e:
                logger.error('refresh %s cache failed', self.scan_key,
                             exc_info=e)
            else:
                logger.info('refresh %s cache success', self.scan_key) 

def auto_persist(self):
        while True:
            gevent.sleep(60 + random.randint(1, 5))
            try:
                self.persist()
            except Exception as e:
                logger.error('incr access_count failed', exc_info=e) 

def refresh(self):
        while True:
            gevent.sleep(300 + random.randint(1, 60))
            logger.info('start refresh strategys or rules')
            try:
                strategys.load_strategys()
                self.load_rules()
            except Exception as e:
                logger.error('refresh strategys or rules failed', exc_info=e)
            else:
                logger.info('refresh strategys or rules success') 

def refresh(self):
        while True:
            gevent.sleep(300 + random.randint(1, 60))
            logger.debug('start refresh sources')
            try:
                self.load_sources()
            except Exception:
                logger.exception('refresh sources failed')
            else:
                logger.debug('refresh sources success') 

def test_lookup():
    with LookupdIntegrationServer() as lookupd_server:
        nsqd_server = NsqdIntegrationServer(lookupd=lookupd_server.tcp_address)
        with NsqdIntegrationServer(lookupd=lookupd_server.tcp_address) as nsqd_server:
            lookupd = gnsq.LookupdClient(lookupd_server.address, lookupd_server.http_port)
            nsqd = gnsq.NsqdHTTPClient(nsqd_server.address, nsqd_server.http_port)
            gevent.sleep(0.1)

            assert len(lookupd.topics()['topics']) == 0
            assert len(lookupd.channels('topic')['channels']) == 0
            assert len(lookupd.nodes()['producers']) == 1

            nsqd.create_topic('topic')
            gevent.sleep(0.1)

            info = lookupd.lookup('topic')
            assert len(info['channels']) == 0
            assert len(info['producers']) == 1
            assert len(lookupd.topics()['topics']) == 1
            assert len(lookupd.channels('topic')['channels']) == 0

            nsqd.create_channel('topic', 'channel')
            gevent.sleep(0.1)

            info = lookupd.lookup('topic')
            assert len(info['channels']) == 1
            assert len(info['producers']) == 1
            assert len(lookupd.topics()['topics']) == 1
            assert len(lookupd.channels('topic')['channels']) == 1