Python redis.RedisError() Examples

def cluster_health(self):
        '''
        Polls redis for current cluster status.
        If a node has failed repeatedly, the apps on it will be offloaded
        '''
        if self.role != 'master':
            return
        self.monaco.refresh(self.r)
        offload = []
        for node_id in self.monaco.node_ids:
            node = schema.MonacoNode(node_id=node_id)
            node.refresh(self.r)
            if node.hostname == self.hostname:
                continue
            try:
                nodecli = redis.StrictRedis(host=node.hostname, port=config['mgmt_port'], socket_connect_timeout=1, socket_timeout=1)
                nodecli.info()
                self.monaco.node_up(node_id, self.r)
                self.logger.debug('Node %s is healthy', node_id)
            except redis.RedisError, exc:
                self.logger.warn('Node %s missed a healthcheck', node_id)
                if self.monaco.node_down(node_id, self.r):
                    self.logger.error('Node %s has been marked down', node_id)
                    self.logger.exception(exc)
                    offload.append(node_id) 

def handle_redis_except(f):
    """Wrapper which handles Redis exceptions."""
    # noqa
    @wraps(f)
    def decorated_function(*args, **kwargs):
        """Represents decorated function."""
        try:
            return f(*args, **kwargs)
        except (RedisError, OSError) as e:
            error_code = REDIS_EXCEPTION_ERROR_CODE

            return jsonify(error={
                'code': error_code,
                'reason': e.messages,
            })

    return decorated_function 

def get_list(self):
        server_list = []
        try:
            server_key_list = self._redis_client.keys(pattern='januscloud:backend_servers:*')
            start = 0
            step = 32
            total = len(server_key_list)
            while True:
                with self._redis_client.pipeline() as p:
                    for server_key in server_key_list[start:start+step]:
                        p.hgetall(server_key)
                    result = p.execute()
                    for rd_server in result:
                        if rd_server:
                            server_list.append(self._from_rd_server(rd_server))
                start += step
                if start >= total:
                    break
        except RedisError as e:
            log.warning('Fail to get backend server list because of Redis client error: {}'.format(e))

        return server_list 

def consume_events(self):
        self._is_running = True
        while self._loop.is_running() and self._is_running:
            try:
                msg = await asyncio.wait_for(self._queue.get(), timeout=1)
                if not msg:
                    self._is_running = False
                    return
                await _ensure_coroutine_function(self._callback)(msg)
                try:
                    self._counter[msg.stream] = msg.sequence
                except pybreaker.CircuitBreakerError:
                    pass
                except redis.RedisError:
                    self._logger.warn("Failed to persist last read event")
            except RejectedMessageException:
                self._logger.warn("%s message %s was rejected and has not been processed",
                                  msg.type, msg.id)
            except asyncio.TimeoutError:
                self._is_running = False
            except:
                self._logger.exception("Failed to process message %s", msg)
                self._is_running = False 

def start_consuming_message(queue_name, consume_function, threads_num=50):
    """
    本例子实现的功能和中间件过于简单，单一函数最好了。
    看不懂有类的代码，不用看上面那个类，看这个函数就可以，使用一个10行代码的函数实现乞丐版分布式函数执行框架。

    """
    pool = ThreadPoolExecutor(threads_num)
    while True:
        try:
            redis_task = redis_db_frame.brpop(queue_name, timeout=60)
            if redis_task:
                task_str = redis_task[1].decode()
                print(f'从redis的 {queue_name} 队列中 取出的消息是： {task_str}')
                pool.submit(consume_function, **json.loads(task_str))
            else:
                print(f'redis的 {queue_name} 队列中没有任务')
        except redis.RedisError as e:
            print(e) 

def hash_write(key, hashdata, r):
    '''
    Atomically overwrite the redis hash at <key>
    Setting it equal to the dict <hashdata>, using client <r>
    '''
    if hasattr(hash_write, 'script'):
        try:
            hash_write.script(args=[key] + [item for tup in hashdata.items() for item in tup], client=r)
            return
        except redis.RedisError:
            pass
    path = os.path.join(os.path.dirname(__file__), 'lua/hash_write.lua')
    with open(path) as scriptfile:
        script = scriptfile.read()
    hash_write.script = r.register_script(script)
    hash_write.script(args=[key] + [item for tup in hashdata.items() for item in tup], client=r) 

def hash_update(key, hashdata, r):
    '''
    Atomically update the redis hash at <key>
    Set the values from the dict <hashdata>, using client <r>
    '''
    if hasattr(hash_update, 'script'):
        try:
            hash_update.script(args=[key] + [item for tup in hashdata.items() for item in tup], client=r)
            return
        except redis.RedisError:
            pass
    path = os.path.join(os.path.dirname(__file__), 'lua/hash_update.lua')
    with open(path) as scriptfile:
        script = scriptfile.read()
    hash_update.script = r.register_script(script)
    hash_update.script(args=[key] + [item for tup in hashdata.items() for item in tup], client=r) 

def publish_event_data(self, event_id, data_obj):
        """
        Publishes the serialized form of the data object for the given event.

        Note that this occurs in a thread to prevent blocking.
        """

        def conduct():
            try:
                self.publish_event_data_sync(event_id, data_obj)
                logger.debug("Published user event %s: %s", event_id, data_obj)
            except redis.RedisError:
                logger.exception("Could not publish user event")

        thread = threading.Thread(target=conduct)
        thread.start() 

def update_subs(self):
        '''
        Subscribes the slave to the channels associated with the
        '''
        try:
            with self.lock:
                channels = self._subscriptions.keys()

                subs = {}
                self.node.refresh(self.r)
                subs[keysub(schema.NODE_APPS_TMPL % self.node.node_id)] = self.node_handler_factory()
                subs[keysub(schema.NODE_TWEMS_TMPL % self.node.node_id)] = self.twem_handler_factory()
                for app_id in self.node.apps:
                    subs[keysub(schema.APP_CLUSTER_TMPL % app_id)] = self.app_handler_factory(app_id)
                    subs[keysub(schema.APP_HASH_TMPL % app_id)] = self.app_config_handler_factory(app_id)
                map(self._remove_subscription, [chan for chan in channels if chan not in subs.keys()])
                for k, v in subs.iteritems():
                    self._add_subscription(k, v)
        except redis.RedisError, err:
            self.logger.exception(err)
            self.r = redis.StrictRedis(port=config['mgmt_port']) 

def acquire(self):
        logger.debug("Acquiring global lock %s", self._lock_name)
        try:
            self._redlock = RedLock(
                self._lock_name, connection_details=[self._redis_info], ttl=self._lock_ttl
            )
            acquired = self._redlock.acquire()
            if not acquired:
                logger.debug("Was unable to not acquire lock %s", self._lock_name)
                return False

            logger.debug("Acquired lock %s", self._lock_name)
            return True
        except RedLockError:
            logger.debug("Could not acquire lock %s", self._lock_name)
            return False
        except RedisError as re:
            logger.debug("Could not connect to Redis for lock %s: %s", self._lock_name, re)
            return False 

def index_documents(docs):
    pipe = DB.pipeline(transaction=False)
    for doc in docs:
        if not doc:
            continue
        if doc.get('_action') in ['delete', 'update']:
            key = keys.document_key(doc['_id']).encode()
            known_doc = get_document(key)
            if known_doc:
                deindex_document(known_doc)
        if doc.get('_action') in ['index', 'update', None]:
            index_document(pipe, doc)
        yield doc
    try:
        pipe.execute()
    except redis.RedisError as e:
        msg = 'Error while importing document:\n{}\n{}'.format(doc, str(e))
        raise ValueError(msg) 

def limit(self, client, ignore_redis_error=False):
        if self._limiter is None:
            return dummy_limit()

        @contextmanager
        def limit():
            try:
                with self._limiter.limit(client):
                    yield
            except RedisError as e:
                if ignore_redis_error:
                    logger.warning(
                        "Ignoring RedisError in rate limiter for %s",
                        self._limiter.resource,
                        exc_info=True,
                    )
                    yield
                else:
                    raise

        return limit() 

def test_retry(monkeypatch):
    from ruskit.cluster import ClusterNode

    monkeypatch.setattr(redis.Redis, "ping", mock.Mock())
    times = [0]

    def execute(*args, **kwargs):
        times[0] += 1
        if times[0] < 4:
            raise redis.RedisError('Err: connection timeout')

    monkeypatch.setattr(redis.Redis, "execute_command", execute)

    node = ClusterNode("localhost", 8000, retry=3)
    node.setslot('NODE', 844, 'abcd')

    assert times[0] == 4 

def load_strategys(self):
        uuid_strategy_map = {}
        conn = get_config_redis_client()
        logger.info('start load strategys from db, current strategy: %s',
                    self.uuid_strategy_map.keys())
        for strategy_cls in _used_strategies:
            try:
                for name in conn.scan_iter(match=strategy_cls.prefix):
                    d = conn.hgetall(name)
                    strategy = strategy_cls(d)
                    uuid_strategy_map[strategy.uuid] = strategy
            except redis.RedisError:
                logger.error('load strategys occur redis conn error')
                return
        self.uuid_strategy_map = uuid_strategy_map
        logger.info('load strategys success, current strategy: %s',
                    self.uuid_strategy_map.keys()) 

def load_raw_source(cls):
        conn = get_config_redis_client()
        name_keys_map = {}

        try:
            d = conn.hgetall(conf.REDIS_SOURCE_MAP)
        except redis.RedisError:
            logger.exception('load raw sources error')
            return

        for name, fields_str in d.items():
            fields = json.loads(fields_str)
            fields.pop('name_show', None)
            keys = {}
            for key_name, key_type in fields.items():
                if key_type in {'string', 'str'}:
                    key_type = 'str'
                try:
                    type_ = getattr(__builtins__, key_type)
                except AttributeError:
                    type_ = __builtins__[key_type]
                keys[key_name] = type_
            name_keys_map[name] = keys

        cls.name_keys_map = name_keys_map 

def _write_one_record(self, zkey, score, member, preserve_time):
        pipeline = self.conn.pipeline()
        try:
            pipeline.zadd(zkey, score, member)
            pipeline.expire(zkey, preserve_time)
            # 这个是为了减少redis存储压力，每次删除部分老旧数据，可以修改此处逻辑
            pipeline.zremrangebyrank(zkey, 0, -128)
            pipeline.execute()
        except redis.RedisError:
            logger.error(
                f'pipeline execute error,'
                f'zkey --> {zkey},'
                f'score --> {score},'
                f'member --> {member},'
                f'preserve_time --> {preserve_time})')
            return False
        return True 

def load_rules(self):
        id_rule_map = {}
        conn = get_config_redis_client()
        logger.info('start load rules, current rule ids: %s',
                    self.id_rule_map.keys())
        try:
            for name in conn.scan_iter(match='rule:*'):
                d = conn.hgetall(name)
                if self._should_load(d):
                    rule = Rule(d)
                    id_rule_map[rule.id] = rule
        except redis.RedisError:
            logger.error('load rules occur redis conn error')
            return
        self.id_rule_map = id_rule_map
        logger.info('load rules success, current rule ids: %s',
                    self.id_rule_map.keys()) 

def __init__(self, nodes):
        self.down_nodes = set()
        all_addrs = set(n.gen_addr() for n in nodes)
        while True:
            former_len = len(all_addrs)
            active_nodes = []
            for n in nodes:
                try:
                    addrs = [node['addr'] for node in n.nodes()]
                    all_addrs.update(addrs)
                    active_nodes.append(n)
                except redis.RedisError:
                    self.down_nodes.add(n.gen_addr())

            nodes = []
            for addr in all_addrs:
                host, port = addr.split(':')
                nodes.append(ClusterNode(host, int(port), retry=NO_RETRY))

            if len(all_addrs) == former_len:
                break

        self.nodes = active_nodes
        self.all_addrs = all_addrs 

def promote_new_masters(self, plan):
        retry = 5
        new_masters = [p['slave'].node for p in plan]
        while len(new_masters) and retry > 0:
            for m in new_masters:
                try:
                    m.failover(takeover=True)
                except redis.RedisError as e:
                    logger.error(str(e))
            self.cluster.wait()
            for n in new_masters:
                n.flush_cache()
            new_masters = [n for n in new_masters if not n.is_master()]
            if len(new_masters) == 0:
                break
            retry -= 1
            logger.warning(
                'The nodes below are still slaves: {}'.format(new_masters)) 

def _pipeline(self):
        pipeline = self.redis.pipeline()
        yield pipeline
        with contextlib.suppress(RedisError):
            pipeline.multi()
        pipeline.ping()
        pipeline.execute() 

def index_ngram_keys(*keys):
    pipe = DB.pipeline(transaction=False)
    for key in keys:
        key = key.decode()
        _, token = key.split('|')
        if token.isdigit():
            continue
        index_edge_ngrams(pipe, token)
    try:
        pipe.execute()
    except redis.RedisError as e:
        msg = 'Error while generating ngrams:\n{}'.format(str(e))
        raise ValueError(msg)
    return keys 

def deleteBanchoSessions(self):
		"""
		Remove all `peppy:sessions:*` redis keys.
		Call at bancho startup to delete old cached sessions

		:return:
		"""
		try:
			# TODO: Make function or some redis meme
			glob.redis.eval("return redis.call('del', unpack(redis.call('keys', ARGV[1])))", 0, "peppy:sessions:*")
		except redis.RedisError:
			pass 

def invalidate_key(self, name, key):
        """Invalidate a cache `key` in users hash set."""
        try:
            self.cache.hdel(name, key)
        except RedisError:
            pass 

def start_consuming_message(self):
        while True:
            try:
                redis_task = redis_db_frame.blpop(self.queue_name, timeout=60)
                if redis_task:
                    task_str = redis_task[1].decode()
                    print(f'从redis的 [{self.queue_name}] 队列中 取出的消息是： {task_str}  ')
                    task_dict = json.loads(task_str)
                    self.pool.submit(self.consume_function, **task_dict)
                else:
                    print(f'redis的 {self.queue_name} 队列中没有任务')
            except redis.RedisError as e:
                print(e) 

def __init__(self, redis_config, username, events=None):
        events = events or set([])
        channels = [self._user_event_key(username, e) for e in events]

        args = dict(redis_config)
        args.update({"socket_connect_timeout": 5, "single_connection_client": True})

        try:
            self._redis = redis.StrictRedis(**args)
            self._pubsub = self._redis.pubsub(ignore_subscribe_messages=True)
            self._pubsub.subscribe(channels)
        except redis.RedisError as re:
            logger.exception("Could not reach user events redis: %s", re)
            raise CannotReadUserEventsException 

def event_stream(self):
        """
        Starts listening for events on the channel(s), yielding for each event found.

        Will yield a "pulse" event (a custom event we've decided) as a heartbeat every few seconds.
        """
        while True:
            pubsub = self._pubsub
            if pubsub is None:
                return

            try:
                item = pubsub.get_message(ignore_subscribe_messages=True, timeout=5)
            except redis.RedisError:
                item = None

            if item is None:
                yield "pulse", {}
            else:
                channel = item["channel"]
                event_id = channel.split("/")[3]  # user/{username}/{events}/{id}
                data = None

                try:
                    data = json.loads(item["data"] or "{}")
                except ValueError:
                    continue

                if data:
                    yield event_id, data 

def get_log_entries(self, build_id, start_index):
        """
        Returns a tuple of the current length of the list and an iterable of the requested log
        entries.
        """
        try:
            llen = self._redis.llen(self._logs_key(build_id))
            log_entries = self._redis.lrange(self._logs_key(build_id), start_index, -1)
            return (llen, (json.loads(entry) for entry in log_entries))
        except redis.RedisError as re:
            raise BuildStatusRetrievalError("Cannot retrieve build logs: %s" % re) 

def get_status(self, build_id):
        """
        Loads the status information for the specified build id.
        """
        try:
            fetched = self._redis.get(self._status_key(build_id))
        except redis.RedisError as re:
            raise BuildStatusRetrievalError("Cannot retrieve build status: %s" % re)

        return json.loads(fetched) if fetched else None 

def execute_command(self, *args, **kwargs):
        if self.before_request_redis:
            self.before_request_redis()

        i = 0
        while True:
            try:
                return self.r.execute_command(*args, **kwargs)
            except redis.RedisError:
                if i > self.retry:
                    raise
                i += 1
                logger.warn("retry %d times", i)
                time.sleep(1) 

def release(self):
        if self._redlock is not None:
            logger.debug("Releasing lock %s", self._lock_name)
            try:
                self._redlock.release()
            except RedLockError:
                logger.debug("Could not release lock %s", self._lock_name)
            except RedisError as re:
                logger.debug(
                    "Could not connect to Redis for releasing lock %s: %s", self._lock_name, re
                )

            logger.debug("Released lock %s", self._lock_name)
            self._redlock = None