Python aioredis.Redis() Examples

def _create_redis_session(self) -> None:
        """
        Create the Redis connection pool, and then open the redis event gate.

        If constants.Redis.use_fakeredis is True, we'll set up a fake redis pool instead
        of attempting to communicate with a real Redis server. This is useful because it
        means contributors don't necessarily need to get Redis running locally just
        to run the bot.

        The fakeredis cache won't have persistence across restarts, but that
        usually won't matter for local bot testing.
        """
        if constants.Redis.use_fakeredis:
            log.info("Using fakeredis instead of communicating with a real Redis server.")
            self.redis_session = await fakeredis.aioredis.create_redis_pool()
        else:
            self.redis_session = await aioredis.create_redis_pool(
                address=(constants.Redis.host, constants.Redis.port),
                password=constants.Redis.password,
            )

        self.redis_closed = False
        self.redis_ready.set() 

def init():
    """Create a connection to the Redis server."""
    global redis_conn
    conn = await aioredis.create_connection(
        "redis://{}:{}".format(
            SETTINGS.get("FLOW_EXECUTOR", {})
            .get("REDIS_CONNECTION", {})
            .get("host", "localhost"),
            SETTINGS.get("FLOW_EXECUTOR", {})
            .get("REDIS_CONNECTION", {})
            .get("port", 56379),
        ),
        db=int(
            SETTINGS.get("FLOW_EXECUTOR", {}).get("REDIS_CONNECTION", {}).get("db", 1)
        ),
    )
    redis_conn = aioredis.Redis(conn) 

def deinit():
    """Close the Redis connection cleanly."""
    redis_conn.close()
    await redis_conn.wait_closed() 

def __init__(self, connection):
        """
        Redis instance constructor

        Constructor takes single argument - a redis host address
        The address can be one of the following:
         * a dict - {'host': 'localhost', 'port': 6379,
                     'db': 0, 'password': 'pass'}
           all keys except host and port will be passed as kwargs to
           the aioredis.create_redis_pool();
         * a Redis URI - "redis://host:6379/0?encoding=utf-8";
         * a (host, port) tuple - ('localhost', 6379);
         * or a unix domain socket path string - "/path/to/redis.sock".
         * a redis connection pool.

        :param connection: redis host address (dict, tuple or str)
        """

        self.connection = connection

        self._pool = None
        self._lock = asyncio.Lock()

        self.set_lock_script = re.sub(r'^\s+', '', self.SET_LOCK_SCRIPT, flags=re.M).strip()
        self.unset_lock_script = re.sub(r'^\s+', '', self.UNSET_LOCK_SCRIPT, flags=re.M).strip() 

def start(self, *args, **kwargs):
        t = threading.Thread(target=block_check, args=(self.loop,))
        t.setDaemon(True)
        t.start()
        self.redis = aioredis.Redis(await aioredis.create_pool("redis://" + self.config.redis_host))
        # self.loop.create_task(self._shards_reader())

        return await super().start(*args, **kwargs) 

def get(self, key: str):
        """Redis GET"""
        # Add a prefix to allow our bot to be friendly with other bots within the same redis DB
        key = f"{Env.currency_name().lower()}{key}"
        redis = await self.get_redis()
        return await redis.get(key) 

def retry_on_redis_connection_failure(fn, *, args=tuple(), retry_delay: int, action: str):
    """Decorator to repeatedly call fn in case of Redis connection problems"""
    while True:
        try:
            return await fn(*args)
        except (ConnectionClosedError, ConnectionResetError):
            # ConnectionClosedError is from aioredis. However, sometimes the connection
            # can die outside of aioredis, in which case we get a builtin ConnectionResetError.
            logger.warning(
                f"Redis connection lost while {action}, reconnecting "
                f"in {retry_delay} seconds..."
            )
            await asyncio.sleep(retry_delay)
        except ConnectionRefusedError:
            logger.warning(
                f"Redis connection refused while {action}, retrying " f"in {retry_delay} seconds..."
            )
            await asyncio.sleep(retry_delay)
        except ReplyError as e:
            if "LOADING" in str(e):
                logger.warning(
                    f"Redis server is still loading, retrying " f"in {retry_delay} seconds..."
                )
                await asyncio.sleep(retry_delay)
            else:
                raise 

def delete(self, key: str):
        """Redis DELETE"""
        key = f"{Env.currency_name().lower()}{key}"
        await self._delete(key) 

def _delete(self, key: str):
        """Redis DELETE"""
        redis = await self.get_redis()
        await redis.delete(key) 

def test_make_subscriber():
    sub, chan = await utils.make_subscriber("test")
    assert sub is not None
    assert chan is not None
    assert isinstance(sub, aioredis.Redis)
    assert isinstance(chan, aioredis.Channel)
    await sub.subscribe("channel:test")
    settings.loop.create_task(reader(chan))
    assert await utils.publish_message("test", {"hello": "world"}) == 1 

def connect(self):
        """
        Get an connection for the self instance
        """

        if isinstance(self.connection, dict):
            # a dict like {'host': 'localhost', 'port': 6379,
            #              'db': 0, 'password': 'pass'}
            kwargs = self.connection.copy()
            address = (
                kwargs.pop('host', 'localhost'),
                kwargs.pop('port', 6379)
            )
            redis_kwargs = kwargs
        elif isinstance(self.connection, aioredis.Redis):
            self._pool = self.connection
        else:
            # a tuple or list ('localhost', 6379)
            # a string "redis://host:6379/0?encoding=utf-8" or
            # a unix domain socket path "/path/to/redis.sock"
            address = self.connection
            redis_kwargs = {}

        if self._pool is None:
            async with self._lock:
                if self._pool is None:
                    self.log.debug('Connecting %s', repr(self))
                    self._pool = await self._create_redis_pool(
                        address, **redis_kwargs,
                        minsize=1, maxsize=100)

        return await self._pool 

def close(self):
        """
        Closes connection and resets pool
        """
        if self._pool is not None and not isinstance(self.connection, aioredis.Redis):
            self._pool.close()
            await self._pool.wait_closed()
        self._pool = None 

def conn(func):
    @functools.wraps(func)
    async def wrapper(self, *args, _conn=None, **kwargs):
        if _conn is None:

            pool = await self._get_pool()
            conn_context = await pool
            with conn_context as _conn:
                if not AIOREDIS_BEFORE_ONE:
                    _conn = aioredis.Redis(_conn)
                return await func(self, *args, _conn=_conn, **kwargs)

        return await func(self, *args, _conn=_conn, **kwargs)

    return wrapper 

def acquire_conn(self):
        await self._get_pool()
        conn = await self._pool.acquire()
        if not AIOREDIS_BEFORE_ONE:
            conn = aioredis.Redis(conn)
        return conn 

def parse_uri_path(self, path):
        """
        Given a uri path, return the Redis specific configuration
        options in that path string according to iana definition
        http://www.iana.org/assignments/uri-schemes/prov/redis

        :param path: string containing the path. Example: "/0"
        :return: mapping containing the options. Example: {"db": "0"}
        """
        options = {}
        db, *_ = path[1:].split("/")
        if db:
            options["db"] = db
        return options 

def connection_manager(self) -> Redis:
        if self._closed:
            # This was first caught when the state plugin tried to send a
            # message to the bus on upon the after_worker_stopped stopped event.
            raise TransportIsClosed(
                "Transport has been closed. Connection to Redis is no longer available."
            )

        if not self._redis_pool:
            self._redis_pool = await aioredis.create_redis_pool(**self.connection_parameters)

        try:
            internal_pool = self._redis_pool._pool_or_conn
            if hasattr(internal_pool, "size") and hasattr(internal_pool, "maxsize"):
                if internal_pool.size == internal_pool.maxsize:
                    logging.critical(
                        "Redis pool has reached maximum size. It is possible that this will recover normally, "
                        "you may have more event listeners than connections available to the Redis pool. "
                        "You can increase the redis pool size by specifying the `maxsize` "
                        "parameter in each of the Redis transport configuration sections. Current maxsize is: {}"
                        "".format(self.connection_parameters.get("maxsize"))
                    )

            return await self._redis_pool
        except aioredis.PoolClosedError:
            raise LightbusShutdownInProgress(
                "Redis connection pool has been closed. Assuming shutdown in progress."
            ) 

def __init__(self, *args, **kwargs):
        if "connector" in kwargs:
            warnings.warn(
                "If login() is called (or the bot is started), the connector will be overwritten "
                "with an internal one"
            )

        super().__init__(*args, **kwargs)

        self.http_session: Optional[aiohttp.ClientSession] = None
        self.redis_session: Optional[aioredis.Redis] = None
        self.redis_ready = asyncio.Event()
        self.redis_closed = False
        self.api_client = api.APIClient(loop=self.loop)

        self._connector = None
        self._resolver = None
        self._guild_available = asyncio.Event()

        statsd_url = constants.Stats.statsd_host

        if DEBUG_MODE:
            # Since statsd is UDP, there are no errors for sending to a down port.
            # For this reason, setting the statsd host to 127.0.0.1 for development
            # will effectively disable stats.
            statsd_url = "127.0.0.1"

        self.stats = AsyncStatsClient(self.loop, statsd_url, 8125, prefix="bot") 

def get_redis(cls) -> aioredis.Redis:
        if cls.redis is not None:
            return cls.redis
        # TODO - we should let them override redis host/port in configuration
        cls.redis = await aioredis.create_redis_pool((os.getenv('REDIS_HOST', 'localhost'), 6379), db=int(os.getenv('REDIS_DB', '1')), encoding='utf-8', minsize=1, maxsize=5)
        return cls.redis 

def redis(self) -> aioredis.Redis:
        if self.closed:
            raise RuntimeError("Redis connection is not opened")
        return self._redis 

def __init__(self, host: str, port: int = 6379, db: int = 0):
        self.host = host
        self.port = port
        self.db = db

        self._redis: Optional[aioredis.Redis] = None 

def redis(self) -> Redis:
        """Retrieve the redis instance of the crawl manager

        :return: The redis instance of the crawl manager
        """
        return self.manager.redis 

def __init__(self) -> None:
        self.redis: Redis = None
        self.session: ClientSession = None
        self.loop: AbstractEventLoop = None
        self.depth: int = env('DEFAULT_DEPTH', type_=int, default=1)
        self.same_domain_depth: int = env(
            'DEFAULT_SAME_DOMAIN_DEPTH', type_=int, default=100
        )

        self.num_browsers: int = env('DEFAULT_NUM_BROWSERS', type_=int, default=2)

        self.flock: str = env('DEFAULT_FLOCK', default='browsers')

        self.shepherd_host: str = env(
            'DEFAULT_SHEPHERD', default='http://shepherd:9020'
        )

        self.browser_api_url: str = f'{self.shepherd_host}/api'
        self.pool: str = env('DEFAULT_POOL', default='')

        self.scan_key: str = 'a:*:info'

        self.container_environ: Dict[str, str] = {
            'URL': 'about:blank',
            'REDIS_URL': env('REDIS_URL', default=DEFAULT_REDIS_URL),
            'WAIT_FOR_Q': '10',
            'TAB_TYPE': 'CrawlerTab',
            'CRAWL_NO_NETCACHE': '0',
            'VNC_PASS': 'pass',
            'IDLE_TIMEOUT': '',
            'BEHAVIOR_API_URL': 'http://behaviors:3030',
            'SCREENSHOT_API_URL': env('SCREENSHOT_API_URL'),
            'EXTRACTED_RAW_DOM_API_URL': env('EXTRACTED_RAW_DOM_API_URL'),
        }

        self.default_browser = None

        if os.environ.get('DEBUG'):
            logger.setLevel(logging.DEBUG)
        else:
            logger.setLevel(logging.INFO) 

def init_redis(redis_url: str, loop: AbstractEventLoop) -> Redis:
    return await create_redis(redis_url, encoding='utf-8', loop=loop) 

def __init__(
        self,
        job_id: str,
        redis: Redis,
        _queue_name: str = default_queue_name,
        _deserializer: Optional[Deserializer] = None,
    ):
        self.job_id = job_id
        self._redis = redis
        self._queue_name = _queue_name
        self._deserializer = _deserializer 

def log_redis_info(redis: Redis, log_func: Callable[[str], Any]) -> None:
    with await redis as r:
        info, key_count = await asyncio.gather(r.info(), r.dbsize())
    log_func(
        f'redis_version={info["server"]["redis_version"]} '
        f'mem_usage={info["memory"]["used_memory_human"]} '
        f'clients_connected={info["clients"]["connected_clients"]} '
        f'db_keys={key_count}'
    ) 

def __init__(self, redis: Redis, key: str, timeout: int = 30, wait_timeout: int = 30, *, token: str = None):
        self.redis = redis
        self.key = key
        self.timeout = timeout
        self.wait_timeout = wait_timeout  # Can be None to wait forever
        self._token = token or str(uuid.uuid4()) 

def test_yield_from_backwards_compatibility(create_redis, server):
    redis = await create_redis(server.tcp_address)

    assert isinstance(redis, Redis)
    # TODO: there should not be warning
    # with pytest.warns(UserWarning):
    with await redis as client:
        assert isinstance(client, Redis)
        assert client is not redis
        assert await client.ping() 

def test_repr(create_redis, server):
    redis = await create_redis(server.tcp_address, db=1)
    assert repr(redis) in {
        '<Redis <RedisConnection [db:1]>>',
        '<Redis <ConnectionsPool [db:1, size:[1:10], free:1]>>',
        }

    redis = await create_redis(server.tcp_address, db=0)
    assert repr(redis) in {
        '<Redis <RedisConnection [db:0]>>',
        '<Redis <ConnectionsPool [db:0, size:[1:10], free:1]>>',
        } 

def _recreate(self) -> None:
        """Re-create the connector, aiohttp session, the APIClient and the Redis session."""
        # Use asyncio for DNS resolution instead of threads so threads aren't spammed.
        # Doesn't seem to have any state with regards to being closed, so no need to worry?
        self._resolver = aiohttp.AsyncResolver()

        # Its __del__ does send a warning but it doesn't always show up for some reason.
        if self._connector and not self._connector._closed:
            log.warning(
                "The previous connector was not closed; it will remain open and be overwritten"
            )

        if self.redis_session and not self.redis_session.closed:
            log.warning(
                "The previous redis pool was not closed; it will remain open and be overwritten"
            )

        # Create the redis session
        self.loop.create_task(self._create_redis_session())

        # Use AF_INET as its socket family to prevent HTTPS related problems both locally
        # and in production.
        self._connector = aiohttp.TCPConnector(
            resolver=self._resolver,
            family=socket.AF_INET,
        )

        # Client.login() will call HTTPClient.static_login() which will create a session using
        # this connector attribute.
        self.http.connector = self._connector

        # Its __del__ does send a warning but it doesn't always show up for some reason.
        if self.http_session and not self.http_session.closed:
            log.warning(
                "The previous session was not closed; it will remain open and be overwritten"
            )

        self.http_session = aiohttp.ClientSession(connector=self._connector)
        self.api_client.recreate(force=True, connector=self._connector) 

def handle_kernel_log(app: web.Application, agent_id: AgentId, event_name: str,
                            raw_kernel_id: str, container_id: str):
    dbpool = app['dbpool']
    redis_conn: aioredis.Redis = await redis.connect_with_retries(
        app['config']['redis']['addr'].as_sockaddr(),
        db=REDIS_STREAM_DB,
        password=(app['config']['redis']['password']
                  if app['config']['redis']['password'] else None),
        encoding=None,
    )
    # The log data is at most 10 MiB.
    log_buffer = BytesIO()
    log_key = f'containerlog.{container_id}'
    try:
        list_size = await redis.execute_with_retries(
            lambda: redis_conn.llen(log_key)
        )
        if list_size is None:
            # The log data is expired due to a very slow event delivery.
            # (should never happen!)
            log.warning('tried to store console logs for cid:{}, but the data is expired',
                        container_id)
            return
        for _ in range(list_size):
            # Read chunk-by-chunk to allow interleaving with other Redis operations.
            chunk = await redis.execute_with_retries(lambda: redis_conn.lpop(log_key))
            log_buffer.write(chunk)
        try:
            async with dbpool.acquire() as conn, conn.begin():
                query = (sa.update(kernels)
                           .values(container_log=log_buffer.getvalue())
                           .where(kernels.c.id == raw_kernel_id))
                await conn.execute(query)
        finally:
            # Clear the log data from Redis when done.
            await redis.execute_with_retries(
                lambda: redis_conn.delete(log_key)
            )
    finally:
        log_buffer.close()
        redis_conn.close()
        await redis_conn.wait_closed()