Python tenacity.retry() Examples

def quota_retry(*args, **kwargs) -> Callable:
        """
        A decorator that provides a mechanism to repeat requests in response to exceeding a temporary quote
        limit.
        """
        def decorator(fun: Callable):
            default_kwargs = {
                'wait': tenacity.wait_exponential(multiplier=1, max=100),
                'retry': retry_if_temporary_quota(),
                'before': tenacity.before_log(log, logging.DEBUG),
                'after': tenacity.after_log(log, logging.DEBUG),
            }
            default_kwargs.update(**kwargs)
            return tenacity.retry(
                *args, **default_kwargs
            )(fun)
        return decorator 

def operation_in_progress_retry(*args, **kwargs) -> Callable:
        """
        A decorator that provides a mechanism to repeat requests in response to
        operation in progress (HTTP 409)
        limit.
        """
        def decorator(fun: Callable):
            default_kwargs = {
                'wait': tenacity.wait_exponential(multiplier=1, max=300),
                'retry': retry_if_operation_in_progress(),
                'before': tenacity.before_log(log, logging.DEBUG),
                'after': tenacity.after_log(log, logging.DEBUG),
            }
            default_kwargs.update(**kwargs)
            return tenacity.retry(
                *args, **default_kwargs
            )(fun)
        return decorator 

def _request(session, method, url, params=None, data=None):
    """Request with params.

    Easy to use requests and auto retry.

    Args:
        session: requests session, login session.
        method: string, 'POST' OR 'GET'.
        url: string, post url.
        params=None: dict, get param.
        data=None: dict, post param.

    Returns:
        requests request.

    Raises:
        AutomataError: method param error.
    """
    if method not in ['POST', 'GET'] or not session:
        raise AutomataError

    req = session.request(method, url, params=params, data=data)
    return req.text 

def clone_git_repo(self, repo_url, repo_dir, rev="origin/branches/default/tip"):
        logger.info(f"Cloning {repo_url}...")

        if not os.path.exists(repo_dir):
            tenacity.retry(
                wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
                stop=tenacity.stop_after_attempt(5),
            )(
                lambda: subprocess.run(
                    ["git", "clone", "--quiet", repo_url, repo_dir], check=True
                )
            )()

        tenacity.retry(
            wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
            stop=tenacity.stop_after_attempt(5),
        )(
            lambda: subprocess.run(
                ["git", "fetch"], cwd=repo_dir, capture_output=True, check=True,
            )
        )()

        subprocess.run(
            ["git", "checkout", rev], cwd=repo_dir, capture_output=True, check=True
        ) 

def test_s3_stresstest():
  with Storage('s3://seunglab-test/cloudvolume/connection_pool/', n_threads=0) as stor:
    stor.put_file('test', 'some string')

  n_trials = 500
  pbar = tqdm(total=n_trials)

  @retry
  def create_conn(interface):
    conn = S3_POOL.get_connection()  
    # assert S3_POOL.total_connections() <= S3_POOL.max_connections * 5
    bucket = conn.get_object(
      Bucket='seunglab-test',
      Key='cloudvolume/connection_pool/test',
    )
    S3_POOL.release_connection(conn)
    pbar.update()

  with ThreadedQueue(n_threads=20) as tq:
    for _ in range(n_trials):
      tq.put(create_conn)

  pbar.close() 

def test_gc_stresstest():
  with Storage('gs://seunglab-test/cloudvolume/connection_pool/', n_threads=0) as stor:
    stor.put_file('test', 'some string')

  n_trials = 500
  pbar = tqdm(total=n_trials)

  @retry
  def create_conn(interface):
    # assert GC_POOL.total_connections() <= GC_POOL.max_connections * 5
    bucket = GC_POOL.get_connection()
    blob = bucket.get_blob('cloudvolume/connection_pool/test')
    blob.download_as_string()
    GC_POOL.release_connection(bucket)
    pbar.update()

  with ThreadedQueue(n_threads=20) as tq:
    for _ in range(n_trials):
      tq.put(create_conn)

  pbar.close() 

def download_from_s3(file_info, context, is_raw_frame=False):
    frame_id = file_info.get('frameid')

    logger.info(f"Downloading file {file_info.get('filename')} from archive. ID: {frame_id}.",
                extra_tags={'filename': file_info.get('filename'),
                            'attempt_number': download_from_s3.retry.statistics['attempt_number']})

    if is_raw_frame:
        url = f'{context.RAW_DATA_FRAME_URL}/{frame_id}'
        archive_auth_token = context.RAW_DATA_AUTH_TOKEN
    else:
        url = f'{context.ARCHIVE_FRAME_URL}/{frame_id}'
        archive_auth_token = context.ARCHIVE_AUTH_TOKEN
    response = requests.get(url, headers=archive_auth_token).json()
    buffer = io.BytesIO()
    buffer.write(requests.get(response['url'], stream=True).content)
    buffer.seek(0)
    return buffer 

def test_attempt_number_is_correct_for_interleaved_coroutines(self):

        attempts = []

        def after(retry_state):
            attempts.append((retry_state.args[0], retry_state.attempt_number))

        thing1 = NoIOErrorAfterCount(3)
        thing2 = NoIOErrorAfterCount(3)

        await asyncio.gather(
            _retryable_coroutine.retry_with(after=after)(thing1),
            _retryable_coroutine.retry_with(after=after)(thing2))

        # There's no waiting on retry, only a wait in the coroutine, so the
        # executions should be interleaved.
        even_thing_attempts = attempts[::2]
        things, attempt_nos1 = zip(*even_thing_attempts)
        assert len(set(things)) == 1
        assert list(attempt_nos1) == [1, 2, 3]

        odd_thing_attempts = attempts[1::2]
        things, attempt_nos2 = zip(*odd_thing_attempts)
        assert len(set(things)) == 1
        assert list(attempt_nos2) == [1, 2, 3] 

def _do_it_with_persistence(func, args, config):
    """Exec func with retries based on provided cli flags

    :param: func: function to attempt to execute
    :param: args: argparser generated cli arguments
    :param: config: configparser object of vaultlocker config
    """
    @tenacity.retry(
        wait=tenacity.wait_fixed(1),
        reraise=True,
        stop=(
            tenacity.stop_after_delay(args.retry) if args.retry > 0
            else tenacity.stop_after_attempt(1)
            ),
        retry=(
            tenacity.retry_if_exception(hvac.exceptions.VaultNotInitialized) |
            tenacity.retry_if_exception(hvac.exceptions.VaultDown)
            )
        )
    def _do_it():
        client = _vault_client(config)
        func(args, client, config)
    _do_it() 

def get_connection_from_config(conf):
    retries = conf.database.max_retries
    url = conf.database.connection
    connection_scheme = urlparse.urlparse(url).scheme
    LOG.debug('looking for %(name)r driver in %(namespace)r',
              {'name': connection_scheme, 'namespace': _NAMESPACE})
    mgr = driver.DriverManager(_NAMESPACE, connection_scheme)

    @tenacity.retry(
        wait=tenacity.wait_fixed(conf.database.retry_interval),
        stop=tenacity.stop_after_attempt(retries if retries >= 0 else 5),
        reraise=True)
    def _get_connection():
        """Return an open connection to the database."""
        return mgr.driver(conf, url)

    return _get_connection() 

def _fetch(self, url, verb='get', **kwargs):
        """return response or None in case of failure, try twice"""
        @retry(stop=stop_after_attempt(2), wait=wait_fixed(2))
        def _inner_fetch(verb='get'):
            headers = {
                'Authorization': 'apiToken %s' % C.DEFAULT_SHIPPABLE_TOKEN
            }

            logging.info(u'%s %s' % (verb, url))
            http_method = getattr(requests, verb)
            resp = http_method(url, headers=headers, **kwargs)
            logging.info(u'shippable status code: %s' % resp.status_code)
            logging.info(u'shippable reason: %s' % resp.reason)

            if resp.status_code not in [200, 302, 400]:
                logging.error(u'RC: %s', resp.status_code)
                raise TryAgain

            return resp

        try:
            logging.debug(u'%s' % url)
            return _inner_fetch(verb=verb)
        except RetryError as e:
            logging.error(e) 

def _action(action, request_session, *args, **kwargs):
        """ Perform a session action and retry if auth fails

        This function dynamically performs a specific type of call
        using the provided session (get, patch, post, etc). It will
        attempt a single re-authentication if the initial request
        fails with a 401.
        """
        _action = getattr(request_session, action)
        try:
            req = _action(*args, **kwargs)
            RegistrySessionHelper.check_status(session=request_session,
                                               request=req)
        except requests.exceptions.HTTPError as e:
            if e.response.status_code == 401:
                req = _action(*args, **kwargs)
                RegistrySessionHelper.check_status(session=request_session,
                                                   request=req)
            else:
                raise
        return req 

def request(self, method, url, *a, **kw):
        def _after_request_attempt(func, tries, secs):
            msg = f'RETRY {tries} attempt(s)'
            if tries > 1:
                interval = humanize_interval(secs)
                msg += f'. Time spent: {interval}'
            msg += ' ...'
            self.logger.warning(msg)

        if self.settings.get('RETRY_ENABLED'):
            retry_kw = {'after': _after_request_attempt}
            retry_kw.update(self.settings.get('RETRY_SETTINGS', {}))
        else:
            retry_kw = {'retry': retry_never}

        @retry(**retry_kw)
        def _request():
            with self.rate_limiter:
                self.logger.info(f'{method} "{url}"')
                response = self._requests.request(method, url, *a, **kw)
                self._update_csrftoken(response)
                return validate_response(self, response)

        return _request() 

def etcd(self):
        endpoint = os.environ.get('PYTHON_ETCD_HTTP_URL')
        timeout = 5
        if endpoint:
            url = urlparse(endpoint)
            with etcd3.client(host=url.hostname,
                              port=url.port,
                              timeout=timeout) as client:
                yield client
        else:
            with etcd3.client() as client:
                yield client

        @retry(wait=wait_fixed(2), stop=stop_after_attempt(3))
        def delete_keys_definitely():
            # clean up after fixture goes out of scope
            etcdctl('del', '--prefix', '/')
            out = etcdctl('get', '--prefix', '/')
            assert 'kvs' not in out

        delete_keys_definitely() 

def download_artifact(artifact_path, task_id, artifact_name):
    if os.path.exists(artifact_path):
        return artifact_path

    # Build artifact public url
    # Use un-authenticated Taskcluster client to avoid taskcluster-proxy rewrite issue
    # https://github.com/taskcluster/taskcluster-proxy/issues/44
    queue = taskcluster.Queue({"rootUrl": "https://firefox-ci-tc.services.mozilla.com"})
    url = queue.buildUrl("getLatestArtifact", task_id, artifact_name)
    logger.debug("Downloading artifact", url=url)

    @tenacity.retry(
        reraise=True,
        wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
        stop=tenacity.stop_after_attempt(5),
    )
    def perform_download():
        r = requests.get(url, stream=True)
        r.raise_for_status()

        with open(artifact_path, "wb") as f:
            r.raw.decode_content = True
            shutil.copyfileobj(r.raw, f)

        if artifact_path.endswith(".zip") and not is_zipfile(artifact_path):
            raise BadZipFile("File is not a zip file")

    perform_download() 

def get(request_session, *args, **kwargs):
        """ Perform a get and retry if auth fails

        This function is designed to be used when we perform a get to
        an authenticated source. This function will attempt a single
        re-authentication request if the first one fails.
        """
        return RegistrySessionHelper._action('get',
                                             request_session,
                                             *args,
                                             **kwargs) 

def patch(request_session, *args, **kwargs):
        """ Perform a patch and retry if auth fails

        This function is designed to be used when we perform a path to
        an authenticated source. This function will attempt a single
        re-authentication request if the first one fails.
        """
        return RegistrySessionHelper._action('patch',
                                             request_session,
                                             *args,
                                             **kwargs) 

def join_group(self, group_id):
        if (not self._coordinator or not self._coordinator.is_started
                or not group_id):
            return

        @tenacity.retry(
            wait=tenacity.wait_exponential(
                multiplier=self.conf.coordination.retry_backoff,
                max=self.conf.coordination.max_retry_interval),
            retry=tenacity.retry_if_exception_type(
                ErrorJoiningPartitioningGroup))
        def _inner():
            try:
                join_req = self._coordinator.join_group(group_id)
                join_req.get()
                LOG.info('Joined partitioning group %s', group_id)
            except tooz.coordination.MemberAlreadyExist:
                return
            except tooz.coordination.GroupNotCreated:
                create_grp_req = self._coordinator.create_group(group_id)
                try:
                    create_grp_req.get()
                except tooz.coordination.GroupAlreadyExist:
                    pass
                raise ErrorJoiningPartitioningGroup()
            except tooz.coordination.ToozError:
                LOG.exception('Error joining partitioning group %s,'
                              ' re-trying', group_id)
                raise ErrorJoiningPartitioningGroup()
            self._groups.add(group_id)

        return _inner() 

def post(request_session, *args, **kwargs):
        """ Perform a post and retry if auth fails

        This function is designed to be used when we perform a post to
        an authenticated source. This function will attempt a single
        re-authentication request if the first one fails.
        """
        return RegistrySessionHelper._action('post',
                                             request_session,
                                             *args,
                                             **kwargs) 

def test_old_tornado(self):
        old_attr = gen.is_coroutine_function
        try:
            del gen.is_coroutine_function

            # is_coroutine_function was introduced in tornado 4.5;
            # verify that we don't *completely* fall over on old versions
            @retry
            def retryable(thing):
                pass
        finally:
            gen.is_coroutine_function = old_attr 

def put(request_session, *args, **kwargs):
        """ Perform a put and retry if auth fails

        This function is designed to be used when we perform a put to
        an authenticated source. This function will attempt a single
        re-authentication request if the first one fails.
        """
        return RegistrySessionHelper._action('put',
                                             request_session,
                                             *args,
                                             **kwargs) 

def get(details=True):
        enforce("get status", {})
        try:
            members_req = pecan.request.coordinator.get_members(
                metricd.MetricProcessor.GROUP_ID)
        except tooz.NotImplemented:
            members_req = None
        try:
            report = pecan.request.incoming.measures_report(
                strtobool("details", details))
        except incoming.ReportGenerationError:
            abort(503, 'Unable to generate status. Please retry.')
        report_dict = {"storage": {"summary": report['summary']}}
        if 'details' in report:
            report_dict["storage"]["measures_to_process"] = report['details']
        report_dict['metricd'] = {}
        if members_req:
            members = members_req.get()
            caps = [
                pecan.request.coordinator.get_member_capabilities(
                    metricd.MetricProcessor.GROUP_ID, member)
                for member in members
            ]
            report_dict['metricd']['processors'] = [
                member.decode() for member in members
            ]
            report_dict['metricd']['statistics'] = {
                member.decode(): cap.get()
                for member, cap in six.moves.zip(members, caps)
            }
        else:
            report_dict['metricd']['processors'] = None
            report_dict['metricd']['statistics'] = {}
        return report_dict 

def validate(self, max_retries=0):
        """Performs basic **connection** validation of a sqlalchemy engine."""

        def _retry_on_exception(exc):
            LOG.warning("Engine connection (validate) failed due to '%s'", exc)
            if isinstance(exc, sa_exc.OperationalError) and \
               _is_db_connection_error(six.text_type(exc.args[0])):
                # We may be able to fix this by retrying...
                return True
            if isinstance(exc, (sa_exc.TimeoutError,
                                sa_exc.ResourceClosedError,
                                sa_exc.DisconnectionError)):
                # We may be able to fix this by retrying...
                return True
            # Other failures we likely can't fix by retrying...
            return False

        @tenacity.retry(
            stop=tenacity.stop_after_attempt(max(0, int(max_retries))),
            wait=tenacity.wait_exponential(),
            reraise=True,
            retry=tenacity.retry_if_exception(_retry_on_exception)
        )
        def _try_connect(engine):
            # See if we can make a connection happen.
            #
            # NOTE(harlowja): note that even though we are connecting
            # once it does not mean that we will be able to connect in
            # the future, so this is more of a sanity test and is not
            # complete connection insurance.
            with contextlib.closing(engine.connect()):
                pass

        _try_connect(self._engine) 

def clone_git_repo(self):
        tenacity.retry(
            wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
            stop=tenacity.stop_after_attempt(5),
        )(
            lambda: subprocess.run(
                ["git", "clone", "--quiet", self.repo_url, self.git_repo_path],
                check=True,
            )
        )()

        try:
            tenacity.retry(
                wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
                stop=tenacity.stop_after_attempt(5),
            )(
                lambda: subprocess.run(
                    ["git", "pull", "--quiet", self.repo_url, "master"],
                    cwd=self.git_repo_path,
                    capture_output=True,
                    check=True,
                )
            )()
        except subprocess.CalledProcessError as e:
            # When the repo is empty.
            if b"Couldn't find remote ref master" in e.stdout:
                pass 

def download(package):
    """
    TODO: Flaky downloader
    Why we need a retry here?

    The issue with the flake here is that:
    - When downloading the first package, the datadog_checks_downloader will also download the signing metadata and
      especially "timestamp.json" which gives the current "version". Anytime a new package is pushed, that version
      gets increased.
    - When downloading package A, datadog_checks_downloader will write to disk the current version of "timestamp.json"
    - When downloading package B, datadog_checks_downloader will check the current version of timestamp.json against
      the local one and will fail here if it finds an older version on the repo.

    Retrying should help test not to fail, but the real issue describe above still need to be solved.
    Users relying on it for automated deploys and install can face the same issue.
    Source: https://github.com/DataDog/integrations-core/pull/6476#issuecomment-619059117
    """
    # -v:     CRITICAL
    # -vv:    ERROR
    # -vvv:   WARNING
    # -vvvv:  INFO
    # -vvvvv: DEBUG
    cmd = ['python', '-m', 'datadog_checks.downloader', '-vvvv', package]
    out = subprocess.check_output(cmd)
    log.debug(' '.join(cmd))
    log.debug(out)
    log.debug('') 

def clone_git_repo(self, repo_url, repo_dir):
        if not os.path.exists(repo_dir):
            tenacity.retry(
                wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
                stop=tenacity.stop_after_attempt(5),
            )(
                lambda: subprocess.run(
                    ["git", "clone", "--quiet", repo_url, repo_dir], check=True
                )
            )()

            logger.info(f"{repo_dir} cloned")

        logger.info(f"Fetching {repo_dir}")

        tenacity.retry(
            wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
            stop=tenacity.stop_after_attempt(5),
        )(
            lambda: subprocess.run(
                ["git", "fetch", "--quiet"],
                cwd=repo_dir,
                capture_output=True,
                check=True,
            )
        )()

        logger.info(f"{repo_dir} fetched") 

def get_connection_from_config():
    retries = CONF.database.max_retries
    url = CONF.database.connection

    try:
        # TOTO(iafek): check why this call randomly fails
        connection_scheme = urlparse.urlparse(url).scheme
        LOG.debug('looking for %(name)r driver in %(namespace)r',
                  {'name': connection_scheme, 'namespace': _NAMESPACE})
        mgr = driver.DriverManager(_NAMESPACE, connection_scheme)

    except Exception:
        LOG.exception('Failed to get scheme %s.' % url)
        return None

    @tenacity.retry(
        wait=tenacity.wait_fixed(CONF.database.retry_interval),
        stop=tenacity.stop_after_attempt(retries),
        after=tenacity.after_log(LOG, log.WARN),
        reraise=True)
    def _get_connection():
        """Return an open connection to the database."""
        conn = mgr.driver(url)
        session = conn._engine_facade.get_session()
        session.execute('SELECT 1;')
        return conn

    return _get_connection() 

def execute(self, amphora, passive_failure=False):
        if self.execute.retry.statistics.get(constants.ATTEMPT_NUMBER, 1) == 1:
            LOG.debug('Compute delete execute for amphora with ID %s and '
                      'compute ID: %s', amphora.id, amphora.compute_id)
        else:
            LOG.warning('Retrying compute delete of %s attempt %s of %s.',
                        amphora.compute_id,
                        self.execute.retry.statistics[
                            constants.ATTEMPT_NUMBER],
                        self.execute.retry.stop.max_attempt_number)
        # Let the Taskflow engine know we are working and alive
        # Don't use get with a default for 'attempt_number', we need to fail
        # if that number is missing.
        self.update_progress(
            self.execute.retry.statistics[constants.ATTEMPT_NUMBER] /
            self.execute.retry.stop.max_attempt_number)

        try:
            self.compute.delete(amphora.compute_id)
        except Exception:
            if (self.execute.retry.statistics[constants.ATTEMPT_NUMBER] !=
                    self.execute.retry.stop.max_attempt_number):
                LOG.warning('Compute delete for amphora id: %s failed. '
                            'Retrying.', amphora.id)
                raise
            if passive_failure:
                LOG.exception('Compute delete for compute ID: %s on amphora '
                              'ID: %s failed. This resource will be abandoned '
                              'and should manually be cleaned up once the '
                              'compute service is functional.',
                              amphora.compute_id, amphora.id)
            else:
                LOG.exception('Compute delete for compute ID: %s on amphora '
                              'ID: %s failed. The compute service has failed. '
                              'Aborting and reverting.', amphora.compute_id,
                              amphora.id)
                raise 

def execute(self, loadbalancer_id):
        """Task to setup SG for LB.

        Task is idempotent and safe to retry.
        """

        LOG.debug("Setup SG for loadbalancer id: %s", loadbalancer_id)

        loadbalancer = self.lb_repo.get(db_apis.get_session(),
                                        id=loadbalancer_id)

        return self.network_driver.update_vip_sg(loadbalancer,
                                                 loadbalancer.vip) 

def _retry_if_tooz_error(exception):
    """Return True if we should retry, False otherwise"""
    return isinstance(exception, tooz.coordination.ToozError)