Python tenacity.wait_exponential() Examples

def retry_api_call(func, config, logger=None, *args, **kwargs):
    retry = tenacity.Retrying(
        retry=retry_if_exception(
            lambda e: getattr(e, "response", {}).get("Error", {}).get("Code", None)
            in config.exceptions
            if e
            else False
        ),
        stop=stop_after_attempt(config.attempt),
        wait=wait_exponential(
            multiplier=config.multiplier,
            max=config.max_delay,
            exp_base=config.exponential_base,
        ),
        after=after_log(logger, logger.level) if logger else None,
        reraise=True,
    )
    return retry(func, *args, **kwargs) 

def operation_in_progress_retry(*args, **kwargs) -> Callable:
        """
        A decorator that provides a mechanism to repeat requests in response to
        operation in progress (HTTP 409)
        limit.
        """
        def decorator(fun: Callable):
            default_kwargs = {
                'wait': tenacity.wait_exponential(multiplier=1, max=300),
                'retry': retry_if_operation_in_progress(),
                'before': tenacity.before_log(log, logging.DEBUG),
                'after': tenacity.after_log(log, logging.DEBUG),
            }
            default_kwargs.update(**kwargs)
            return tenacity.retry(
                *args, **default_kwargs
            )(fun)
        return decorator 

def quota_retry(*args, **kwargs) -> Callable:
        """
        A decorator that provides a mechanism to repeat requests in response to exceeding a temporary quote
        limit.
        """
        def decorator(fun: Callable):
            default_kwargs = {
                'wait': tenacity.wait_exponential(multiplier=1, max=100),
                'retry': retry_if_temporary_quota(),
                'before': tenacity.before_log(log, logging.DEBUG),
                'after': tenacity.after_log(log, logging.DEBUG),
            }
            default_kwargs.update(**kwargs)
            return tenacity.retry(
                *args, **default_kwargs
            )(fun)
        return decorator 

def _wait_for_init_container(self, context, container, timeout=3600):
        def retry_if_result_is_false(result):
            return result is False

        def check_init_container_stopped():
            status = self.show(context, container).status
            if status == consts.STOPPED:
                return True
            elif status == consts.RUNNING:
                return False
            else:
                raise exception.ZunException(
                    _("Container has unexpected status: %s") % status)

        r = tenacity.Retrying(
            stop=tenacity.stop_after_delay(timeout),
            wait=tenacity.wait_exponential(),
            retry=tenacity.retry_if_result(retry_if_result_is_false))
        r.call(check_init_container_stopped) 

def clone_git_repo(self, repo_url, repo_dir, rev="origin/branches/default/tip"):
        logger.info(f"Cloning {repo_url}...")

        if not os.path.exists(repo_dir):
            tenacity.retry(
                wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
                stop=tenacity.stop_after_attempt(5),
            )(
                lambda: subprocess.run(
                    ["git", "clone", "--quiet", repo_url, repo_dir], check=True
                )
            )()

        tenacity.retry(
            wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
            stop=tenacity.stop_after_attempt(5),
        )(
            lambda: subprocess.run(
                ["git", "fetch"], cwd=repo_dir, capture_output=True, check=True,
            )
        )()

        subprocess.run(
            ["git", "checkout", rev], cwd=repo_dir, capture_output=True, check=True
        ) 

def _wait_for_init_container(self, context, container, timeout=3600):
        def retry_if_result_is_false(result):
            return result is False

        def check_init_container_stopped():
            status = self._show_container(context, container).status
            if status == consts.STOPPED:
                return True
            elif status == consts.RUNNING:
                return False
            else:
                raise exception.ZunException(
                    _("Container has unexpected status: %s") % status)

        r = tenacity.Retrying(
            stop=tenacity.stop_after_delay(timeout),
            wait=tenacity.wait_exponential(),
            retry=tenacity.retry_if_result(retry_if_result_is_false))
        r.call(check_init_container_stopped) 

def clone_git_repo(self):
        tenacity.retry(
            wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
            stop=tenacity.stop_after_attempt(5),
        )(
            lambda: subprocess.run(
                ["git", "clone", "--quiet", self.repo_url, self.git_repo_path],
                check=True,
            )
        )()

        try:
            tenacity.retry(
                wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
                stop=tenacity.stop_after_attempt(5),
            )(
                lambda: subprocess.run(
                    ["git", "pull", "--quiet", self.repo_url, "master"],
                    cwd=self.git_repo_path,
                    capture_output=True,
                    check=True,
                )
            )()
        except subprocess.CalledProcessError as e:
            # When the repo is empty.
            if b"Couldn't find remote ref master" in e.stdout:
                pass 

def download_artifact(artifact_path, task_id, artifact_name):
    if os.path.exists(artifact_path):
        return artifact_path

    # Build artifact public url
    # Use un-authenticated Taskcluster client to avoid taskcluster-proxy rewrite issue
    # https://github.com/taskcluster/taskcluster-proxy/issues/44
    queue = taskcluster.Queue({"rootUrl": "https://firefox-ci-tc.services.mozilla.com"})
    url = queue.buildUrl("getLatestArtifact", task_id, artifact_name)
    logger.debug("Downloading artifact", url=url)

    @tenacity.retry(
        reraise=True,
        wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
        stop=tenacity.stop_after_attempt(5),
    )
    def perform_download():
        r = requests.get(url, stream=True)
        r.raise_for_status()

        with open(artifact_path, "wb") as f:
            r.raw.decode_content = True
            shutil.copyfileobj(r.raw, f)

        if artifact_path.endswith(".zip") and not is_zipfile(artifact_path):
            raise BadZipFile("File is not a zip file")

    perform_download() 

def join_group(self, group_id):
        if (not self._coordinator or not self._coordinator.is_started
                or not group_id):
            return

        @tenacity.retry(
            wait=tenacity.wait_exponential(
                multiplier=self.conf.coordination.retry_backoff,
                max=self.conf.coordination.max_retry_interval),
            retry=tenacity.retry_if_exception_type(
                ErrorJoiningPartitioningGroup))
        def _inner():
            try:
                join_req = self._coordinator.join_group(group_id)
                join_req.get()
                LOG.info('Joined partitioning group %s', group_id)
            except tooz.coordination.MemberAlreadyExist:
                return
            except tooz.coordination.GroupNotCreated:
                create_grp_req = self._coordinator.create_group(group_id)
                try:
                    create_grp_req.get()
                except tooz.coordination.GroupAlreadyExist:
                    pass
                raise ErrorJoiningPartitioningGroup()
            except tooz.coordination.ToozError:
                LOG.exception('Error joining partitioning group %s,'
                              ' re-trying', group_id)
                raise ErrorJoiningPartitioningGroup()
            self._groups.add(group_id)

        return _inner() 

def test_exponential_with_min_wait_and_multiplier(self):
        r = Retrying(wait=tenacity.wait_exponential(
            min=20, multiplier=2))
        self.assertEqual(r.wait(1, 0), 20)
        self.assertEqual(r.wait(2, 0), 20)
        self.assertEqual(r.wait(3, 0), 20)
        self.assertEqual(r.wait(4, 0), 20)
        self.assertEqual(r.wait(5, 0), 32)
        self.assertEqual(r.wait(6, 0), 64)
        self.assertEqual(r.wait(7, 0), 128)
        self.assertEqual(r.wait(8, 0), 256)
        self.assertEqual(r.wait(20, 0), 1048576) 

def test_exponential_with_max_wait_and_multiplier(self):
        r = Retrying(wait=tenacity.wait_exponential(
            max=50, multiplier=1))
        self.assertEqual(r.wait(1, 0), 1)
        self.assertEqual(r.wait(2, 0), 2)
        self.assertEqual(r.wait(3, 0), 4)
        self.assertEqual(r.wait(4, 0), 8)
        self.assertEqual(r.wait(5, 0), 16)
        self.assertEqual(r.wait(6, 0), 32)
        self.assertEqual(r.wait(7, 0), 50)
        self.assertEqual(r.wait(8, 0), 50)
        self.assertEqual(r.wait(50, 0), 50) 

def test_exponential_with_min_wait(self):
        r = Retrying(wait=tenacity.wait_exponential(min=20))
        self.assertEqual(r.wait(1, 0), 20)
        self.assertEqual(r.wait(2, 0), 20)
        self.assertEqual(r.wait(3, 0), 20)
        self.assertEqual(r.wait(4, 0), 20)
        self.assertEqual(r.wait(5, 0), 20)
        self.assertEqual(r.wait(6, 0), 32)
        self.assertEqual(r.wait(7, 0), 64)
        self.assertEqual(r.wait(8, 0), 128)
        self.assertEqual(r.wait(20, 0), 524288) 

def test_exponential_with_max_wait(self):
        r = Retrying(wait=tenacity.wait_exponential(max=40))
        self.assertEqual(r.wait(1, 0), 1)
        self.assertEqual(r.wait(2, 0), 2)
        self.assertEqual(r.wait(3, 0), 4)
        self.assertEqual(r.wait(4, 0), 8)
        self.assertEqual(r.wait(5, 0), 16)
        self.assertEqual(r.wait(6, 0), 32)
        self.assertEqual(r.wait(7, 0), 40)
        self.assertEqual(r.wait(8, 0), 40)
        self.assertEqual(r.wait(50, 0), 40) 

def test_exponential(self):
        r = Retrying(wait=tenacity.wait_exponential())
        self.assertEqual(r.wait(1, 0), 1)
        self.assertEqual(r.wait(2, 0), 2)
        self.assertEqual(r.wait(3, 0), 4)
        self.assertEqual(r.wait(4, 0), 8)
        self.assertEqual(r.wait(5, 0), 16)
        self.assertEqual(r.wait(6, 0), 32)
        self.assertEqual(r.wait(7, 0), 64)
        self.assertEqual(r.wait(8, 0), 128) 

def validate(self, max_retries=0):
        """Performs basic **connection** validation of a sqlalchemy engine."""

        def _retry_on_exception(exc):
            LOG.warning("Engine connection (validate) failed due to '%s'", exc)
            if isinstance(exc, sa_exc.OperationalError) and \
               _is_db_connection_error(six.text_type(exc.args[0])):
                # We may be able to fix this by retrying...
                return True
            if isinstance(exc, (sa_exc.TimeoutError,
                                sa_exc.ResourceClosedError,
                                sa_exc.DisconnectionError)):
                # We may be able to fix this by retrying...
                return True
            # Other failures we likely can't fix by retrying...
            return False

        @tenacity.retry(
            stop=tenacity.stop_after_attempt(max(0, int(max_retries))),
            wait=tenacity.wait_exponential(),
            reraise=True,
            retry=tenacity.retry_if_exception(_retry_on_exception)
        )
        def _try_connect(engine):
            # See if we can make a connection happen.
            #
            # NOTE(harlowja): note that even though we are connecting
            # once it does not mean that we will be able to connect in
            # the future, so this is more of a sanity test and is not
            # complete connection insurance.
            with contextlib.closing(engine.connect()):
                pass

        _try_connect(self._engine) 

def run_with_advanced_retry(self, _retry_args, *args, **kwargs):
        """
        Runs Hook.run() with a Tenacity decorator attached to it. This is useful for
        connectors which might be disturbed by intermittent issues and should not
        instantly fail.

        :param _retry_args: Arguments which define the retry behaviour.
            See Tenacity documentation at https://github.com/jd/tenacity
        :type _retry_args: dict


        .. code-block:: python

            hook = HttpHook(http_conn_id='my_conn',method='GET')
            retry_args = dict(
                 wait=tenacity.wait_exponential(),
                 stop=tenacity.stop_after_attempt(10),
                 retry=requests.exceptions.ConnectionError
             )
             hook.run_with_advanced_retry(
                     endpoint='v1/test',
                     _retry_args=retry_args
                 )

        """
        self._retry_obj = tenacity.Retrying(
            **_retry_args
        )

        return self._retry_obj(self.run, *args, **kwargs) 

def retry_on_exception_and_log(msg):
    return tenacity.Retrying(
        wait=wait_exponential, retry=_retry_on_exception_and_log(msg)).wraps 

def clone_git_repo(self, repo_url, repo_dir):
        if not os.path.exists(repo_dir):
            tenacity.retry(
                wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
                stop=tenacity.stop_after_attempt(5),
            )(
                lambda: subprocess.run(
                    ["git", "clone", "--quiet", repo_url, repo_dir], check=True
                )
            )()

            logger.info(f"{repo_dir} cloned")

        logger.info(f"Fetching {repo_dir}")

        tenacity.retry(
            wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
            stop=tenacity.stop_after_attempt(5),
        )(
            lambda: subprocess.run(
                ["git", "fetch", "--quiet"],
                cwd=repo_dir,
                capture_output=True,
                check=True,
            )
        )()

        logger.info(f"{repo_dir} fetched") 

def download_bugs(bug_ids, products=None, security=False):
    old_bug_count = 0
    new_bug_ids = set(int(bug_id) for bug_id in bug_ids)
    for bug in get_bugs(include_invalid=True):
        old_bug_count += 1
        if int(bug["id"]) in new_bug_ids:
            new_bug_ids.remove(bug["id"])

    print(f"Loaded {old_bug_count} bugs.")

    new_bug_ids = sorted(list(new_bug_ids))

    CHUNK_SIZE = 100

    chunks = (
        new_bug_ids[i : (i + CHUNK_SIZE)]
        for i in range(0, len(new_bug_ids), CHUNK_SIZE)
    )

    @tenacity.retry(
        stop=tenacity.stop_after_attempt(7),
        wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
    )
    def get_chunk(chunk):
        new_bugs = get(chunk)

        if not security:
            new_bugs = [bug for bug in new_bugs.values() if len(bug["groups"]) == 0]

        if products is not None:
            new_bugs = [bug for bug in new_bugs.values() if bug["product"] in products]

        return new_bugs

    with tqdm(total=len(new_bug_ids)) as progress_bar:
        for chunk in chunks:
            new_bugs = get_chunk(chunk)

            progress_bar.update(len(chunk))

            db.append(BUGS_DB, new_bugs) 

def send_batch(
    bq,
    table,
    template_suffix,
    batch,
    *args,
    retry_max_attempts=None,
    retry_max_wait=None,
    retry_multiplier=None,
    **kwargs
):
    if retry_max_attempts is None:
        retry_max_attempts = 10
    if retry_max_wait is None:
        retry_max_wait = 60
    if retry_multiplier is None:
        retry_multiplier = 0.5

    # We split up send_batch and actually_send_batch so that we can use tenacity to
    # handle retries for us, while still getting to use the Nurser.start_soon interface.
    # This also makes it easier to deal with the error handling aspects of sending a
    # batch, from the work of actually sending. The general rule here is that errors
    # shoudl not escape from this function.

    send = actually_send_batch.retry_with(
        wait=tenacity.wait_exponential(multiplier=retry_multiplier, max=retry_max_wait),
        stop=tenacity.stop_after_attempt(retry_max_attempts),
    )

    try:
        await send(bq, table, template_suffix, batch, *args, **kwargs)
    # We've tried to send this batch to BigQuery, however for one reason or another
    # we were unable to do so. We should log this error, but otherwise we're going
    # to just drop this on the floor because there's not much else we can do here
    # except buffer it forever (which is not a great idea).
    except trio.TooSlowError:
        logger.error("Timed out sending %d items; Dropping them.", len(batch))
    except Exception:
        logger.exception("Error sending %d items; Dropping them.", len(batch)) 

def generate(self):
        db_path = os.path.join("data", self.git_repo_path)
        db.register(
            db_path,
            "https://s3-us-west-2.amazonaws.com/communitytc-bugbug/data/",
            VERSION,
        )

        is_old_version = db.is_old_schema(db_path)

        with ThreadPoolExecutorResult(max_workers=2) as executor:
            cloner = executor.submit(repository.clone, self.repo_dir)
            cloner.add_done_callback(
                lambda future: logger.info("mozilla-central cloned")
            )

            git_user = get_secret("GIT_USER")
            git_password = get_secret("GIT_PASSWORD")

            repo_push_url = self.repo_url.replace(
                "https://", f"https://{git_user}:{git_password}@"
            )

            if not is_old_version:
                executor.submit(self.clone_git_repo)
            else:
                executor.submit(self.init_git_repo)

        subprocess.run(
            ["git", "config", "--global", "http.postBuffer", "12M"], check=True
        )

        push_args = ["git", "push", repo_push_url, "master"]
        if is_old_version:
            push_args.append("--force")

        done = False
        while not done:
            done = generator.generate(
                self.repo_dir,
                self.git_repo_path,
                limit=COMMITS_STEP,
                tokenize=self.tokenize,
                remove_comments=self.remove_comments,
            )

            tenacity.retry(
                wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
                stop=tenacity.stop_after_attempt(5),
            )(lambda: subprocess.run(push_args, cwd=self.git_repo_path, check=True))()

            # We are not using db.upload as we don't need to upload the git repo.
            upload_s3([f"{db_path}.version"]) 

def get_columns(self, connection, table_name, schema=None, **kw):
        raw_connection = self._raw_connection(connection)
        schema = schema if schema else raw_connection.schema_name
        query = """
                SELECT
                  table_schema,
                  table_name,
                  column_name,
                  data_type,
                  is_nullable,
                  column_default,
                  ordinal_position,
                  comment
                FROM information_schema.columns
                WHERE table_schema = '{schema}'
                AND table_name = '{table}'
                """.format(
            schema=schema, table=table_name
        )
        retry_config = raw_connection.retry_config
        retry = tenacity.Retrying(
            retry=retry_if_exception(
                lambda exc: self._retry_if_data_catalog_exception(
                    exc, schema, table_name
                )
            ),
            stop=stop_after_attempt(retry_config.attempt),
            wait=wait_exponential(
                multiplier=retry_config.multiplier,
                max=retry_config.max_delay,
                exp_base=retry_config.exponential_base,
            ),
            reraise=True,
        )
        try:
            return [
                {
                    "name": row.column_name,
                    "type": _TYPE_MAPPINGS.get(
                        self._get_column_type(row.data_type), NULLTYPE
                    ),
                    "nullable": True if row.is_nullable == "YES" else False,
                    "default": row.column_default
                    if not self._is_nan(row.column_default)
                    else None,
                    "ordinal_position": row.ordinal_position,
                    "comment": row.comment,
                }
                for row in retry(connection.execute, query).fetchall()
            ]
        except OperationalError as e:
            if not self._retry_if_data_catalog_exception(e, schema, table_name):
                raise_from(NoSuchTableError(table_name), e)
            else:
                raise e