Python oslo_db.exception.RetryRequest() Examples

def __init__(self, retry_interval=1, max_retries=20,
                 inc_retry_interval=True,
                 max_retry_interval=10, retry_on_disconnect=False,
                 retry_on_deadlock=False,
                 exception_checker=lambda exc: False,
                 jitter=False):
        super(wrap_db_retry, self).__init__()

        self.jitter = jitter
        self.db_error = (exception.RetryRequest, )
        # default is that we re-raise anything unexpected
        self.exception_checker = exception_checker
        if retry_on_disconnect:
            self.db_error += (exception.DBConnectionError, )
        if retry_on_deadlock:
            self.db_error += (exception.DBDeadlock, )
        self.retry_interval = retry_interval
        self.max_retries = max_retries
        self.inc_retry_interval = inc_retry_interval
        self.max_retry_interval = max_retry_interval 

def exc_to_retry(etypes):
    """Contextually reraise Exceptions as a RetryRequests.

    :param etypes: The class type to check the exception for.
    :returns: None
    :raises: A RetryRequest if any exception is caught in the context
        is a nested instance of etypes.
    """
    try:
        yield
    except Exception as e:
        with excutils.save_and_reraise_exception() as ctx:
            if _is_nested_instance(e, etypes):
                ctx.reraise = False
                raise db_exc.RetryRequest(e)


# for convenient access as decorators 

def reraise_as_retryrequest(function):
    """Wrap the said function with a RetryRequest upon error.

    :param function: The function to wrap/decorate.
    :returns: The 'function' wrapped in a try block that will reraise any
        Exception's as a RetryRequest.
    :raises RetryRequest: If the wrapped function raises retriable exception.
    """
    @functools.wraps(function)
    def _wrapped(*args, **kwargs):
        try:
            return function(*args, **kwargs)
        except Exception as e:
            with excutils.save_and_reraise_exception() as ctx:
                if is_retriable(e):
                    ctx.reraise = False
                    raise db_exc.RetryRequest(e)
    return _wrapped 

def is_retriable(exception):
    """Determine if the said exception is retriable.

    :param exception: The exception to check.
    :returns: True if 'exception' is retriable, otherwise False.
    """
    if _is_nested_instance(exception,
                           (db_exc.DBDeadlock, exc.StaleDataError,
                            db_exc.DBConnectionError,
                            db_exc.DBDuplicateEntry, db_exc.RetryRequest)):
        return True
    # Look for savepoints mangled by deadlocks. See bug/1590298 for details.
    return (_is_nested_instance(exception, db_exc.DBError) and
            '1305' in str(exception)) 

def test_notify_handle_retriable_exception(self):
        self.manager.subscribe(
            callback_raise_retriable, resources.PORT, events.BEFORE_CREATE)
        self.assertRaises(db_exc.RetryRequest, self.manager.notify,
                          resources.PORT, events.BEFORE_CREATE, self) 

def test_multi_exception_contains_retry(self):
        e = exceptions.MultipleExceptions(
            [ValueError(), db_exc.RetryRequest(TypeError())])
        self.assertIsNone(self._decorated_function(1, e)) 

def test_retries_on_multi_exception_containing_target(self):
        with testtools.ExpectedException(db_exc.RetryRequest):
            with db_api.exc_to_retry(ValueError):
                e = exceptions.MultipleExceptions([ValueError(), TypeError()])
                raise e 

def test_inner_exception_preserved_in_retryrequest(self):
        try:
            exc = ValueError('test')
            with db_api.exc_to_retry(ValueError):
                raise exc
        except db_exc.RetryRequest as e:
            self.assertEqual(exc, e.inner_exc) 

def test_translates_DBerror_inner_exception(self):
        with testtools.ExpectedException(db_exc.RetryRequest):
            with db_api.exc_to_retry(ValueError):
                raise db_exc.DBError(ValueError()) 

def test_translates_single_exception(self):
        with testtools.ExpectedException(db_exc.RetryRequest):
            with db_api.exc_to_retry(ValueError):
                raise ValueError() 

def is_retriable(e):
    """Determine if the exception is retriable.

    :param e: The exception to check.
    :returns: True if e is retriable and False otherwise.
    """
    if getattr(e, '_RETRY_EXCEEDED', False):
        return False
    if _is_nested_instance(e, (db_exc.DBDeadlock, exc.StaleDataError,
                               db_exc.DBConnectionError,
                               db_exc.DBDuplicateEntry, db_exc.RetryRequest,
                               obj_exc.NeutronDbObjectDuplicateEntry)):
        return True
    # looking savepoints mangled by deadlocks. see bug/1590298 for details.
    return _is_nested_instance(e, db_exc.DBError) and '1305' in str(e) 

def test_health_check_stale_amphora(self, session_mock, get_stale_amp_mock,
                                        failover_mockv2, failover_mock,
                                        db_wait_mock):
        conf = oslo_fixture.Config(cfg.CONF)
        conf.config(group="health_manager", heartbeat_timeout=5)
        amphora_health = mock.MagicMock()
        amphora_health.amphora_id = AMPHORA_ID

        get_stale_amp_mock.side_effect = [amphora_health, None]

        exit_event = threading.Event()
        hm = healthmanager.HealthManager(exit_event)

        hm.health_check()

        # Test DBDeadlock and RetryRequest exceptions
        session_mock.reset_mock()
        get_stale_amp_mock.reset_mock()
        mock_session = mock.MagicMock()
        session_mock.return_value = mock_session
        get_stale_amp_mock.side_effect = [
            db_exc.DBDeadlock,
            db_exc.RetryRequest(Exception('retry_test')),
            db_exc.DBConnectionError,
            TestException('test')]
        # Test that a DBDeadlock does not raise an exception
        self.assertIsNone(hm.health_check())
        # Test that a RetryRequest does not raise an exception
        self.assertIsNone(hm.health_check())
        # Test that a DBConnectionError does not raise an exception
        self.assertIsNone(hm.health_check())
        # ... and that it waits for DB reconnection
        db_wait_mock.assert_called_once()
        # Other exceptions should raise
        self.assertRaises(TestException, hm.health_check)
        self.assertEqual(4, mock_session.rollback.call_count) 

def test_retry_wrapper_reaches_limit(self):
        max_retries = 2

        @api.wrap_db_retry(max_retries=max_retries)
        def some_method(res):
            res['result'] += 1
            raise exception.RetryRequest(ValueError())

        res = {'result': 0}
        self.assertRaises(ValueError, some_method, res)
        self.assertEqual(max_retries + 1, res['result']) 

def _is_exception_expected(self, exc):
        if isinstance(exc, self.db_error):
            # RetryRequest is application-initated exception
            # and not an error condition in case retries are
            # not exceeded
            if not isinstance(exc, exception.RetryRequest):
                LOG.debug('DB error: %s', exc)
            return True
        return self.exception_checker(exc) 

def retry_on_request(f):
    """Retry a DB API call if RetryRequest exception was received.

    wrap_db_entry will be applied to all db.api functions marked with this
    decorator.
    """
    f.__dict__['enable_retry_on_request'] = True
    return f 

def _lock_free_update(session, uuid, lock_state=False, session_id=0):
    """Implement lock-free atomic update for the distributed lock

    :param session:    the db session
    :type session:     DB Session object
    :param uuid:         the lock uuid
    :type uuid:          string
    :param lock_state: the lock state to update
    :type lock_state:  boolean
    :param session_id: the API session ID to update
    :type session_id:  string
    :raises RetryRequest(): when the lock failed to update
    """
    if not lock_state:
        # acquire lock
        search_params = {'object_uuid': uuid, 'lock': lock_state}
        update_params = {'lock': not lock_state, 'session_id': session_id}
    else:
        # release or reset lock
        search_params = {'object_uuid': uuid, 'lock': lock_state,
                         'session_id': session_id}
        update_params = {'lock': not lock_state, 'session_id': 0}

    rows_update = session.query(models.DFLockedObjects).\
        filter_by(**search_params).\
        update(update_params, synchronize_session='fetch')

    if not rows_update:
        LOG.debug('The lock for object %(id)s in session '
                  '%(sid)s cannot be updated.', {'id': uuid,
                                                 'sid': session_id})
        raise db_exc.RetryRequest(df_exc.DBLockFailed(oid=uuid,
                                                      sid=session_id)) 

def test_record_triggers_retry_on_reference_error(self, mock_create_row,
                                                      mock_calculate):
        args = [mock.Mock(unsafe=True)] * 5
        self.assertRaises(exception.RetryRequest, journal.record, *args) 

def record(plugin_context, object_type, object_uuid, operation, data,
           ml2_context=None):
    if (object_type == odl_const.ODL_PORT and
            operation in (odl_const.ODL_CREATE, odl_const.ODL_UPDATE)):
        data = _enrich_port(
            plugin_context, ml2_context, object_type, operation, data)

    # Calculate depending_on on other journal entries
    depending_on = dependency_validations.calculate(
        plugin_context, operation, object_type, object_uuid, data)

    # NOTE(mpeterson): Between the moment that a dependency is calculated and
    # the new entry is recorded in the journal, an operation can ocurr that
    # would make the dependency irrelevant. In that case we request a retry.
    # For more details, read the commit message that introduced this comment.
    try:
        entry = db.create_pending_row(
            plugin_context, object_type, object_uuid, operation, data,
            depending_on=depending_on)
    except exception.DBReferenceError as e:
        raise exception.RetryRequest(e)

    _log_entry(LOG_RECORDED, entry)
    LOG.debug('Entry with ID %(entry_id)s depends on these entries: '
              '%(depending_on)s',
              {'entry_id': entry.seqnum,
               'depending_on': [d.seqnum for d in depending_on]}) 

def __call__(self, f):
        @functools.wraps(f)
        def wrapper(*args, **kwargs):
            sleep_time = next_interval = self.retry_interval
            remaining = self.max_retries

            while True:
                try:
                    return f(*args, **kwargs)
                except Exception as e:
                    with excutils.save_and_reraise_exception() as ectxt:
                        expected = self._is_exception_expected(e)
                        if remaining > 0:
                            ectxt.reraise = not expected
                        else:
                            if expected:
                                LOG.exception('DB exceeded retry limit.')
                            # if it's a RetryRequest, we need to unpack it
                            if isinstance(e, exception.RetryRequest):
                                ectxt.type_ = type(e.inner_exc)
                                ectxt.value = e.inner_exc
                    LOG.debug("Performing DB retry for function %s",
                              reflection.get_callable_name(f))
                    # NOTE(vsergeyev): We are using patched time module, so
                    #                  this effectively yields the execution
                    #                  context to another green thread.
                    time.sleep(sleep_time)
                    if self.inc_retry_interval:
                        # NOTE(jiangyikun): In order to minimize the chance of
                        # regenerating a deadlock and reduce the average sleep
                        # time, we are using jitter by default when the
                        # deadlock is detected. With the jitter,
                        # sleep_time = [0, next_interval), otherwise, without
                        # the jitter, sleep_time = next_interval.
                        if isinstance(e, exception.DBDeadlock):
                            jitter = True
                        else:
                            jitter = self.jitter
                        sleep_time, next_interval = self._get_inc_interval(
                            next_interval, jitter)
                    remaining -= 1

        return wrapper