Python oslo_db.exception.DBDeadlock() Examples

def _deadlock_error(operational_error, match, engine_name, is_disconnect):
    """Filter for MySQL or Postgresql deadlock error.

    NOTE(comstud): In current versions of DB backends, Deadlock violation
    messages follow the structure:

    mysql+mysqldb:
    (OperationalError) (1213, 'Deadlock found when trying to get lock; try '
                         'restarting transaction') <query_str> <query_args>

    mysql+mysqlconnector:
    (InternalError) 1213 (40001): Deadlock found when trying to get lock; try
                         restarting transaction

    postgresql:
    (TransactionRollbackError) deadlock detected <deadlock_details>


    ibm_db_sa:
    SQL0911N The current transaction has been rolled back because of a
    deadlock or timeout <deadlock details>

    """
    raise exception.DBDeadlock(operational_error) 

def __init__(self, retry_interval=1, max_retries=20,
                 inc_retry_interval=True,
                 max_retry_interval=10, retry_on_disconnect=False,
                 retry_on_deadlock=False,
                 exception_checker=lambda exc: False,
                 jitter=False):
        super(wrap_db_retry, self).__init__()

        self.jitter = jitter
        self.db_error = (exception.RetryRequest, )
        # default is that we re-raise anything unexpected
        self.exception_checker = exception_checker
        if retry_on_disconnect:
            self.db_error += (exception.DBConnectionError, )
        if retry_on_deadlock:
            self.db_error += (exception.DBDeadlock, )
        self.retry_interval = retry_interval
        self.max_retries = max_retries
        self.inc_retry_interval = inc_retry_interval
        self.max_retry_interval = max_retry_interval 

def is_retriable(exception):
    """Determine if the said exception is retriable.

    :param exception: The exception to check.
    :returns: True if 'exception' is retriable, otherwise False.
    """
    if _is_nested_instance(exception,
                           (db_exc.DBDeadlock, exc.StaleDataError,
                            db_exc.DBConnectionError,
                            db_exc.DBDuplicateEntry, db_exc.RetryRequest)):
        return True
    # Look for savepoints mangled by deadlocks. See bug/1590298 for details.
    return (_is_nested_instance(exception, db_exc.DBError) and
            '1305' in str(exception)) 

def callback_raise_retriable(*args, **kwargs):
    raise db_exc.DBDeadlock() 

def test_retry_if_session_inactive_no_retry_in_active_session(self):
        context = mock.Mock()
        context.session.is_active = True
        with testtools.ExpectedException(db_exc.DBDeadlock):
            # retry decorator should have no effect in an active session
            self._context_function(context, [], {1: 2},
                                   fail_count=1,
                                   exc_to_raise=db_exc.DBDeadlock()) 

def test_retry_if_session_inactive_kwargs_not_mutated_after_retries(self):
        context = mock.Mock()
        context.session.is_active = False
        list_arg = [1, 2, 3, 4]
        dict_arg = {1: 'a', 2: 'b'}
        l, d = self._context_function(context, list_arg=list_arg,
                                      dict_arg=dict_arg,
                                      fail_count=5,
                                      exc_to_raise=db_exc.DBDeadlock())
        # even though we had 5 failures the list and dict should only
        # be mutated once
        self.assertEqual(5, len(l))
        self.assertEqual(3, len(d)) 

def test_retry_if_session_inactive_args_not_mutated_after_retries(self):
        context = mock.Mock()
        context.session.is_active = False
        list_arg = [1, 2, 3, 4]
        dict_arg = {1: 'a', 2: 'b'}
        l, d = self._context_function(context, list_arg, dict_arg,
                                      5, db_exc.DBDeadlock())
        # even though we had 5 failures the list and dict should only
        # be mutated once
        self.assertEqual(5, len(l))
        self.assertEqual(3, len(d)) 

def test_all_deadlock_time_elapsed(self):
        self._test_retry_time_cost(db_exc.DBDeadlock) 

def test_multi_exception_raised_on_exceed(self):
        # limit retries so this doesn't take 40 seconds
        retry_fixture = fixture.DBRetryErrorsFixture(max_retries=2)
        retry_fixture.setUp()
        e = exceptions.MultipleExceptions([ValueError(), db_exc.DBDeadlock()])
        with testtools.ExpectedException(exceptions.MultipleExceptions):
            self._decorated_function(db_api.MAX_RETRIES + 1, e)
        retry_fixture.cleanUp() 

def test_multi_nested_exception_contains_deadlock(self):
        i = exceptions.MultipleExceptions([ValueError(), db_exc.DBDeadlock()])
        e = exceptions.MultipleExceptions([ValueError(), i])
        self.assertIsNone(self._decorated_function(1, e)) 

def test_multi_exception_contains_deadlock(self):
        e = exceptions.MultipleExceptions([ValueError(), db_exc.DBDeadlock()])
        self.assertIsNone(self._decorated_function(1, e)) 

def is_retriable(e):
    """Determine if the exception is retriable.

    :param e: The exception to check.
    :returns: True if e is retriable and False otherwise.
    """
    if getattr(e, '_RETRY_EXCEEDED', False):
        return False
    if _is_nested_instance(e, (db_exc.DBDeadlock, exc.StaleDataError,
                               db_exc.DBConnectionError,
                               db_exc.DBDuplicateEntry, db_exc.RetryRequest,
                               obj_exc.NeutronDbObjectDuplicateEntry)):
        return True
    # looking savepoints mangled by deadlocks. see bug/1590298 for details.
    return _is_nested_instance(e, db_exc.DBError) and '1305' in str(e) 

def test_health_check_stale_amphora(self, session_mock, get_stale_amp_mock,
                                        failover_mockv2, failover_mock,
                                        db_wait_mock):
        conf = oslo_fixture.Config(cfg.CONF)
        conf.config(group="health_manager", heartbeat_timeout=5)
        amphora_health = mock.MagicMock()
        amphora_health.amphora_id = AMPHORA_ID

        get_stale_amp_mock.side_effect = [amphora_health, None]

        exit_event = threading.Event()
        hm = healthmanager.HealthManager(exit_event)

        hm.health_check()

        # Test DBDeadlock and RetryRequest exceptions
        session_mock.reset_mock()
        get_stale_amp_mock.reset_mock()
        mock_session = mock.MagicMock()
        session_mock.return_value = mock_session
        get_stale_amp_mock.side_effect = [
            db_exc.DBDeadlock,
            db_exc.RetryRequest(Exception('retry_test')),
            db_exc.DBConnectionError,
            TestException('test')]
        # Test that a DBDeadlock does not raise an exception
        self.assertIsNone(hm.health_check())
        # Test that a RetryRequest does not raise an exception
        self.assertIsNone(hm.health_check())
        # Test that a DBConnectionError does not raise an exception
        self.assertIsNone(hm.health_check())
        # ... and that it waits for DB reconnection
        db_wait_mock.assert_called_once()
        # Other exceptions should raise
        self.assertRaises(TestException, hm.health_check)
        self.assertEqual(4, mock_session.rollback.call_count) 

def _retry_on_deadlock(f):
    """Decorator to retry a DB API call if Deadlock was received."""
    @functools.wraps(f)
    def wrapped(*args, **kwargs):
        while True:
            try:
                return f(*args, **kwargs)
            except db_exc.DBDeadlock:
                LOG.warning("Deadlock detected when running '%(func_name)s':"
                            " Retrying...", dict(func_name=f.__name__))
                # Retry!
                time.sleep(0.5)
                continue
    functools.update_wrapper(wrapped, f)
    return wrapped 

def test_retry_wrapper_deadlock(self, mock_sleep):

        # Tests that jitter is False, if the retry wrapper hits a
        # non-deadlock error
        @api.wrap_db_retry(max_retries=1, retry_on_deadlock=True)
        def some_method_no_deadlock():
            raise exception.RetryRequest(ValueError())
        with mock.patch(
                'oslo_db.api.wrap_db_retry._get_inc_interval') as mock_get:
            mock_get.return_value = 2, 2
            self.assertRaises(ValueError, some_method_no_deadlock)
            mock_get.assert_called_once_with(1, False)

        # Tests that jitter is True, if the retry wrapper hits a deadlock
        # error.
        @api.wrap_db_retry(max_retries=1, retry_on_deadlock=True)
        def some_method_deadlock():
            raise exception.DBDeadlock('test')
        with mock.patch(
                'oslo_db.api.wrap_db_retry._get_inc_interval') as mock_get:
            mock_get.return_value = 0.1, 2
            self.assertRaises(exception.DBDeadlock, some_method_deadlock)
            mock_get.assert_called_once_with(1, True)

        # Tests that jitter is True, if the jitter is enable by user
        @api.wrap_db_retry(max_retries=1, retry_on_deadlock=True, jitter=True)
        def some_method_no_deadlock_exp():
            raise exception.RetryRequest(ValueError())
        with mock.patch(
                'oslo_db.api.wrap_db_retry._get_inc_interval') as mock_get:
            mock_get.return_value = 0.1, 2
            self.assertRaises(ValueError, some_method_no_deadlock_exp)
            mock_get.assert_called_once_with(1, True) 

def _run_deadlock_detect_test(
        self, dialect_name, message,
        orig_exception_cls=TestsExceptionFilter.OperationalError):
        self._run_test(
            dialect_name, self.statement,
            orig_exception_cls(message),
            exception.DBDeadlock,
            params=self.params
        ) 

def _retry_on_deadlock(exc):
    """Filter to trigger retry a when a Deadlock is received."""
    # TODO(kiall): This is a total leak of the SQLA Driver, we'll need a better
    #              way to handle this.
    if isinstance(exc, db_exception.DBDeadlock):
        LOG.warning("Deadlock detected. Retrying...")
        return True
    return False 

def test_update_recordset_deadlock_retry(self):
        # Create a zone
        zone = self.create_zone()

        # Create a recordset
        recordset = self.create_recordset(zone)

        # Update the recordset
        recordset.ttl = 1800

        # Due to Python's scoping of i - we need to make it a mutable type
        # for the counter to work.. In Py3, we can use the nonlocal keyword.
        i = [False]

        def fail_once_then_pass():
            if i[0] is True:
                return self.central_service.storage.session.commit()
            else:
                i[0] = True
                raise db_exception.DBDeadlock()

        with mock.patch.object(self.central_service.storage, 'commit',
                               side_effect=fail_once_then_pass):
            # Perform the update
            recordset = self.central_service.update_recordset(
                self.admin_context, recordset)

        # Ensure i[0] is True, indicating the side_effect code above was
        # triggered
        self.assertTrue(i[0])

        # Ensure the recordset was updated correctly
        self.assertEqual(1800, recordset.ttl) 

def test_update_zone_deadlock_retry(self):
        # Create a zone
        zone = self.create_zone(name='example.org.')
        original_serial = zone.serial

        # Update the Object
        zone.email = 'info@example.net'

        # Due to Python's scoping of i - we need to make it a mutable type
        # for the counter to work.. In Py3, we can use the nonlocal keyword.
        i = [False]

        def fail_once_then_pass():
            if i[0] is True:
                return self.central_service.storage.session.commit()
            else:
                i[0] = True
                raise db_exception.DBDeadlock()

        with mock.patch.object(self.central_service.storage, 'commit',
                          side_effect=fail_once_then_pass):
            # Perform the update
            zone = self.central_service.update_zone(
                self.admin_context, zone)

        # Ensure i[0] is True, indicating the side_effect code above was
        # triggered
        self.assertTrue(i[0])

        # Ensure the zone was updated correctly
        self.assertGreater(zone.serial, original_serial)
        self.assertEqual('info@example.net', zone.email) 

def __call__(self, f):
        @functools.wraps(f)
        def wrapper(*args, **kwargs):
            sleep_time = next_interval = self.retry_interval
            remaining = self.max_retries

            while True:
                try:
                    return f(*args, **kwargs)
                except Exception as e:
                    with excutils.save_and_reraise_exception() as ectxt:
                        expected = self._is_exception_expected(e)
                        if remaining > 0:
                            ectxt.reraise = not expected
                        else:
                            if expected:
                                LOG.exception('DB exceeded retry limit.')
                            # if it's a RetryRequest, we need to unpack it
                            if isinstance(e, exception.RetryRequest):
                                ectxt.type_ = type(e.inner_exc)
                                ectxt.value = e.inner_exc
                    LOG.debug("Performing DB retry for function %s",
                              reflection.get_callable_name(f))
                    # NOTE(vsergeyev): We are using patched time module, so
                    #                  this effectively yields the execution
                    #                  context to another green thread.
                    time.sleep(sleep_time)
                    if self.inc_retry_interval:
                        # NOTE(jiangyikun): In order to minimize the chance of
                        # regenerating a deadlock and reduce the average sleep
                        # time, we are using jitter by default when the
                        # deadlock is detected. With the jitter,
                        # sleep_time = [0, next_interval), otherwise, without
                        # the jitter, sleep_time = next_interval.
                        if isinstance(e, exception.DBDeadlock):
                            jitter = True
                        else:
                            jitter = self.jitter
                        sleep_time, next_interval = self._get_inc_interval(
                            next_interval, jitter)
                    remaining -= 1

        return wrapper