Python pymongo.errors.OperationFailure() Examples

def _ensure_index(collection):
        try:
            collection.create_index([('symbol', pymongo.HASHED)], background=True)
            # We keep it only for its uniqueness
            collection.create_index([('symbol', pymongo.ASCENDING),
                                     ('sha', pymongo.ASCENDING)], unique=True, background=True)
            # TODO: When/if we remove the segments->versions pointers implementation and keep only the forward pointers,
            #       we can remove the 'parent' from the index.
            collection.create_index([('symbol', pymongo.ASCENDING),
                                     ('parent', pymongo.ASCENDING),
                                     ('segment', pymongo.ASCENDING)], unique=True, background=True)
            # Used for efficient SHA-based read queries that have index ranges
            collection.create_index([('symbol', pymongo.ASCENDING),
                                     ('sha', pymongo.ASCENDING),
                                     ('segment', pymongo.ASCENDING)], unique=True, background=True)
        except OperationFailure as e:
            if "can't use unique indexes" in str(e):
                return
            raise 

def test_retry_nested():
    error = OperationFailure('error')
    with patch('arctic.decorators._log_exception', autospec=True) as le:
        @mongo_retry
        def foo():
            @mongo_retry
            def bar():
                raise error
            try:
                bar()
            except:
                raise error
        with pytest.raises(OperationFailure):
            foo()
    assert le.call_count == 15
    assert le.call_args[0][0] == 'bar'
    assert le.call_args[0][1] == error 

def _test_query_falls_back_to_primary(library_secondary, library_name, fw_pointers_cfg):
    with FwPointersCtx(fw_pointers_cfg):
        allow_secondary = [True]
        def _query(options, *args, **kwargs):
            # If we're allowing secondary read and an error occurs when reading a chunk.
            # We should attempt a call to primary only subsequently.
            # In newer MongoDBs we query <database>.$cmd rather than <database>.<collection>
            if args[0].startswith('arctic_{}.'.format(library_name.split('.')[0])) and \
                        bool(options & _QUERY_OPTIONS['slave_okay']) == True:
                allow_secondary[0] = False
                raise OperationFailure("some_error")
            return __query(options, *args, **kwargs)

        library_secondary.write(symbol, ts1)
        with patch('pymongo.message.query', side_effect=_query), \
             patch('pymongo.server_description.ServerDescription.server_type', SERVER_TYPE.Mongos):
            assert library_secondary.read(symbol) is not None
        # We raised at least once on a secondary read
        assert allow_secondary[0] == False 

def test_prune_previous_versions_retries_on_cleanup_error(library, fw_pointers_cfg):
    with FwPointersCtx(fw_pointers_cfg):
        original_cleanup = _version_store_utils.cleanup
        def _cleanup(*args, **kwargs):
            if _cleanup.first_try:
                _cleanup.first_try = False
                raise OperationFailure(0)
            else:
                return original_cleanup(*args, **kwargs)
        _cleanup.first_try = True

        library.write(symbol, ts1)
        library.write(symbol, ts2)

        with patch("arctic.store.version_store.cleanup", side_effect=_cleanup) as cleanup:
            cleanup.__name__ = "cleanup"  # required by functools.wraps
            library._prune_previous_versions(symbol, keep_mins=0)

        assert len(list(library._arctic_lib.get_top_level_collection().find({'symbol': symbol}))) == 1 

def search(self, uri_query, type_search):
        """
        Search endpoints or resources.
        :param uri_query: parameters to search
        :param type_search: it's equal to ep if the search is for endpoints or res if it is for resources
        :return: the string of results or an error code
        """
        if (type_search != "ep") and (type_search != "res"):
            return defines.Codes.BAD_REQUEST.number
        if len(uri_query) <= 0:
            uri_query = "ep=*"
        query = self.parse_uri_query(uri_query)
        query_rdp, query_res = self.split_queries(query)
        try:
            query = [{"$match": {"$and": [query_rdp, {"$expr": {"$gt": [{"$sum": ["$lt", "$time"]}, int(time())]}}]}},
                     {"$unwind": "$links"}, {"$match": query_res}]
            result = self.collection.aggregate(query)
            link = self.serialize_core_link_format(result, type_search)
            return link
        except ConnectionFailure:
            logger.error("Connection to the database cannot be made or is lost.")
            return defines.Codes.SERVICE_UNAVAILABLE.number
        except OperationFailure as e:
            logger.error("Search operation failure with type of search " + type_search + " and uri query " + uri_query + " " +e.message)
            return defines.Codes.SERVICE_UNAVAILABLE.number 

def update(self, resource, uri_query):
        """
        Update a registration resource.
        :param resource: the resource to update
        :param uri_query: the parameters of the registration resource to update
        :return: the code of the response
        """
        if len(resource) <= 0:
            return defines.Codes.BAD_REQUEST.number
        data = {}
        if len(uri_query) > 0:
            data = self.parse_uri_query(uri_query)
        res = {'res': resource}
        try:
            data.update({"time": int(time())})
            result = self.collection.update_one(res, {"$set": data})
            if not result.matched_count:
                return defines.Codes.NOT_FOUND.number
            return defines.Codes.CHANGED.number
        except ConnectionFailure:
            logger.error("Connection to the database cannot be made or is lost.")
            return defines.Codes.SERVICE_UNAVAILABLE.number
        except OperationFailure:
            logger.error("Update operation failure on resource " + resource + " and with uri query " + uri_query)
            return defines.Codes.SERVICE_UNAVAILABLE.number 

def delete(self, resource):
        """
        Delete an endpoint and all its resources
        :param resource: the registration resource to delete
        :return: the code of the response
        """
        res = {'res': resource}
        try:
            result = self.collection.delete_one(res)
            if not result.deleted_count:
                return defines.Codes.NOT_FOUND.number
            return defines.Codes.DELETED.number
        except ConnectionFailure:
            logger.error("Connection to the database cannot be made or is lost.")
            return defines.Codes.SERVICE_UNAVAILABLE.number
        except OperationFailure:
            logger.error("Delete operation failure on resource " + resource)
            return defines.Codes.SERVICE_UNAVAILABLE.number 

def initialize_library(cls, arctic_lib, hashed=True, **kwargs):
        c = arctic_lib.get_top_level_collection()

        if 'strict_write_handler' in kwargs:
            arctic_lib.set_library_metadata('STRICT_WRITE_HANDLER_MATCH',
                                            bool(kwargs.pop('strict_write_handler')))

        for th in _TYPE_HANDLERS:
            th.initialize_library(arctic_lib, **kwargs)
        VersionStore._bson_handler.initialize_library(arctic_lib, **kwargs)
        VersionStore(arctic_lib)._ensure_index()

        logger.info("Trying to enable sharding...")
        try:
            enable_sharding(arctic_lib.arctic, arctic_lib.get_name(), hashed=hashed)
        except OperationFailure as e:
            logger.warning("Library created, but couldn't enable sharding: %s. This is OK if you're not 'admin'" % str(e)) 

def test_mongo_retry_hook_changes():
    retries = [2]
    self = MagicMock()
    hook1 = Mock()
    register_log_exception_hook(hook1)
    hook2 = Mock()

    @mongo_retry
    def foo(self):
        if retries[0] == 2:
            retries[0] -= 1
            raise OperationFailure('error')
        elif retries[0] == 1:
            register_log_exception_hook(hook2)
            retries[0] -= 1
            raise AutoReconnect('error')
        return "success"
    foo(self)

    assert hook1.call_count == 1
    assert hook2.call_count == 1 

def test_read_handles_operation_failure():
    self = Mock(spec=VersionStore)
    self._read_preference.return_value = sentinel.read_preference
    self._collection = create_autospec(Collection)
    self._read_metadata.side_effect = [sentinel.meta1, sentinel.meta2]
    self._read_metadata.__name__ = 'name'
    self._do_read.__name__ = 'name'  # feh: mongo_retry decorator cares about this
    self._do_read.side_effect = [OperationFailure('error'), sentinel.read]
    VersionStore.read(self, sentinel.symbol, sentinel.as_of,
                      from_version=sentinel.from_version,
                      date_range=sentinel.date_range,
                      other_kwarg=sentinel.other_kwarg)
    # Assert that, for the two read calls, the second uses the new metadata
    assert self._do_read.call_args_list == [call(sentinel.symbol, sentinel.meta1, 
                                                 sentinel.from_version,
                                                 date_range=sentinel.date_range,
                                                 other_kwarg=sentinel.other_kwarg,
                                                 read_preference=sentinel.read_preference)]
    assert self._do_read_retry.call_args_list == [call(sentinel.symbol, sentinel.meta2,
                                                       sentinel.from_version,
                                                       date_range=sentinel.date_range,
                                                       other_kwarg=sentinel.other_kwarg,
                                                       read_preference=ReadPreference.PRIMARY)] 

def test_mongo_retry():
    retries = [2]
    self = MagicMock()
    self._arctic_lib.arctic.mongo_host = sentinel.host
    self._collection.database.client.nodes = set([('a', 12)])
    self._arctic_lib.get_name.return_value = sentinel.lib_name
    with patch('arctic.decorators._handle_error', autospec=True) as he:
        @mongo_retry
        def foo(self):
            if retries[0] == 2:
                retries[0] -= 1
                raise OperationFailure('error')
            elif retries[0] == 1:
                retries[0] -= 1
                raise AutoReconnect('error')
            return "success"
        foo(self)
    assert he.call_count == 2
    assert isinstance(he.call_args_list[0][0][1], OperationFailure)
    assert he.call_args_list[0][0][2] == 1
    assert he.call_args_list[0][1] == {'mnodes': ['a:12'],
                                       'mhost': 'sentinel.host',
                                       'l': sentinel.lib_name}
    assert isinstance(he.call_args_list[1][0][1], AutoReconnect)
    assert he.call_args_list[1][0][2] == 2 

def __init__(self, **kwargs):
        super().__init__(**kwargs)
        from pymongo import MongoClient
        from pymongo.errors import OperationFailure

        self.database_uri = kwargs.get(
            'database_uri', 'mongodb://localhost:27017/chatterbot-database'
        )

        # Use the default host and port
        self.client = MongoClient(self.database_uri)

        # Increase the sort buffer to 42M if possible
        try:
            self.client.admin.command({'setParameter': 1, 'internalQueryExecMaxBlockingSortBytes': 44040192})
        except OperationFailure:
            pass

        # Specify the name of the database
        self.database = self.client.get_database()

        # The mongo collection of statement documents
        self.statements = self.database['statements'] 

def test_write_error_clean_retry():
    write_handler = Mock(write=Mock(__name__=""))
    write_handler.write.side_effect = [OperationFailure("mongo failure"), None]
    vs = create_autospec(VersionStore, instance=True,
                         _collection=Mock(),
                         _version_nums=Mock(find_one_and_update=Mock(return_value={'version': 1})),
                         _versions=Mock(insert_one=Mock(__name__="insert_one"), find_one=Mock(__name__="find_one")),
                         _arctic_lib=create_autospec(ArcticLibraryBinding,
                                                     arctic=create_autospec(Arctic, mongo_host='some_host')))
    vs._insert_version = lambda version: VersionStore._insert_version(vs, version)
    vs._collection.database.connection.nodes = []
    vs._write_handler.return_value = write_handler
    VersionStore.write(vs, 'sym', sentinel.data, prune_previous_version=False)
    assert vs._version_nums.find_one_and_update.call_count == 2
    assert vs._versions.find_one.call_count == 2
    assert write_handler.write.call_count == 2
    assert vs._versions.insert_one.call_count == 1 

def test_append_error_clean_retry():
    read_handler = Mock(append=Mock(__name__=""))
    read_handler.append.side_effect = [OperationFailure("mongo failure"), None]
    previous_version = TPL_VERSION.copy()
    previous_version['version'] = 1
    vs = create_autospec(VersionStore, instance=True,
                         _collection=Mock(),
                         _version_nums=Mock(find_one_and_update=Mock(return_value={'version': previous_version['version']+1})),
                         _versions=Mock(insert_one=Mock(__name__="insert_one"), find_one=Mock(__name__="find_one", return_value=previous_version)),
                         _arctic_lib=create_autospec(ArcticLibraryBinding,
                                                     arctic=create_autospec(Arctic, mongo_host='some_host')))
    vs._insert_version = lambda version: VersionStore._insert_version(vs, version)
    vs._collection.database.connection.nodes = []
    vs._read_handler.return_value = read_handler
    VersionStore.append(vs, 'sym', [1, 2, 3], prune_previous_version=False, upsert=False)
    assert vs._version_nums.find_one_and_update.call_count == 2
    assert vs._versions.find_one.call_count == 2
    assert read_handler.append.call_count == 2
    assert vs._versions.insert_one.call_count == 1 

def test_append_insert_version_operror():
    read_handler = Mock(append=Mock(__name__=""))
    previous_version = TPL_VERSION.copy()
    previous_version['version'] = 1
    vs = create_autospec(VersionStore, instance=True,
                         _collection=Mock(),
                         _version_nums=Mock(find_one_and_update=Mock(return_value={'version': previous_version['version']+1})),
                         _versions=Mock(insert_one=Mock(__name__="insert_one"), find_one=Mock(__name__="find_one", return_value=previous_version)),
                         _arctic_lib=create_autospec(ArcticLibraryBinding,
                                                     arctic=create_autospec(Arctic, mongo_host='some_host')))
    vs._insert_version = lambda version: VersionStore._insert_version(vs, version)
    vs._versions.insert_one.side_effect = [OperationFailure("mongo op error"), None]
    vs._collection.database.connection.nodes = []
    vs._read_handler.return_value = read_handler
    VersionStore.append(vs, 'sym', [1, 2, 3], prune_previous_version=False, upsert=False)
    assert vs._version_nums.find_one_and_update.call_count == 1
    assert vs._versions.find_one.call_count == 1
    assert read_handler.append.call_count == 1
    assert vs._versions.insert_one.call_count == 2 

def test_ArcticTransaction_writes_if_base_data_corrupted():

    vs = Mock(spec=VersionStore)
    ts1 = pd.DataFrame(index=[1, 2], data={'a': [1.0, 2.0]})
    vs.read.side_effect = OperationFailure('some failure')
    vs.write.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=2,
                                          metadata=None, data=None, host=sentinel.host)
    vs.read_metadata.return_value = VersionedItem(symbol=sentinel.symbol, library=sentinel.library, version=1,
                                                  metadata=None, data=None, host=sentinel.host)
    vs.list_versions.return_value = [{'version': 2}, {'version': 1}]

    with ArcticTransaction(vs, sentinel.symbol, sentinel.user, sentinel.log) as cwb:
        cwb.write(sentinel.symbol, ts1, metadata={1: 2})

    vs.write.assert_called_once_with(sentinel.symbol, ANY, prune_previous_version=True, metadata={1: 2})
    assert vs.list_versions.call_args_list == [call(sentinel.symbol)] 

def get(self, key, newer_than_secs=None):
        """

        :param key: Key for the dataset. eg. list_libraries.
        :param newer_than_secs: None to indicate use cache if available. Used to indicate what level of staleness
        in seconds is tolerable.
        :return: None unless if there is non stale data present in the cache.
        """
        try:
            if not self._cachecol:
                # Collection not created or no permissions to read from it.
                return None
            cached_data = self._cachecol.find_one({"type": key})
            # Check that there is data in cache and it's not stale.
            if cached_data and self._is_not_expired(cached_data, newer_than_secs):
                return cached_data['data']
        except OperationFailure as op:
            logging.warning("Could not read from cache due to: %s. Ask your admin to give read permissions on %s:%s",
                            op, CACHE_DB, CACHE_COLL)

        return None 

def _refresh(self):
        """Refreshes the cursor with more data from the server.

        Returns the length of self.__data after refresh. Will exit early if
        self.__data is already non-empty. Raises OperationFailure when the
        cursor cannot be refreshed due to an error on the query.
        """
        if len(self.__data) or self.__killed:
            return len(self.__data)

        if self.__id:  # Get More
            dbname, collname = self.__ns.split('.', 1)
            self.__send_message(
                _GetMore(dbname,
                         collname,
                         self.__batch_size,
                         self.__id,
                         self.__collection.codec_options))

        else:  # Cursor id is zero nothing else to return
            self.__killed = True

        return len(self.__data) 

def init_database(self):
        """
        Initializes the database connection to MongoDB.
        Also pre-caches potentially heavy resources from the DB.
        :return:
        """
        self.log.info('Connecting to Database...')
        self.db = Database(self, self.cfg.db)
        try:
            await self.db[self.db.db_nam].collection.find_one({})
            if self.cfg.cache.type not in ['redis', 'mixed']:
                await self.db.precache_settings()
                await self.db.precache_profiles()
                await self.db.precache_resources()
            set_color_cache_coll(self.db[self.db.db_nam].ColorCache)
        except ServerSelectionTimeoutError:
            self.log.error('A Connection To The Database Host Failed!')
            exit(errno.ETIMEDOUT)
        except OperationFailure:
            self.log.error('Database Access Operation Failed!')
            exit(errno.EACCES)
        self.log.info('Successfully Connected to Database') 

def write_command(self, request_id, msg, docs):
        """A proxy for SocketInfo.write_command that handles event publishing.
        """
        if self.publish:
            duration = datetime.datetime.now() - self.start_time
            self._start(request_id, docs)
            start = datetime.datetime.now()
        try:
            reply = self.sock_info.write_command(request_id, msg)
            if self.publish:
                duration = (datetime.datetime.now() - start) + duration
                self._succeed(request_id, reply, duration)
        except OperationFailure as exc:
            if self.publish:
                duration = (datetime.datetime.now() - start) + duration
                self._fail(request_id, exc.details, duration)
            raise
        finally:
            self.start_time = datetime.datetime.now()
        return reply 

def checkOrCreateReplicaSet(self, cluster_object: V1MongoClusterConfiguration) -> None:
        """
        Checks that the replica set is initialized, or initializes it otherwise.
        :param cluster_object: The cluster object from the YAML file.
        :raise ValueError: In case we receive an unexpected response from Mongo.
        :raise ApiException: In case we receive an unexpected response from Kubernetes.
        """
        cluster_name = cluster_object.metadata.name
        namespace = cluster_object.metadata.namespace
        replicas = cluster_object.spec.mongodb.replicas

        create_status_command = MongoResources.createStatusCommand()

        try:
            create_status_response = self._executeAdminCommand(cluster_object, create_status_command)
            logging.debug("Checking replicas, received %s", repr(create_status_response))

            # The replica set could not be checked
            if create_status_response["ok"] != 1:
                raise ValueError("Unexpected response trying to check replicas: '{}'".format(
                    repr(create_status_response)))

            logging.info("The replica set %s @ ns/%s seems to be working properly with %s/%s pods.",
                         cluster_name, namespace, len(create_status_response["members"]), replicas)

            # The amount of replicas is not the same as configured, we need to fix this
            if replicas != len(create_status_response["members"]):
                self._reconfigureReplicaSet(cluster_object)

        except OperationFailure as err:
            if str(err) != self.NO_REPLICA_SET_RESPONSE:
                raise

            # If the replica set is not initialized yet, we initialize it
            self._initializeReplicaSet(cluster_object) 

def update_one(self, *args, **kwargs):  # skipcq: PYL-R0201, PYL-W0613
        raise OperationFailure(Exception()) 

def remove_host(module, client, host_name, timeout=180):
    start_time = dtdatetime.now()
    while True:
        try:
            admin_db = client['admin']
            local_db = client['local']

            if local_db.system.replset.count() > 1:
                module.fail_json(msg='local.system.replset has unexpected contents')

            cfg = local_db.system.replset.find_one()
            if not cfg:
                module.fail_json(msg='no config object retrievable from local.system.replset')

            cfg['version'] += 1

            if len(cfg['members']) == 1:
                module.fail_json(msg="You can't delete last member of replica set")
            for member in cfg['members']:
                if host_name in member['host']:
                    cfg['members'].remove(member)
                else:
                    fail_msg = "couldn't find member with hostname: {0} in replica set members list".format(host_name)
                    module.fail_json(msg=fail_msg)
        except (OperationFailure, AutoReconnect) as e:
            if (dtdatetime.now() - start_time).seconds > timeout:
                module.fail_json(msg='reached timeout while waiting for rs.reconfig(): %s' % str(e))
            time.sleep(5) 

def is_master(self, force=False):
        try:
            if force or not self._is_master:
                self._is_master = self.admin_command('isMaster', True)
        except OperationFailure, e:
            raise DBOperationError("Unable to run isMaster command! Error: %s" % e) 

def admin_command(self, admin_command, quiet=False):
        tries  = 0
        status = None
        while not status and tries < self.retries:
            try:
                status = self._conn['admin'].command(admin_command)
            except OperationFailure, e:
                if not quiet:
                    logging.error("Error running admin command '%s': %s" % (admin_command, e))
                tries += 1
                sleep(1) 

def auth_if_required(self):
        if self.username is not None and self.password is not None:
            try:
                logging.debug("Authenticating connection with username: %s" % self.username)
                self._conn[self.authdb].authenticate(self.username, self.password)
            except OperationFailure, e:
                logging.fatal("Unable to authenticate with host %s: %s" % (self.uri, e))
                raise DBAuthenticationError(e) 

def connect(self):
        try:
            logging.debug("Getting MongoDB connection to %s (replicaSet=%s, readPreference=%s, readPreferenceTags=%s, ssl=%s)" % (
                self.uri,
                self.replset,
                self.read_pref,
                self.do_rp_tags,
                self.do_ssl(),
            ))
            conn = MongoClient(**self.client_opts())
            if self.do_connect:
                conn['admin'].command({"ping": 1})
        except (ConfigurationError, ConnectionFailure, OperationFailure, ServerSelectionTimeoutError), e:
            logging.error("Unable to connect to %s! Error: %s" % (self.uri, e))
            raise DBConnectionError(e) 

def mark_entry_not_calculated(self, key):
        try:
            self.mongo_collection.update_one(
                filter={
                    'func': _MongoCore._get_func_str(self.func),
                    'key': key
                },
                update={
                    '$set': {'being_calculated': False}
                },
                upsert=False  # should not insert in this case
            )
        except OperationFailure:
            pass  # don't care in this case 

def test_mongo_write_failure():
    """Testing MongoDB core handling of writing failure scenarios."""
    with pytest.raises(OperationFailure):
        val1 = _func_w_bad_mongo(1, 2)
        val2 = _func_w_bad_mongo(1, 2)
        assert val1 == val2 

def check_connection(
    host="localhost", port=27017, username=None, password=None, authdb=None, max_delay=1
):
    """Check if a connection could be made to the mongo process specified

    Args:
        host(str)
        port(int)
        username(str)
        password(str)
        authdb (str): database to to for authentication
        max_delay(int): Number of milliseconds to wait for connection

    Returns:
        bool: If connection could be established
    """
    # uri looks like:
    # mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]]
    if username and password:
        uri = "mongodb://{}:{}@{}:{}/{}".format(
            quote_plus(username), quote_plus(password), host, port, authdb
        )
        log_uri = "mongodb://{}:****@{}:{}/{}".format(quote_plus(username), host, port, authdb)
    else:
        log_uri = uri = "mongodb://%s:%s" % (host, port)

    LOG.info("Test connection with uri: %s", log_uri)
    client = MongoClient(uri, serverSelectionTimeoutMS=max_delay)
    try:
        client.server_info()
    except (ServerSelectionTimeoutError, OperationFailure) as err:
        LOG.warning(err)
        return False

    return True