Python pymongo.errors.AutoReconnect() Examples

def check_server_up(self):
        """Test connection to the server."""
        import pymongo
        from pymongo.errors import AutoReconnect, ConnectionFailure

        # Hostname must exist before continuing
        # Some server class (e.g. Docker) will only allocate an IP after the
        # container has started.
        if not self.hostname:
            return False

        log.info("Connecting to Mongo at %s:%s" % (self.hostname, self.port))
        try:
            self.api = pymongo.MongoClient(self.hostname, self.port,
                                           serverselectiontimeoutms=200)
            self.api.list_database_names()
            # Configure the client with default timeouts in case the server goes slow
            self.api = pymongo.MongoClient(self.hostname, self.port)
            return True
        except (AutoReconnect, ConnectionFailure) as e:
            pass
        return False 

def __call__(self, *args, **kwargs):
        """ Call the method and handle the AutoReconnect exception gracefully """
        start_time = time.time()

        for attempt in count():
            try:
                return self._method(*args, **kwargs)
            except AutoReconnect:
                duration = time.time() - start_time

                if duration >= WAIT_TIME:
                    break

                logger.warning(
                    'Reconnecting to MongoDB, attempt {} ({:.3f} seconds elapsed)'.
                    format(attempt, duration))

                time.sleep(self.calc_sleep(attempt))

        return self._method(*args, **kwargs) 

def update(self, identifier, updates):
        updates = copy(updates)
        unset = {}
        for key, value in [(k, v) for k, v in updates.items()]:
            if value is None:
                del updates[key]
                unset[key] = ''
        doc = {}
        if updates:
            doc['$set'] = updates
        if unset:
            doc['$unset'] = unset
        if not doc:
            return
        while True:
            try:
                self.collection.update_one({'id': identifier}, doc)
                return
            except AutoReconnect:  # pragma: no cover
                log.exception('update failure, retrying')
                time.sleep(1) 

def _receive_data_on_socket(sock, length):
    msg = b""
    while length:
        try:
            chunk = sock.recv(length)
        except (IOError, OSError) as exc:
            err = None
            if hasattr(exc, 'errno'):
                err = exc.errno
            elif exc.args:
                err = exc.args[0]
            if err == errno.EINTR:
                continue
            raise
        if chunk == b"":
            raise AutoReconnect("connection closed")

        length -= len(chunk)
        msg += chunk

    return msg 

def get_library(self, library):
        """
        Return the library instance.  Can generally use slicing to return the library:
            arctic_store[library]

        Parameters
        ----------
        library : `str`
            The name of the library. e.g. 'library' or 'user.library'
        """
        if library in self._library_cache:
            return self._library_cache[library]

        try:
            error = None
            lib = ArcticLibraryBinding(self, library)
            lib_type = lib.get_library_type()
        except (OperationFailure, AutoReconnect) as e:
            error = e

        if error:
            raise LibraryNotFoundException("Library %s was not correctly initialized in %s.\nReason: %r)" %
                                           (library, self, error))
        elif not lib_type:
            raise LibraryNotFoundException("Library %s was not correctly initialized in %s." %
                                           (library, self))
        elif lib_type not in LIBRARY_TYPES:
            raise LibraryNotFoundException("Couldn't load LibraryType '%s' for '%s' (has the class been registered?)" %
                                           (lib_type, library))
        instance = LIBRARY_TYPES[lib_type](lib)
        self._library_cache[library] = instance
        # The library official name may be different from 'library': e.g. 'library' vs 'user.library'
        self._library_cache[lib.get_name()] = instance
        return self._library_cache[library] 

def dbUpdate(self, dbName, collectionName, d, flt, upsert=False):
        """向MongoDB中更新数据，d是具体数据，flt是过滤条件，upsert代表若无是否要插入"""
        try:
            if self.dbClient:
                db = self.dbClient[dbName]
                collection = db[collectionName]
                collection.replace_one(flt, d, upsert)
            else:
                self.writeLog(text.DATA_UPDATE_FAILED)
                if self.db_has_connected:
                    self.writeLog(u'重新尝试连接数据库')
                    self.dbConnect()

        except AutoReconnect as ex:
            self.writeError(u'数据库连接断开重连:{}'.format(str(ex)))
            time.sleep(1)
        except ConnectionFailure:
            self.dbClient = None
            self.writeError(u'数据库连接断开')
            if self.db_has_connected:
                self.writeLog(u'重新尝试连接数据库')
                self.dbConnect()
        except Exception as ex:
            self.writeError(u'dbUpdate exception:{}'.format(str(ex))) 

def test_mongo_retry():
    retries = [2]
    self = MagicMock()
    self._arctic_lib.arctic.mongo_host = sentinel.host
    self._collection.database.client.nodes = set([('a', 12)])
    self._arctic_lib.get_name.return_value = sentinel.lib_name
    with patch('arctic.decorators._handle_error', autospec=True) as he:
        @mongo_retry
        def foo(self):
            if retries[0] == 2:
                retries[0] -= 1
                raise OperationFailure('error')
            elif retries[0] == 1:
                retries[0] -= 1
                raise AutoReconnect('error')
            return "success"
        foo(self)
    assert he.call_count == 2
    assert isinstance(he.call_args_list[0][0][1], OperationFailure)
    assert he.call_args_list[0][0][2] == 1
    assert he.call_args_list[0][1] == {'mnodes': ['a:12'],
                                       'mhost': 'sentinel.host',
                                       'l': sentinel.lib_name}
    assert isinstance(he.call_args_list[1][0][1], AutoReconnect)
    assert he.call_args_list[1][0][2] == 2 

def test__conn_auth_issue():
    auth_timeout = [0]

    a = Arctic("host:12345")
    sentinel.creds = Mock()

    def flaky_auth(*args, **kwargs):
        if not auth_timeout[0]:
            auth_timeout[0] = 1
            raise AutoReconnect()

    with patch('arctic.arctic.authenticate', flaky_auth), \
    patch('arctic.arctic.get_auth', return_value=sentinel.creds), \
    patch('arctic._cache.Cache.__init__', autospec=True, return_value=None), \
    patch('arctic.decorators._handle_error') as he:
        a._conn
        assert he.call_count == 1
        assert auth_timeout[0] 

def test_mongo_retry_hook_changes():
    retries = [2]
    self = MagicMock()
    hook1 = Mock()
    register_log_exception_hook(hook1)
    hook2 = Mock()

    @mongo_retry
    def foo(self):
        if retries[0] == 2:
            retries[0] -= 1
            raise OperationFailure('error')
        elif retries[0] == 1:
            register_log_exception_hook(hook2)
            retries[0] -= 1
            raise AutoReconnect('error')
        return "success"
    foo(self)

    assert hook1.call_count == 1
    assert hook2.call_count == 1 

def dbInsert(self, dbName, collectionName, d):
        """向MongoDB中插入数据，d是具体数据"""
        try:
            # mongodb客户端
            if self.dbClient:
                db = self.dbClient[dbName]
                collection = db[collectionName]
                collection.insert_one(d)
            else:
                self.writeLog(text.DATA_INSERT_FAILED)
                # 判断是否连接
                if self.db_has_connected:
                    self.writeLog(u'重新尝试连接数据库')
                    # 重新连接
                    self.dbConnect()

        except AutoReconnect as ex:
            self.writeError(u'数据库连接断开重连:{}'.format(str(ex)))
            time.sleep(1)
        except ConnectionFailure:
            self.dbClient = None
            self.writeError(u'数据库连接断开')
            if self.db_has_connected:
                self.writeLog(u'重新尝试连接数据库')
                self.dbConnect()
        except Exception as ex:
            self.writeError(u'dbInsert exception:{}'.format(str(ex))) 

def mongodb_stats(host, p, database, username, password):
    port = int(p)
    try:
        if username and password and database:
            c = Client("mongodb://"+username+":"+password+"@"+host+"/"+database, port)
        elif username and password:
            c = Client('mongodb://'+username+':'+password+'@'+host+'/', port)
        elif database:
            c = Client('mongodb://'+host+'/'+database, port)
        else:
            c = Client(host, port)
    except ConnectionFailure, AutoReconnect:
        return None 

def delete(self, identifier):
        while True:
            try:
                result = self.collection.remove({'id': identifier})
                if result['n'] == 0:
                    raise KeyError('No such canary {}'.format(identifier))
                return
            except AutoReconnect:  # pragma: no cover
                log.exception('remove failure, retrying')
                time.sleep(1) 

def list(self, *, verbose=False, paused=None, late=None, search=None,
             order_by=None):
        if verbose:
            fields = {'_id': False}
        else:
            fields = {'_id': False, 'name': True, 'id': True}

        spec = {}

        if paused is not None:
            spec['paused'] = paused

        if late is not None:
            spec['late'] = late

        if order_by is not None:
            order_by = [(order_by, ASCENDING)]

        if search is not None:
            search = re.compile(search)
            spec['$or'] = [{'name': search}, {'slug': search}, {'id': search},
                           {'emails': search}]

        skip = 0

        while True:
            try:
                for canary in self.collection.find(spec, projection=fields,
                                                   sort=order_by, skip=skip):
                    yield canary
                break
            except AutoReconnect:  # pragma: no cover
                log.exception('find failure, retrying')
                time.sleep(1) 

def get(self, identifier):
        while True:
            try:
                canary = self.collection.find_one({'id': identifier},
                                                  projection={'_id': False})
                if not canary:
                    raise KeyError('No such canary {}'.format(identifier))
                return canary
            except AutoReconnect:  # pragma: no cover
                log.exception('find_one failure, retrying')
                time.sleep(1) 

def create(self, canary):
        canary['_id'] = bson.ObjectId()
        while True:
            try:
                self.collection.insert_one(canary)
                del canary['_id']
                break
            except AutoReconnect:  # pragma: no cover
                log.exception('save failure, retrying')
                time.sleep(1) 

def rs_reconfigure(client, rs_config):
    try:
        client.admin.command('replSetReconfig', rs_config)
    except AutoReconnect:
        pass 

def remove_host(module, client, host_name, timeout=180):
    start_time = dtdatetime.now()
    while True:
        try:
            admin_db = client['admin']
            local_db = client['local']

            if local_db.system.replset.count() > 1:
                module.fail_json(msg='local.system.replset has unexpected contents')

            cfg = local_db.system.replset.find_one()
            if not cfg:
                module.fail_json(msg='no config object retrievable from local.system.replset')

            cfg['version'] += 1

            if len(cfg['members']) == 1:
                module.fail_json(msg="You can't delete last member of replica set")
            for member in cfg['members']:
                if host_name in member['host']:
                    cfg['members'].remove(member)
                else:
                    fail_msg = "couldn't find member with hostname: {0} in replica set members list".format(host_name)
                    module.fail_json(msg=fail_msg)
        except (OperationFailure, AutoReconnect) as e:
            if (dtdatetime.now() - start_time).seconds > timeout:
                module.fail_json(msg='reached timeout while waiting for rs.reconfig(): %s' % str(e))
            time.sleep(5) 

def __init__(self, obj, methods):
        """ Initialize the MongoReconnectProxy.

        Args:
            obj: The object for which all calls should be wrapped in the AutoReconnect
                 exception handling block.
            methods (set): The list of method names that should be wrapped.
        """
        self._unproxied_object = obj
        self._methods = methods 

def __send_message(self, msg):
        """Send a getmore message and handle the response.
        """
        client = self.__collection.database.connection
        try:
            res = client._send_message_with_response(
                msg, _connection_to_use=self.__conn_id)
            self.__conn_id, (response, dummy0, dummy1) = res
        except AutoReconnect:
            # Don't try to send kill cursors on another socket
            # or to another server. It can cause a _pinValue
            # assertion on some server releases if we get here
            # due to a socket timeout.
            self.__killed = True
            raise

        try:
            response = helpers._unpack_response(response,
                                                self.__id,
                                                *self.__decode_opts)
        except CursorNotFound:
            self.__killed = True
            raise
        except AutoReconnect:
            # Don't send kill cursors to another server after a "not master"
            # error. It's completely pointless.
            self.__killed = True
            client.disconnect()
            raise
        self.__id = response["cursor_id"]

        assert response["starting_from"] == self.__retrieved, (
            "Result batch started from %s, expected %s" % (
                response['starting_from'], self.__retrieved))

        self.__retrieved += response["number_returned"]
        self.__data = deque(response["data"]) 

def dbQueryBySort(self, dbName, collectionName, d, sortName, sortType, limitNum=0):
        """从MongoDB中读取数据，d是查询要求，sortName是排序的字段,sortType是排序类型
          返回的是数据库查询的指针"""
        try:
            if self.dbClient:
                db = self.dbClient[dbName]
                collection = db[collectionName]
                if limitNum > 0:
                    cursor = collection.find(d).sort(sortName, sortType).limit(limitNum)
                else:
                    cursor = collection.find(d).sort(sortName, sortType)
                if cursor:
                    return list(cursor)
                else:
                    return []
            else:
                self.writeLog(text.DATA_QUERY_FAILED)
                if self.db_has_connected:
                    self.writeLog(u'重新尝试连接数据库')
                    self.dbConnect()

        except AutoReconnect as ex:
            self.writeError(u'数据库连接断开重连:{}'.format(str(ex)))
            time.sleep(1)
        except ConnectionFailure:
            self.dbClient = None
            self.writeError(u'数据库连接断开')
            if self.db_has_connected:
                self.writeLog(u'重新尝试连接数据库')
                self.dbConnect()
        except Exception as ex:
            self.writeError(u'dbQueryBySort exception:{}'.format(str(ex)))

        return []

    # ---------------------------------------------------------------------- 

def dbQuery(self, dbName, collectionName, d):
        """从MongoDB中读取数据，d是查询要求，返回的是数据库查询的指针"""
        try:
            if self.dbClient:
                db = self.dbClient[dbName]
                collection = db[collectionName]
                cursor = collection.find(d)
                if cursor:
                    return list(cursor)
                else:
                    return []
            else:
                self.writeLog(text.DATA_QUERY_FAILED)
                if self.db_has_connected:
                    self.writeLog(u'重新尝试连接数据库')
                    self.dbConnect()

        except AutoReconnect as ex:
            self.writeError(u'数据库连接断开重连:{}'.format(str(ex)))
            time.sleep(1)
        except ConnectionFailure:
            self.dbClient = None
            self.writeError(u'数据库连接断开')
            if self.db_has_connected:
                self.writeLog(u'重新尝试连接数据库')
                self.dbConnect()
        except Exception as ex:
            self.writeError(u'dbQuery exception:{}'.format(str(ex)))

        return [] 

def dbInsertMany(self, dbName, collectionName, data_list, ordered=True):
        """
        向MongoDB中插入数据，data_list是具体数据 列表
        :param dbName:
        :param collectionName:
        :param data_list:
        :param ordered: 是否忽略insert error
        :return:
        """
        if not isinstance(data_list, list):
            self.writeLog(text.DATA_INSERT_FAILED)
            return
        try:
            if self.dbClient:
                db = self.dbClient[dbName]
                collection = db[collectionName]
                collection.insert_many(data_list, ordered=ordered)
            else:
                self.writeLog(text.DATA_INSERT_FAILED)
                if self.db_has_connected:
                    self.writeLog(u'重新尝试连接数据库')
                    self.dbConnect()

        except AutoReconnect as ex:
            self.writeError(u'数据库连接断开重连:{}'.format(str(ex)))
            time.sleep(1)
        except ConnectionFailure:
            self.dbClient = None
            self.writeError(u'数据库连接断开')
            if self.db_has_connected:
                self.writeLog(u'重新尝试连接数据库')
                self.dbConnect()
        except Exception as ex:
            self.writeError(u'dbInsertMany exception:{}'.format(str(ex)))

    # ---------------------------------------------------------------------- 

def _send_message_with_response(self, message, _connection_to_use=None,
                                    _must_use_master=False, **kwargs):
        """Receive a message from Mongo.

        Sends the given message and returns a (connection_id, response) pair.

        :Parameters:
          - `operation`: opcode of the message to send
          - `data`: data to send
        """
        if _connection_to_use is not None:
            if _connection_to_use == -1:
                member = self.__master
                conn = -1
            else:
                member = self.__slaves[_connection_to_use]
                conn = _connection_to_use
            return (conn,
                    member._send_message_with_response(message, **kwargs)[1])

        # _must_use_master is set for commands, which must be sent to the
        # master instance. any queries in a request must be sent to the
        # master since that is where writes go.
        if _must_use_master or self.in_request():
            return (-1, self.__master._send_message_with_response(message,
                                                                  **kwargs)[1])

        # Iterate through the slaves randomly until we have success. Raise
        # reconnect if they all fail.
        for connection_id in helpers.shuffled(range(len(self.__slaves))):
            try:
                slave = self.__slaves[connection_id]
                return (connection_id,
                        slave._send_message_with_response(message,
                                                          **kwargs)[1])
            except AutoReconnect:
                pass

        raise AutoReconnect("failed to connect to slaves") 

def add_host(module, client, host_name, host_port, host_type, timeout=180, **kwargs):
    start_time = dtdatetime.now()
    while True:
        try:
            admin_db = client['admin']
            local_db = client['local']

            if local_db.system.replset.count() > 1:
                module.fail_json(msg='local.system.replset has unexpected contents')

            cfg = local_db.system.replset.find_one()
            if not cfg:
                module.fail_json(msg='no config object retrievable from local.system.replset')

            cfg['version'] += 1
            max_id = max(cfg['members'], key=lambda x:x['_id'])
            new_host = { '_id': max_id['_id'] + 1, 'host': "{0}:{1}".format(host_name, host_port) }
            if host_type == 'arbiter':
                new_host['arbiterOnly'] = True

            if not kwargs['build_indexes']:
                new_host['buildIndexes'] = False

            if kwargs['hidden']:
                new_host['hidden'] = True

            if kwargs['priority'] != 1.0:
                new_host['priority'] = kwargs['priority']

            if kwargs['slave_delay'] != 0:
                new_host['slaveDelay'] = kwargs['slave_delay']

            if kwargs['votes'] != 1:
                new_host['votes'] = kwargs['votes']

            cfg['members'].append(new_host)
            admin_db.command('replSetReconfig', cfg)
            return
        except (OperationFailure, AutoReconnect) as e:
            if (dtdatetime.now() - start_time).seconds > timeout:
                module.fail_json(msg='reached timeout while waiting for rs.reconfig(): %s' % str(e))
            time.sleep(5) 

def run(self):
        try:
            logging.info("Tailing oplog on %s for changes" % self.uri)
            self.timer.start(self.timer_name)

            self.state.set('running', True)
            self.connect()
            oplog = self.oplog()
            while not self.tail_stop.is_set() and not self.backup_stop.is_set():
                try:
                    self._cursor = self.db.get_oplog_cursor_since(self.__class__, self.last_ts)
                    while self.check_cursor():
                        try:
                            # get the next oplog doc and write it
                            doc = self._cursor.next()
                            if self.last_ts and self.last_ts >= doc['ts']:
                                continue
                            oplog.add(doc)

                            # update states
                            self.count  += 1
                            self.last_ts = doc['ts']
                            if self.first_ts is None:
                                self.first_ts = self.last_ts
                            update = {
                                'count':    self.count,
                                'first_ts': self.first_ts,
                                'last_ts':  self.last_ts
                            }
                            self.state.set(None, update, True)

                            # print status report every N seconds
                            self.status()
                        except NotMasterError:
                            # pymongo.errors.NotMasterError means a RECOVERING-state when connected to secondary (which should be true)
                            self.backup_stop.set()
                            logging.error("Node %s is in RECOVERING state! Stopping tailer thread" % self.uri)
                            raise OperationError("Node %s is in RECOVERING state! Stopping tailer thread" % self.uri)
                        except CursorNotFound:
                            self.backup_stop.set()
                            logging.error("Cursor disappeared on server %s! Stopping tailer thread" % self.uri)
                            raise OperationError("Cursor disappeared on server %s! Stopping tailer thread" % self.uri)
                        except (AutoReconnect, ConnectionFailure, ExceededMaxWaiters, ExecutionTimeout, NetworkTimeout), e:
                            logging.error("Tailer %s received %s exception: %s. Attempting retry" % (self.uri, type(e).__name__, e))
                            if self._tail_retry > self._tail_retry_max:
                                self.backup_stop.set()
                                logging.error("Reconnected to %s %i/%i times, stopping backup!" % (self.uri, self._tail_retry, self._tail_retry_max))
                                raise OperationError("Reconnected to %s %i/%i times, stopping backup!" % (self.uri, self._tail_retry, self._tail_retry_max))
                            self._tail_retry += 1
                        except StopIteration:
                            continue
                    sleep(1)
                finally:
                    if self._cursor:
                        logging.debug("Stopping oplog cursor on %s" % self.uri)
                        self._cursor.close() 

def read(self, symbol, as_of=None, date_range=None, from_version=None, allow_secondary=None, **kwargs):
        """
        Read data for the named symbol.  Returns a VersionedItem object with
        a data and metdata element (as passed into write).

        Parameters
        ----------
        symbol : `str`
            symbol name for the item
        as_of : `str` or `int` or `datetime.datetime`
            Return the data as it was as_of the point in time.
            `int` : specific version number
            `str` : snapshot name which contains the version
            `datetime.datetime` : the version of the data that existed as_of the requested point in time
        date_range: `arctic.date.DateRange`
            DateRange to read data for.  Applies to Pandas data, with a DateTime index
            returns only the part of the data that falls in the DateRange.
        allow_secondary : `bool` or `None`
            Override the default behavior for allowing reads from secondary members of a cluster:
            `None` : use the settings from the top-level `Arctic` object used to query this version store.
            `True` : allow reads from secondary members
            `False` : only allow reads from primary members

        Returns
        -------
        VersionedItem namedtuple which contains a .data and .metadata element
        """
        try:
            read_preference = self._read_preference(allow_secondary)
            _version = self._read_metadata(symbol, as_of=as_of, read_preference=read_preference)
            return self._do_read(symbol, _version, from_version,
                                 date_range=date_range, read_preference=read_preference, **kwargs)
        except (OperationFailure, AutoReconnect) as e:
            # Log the exception so we know how often this is happening
            log_exception('read', e, 1)
            # If we've failed to read from the secondary, then it's possible the
            # secondary has lagged.  In this case direct the query to the primary.
            _version = mongo_retry(self._read_metadata)(symbol, as_of=as_of,
                                                        read_preference=ReadPreference.PRIMARY)
            return self._do_read_retry(symbol, _version, from_version,
                                       date_range=date_range,
                                       read_preference=ReadPreference.PRIMARY,
                                       **kwargs)
        except Exception as e:
            log_exception('read', e, 1)
            raise