Python bson.ObjectId() Examples

def test_prunes_multiple_versions_ts(library, fw_pointers_cfg):
    with FwPointersCtx(fw_pointers_cfg):
        coll = library._collection
        a = ts1
        c = ts2
        # Create an ObjectId
        now = dt.utcnow()
        with patch("bson.ObjectId", return_value=bson.ObjectId.from_datetime(now - dtd(minutes=125))):
            library.write(symbol, a, prune_previous_version=False)
        with patch("bson.ObjectId", return_value=bson.ObjectId.from_datetime(now - dtd(minutes=122))):
            library.write(symbol, c, prune_previous_version=False)
        with patch("bson.ObjectId", return_value=bson.ObjectId.from_datetime(now - dtd(minutes=121))):
            library.write(symbol, a, prune_previous_version=False)
        with patch("bson.ObjectId", return_value=bson.ObjectId.from_datetime(now - dtd(minutes=119))):
            library.write(symbol, c, prune_previous_version=False)
        assert mongo_count(coll.versions) == 4

        # Prunes all versions older than the most recent version that's older than 10 mins
        library.write(symbol, a, prune_previous_version=True)
        assert mongo_count(coll.versions) == 3
        assert_frame_equal(library.read(symbol, as_of=3).data, a)
        assert_frame_equal(library.read(symbol, as_of=4).data, c)
        assert_frame_equal(library.read(symbol, as_of=5).data, a) 

def _write_audit(self, user, message, changed_version):
        """
        Creates an audit entry, which is much like a snapshot in that
        it references versions and provides some history of the changes made.
        """
        audit = {'_id': bson.ObjectId(),
                 'user': user,
                 'message': message,
                 'symbol': changed_version.symbol
                 }
        orig_version = changed_version.orig_version.version
        new_version = changed_version.new_version.version
        audit['orig_v'] = orig_version
        audit['new_v'] = new_version
        # Update the versions to contain the audit
        mongo_retry(self._versions.update_many)({'symbol': changed_version.symbol,
                                                 'version': {'$in': [orig_version, new_version]}
                                                 },
                                                {'$addToSet': {'parent': audit['_id']}})
        # Create the audit entry
        mongo_retry(self._audit.insert_one)(audit) 

def test_400_on_bad_recipient(self):
        user = self.login()
        dataset = DatasetFactory(owner=user)
        comment = faker.sentence()

        response = self.post(url_for('api.transfers'), {
            'subject': {
                'class': 'Dataset',
                'id': str(dataset.id),
            },
            'recipient': {
                'class': 'User',
                'id': str(ObjectId()),
            },
            'comment': comment
        })

        self.assert400(response)

        data = response.json

        self.assertIn('recipient', data['errors']) 

def test_400_on_bad_subject(self):
        user = self.login()
        recipient = UserFactory()
        comment = faker.sentence()

        response = self.post(url_for('api.transfers'), {
            'subject': {
                'class': 'Dataset',
                'id': str(ObjectId()),
            },
            'recipient': {
                'class': 'User',
                'id': str(recipient.id),
            },
            'comment': comment
        })

        self.assert400(response)

        data = response.json

        self.assertIn('subject', data['errors']) 

def test_list_version(library, fw_pointers_cfg):
    with FwPointersCtx(fw_pointers_cfg):
        assert len(list(library.list_versions(symbol))) == 0
        dates = [None, None, None]
        now = dt.utcnow().replace(tzinfo=mktz('UTC'))
        for x in six.moves.xrange(len(dates)):
            dates[x] = now - dtd(minutes=130 - x)
            with patch("bson.ObjectId", return_value=bson.ObjectId.from_datetime(dates[x])):
                library.write(symbol, ts1, prune_previous_version=False)
        assert len(list(library.list_versions(symbol))) == 3

        library.write(symbol, ts1, prune_previous_version=True)
        assert len(list(library.list_versions(symbol))) >= 2

        versions = list(library.list_versions(symbol))
        for i, x in enumerate([4, 3]):
            assert versions[i]['symbol'] == symbol
            assert versions[i]['date'] >= dates[i]
            assert versions[i]['version'] == x 

def test_list_version_latest_only(library):
    assert len(list(library.list_versions(symbol))) == 0
    dates = [None, None, None]
    now = dt.utcnow().replace(tzinfo=mktz('UTC'))
    for x in six.moves.xrange(len(dates)):
        dates[x] = now - dtd(minutes=20 - x)
        with patch("bson.ObjectId", return_value=bson.ObjectId.from_datetime(dates[x])):
            library.write(symbol, ts1, prune_previous_version=False)
    assert len(list(library.list_versions(symbol))) == 3

    library.write(symbol, ts1, prune_previous_version=True)
    assert len(list(library.list_versions(symbol, latest_only=True))) == 1

    versions = list(library.list_versions(symbol))
    for i, x in enumerate([4, ]):
        assert versions[i]['symbol'] == symbol
        assert versions[i]['date'] >= dates[i]
        assert versions[i]['version'] == x 

def test_prunes_multiple_versions(library, fw_pointers_cfg):
    with FwPointersCtx(fw_pointers_cfg):
        coll = library._collection

        a = [{'a': 'b'}]
        c = [{'c': 'd'}]
        # Create an ObjectId
        now = dt.utcnow()
        with patch("bson.ObjectId", return_value=bson.ObjectId.from_datetime(now - dtd(minutes=125))):
            library.write(symbol, a, prune_previous_version=False)
        with patch("bson.ObjectId", return_value=bson.ObjectId.from_datetime(now - dtd(minutes=122))):
            library.write(symbol, c, prune_previous_version=False)
        with patch("bson.ObjectId", return_value=bson.ObjectId.from_datetime(now - dtd(minutes=121))):
            library.write(symbol, a, prune_previous_version=False)
        with patch("bson.ObjectId", return_value=bson.ObjectId.from_datetime(now - dtd(minutes=119))):
            library.write(symbol, c, prune_previous_version=False)
        assert mongo_count(coll.versions) == 4

        # Prunes all versions older than the most recent version that's older than 10 mins
        library.write(symbol, a, prune_previous_version=True)
        assert mongo_count(coll.versions) == 3
        assert library.read(symbol, as_of=3).data == a
        assert library.read(symbol, as_of=4).data == c
        assert library.read(symbol, as_of=5).data == a 

def test_prune_previous_versions_0_timeout():
    self = create_autospec(VersionStore, _versions=Mock())
    self.name = sentinel.name
    self._versions = create_autospec(Collection)
    self._versions.with_options.return_value.find.__name__ = 'find'
    self._versions.with_options.return_value.find.return_value = []
    with patch('arctic.store.version_store.dt') as dt:
        dt.utcnow.return_value = datetime.datetime(2013, 10, 1)
        VersionStore._find_prunable_version_ids(self, sentinel.symbol, keep_mins=0)
    assert self._versions.with_options.call_args_list == [call(read_preference=ReadPreference.PRIMARY)]
    assert self._versions.with_options.return_value.find.call_args_list == [
                                                  call({'$or': [{'parent': {'$exists': False}},
                                                                {'parent': []}],
                                                        'symbol': sentinel.symbol,
                                                        '_id': {'$lt': bson.ObjectId('524a10810000000000000000')}},
                                                       sort=[('version', -1)],
                                                       skip=1,
                                                       projection={'FW_POINTERS_CONFIG': 1, '_id': 1, 'SEGMENT_SHAS': 1}
                                                       )] 

def generic_in(self, **kwargs):
        '''Bypass buggy GenericReferenceField querying issue'''
        query = {}
        for key, value in kwargs.items():
            if not value:
                continue
            # Optimize query for when there is only one value
            if isinstance(value, (list, tuple)) and len(value) == 1:
                value = value[0]
            if isinstance(value, (list, tuple)):
                if all(isinstance(v, str) for v in value):
                    ids = [ObjectId(v) for v in value]
                    query['{0}._ref.$id'.format(key)] = {'$in': ids}
                elif all(isinstance(v, DBRef) for v in value):
                    query['{0}._ref'.format(key)] = {'$in': value}
                elif all(isinstance(v, ObjectId) for v in value):
                    query['{0}._ref.$id'.format(key)] = {'$in': value}
            elif isinstance(value, ObjectId):
                query['{0}._ref.$id'.format(key)] = value
            elif isinstance(value, str):
                query['{0}._ref.$id'.format(key)] = ObjectId(value)
            else:
                self.error('expect a list of string, ObjectId or DBRef')
        return self(__raw__=query) 

def test_list_symbols_newer_version_with_lower_id(library, fw_pointers_cfg):
    with FwPointersCtx(fw_pointers_cfg):
        now = struct.pack(">i", int(time.time()))
        old_id = bson.ObjectId(now + b"\x00\x00\x00\x00\x00\x00\x00\x00")
        new_id = bson.ObjectId(now + b"\x00\x00\x00\x00\x00\x00\x00\x01")
        object_id_class = Mock()
        object_id_class.from_datetime = bson.ObjectId.from_datetime

        object_id_class.return_value = new_id
        with patch("bson.ObjectId", object_id_class):
            library.write(symbol, ts1)

        library.snapshot('s1')

        object_id_class.return_value = old_id
        with patch("bson.ObjectId", object_id_class):
            library.delete(symbol)

        assert symbol not in library.list_symbols() 

def test_list_versions_localTime():
    # Object ID's are stored in UTC. We need to ensure that the returned times
    # for versions are in the local  TimeZone
    vs = create_autospec(VersionStore, instance=True, _versions=Mock(), _snapshots=Mock())
    mocked_snap_resp = [{'_id': 'abcde', 'name': 'snap'}]
    vs._snapshots.find.return_value = mocked_snap_resp
    vs._snapshots.find_one.return_value = mocked_snap_resp
    date = dt(2013, 4, 1, 9, 0)
    vs._versions.find.return_value = [{'_id': bson.ObjectId.from_datetime(date),
                                       'symbol': 's',
                                       'version': 10,
                                       'metadata': None,
                                       'parent': [mocked_snap_resp[0]['_id']]}]
    version = list(VersionStore.list_versions(vs, "symbol"))[0]
    local_date = date.replace(tzinfo=mktz("UTC"))
    assert version == {'symbol': version['symbol'], 'version': version['version'],
                       # We return naive datetimes in 'default' time, which is London for us
                       'date': local_date,
                       'snapshots': ['snap'],
                       'deleted': False} 

def test_list_versions_no_snapshot():
    vs = create_autospec(VersionStore, instance=True, _versions=Mock(), _snapshots=Mock())
    vs._snapshots.find.return_value = []
    vs._snapshots.find_one.return_value = []
    date = dt(2013, 4, 1, 9, 0)
    vs._versions.find.return_value = [{'_id': bson.ObjectId.from_datetime(date),
                                       'symbol': 's',
                                       'version': 10,
                                       'metadata': None,
                                       'parent': []}]
    version = list(VersionStore.list_versions(vs, "symbol"))[0]
    local_date = date.replace(tzinfo=mktz("UTC"))
    assert version == {'symbol': version['symbol'],
                       'version': version['version'],
                       'date': local_date,
                       'snapshots': [],
                       'deleted': False} 

def test_write_metadata_with_previous_data():
    vs = _create_mock_versionstore()

    expected_new_version = TPL_VERSION.copy()
    expected_new_version.update({'_id': MOCK_OBJID,
                                 'version': TPL_VERSION['version'] + 1,
                                 'metadata': META_TO_WRITE})

    expected_ret_val = VersionedItem(symbol=TEST_SYMBOL,
                                     library=vs._arctic_lib.get_name(),
                                     host=vs._arctic_lib.arctic.mongo_host,
                                     version=TPL_VERSION['version'] + 1,
                                     metadata=META_TO_WRITE,
                                     data=None)

    with patch('arctic.store.version_store.bson.ObjectId') as mock_objId,\
            patch('arctic.store.version_store.mongo_retry') as mock_retry:
        mock_objId.return_value = MOCK_OBJID
        mock_retry.side_effect = lambda f: f
        assert expected_ret_val == VersionStore.write_metadata(vs, symbol=TEST_SYMBOL, metadata=META_TO_WRITE)
        assert vs._versions.insert_one.call_args_list == [call(expected_new_version)]
        assert vs._versions.delete_one.called is False
        assert vs.write.called is False 

def test_restore_last_version():
    vs = _create_mock_versionstore()
    vs._version_nums.find_one.return_value = {'version': TPL_VERSION['version']}

    vs._read_metadata.side_effect = [TPL_VERSION, TPL_VERSION]

    with patch('arctic.store.version_store.bson.ObjectId') as mock_objId, \
            patch('arctic.store.version_store.mongo_retry') as mock_retry:
        mock_objId.return_value = MOCK_OBJID
        mock_retry.side_effect = lambda f: f

        ret_item = VersionStore.restore_version(vs, symbol=TEST_SYMBOL, as_of=TPL_VERSION['version'],
                                               prune_previous_version=True)

        assert ret_item.version == TPL_VERSION['version']
        assert ret_item.metadata == TPL_VERSION.get('metadata')
        assert vs._read_metadata.call_args_list == [call(TEST_SYMBOL, as_of=TPL_VERSION['version'])]
        assert vs._version_nums.find_one.call_args_list == [call({'symbol': TEST_SYMBOL})]
        assert not vs.read.called
        assert not vs.write.called 

def get_detail(cls, **kwargs):
        data_type = kwargs['data_type']
        if data_type == 'request':
            model_cls = AccessLogRequest
        else:
            model_cls = AccessLogResponse

        entry = model_cls()
        body_id = kwargs.get('body_id')
        if body_id and body_id != 'None':
            entry.body.grid_id = ObjectId(body_id)
            body = entry.body.read()
            return body

        headers_id = kwargs.get('headers_id')
        if headers_id and headers_id != 'None':
            entry.headers.grid_id = ObjectId(headers_id)
            headers = entry.headers.read()
            return headers

        return '' 

def from_id(cls, ctx, _id, meta_only=False):
        if not isinstance(_id, ObjectId):
            _id = ObjectId(_id)

        if meta_only:
            obj = await cls.data_coll(ctx).find_one({"_id": _id}, ['_id', 'name', 'owner', 'public'])
        else:
            obj = await cls.data_coll(ctx).find_one({"_id": _id})
        if obj is None:
            raise NoActiveBrew()

        if not meta_only:
            return cls.from_dict(obj)
        return obj

    # helpers 

def get_user_id_via_auth_cookie(request: Request) -> Optional[bson.ObjectId]:
    if auth_cookie_name not in request.cookies:
        return None

    val = request.cookies[auth_cookie_name]
    parts = val.split(':')
    if len(parts) != 2:
        return None

    user_id = parts[0]
    hash_val = parts[1]
    hash_val_check = __hash_text(user_id)
    if hash_val != hash_val_check:
        print("Warning: Hash mismatch, invalid cookie value")
        return None

    try:
        return bson.ObjectId(user_id)
    except:
        return None 

def add_message(self, request):
        """Registers a new message for the user."""
        session = await get_session(request)
        user_id = session.get('user_id')
        if not user_id:
            raise web.HTTPNotAuthorized()
        form = await request.post()

        if form.get('text'):
            user = await self.mongo.user.find_one(
                {'_id': ObjectId(session['user_id'])},
                {'email': 1, 'username': 1})

            await self.mongo.message.insert_one(
                {'author_id': ObjectId(user_id),
                 'email': user['email'],
                 'username': user['username'],
                 'text': form['text'],
                 'pub_date': datetime.datetime.utcnow()})
        return redirect(request, 'timeline') 

def unfollow_user(self, request):
        """Removes the current user as follower of the given user."""
        username = request.match_info['username']
        session = await get_session(request)
        user_id = session.get('user_id')
        if not user_id:
            raise web.HTTPNotAuthorized()

        whom_id = await db.get_user_id(self.mongo.user, username)
        if whom_id is None:
            raise web.HTTPFound()

        await self.mongo.follower.update_many(
            {'who_id': ObjectId(session['user_id'])},
            {'$pull': {'whom_id': whom_id}})
        return redirect(request, 'user_timeline', username=username) 

def test_not_found(self):
        form = self.form.from_json({'nested': {
            'class': 'Nested',
            'id': str(ObjectId())
        }})

        form.validate()
        self.assertIn('nested', form.errors)
        self.assertEqual(len(form.errors['nested']), 1) 

def generate_followers(mongo, schema, rows, fake, user_ids):
    values = []
    for user_id in user_ids:
        entry = schema({'_id': ObjectId(),
                        'who_id': user_id,
                        'whom_id': []})
        for i in range(rows):
            entry['whom_id'].append(random.choice(user_ids))
        values.append(entry)
    await insert_data(mongo, values) 

def generate_messages(mongo, schema, rows, fake, users):
    values = []
    for user in users:
        for i in range(rows):
            values.append(schema({
                '_id': ObjectId(),
                'author_id': ObjectId(user['_id']),
                'username': user['username'],
                'text': fake.text(max_nb_chars=140),
                'pub_date': fake.iso8601(),
            }))

    ids = await insert_data(mongo, values)
    return ids 

def delete(self, id):
        store.config_blocks.remove(ObjectId(id))

        return 'ok' 

def test_jsonify_object_id():
    obj = {'foo': ObjectId('1' * 24)}
    jsoned = jsonify(obj)
    assert jsoned == '{"foo": "111111111111111111111111"}' 

def set_auth(request: Request, user_id: bson.ObjectId):
    hash_val = __hash_text(str(user_id))
    val = "{}:{}".format(user_id, hash_val)

    request.add_response_callback(lambda req, resp: __add_cookie_callback(
        req, resp, auth_cookie_name, val
    )) 

def put(self):
        '''Set the homepage reuses editorial selection'''
        if not isinstance(request.json, list):
            api.abort(400, 'Expect a list of reuse IDs')
        ids = [ObjectId(id) for id in request.json]
        current_site.settings.home_reuses = Reuse.objects.bulk_list(ids)
        current_site.save()
        return current_site.settings.home_reuses 

def default(self, obj):
        if isinstance(obj, ObjectId):
            return force_text(obj)
        return super(MongoDBJSONEncoder, self).default(obj) 

def getObjectID(self, _id):
        return ObjectId(_id) 

def get_or_404(objectmanager, *args, **kwargs):
    if '_id' in kwargs:
        kwargs['_id'] = ObjectId(kwargs['_id'])

    result = objectmanager.find_one(kwargs)
    if result:
        return result
    else:
        abort(404) 

def setUp(self):
        """Set up."""
        self.doc = NormalSchema(uid=ObjectId(), name="test")
        self.expected = {
            "_id": {"$oid": str(self.doc.uid)},
            "name": self.doc.name,
        }