Python bson.SON Examples

def objify(self, doc, columns=None):
        """
        Decode a Pymongo SON object into an Pandas DataFrame
        """
        cols = columns or doc[METADATA][COLUMNS]
        data = {}

        for col in cols:
            # if there is missing data in a chunk, we can default to NaN
            # and pandas will autofill the missing values to the correct length
            if col not in doc[METADATA][LENGTHS]:
                d = [np.nan]
            else:
                d = decompress(doc[DATA][doc[METADATA][LENGTHS][col][0]: doc[METADATA][LENGTHS][col][1] + 1])
                # d is ready-only but that's not an issue since DataFrame will copy the data anyway.
                d = np.frombuffer(d, doc[METADATA][DTYPE][col])

                if MASK in doc[METADATA] and col in doc[METADATA][MASK]:
                    mask_data = decompress(doc[METADATA][MASK][col])
                    mask = np.frombuffer(mask_data, 'bool')
                    d = ma.masked_array(d, mask)
            data[col] = d

        # Copy into
        return pd.DataFrame(data, columns=cols, copy=True)[cols] 

def count_by_state_unsynced(self, arg):
        exp_key = self._exp_key
        # TODO: consider searching by SON rather than dict
        if isinstance(arg, int):
            if arg not in JOB_STATES:
                raise ValueError('invalid state', arg)
            query = dict(state=arg)
        else:
            assert hasattr(arg, '__iter__')
            states = list(arg)
            assert all([x in JOB_STATES for x in states])
            query = dict(state={'$in': states})
        if exp_key != None:
            query['exp_key'] = exp_key
        rval = self.handle.jobs.find(query).count()
        return rval 

def dumps(obj, *args, **kwargs):
    """Helper function that wraps :func:`json.dumps`.

    Recursive function that handles all BSON types including
    :class:`~bson.binary.Binary` and :class:`~bson.code.Code`.

    :Parameters:
      - `json_options`: A :class:`JSONOptions` instance used to modify the
        encoding of MongoDB Extended JSON types. Defaults to
        :const:`DEFAULT_JSON_OPTIONS`.

    .. versionchanged:: 3.4
       Accepts optional parameter `json_options`. See :class:`JSONOptions`.

    .. versionchanged:: 2.7
       Preserves order when rendering SON, Timestamp, Code, Binary, and DBRef
       instances.
    """
    json_options = kwargs.pop("json_options", DEFAULT_JSON_OPTIONS)
    return json.dumps(_json_convert(obj, json_options), *args, **kwargs) 

def dumps(obj, *args, **kwargs):
    """Helper function that wraps :func:`json.dumps`.

    Recursive function that handles all BSON types including
    :class:`~bson.binary.Binary` and :class:`~bson.code.Code`.

    :Parameters:
      - `json_options`: A :class:`JSONOptions` instance used to modify the
        encoding of MongoDB Extended JSON types. Defaults to
        :const:`DEFAULT_JSON_OPTIONS`.

    .. versionchanged:: 3.4
       Accepts optional parameter `json_options`. See :class:`JSONOptions`.

    .. versionchanged:: 2.7
       Preserves order when rendering SON, Timestamp, Code, Binary, and DBRef
       instances.
    """
    json_options = kwargs.pop("json_options", DEFAULT_JSON_OPTIONS)
    return json.dumps(_json_convert(obj, json_options), *args, **kwargs) 

def _match_map(self, doc, other_doc):
        for key, val in doc.items():
            if val is absent:
                if key in other_doc:
                    return False
            elif not self._match_val(val, other_doc.get(key, None)):
                return False

        if isinstance(doc, (OrderedDict, bson.SON)):
            if not isinstance(other_doc, (OrderedDict, bson.SON)):
                raise TypeError(
                    "Can't compare ordered and unordered document types:"
                    " %r, %r" % (doc, other_doc))
            keys = [key for key, val in doc.items()
                    if val is not absent]
            if not seq_match(keys, list(other_doc.keys())):
                return False

        return True 

def dumps(obj, *args, **kwargs):
    """Helper function that wraps :func:`json.dumps`.

    Recursive function that handles all BSON types including
    :class:`~bson.binary.Binary` and :class:`~bson.code.Code`.

    :Parameters:
      - `json_options`: A :class:`JSONOptions` instance used to modify the
        encoding of MongoDB Extended JSON types. Defaults to
        :const:`DEFAULT_JSON_OPTIONS`.

    .. versionchanged:: 3.4
       Accepts optional parameter `json_options`. See :class:`JSONOptions`.

    .. versionchanged:: 2.7
       Preserves order when rendering SON, Timestamp, Code, Binary, and DBRef
       instances.
    """
    json_options = kwargs.pop("json_options", DEFAULT_JSON_OPTIONS)
    return json.dumps(_json_convert(obj, json_options), *args, **kwargs) 

def validate_is_mapping(option, value):
    """Validate the type of method arguments that expect a document."""
    if not isinstance(value, abc.Mapping):
        raise TypeError("%s must be an instance of dict, bson.son.SON, or "
                        "other type that inherits from "
                        "collections.Mapping" % (option,)) 

def connection_with_tunnel(dbname, host='localhost',
                           auth_dbname=None, port=27017,
                           ssh=False, user='hyperopt', pw=None):
    if ssh:
        local_port = numpy.random.randint(low=27500, high=28000)
        # -- forward from local to remote machine
        ssh_tunnel = subprocess.Popen(
            ['ssh', '-NTf', '-L',
                    '%i:%s:%i' % (local_port, '127.0.0.1', port),
                    host],
        )
        # -- give the subprocess time to set up
        time.sleep(.5)
        connection = pymongo.MongoClient('127.0.0.1', local_port,
                                         document_class=SON, w=1, j=True)
    else:
        connection = pymongo.MongoClient(host, port, document_class=SON, w=1, j=True)
        if user:
            if not pw:
                pw = read_pw()
                
            if user == 'hyperopt' and not auth_dbname:
                auth_dbname = 'admin'
                    
            connection[dbname].authenticate(user, pw, source=auth_dbname)
            
        ssh_tunnel = None

    # Note that the w=1 and j=True args to MongoClient above should:
    # -- Ensure that changes are written to at least one server.
    # -- Ensure that changes are written to the journal if there is one.

    return connection, ssh_tunnel 

def _json_convert(obj, json_options=DEFAULT_JSON_OPTIONS):
    """Recursive helper method that converts BSON types so they can be
    converted into json.
    """
    if hasattr(obj, 'iteritems') or hasattr(obj, 'items'):  # PY3 support
        return SON(((k, _json_convert(v, json_options))
                    for k, v in iteritems(obj)))
    elif hasattr(obj, '__iter__') and not isinstance(obj, (text_type, bytes)):
        return list((_json_convert(v, json_options) for v in obj))
    try:
        return default(obj, json_options)
    except TypeError:
        return obj 

def deserialize(self, data, columns=None):
        """
        Deserializes SON to a DataFrame

        Parameters
        ----------
        data: SON data
        columns: None, or list of strings
            optionally you can deserialize a subset of the data in the SON. Index
            columns are ALWAYS deserialized, and should not be specified

        Returns
        -------
        pandas dataframe or series
        """
        if not data:
            return pd.DataFrame()

        meta = data[0][METADATA] if isinstance(data, list) else data[METADATA]
        index = INDEX in meta

        if columns:
            if index:
                columns = columns[:]
                columns.extend(meta[INDEX])
            if len(columns) > len(set(columns)):
                raise Exception("Duplicate columns specified, cannot de-serialize")

        if not isinstance(data, list):
            df = self.converter.objify(data, columns)
        else:
            df = pd.concat([self.converter.objify(d, columns) for d in data], ignore_index=not index)

        if index:
            df = df.set_index(meta[INDEX])
        if meta[TYPE] == 'series':
            return df[df.columns[0]]
        return df 

def _encode_binary(data, subtype, json_options):
    if json_options.json_mode == JSONMode.LEGACY:
        return SON([
            ('$binary', base64.b64encode(data).decode()),
            ('$type', "%02x" % subtype)])
    return {'$binary': SON([
        ('base64', base64.b64encode(data).decode()),
        ('subType', "%02x" % subtype)])} 

def dumps(obj, *args, **kwargs):
    """Helper function that wraps :class:`json.dumps`.

    Recursive function that handles all BSON types including
    :class:`~bson.binary.Binary` and :class:`~bson.code.Code`.

    .. versionchanged:: 2.7
       Preserves order when rendering SON, Timestamp, Code, Binary, and DBRef
       instances. (But not in Python 2.4.)
    """
    if not json_lib:
        raise Exception("No json library available")
    return json.dumps(_json_convert(obj), *args, **kwargs) 

def validate_is_document_type(option, value):
    """Validate the type of method arguments that expect a MongoDB document."""
    if not isinstance(value, (abc.MutableMapping, RawBSONDocument)):
        raise TypeError("%s must be an instance of dict, bson.son.SON, "
                        "bson.raw_bson.RawBSONDocument, or "
                        "a type that inherits from "
                        "collections.MutableMapping" % (option,)) 

def validate_is_mapping(option, value):
    """Validate the type of method arguments that expect a document."""
    if not isinstance(value, abc.Mapping):
        raise TypeError("%s must be an instance of dict, bson.son.SON, or "
                        "any other type that inherits from "
                        "collections.Mapping" % (option,)) 

def validate_list_or_mapping(option, value):
    """Validates that 'value' is a list or a document."""
    if not isinstance(value, (abc.Mapping, list)):
        raise TypeError("%s must either be a list or an instance of dict, "
                        "bson.son.SON, or any other type that inherits from "
                        "collections.Mapping" % (option,)) 

def validate_document_class(option, value):
    """Validate the document_class option."""
    if not issubclass(value, (abc.MutableMapping, RawBSONDocument)):
        raise TypeError("%s must be dict, bson.son.SON, "
                        "bson.raw_bson.RawBSONDocument, or a "
                        "sublass of collections.MutableMapping" % (option,))
    return value 

def _json_convert(obj):
    """Recursive helper method that converts BSON types so they can be
    converted into json.
    """
    if hasattr(obj, 'iteritems') or hasattr(obj, 'items'):  # PY3 support
        return SON(((k, _json_convert(v)) for k, v in obj.items()))
    elif hasattr(obj, '__iter__') and not isinstance(obj, string_types):
        return list((_json_convert(v) for v in obj))
    try:
        return default(obj)
    except TypeError:
        return obj 

def _json_convert(obj):
    """Recursive helper method that converts BSON types so they can be
    converted into json.
    """
    if hasattr(obj, 'iteritems') or hasattr(obj, 'items'):  # PY3 support
        return SON(((k, _json_convert(v)) for k, v in iteritems(obj)))
    elif hasattr(obj, '__iter__') and not isinstance(obj, (text_type, bytes)):
        return list((_json_convert(v) for v in obj))
    try:
        return default(obj)
    except TypeError:
        return obj 

def dumps(obj, *args, **kwargs):
    """Helper function that wraps :class:`json.dumps`.

    Recursive function that handles all BSON types including
    :class:`~bson.binary.Binary` and :class:`~bson.code.Code`.

    .. versionchanged:: 2.7
       Preserves order when rendering SON, Timestamp, Code, Binary, and DBRef
       instances.
    """
    return json.dumps(_json_convert(obj), *args, **kwargs) 

def to_mongo(self, *args, **kwargs) -> SON:
        """Wrap to_mongo."""
        raw = kwargs.pop("raw", False)
        son = super().to_mongo(*args, **kwargs)
        if all([not raw, "_id" in son, "id" not in son]):
            son["id"] = son.pop("_id")
        return son 

def _json_convert(obj, json_options=DEFAULT_JSON_OPTIONS):
    """Recursive helper method that converts BSON types so they can be
    converted into json.
    """
    if hasattr(obj, 'iteritems') or hasattr(obj, 'items'):  # PY3 support
        return SON(((k, _json_convert(v, json_options))
                    for k, v in iteritems(obj)))
    elif hasattr(obj, '__iter__') and not isinstance(obj, (text_type, bytes)):
        return list((_json_convert(v, json_options) for v in obj))
    try:
        return default(obj, json_options)
    except TypeError:
        return obj 

def validate_is_document_type(option, value):
    """Validate the type of method arguments that expect a MongoDB document."""
    if not isinstance(value, (abc.MutableMapping, RawBSONDocument)):
        raise TypeError("%s must be an instance of dict, bson.son.SON, "
                        "bson.raw_bson.RawBSONDocument, or "
                        "a type that inherits from "
                        "collections.MutableMapping" % (option,)) 

def validate_document_class(option, value):
    """Validate the document_class option."""
    if not issubclass(value, (abc.MutableMapping, RawBSONDocument)):
        raise TypeError("%s must be dict, bson.son.SON, "
                        "bson.raw_bson.RawBSONDocument, or a "
                        "sublass of collections.MutableMapping" % (option,))
    return value 

def get_endpoint_ratio(request):
    """
    获取今天 endpoint 访问占比
    :param request:
    :return:
    """
    success, msg, data = False, '', []
    post_data = json_loads(request.body)
    model_cls, filter_dict = parse_ratio_post_data(post_data)

    pipeline = [
        {
            "$group": {
                "_id": "$endpoint_id",
                "count": {"$sum": "$count"}
            },
        },
        {
            "$sort": SON([("count", -1), ("_id", -1)])
        }
    ]

    count_list = model_cls.objects(**filter_dict).aggregate(*pipeline)
    count_list = list(count_list)
    endpoint_id_list = [t['_id'] for t in count_list]
    endpoints = Endpoint.objects.filter(id__in=endpoint_id_list).values('unique_name', 'id')
    endpoint_dict = {}
    for t in endpoints:
        endpoint_dict[t['id']] = t['unique_name']

    legend = []
    y_data = []
    # 因为数据导入导出的原因，有可能导致出现 id 不匹配的问题
    new_count_list = []
    for t in count_list:
        if t['_id'] in endpoint_dict:
            new_count_list.append(t)
    count_list = new_count_list

    # 因为饼图显示的问题，只显示前几项
    count_list = sorted(count_list, key=lambda x: x['count'], reverse=True)
    count_list = count_list[:ECHARTS_PIPE_PLOT_MAX_NUM]
    for t in count_list:
        name = endpoint_dict.get(t['_id'])
        if name:
            legend.append(name)
            y_data.append({'value': t['count'], 'name': name})

    data = {
        'legend': legend,
        'y_data': y_data
    }
    return http_response_json({'success': True, 'msg': msg, 'data': data}) 

def get_client_ratio(request):
    """
    获取今天 client 访问占比
    :param request:
    :return:
    """
    success, msg, data = False, '', []
    post_data = json_loads(request.body)
    model_cls, filter_dict = parse_ratio_post_data(post_data)
    pipeline = [
        {
            "$group": {
                "_id": "$client_id",
                "count": {"$sum": "$count"}
            },
        },
        {
            "$sort": SON([("count", -1), ("_id", -1)])
        }
    ]

    count_list = model_cls.objects(**filter_dict).aggregate(*pipeline)
    count_list = list(count_list)
    client_id_list = [t['_id'] for t in count_list]
    clients = Client.objects.filter(id__in=client_id_list).values('name', 'id')
    client_dict = {}
    for t in clients:
        client_dict[t['id']] = t['name']

    legend = []
    y_data = []

    # 因为数据导入导出的原因，有可能导致出现 id 不匹配的问题
    new_count_list = []
    for t in count_list:
        if t['_id'] in client_dict:
            new_count_list.append(t)
    count_list = new_count_list

    count_list = sorted(count_list, key=lambda x: x['count'], reverse=True)
    count_list = count_list[:ECHARTS_PIPE_PLOT_MAX_NUM]
    for t in count_list:
        name = client_dict.get(t['_id'])
        if name:
            legend.append(name)
            y_data.append({'value': t['count'], 'name': name})

    data = {
        'legend': legend,
        'y_data': y_data
    }
    return http_response_json({'success': True, 'msg': msg, 'data': data}) 

def default(obj):
    # We preserve key order when rendering SON, DBRef, etc. as JSON by
    # returning a SON for those types instead of a dict.
    if isinstance(obj, ObjectId):
        return {"$oid": str(obj)}
    if isinstance(obj, DBRef):
        return _json_convert(obj.as_doc())
    if isinstance(obj, datetime.datetime):
        # TODO share this code w/ bson.py?
        if obj.utcoffset() is not None:
            obj = obj - obj.utcoffset()
        millis = int(calendar.timegm(obj.timetuple()) * 1000 +
                     obj.microsecond / 1000)
        return {"$date": millis}
    if isinstance(obj, (RE_TYPE, Regex)):
        flags = ""
        if obj.flags & re.IGNORECASE:
            flags += "i"
        if obj.flags & re.LOCALE:
            flags += "l"
        if obj.flags & re.MULTILINE:
            flags += "m"
        if obj.flags & re.DOTALL:
            flags += "s"
        if obj.flags & re.UNICODE:
            flags += "u"
        if obj.flags & re.VERBOSE:
            flags += "x"
        if isinstance(obj.pattern, text_type):
            pattern = obj.pattern
        else:
            pattern = obj.pattern.decode('utf-8')
        return SON([("$regex", pattern), ("$options", flags)])
    if isinstance(obj, MinKey):
        return {"$minKey": 1}
    if isinstance(obj, MaxKey):
        return {"$maxKey": 1}
    if isinstance(obj, Timestamp):
        return {"$timestamp": SON([("t", obj.time), ("i", obj.inc)])}
    if isinstance(obj, Code):
        return SON([('$code', str(obj)), ('$scope', obj.scope)])
    if isinstance(obj, Binary):
        return SON([
            ('$binary', base64.b64encode(obj).decode()),
            ('$type', "%02x" % obj.subtype)])
    if PY3 and isinstance(obj, bytes):
        return SON([
            ('$binary', base64.b64encode(obj).decode()),
            ('$type', "00")])
    if isinstance(obj, uuid.UUID):
        return {"$uuid": obj.hex}
    raise TypeError("%r is not JSON serializable" % obj) 

def default(obj):
    # We preserve key order when rendering SON, DBRef, etc. as JSON by
    # returning a SON for those types instead of a dict. This works with
    # the "json" standard library in Python 2.6+ and with simplejson
    # 2.1.0+ in Python 2.5+, because those libraries iterate the SON
    # using PyIter_Next. Python 2.4 must use simplejson 2.0.9 or older,
    # and those versions of simplejson use the lower-level PyDict_Next,
    # which bypasses SON's order-preserving iteration, so we lose key
    # order in Python 2.4.
    if isinstance(obj, ObjectId):
        return {"$oid": str(obj)}
    if isinstance(obj, DBRef):
        return _json_convert(obj.as_doc())
    if isinstance(obj, datetime.datetime):
        # TODO share this code w/ bson.py?
        if obj.utcoffset() is not None:
            obj = obj - obj.utcoffset()
        millis = int(calendar.timegm(obj.timetuple()) * 1000 +
                     obj.microsecond / 1000)
        return {"$date": millis}
    if isinstance(obj, (RE_TYPE, Regex)):
        flags = ""
        if obj.flags & re.IGNORECASE:
            flags += "i"
        if obj.flags & re.LOCALE:
            flags += "l"
        if obj.flags & re.MULTILINE:
            flags += "m"
        if obj.flags & re.DOTALL:
            flags += "s"
        if obj.flags & re.UNICODE:
            flags += "u"
        if obj.flags & re.VERBOSE:
            flags += "x"
        if isinstance(obj.pattern, str):
            pattern = obj.pattern
        else:
            pattern = obj.pattern.decode('utf-8')
        return SON([("$regex", pattern), ("$options", flags)])
    if isinstance(obj, MinKey):
        return {"$minKey": 1}
    if isinstance(obj, MaxKey):
        return {"$maxKey": 1}
    if isinstance(obj, Timestamp):
        return {"$timestamp": SON([("t", obj.time), ("i", obj.inc)])}
    if isinstance(obj, Code):
        return SON([('$code', str(obj)), ('$scope', obj.scope)])
    if isinstance(obj, Binary):
        return SON([
            ('$binary', base64.b64encode(obj).decode()),
            ('$type', "%02x" % obj.subtype)])
    if PY3 and isinstance(obj, binary_type):
        return SON([
            ('$binary', base64.b64encode(obj).decode()),
            ('$type', "00")])
    if bson.has_uuid() and isinstance(obj, bson.uuid.UUID):
        return {"$uuid": obj.hex}
    raise TypeError("%r is not JSON serializable" % obj) 

def docify(self, df):
        """
        Convert a Pandas DataFrame to SON.

        Parameters
        ----------
        df:  DataFrame
            The Pandas DataFrame to encode
        """
        dtypes = {}
        masks = {}
        lengths = {}
        columns = []
        data = Binary(b'')
        start = 0

        arrays = []
        for c in df:
            try:
                columns.append(str(c))
                arr, mask = self._convert_types(df[c].values)
                dtypes[str(c)] = arr.dtype.str
                if mask is not None:
                    masks[str(c)] = Binary(compress(mask.tostring()))
                arrays.append(arr.tostring())
            except Exception as e:
                typ = infer_dtype(df[c], skipna=False)
                msg = "Column '{}' type is {}".format(str(c), typ)
                logging.warning(msg)
                raise e

        arrays = compress_array(arrays)
        for index, c in enumerate(df):
            d = Binary(arrays[index])
            lengths[str(c)] = (start, start + len(d) - 1)
            start += len(d)
            data += d

        doc = SON({DATA: data, METADATA: {}})
        doc[METADATA] = {COLUMNS: columns,
                         MASK: masks,
                         LENGTHS: lengths,
                         DTYPE: dtypes
                         }

        return doc