Python bson.decode_all() Examples

def unpack_response(self, cursor_id=None,
                        codec_options=_UNICODE_REPLACE_CODEC_OPTIONS):
        """Unpack a response from the database and decode the BSON document(s).

        Check the response for errors and unpack, returning a dictionary
        containing the response data.

        Can raise CursorNotFound, NotMasterError, ExecutionTimeout, or
        OperationFailure.

        :Parameters:
          - `cursor_id` (optional): cursor_id we sent to get this response -
            used for raising an informative exception when we get cursor id not
            valid at server response
          - `codec_options` (optional): an instance of
            :class:`~bson.codec_options.CodecOptions`
        """
        self.raw_response(cursor_id)
        return bson.decode_all(self.documents, codec_options) 

def unpack_response(self, cursor_id=None,
                        codec_options=_UNICODE_REPLACE_CODEC_OPTIONS,
                        user_fields=None, legacy_response=False):
        """Unpack a response from the database and decode the BSON document(s).

        Check the response for errors and unpack, returning a dictionary
        containing the response data.

        Can raise CursorNotFound, NotMasterError, ExecutionTimeout, or
        OperationFailure.

        :Parameters:
          - `cursor_id` (optional): cursor_id we sent to get this response -
            used for raising an informative exception when we get cursor id not
            valid at server response
          - `codec_options` (optional): an instance of
            :class:`~bson.codec_options.CodecOptions`
        """
        self.raw_response(cursor_id)
        if legacy_response:
            return bson.decode_all(self.documents, codec_options)
        return bson._decode_all_selective(
            self.documents, codec_options, user_fields) 

def _recv_dict(self):
        if self.connection:
            packet_size = self.connection.recv(4)
            packet_size_int = struct.unpack('<l', packet_size)[0]
            # First part of packet will be the 4-byte packet size
            packet = packet_size
            while len(packet) < packet_size_int:
                partial_packet = self.connection.recv(
                    packet_size_int - len(packet))
                if not partial_packet:
                    break
                packet += partial_packet
            return bson.decode_all(packet)[0] 

def unpack_response(self, cursor_id=None,
                        codec_options=_UNICODE_REPLACE_CODEC_OPTIONS):
        """Unpack a OP_MSG command response.

        :Parameters:
          - `cursor_id` (optional): Ignored, for compatibility with _OpReply.
          - `codec_options` (optional): an instance of
            :class:`~bson.codec_options.CodecOptions`
        """
        return bson.decode_all(self.payload_document, codec_options) 

def load(self, load_one=False, filename=None):
        f = None
        if not filename:
            filename = self.state_file
        try:
            f = open(filename, "r")
            data = decode_all(f.read())
            if load_one and len(data) > 0:
                return data[0]
            return data
        except Exception, e:
            raise e 

def unpack(cls, msg, client, server, request_id):
        """Parse message and return an `OpQuery` or `Command`.

        Takes the client message as bytes, the client and server socket objects,
        and the client request id.
        """
        flags, = _UNPACK_INT(msg[:4])
        namespace, pos = _get_c_string(msg, 4)
        is_command = namespace.endswith('.$cmd')
        num_to_skip, = _UNPACK_INT(msg[pos:pos + 4])
        pos += 4
        num_to_return, = _UNPACK_INT(msg[pos:pos + 4])
        pos += 4
        docs = bson.decode_all(msg[pos:], CODEC_OPTIONS)
        if is_command:
            assert len(docs) == 1
            command_ns = namespace[:-len('.$cmd')]
            return Command(docs, namespace=command_ns, flags=flags,
                           _client=client, request_id=request_id,
                           _server=server)
        else:
            if len(docs) == 1:
                fields = None
            else:
                assert len(docs) == 2
                fields = docs[1]
            return OpQuery(docs[0], fields=fields, namespace=namespace,
                           flags=flags, num_to_skip=num_to_skip,
                           num_to_return=num_to_return, _client=client,
                           request_id=request_id, _server=server) 

def unpack(cls, msg, client, server, request_id):
        """Parse message and return an `OpInsert`.

        Takes the client message as bytes, the client and server socket objects,
        and the client request id.
        """
        flags, = _UNPACK_INT(msg[:4])
        namespace, pos = _get_c_string(msg, 4)
        docs = bson.decode_all(msg[pos:], CODEC_OPTIONS)
        return cls(*docs, namespace=namespace, flags=flags, _client=client,
                   request_id=request_id, _server=server) 

def unpack(cls, msg, client, server, request_id):
        """Parse message and return an `OpUpdate`.

        Takes the client message as bytes, the client and server socket objects,
        and the client request id.
        """
        # First 4 bytes of OP_UPDATE are "reserved".
        namespace, pos = _get_c_string(msg, 4)
        flags, = _UNPACK_INT(msg[pos:pos + 4])
        docs = bson.decode_all(msg[pos + 4:], CODEC_OPTIONS)
        return cls(*docs, namespace=namespace, flags=flags, _client=client,
                   request_id=request_id, _server=server) 

def unpack(cls, msg, client, server, request_id):
        """Parse message and return an `OpDelete`.

        Takes the client message as bytes, the client and server socket objects,
        and the client request id.
        """
        # First 4 bytes of OP_DELETE are "reserved".
        namespace, pos = _get_c_string(msg, 4)
        flags, = _UNPACK_INT(msg[pos:pos + 4])
        docs = bson.decode_all(msg[pos + 4:], CODEC_OPTIONS)
        return cls(*docs, namespace=namespace, flags=flags, _client=client,
                   request_id=request_id, _server=server) 

def montyrestore(database, collection, dumpfile):
    """Loads a binary database dump into a MontyCollection instance

    Should be able to accept the dump created by `mongodump`.

    Example:
        >>> from montydb import open_repo, utils
        >>> with open_repo("foo/bar"):
        >>>     utils.montyrestore("db", "col", "/data/dump.bson")
        >>>

    Args:
        database (str): Database name
        collection (str): Collection name to load into
        dumpfile (str): File path to load from

    """
    collection = _collection(database, collection)

    with open(dumpfile, "rb") as fp:
        raw = fp.read()

    try:
        collection.insert_many(decode_all(raw))
    except DuplicateKeyError:
        pass 

def get_details(self):
        """Get machine details information.

        :returns: Mapping of hardware details.
        """
        data = await self._handler.details(system_id=self.system_id)
        return bson.decode_all(data)[0] 

def _unpack_response(response, cursor_id=None, as_class=dict,
                     tz_aware=False, uuid_subtype=OLD_UUID_SUBTYPE,
                     compile_re=True):
    """Unpack a response from the database.

    Check the response for errors and unpack, returning a dictionary
    containing the response data.

    :Parameters:
      - `response`: byte string as returned from the database
      - `cursor_id` (optional): cursor_id we sent to get this response -
        used for raising an informative exception when we get cursor id not
        valid at server response
      - `as_class` (optional): class to use for resulting documents
    """
    response_flag = struct.unpack("<i", response[:4])[0]
    if response_flag & 1:
        # Shouldn't get this response if we aren't doing a getMore
        assert cursor_id is not None

        raise CursorNotFound("cursor id '%s' not valid at server" %
                             cursor_id)
    elif response_flag & 2:
        error_object = bson.BSON(response[20:]).decode()
        if error_object["$err"].startswith("not master"):
            raise AutoReconnect(error_object["$err"])
        elif error_object.get("code") == 50:
            raise ExecutionTimeout(error_object.get("$err"),
                                   error_object.get("code"),
                                   error_object)
        raise OperationFailure("database error: %s" %
                               error_object.get("$err"),
                               error_object.get("code"),
                               error_object)

    result = {}
    result["cursor_id"] = struct.unpack("<q", response[4:12])[0]
    result["starting_from"] = struct.unpack("<i", response[12:16])[0]
    result["number_returned"] = struct.unpack("<i", response[16:20])[0]
    result["data"] = bson.decode_all(response[20:],
                                     as_class, tz_aware, uuid_subtype,
                                     compile_re)
    assert len(result["data"]) == result["number_returned"]
    return result 

def handle_io_in(self, data):
        offset = 0
        while len(data) - offset >= 16:
            h = packets.MsgHeader(data[offset:offset+16])
            # print(h.messageLength)
            # print(h.opCode)
            if len(data) - offset < h.messageLength:
                break
            if h.opCode == 2004:
                msg = packets.MsgQuery(data[offset+16:offset+h.messageLength])
                # print(h.show())
                # print(msg.show())
                query = None
                field_selectors = []
                if bson:
                    for doc in bson.decode_all(msg.payload.load):
                        if query is None:
                            query = doc
                        else:
                            field_selectors.append(doc)
                res = self._handle_query(fullCollectionName=msg.fullCollectionName, query=query, field_selectors=field_selectors)

                # print(msg)
                # print(msg.payload)
                # print(msg.payload.load)
                payload = b""
                for doc in res:
                    payload += bson.BSON.encode(doc)

                pkg = packets.MsgHeader(
                    responseTo=h.requestID,
                    opCode=1
                ) / packets.MsgReply(
                    numberReturned=len(res)
                ) / Raw(payload)
                pkg.show()
                self.send(pkg.build())
            elif h.opCode == 2010:
                msg = packets.MsgCommand(data[offset + 16:offset + h.messageLength])

                docs = bson.decode_all(msg.payload.load)
                res = self._handle_command(msg.database, msg.commandName, docs[0], docs[1], docs[1:])

                payload = b""
                for doc in res:
                    payload += bson.BSON.encode(doc)

                pkg = packets.MsgHeader(
                    responseTo=h.requestID,
                    opCode=2011
                ) / packets.MsgCommandReply(
                ) / Raw(payload)
                pkg.show()
                self.send(pkg.build())

            # print(h.payload)

            # ToDo: check length
            offset = offset + h.messageLength

        return offset 

def _unpack_response(response, cursor_id=None, codec_options=CodecOptions()):
    """Unpack a response from the database.

    Check the response for errors and unpack, returning a dictionary
    containing the response data.

    Can raise CursorNotFound, NotMasterError, ExecutionTimeout, or
    OperationFailure.

    :Parameters:
      - `response`: byte string as returned from the database
      - `cursor_id` (optional): cursor_id we sent to get this response -
        used for raising an informative exception when we get cursor id not
        valid at server response
      - `codec_options` (optional): an instance of
        :class:`~bson.codec_options.CodecOptions`
    """
    response_flag = struct.unpack("<i", response[:4])[0]
    if response_flag & 1:
        # Shouldn't get this response if we aren't doing a getMore
        assert cursor_id is not None

        # Fake a getMore command response. OP_GET_MORE provides no document.
        msg = "Cursor not found, cursor id: %d" % (cursor_id,)
        errobj = {"ok": 0, "errmsg": msg, "code": 43}
        raise CursorNotFound(msg, 43, errobj)
    elif response_flag & 2:
        error_object = bson.BSON(response[20:]).decode()
        # Fake the ok field if it doesn't exist.
        error_object.setdefault("ok", 0)
        if error_object["$err"].startswith("not master"):
            raise NotMasterError(error_object["$err"], error_object)
        elif error_object.get("code") == 50:
            raise ExecutionTimeout(error_object.get("$err"),
                                   error_object.get("code"),
                                   error_object)
        raise OperationFailure("database error: %s" %
                               error_object.get("$err"),
                               error_object.get("code"),
                               error_object)

    result = {"cursor_id": struct.unpack("<q", response[4:12])[0],
              "starting_from": struct.unpack("<i", response[12:16])[0],
              "number_returned": struct.unpack("<i", response[16:20])[0],
              "data": bson.decode_all(response[20:], codec_options)}

    assert len(result["data"]) == result["number_returned"]
    return result 

def unpack(cls, msg, client, server, request_id):
        """Parse message and return an `OpMsg`.

        Takes the client message as bytes, the client and server socket objects,
        and the client request id.
        """
        payload_document = OrderedDict()
        flags, = _UNPACK_UINT(msg[:4])
        pos = 4
        if flags & ~_ALL_OP_MSG_FLAGS:
            raise ValueError(
                'OP_MSG flags has reserved bits set.'
                ' Allowed flags: 0x%x. Provided flags: 0x%x' % (
                    _ALL_OP_MSG_FLAGS, flags))

        checksum_present = flags & OP_MSG_FLAGS['checksumPresent']
        checksum = None
        if checksum_present:
            msg_len_without_checksum = len(msg) - 4
        else:
            msg_len_without_checksum = len(msg)

        while pos < msg_len_without_checksum:
            payload_type, = _UNPACK_BYTE(msg[pos:pos + 1])
            pos += 1
            payload_size, = _UNPACK_INT(msg[pos:pos + 4])
            if payload_type == 0:
                doc = bson.decode_all(msg[pos:pos + payload_size],
                                      CODEC_OPTIONS)[0]
                payload_document.update(doc)
                pos += payload_size
            elif payload_type == 1:
                section_size, = _UNPACK_INT(msg[pos:pos + 4])
                pos += 4
                identifier, pos = _get_c_string(msg, pos)
                # Section starts w/ 4-byte size prefix, identifier ends w/ nil.
                documents_len = section_size - len(identifier) - 1 - 4
                documents = bson.decode_all(msg[pos:pos + documents_len],
                                            CODEC_OPTIONS)
                payload_document[identifier] = documents
                pos += documents_len

        remaining = len(msg) - pos
        if checksum_present:
            if remaining != 4:
                raise ValueError(
                    'OP_MSG has checksumPresent flag set, expected 4 bytes'
                    ' remaining but have %d bytes remaining' % (remaining,))

            checksum = _UNPACK_UINT(msg[pos:pos+4])[0]
        else:
            if remaining != 0:
                raise ValueError(
                    'OP_MSG has no checksumPresent flag, expected 0 bytes'
                    ' remaining but have %d bytes remaining' % (remaining,))

        database = payload_document['$db']
        return OpMsg(payload_document, namespace=database, flags=flags,
                     _client=client, request_id=request_id, checksum=checksum,
                     _server=server) 

def _unpack_response(response,
                     cursor_id=None,
                     codec_options=_UNICODE_REPLACE_CODEC_OPTIONS):
    """Unpack a response from the database.

    Check the response for errors and unpack, returning a dictionary
    containing the response data.

    Can raise CursorNotFound, NotMasterError, ExecutionTimeout, or
    OperationFailure.

    :Parameters:
      - `response`: byte string as returned from the database
      - `cursor_id` (optional): cursor_id we sent to get this response -
        used for raising an informative exception when we get cursor id not
        valid at server response
      - `codec_options` (optional): an instance of
        :class:`~bson.codec_options.CodecOptions`
    """
    response_flag = struct.unpack("<i", response[:4])[0]
    if response_flag & 1:
        # Shouldn't get this response if we aren't doing a getMore
        if cursor_id is None:
            raise ProtocolError("No cursor id for getMore operation")

        # Fake a getMore command response. OP_GET_MORE provides no document.
        msg = "Cursor not found, cursor id: %d" % (cursor_id,)
        errobj = {"ok": 0, "errmsg": msg, "code": 43}
        raise CursorNotFound(msg, 43, errobj)
    elif response_flag & 2:
        error_object = bson.BSON(response[20:]).decode()
        # Fake the ok field if it doesn't exist.
        error_object.setdefault("ok", 0)
        if error_object["$err"].startswith("not master"):
            raise NotMasterError(error_object["$err"], error_object)
        elif error_object.get("code") == 50:
            raise ExecutionTimeout(error_object.get("$err"),
                                   error_object.get("code"),
                                   error_object)
        raise OperationFailure("database error: %s" %
                               error_object.get("$err"),
                               error_object.get("code"),
                               error_object)

    result = {"cursor_id": struct.unpack("<q", response[4:12])[0],
              "starting_from": struct.unpack("<i", response[12:16])[0],
              "number_returned": struct.unpack("<i", response[16:20])[0],
              "data": bson.decode_all(response[20:], codec_options)}

    assert len(result["data"]) == result["number_returned"]
    return result