Python msgpack.loads() Examples

def send(hub, worker, payload):
    '''
    Send the given payload to the given worker, yield iterations based on the
    returns from the remote.
    '''
    mp = msgpack.dumps(payload, use_bin_type=True)
    mp += hub.proc.DELIM
    reader, writer = await asyncio.open_unix_connection(path=worker['path'])
    writer.write(mp)
    await writer.drain()
    final_ret = True
    while True:
        ret = await reader.readuntil(hub.proc.DELIM)
        p_ret = ret[:-len(hub.proc.DELIM)]
        i_flag = p_ret[-1:]
        ret = msgpack.loads(p_ret[:-1], raw=False)
        if i_flag == hub.proc.D_FLAG:
            # break for the end of the sequence
            break
        yield ret
        final_ret = False
    if final_ret:
        yield ret 

def add_to_refmap(self, result: ResultStorer) -> None:
        """
        :param result: the result of the compare function

        This method verifies whether the comparison in input is the best available for the
        candidate reference gene, and loads it into the refmap dictionary if such is the case.

        """

        if result is not None and result.ccode != ("u",):
            gid, tid = result.ref_gene[0], result.ref_id[0]
            if gid not in self.gene_matches:
                self.gene_matches[gid] = dict()
            if tid not in self.gene_matches[gid]:
                self.gene_matches[gid][tid] = list()
            self.gene_matches[gid][tid].append(result)
        return 

def load(cls, bmessage):
        """
        From a bytestring given by a (distant) call to Message.dump(), retrieve the original message
        :param bmessage: bytestring given by a .dump() call on a message
        :return: the original message
        """
        message_dict = msgpack.loads(bmessage, use_list=False)

        try:
            obj = MessageMeta._registered_messages[message_dict["type"]].__new__(MessageMeta._registered_messages[message_dict["type"]])
            object.__setattr__(obj, "__dict__", message_dict)
        except:
            raise TypeError("Unknown message type") from None

        if not obj._verify():  # pylint: disable=protected-access
            raise TypeError("Invalid message content")

        return obj 

def un(byts):
    '''
    Use msgpack to de-serialize a python object.

    Args:
        byts (bytes): The bytes to de-serialize

    Notes:
        String objects are decoded using utf8 encoding.  In order to handle
        potentially malformed input, ``unicode_errors='surrogatepass'`` is set
        to allow decoding bad input strings.

    Returns:
        obj: The de-serialized object
    '''
    # This uses a subset of unpacker_kwargs
    return msgpack.loads(byts, use_list=False, raw=False, unicode_errors='surrogatepass') 

def test_mutations(self):
        tables = tskit.TableCollection(sequence_length=1)
        metadata = ExampleMetadata(one="node1", two="node2")
        pickled = pickle.dumps(metadata)
        tables.nodes.add_row(time=0)
        tables.sites.add_row(position=0.1, ancestral_state="A")
        tables.mutations.add_row(site=0, node=0, derived_state="T", metadata=pickled)
        ts = tables.tree_sequence()
        mutation = ts.site(0).mutations[0]
        self.assertEqual(mutation.site, 0)
        self.assertEqual(mutation.node, 0)
        self.assertEqual(mutation.derived_state, "T")
        self.assertEqual(mutation.metadata, pickled)
        unpickled = pickle.loads(mutation.metadata)
        self.assertEqual(unpickled.one, metadata.one)
        self.assertEqual(unpickled.two, metadata.two) 

def loads(self, data):
		"""
		Deserialize the data into it's original python object.

		:param bytes data: The serialized object to load.
		:return: The original python object.
		"""
		if not isinstance(data, bytes):
			raise TypeError("loads() argument 1 must be bytes, not {0}".format(type(data).__name__))
		if self._compression == 'zlib':
			data = zlib.decompress(data)
		if sys.version_info[0] == 3 and self.name.startswith('application/'):
			data = data.decode(self._charset)
		data = g_serializer_drivers[self.name]['loads'](data, (self._charset if sys.version_info[0] == 3 else None))
		if isinstance(data, list):
			data = tuple(data)
		return data 

def do_POST(self):
		if not self.check_authorization():
			self.respond_unauthorized(request_authentication=True)
			return
		content_length = int(self.headers.get('content-length', 0))
		data = self.rfile.read(content_length)
		self.raw_query_data = data
		content_type = self.headers.get('content-type', '')
		content_type = content_type.split(';', 1)[0]
		self.query_data = {}
		try:
			if not isinstance(data, str):
				data = data.decode(self.get_content_type_charset())
			if content_type.startswith('application/json'):
				data = json.loads(data)
				if isinstance(data, dict):
					self.query_data = dict([(i[0], [i[1]]) for i in data.items()])
			else:
				self.query_data = urllib.parse.parse_qs(data, keep_blank_values=1)
		except Exception:
			self.respond_server_error(400)
		else:
			self.dispatch_handler(self.query_data)
		return 

def load(self):
        try:
            with open(self.store_file, "rb") as fp:
                data = fp.read()
                data = msgpack.loads(data)
                self.plans = {}
                for plan in data.get(b"plans", []):
                    plan = Plan.loads(plan)
                    self.plans[plan.key] = plan

                self.time_plans = defaultdict(dict)
                for key, plans in data.get(b"time_plans", {}).items():
                    self.time_plans[key] = {}
                    for plan_key in (plans or []):
                        if not isinstance(plan_key, unicode_type):
                            plan_key = unicode_type(plan_key, "utf-8")

                        if plan_key in self.plans:
                            self.time_plans[key][plan_key] = self.plans[plan_key]

                current_time = data.get(b"current_time")
                self.current_time = current_time if current_time and current_time > 0 else self.current_time
            logging.info("store mem load session %s", self.store_file)
        except Exception as e:
            logging.error("store mem load session error: %s %s", self.store_file, e) 

def test_json(self):
        ts = msprime.simulate(10, random_seed=1)
        tables = ts.dump_tables()
        nodes = tables.nodes
        # For each node, we create some Python metadata that can be JSON encoded.
        metadata = [
            {"one": j, "two": 2 * j, "three": list(range(j))} for j in range(len(nodes))
        ]
        encoded, offset = tskit.pack_strings(map(json.dumps, metadata))
        nodes.set_columns(
            flags=nodes.flags,
            time=nodes.time,
            population=nodes.population,
            metadata_offset=offset,
            metadata=encoded,
        )
        self.assertTrue(np.array_equal(nodes.metadata_offset, offset))
        self.assertTrue(np.array_equal(nodes.metadata, encoded))
        ts1 = tables.tree_sequence()
        for j, node in enumerate(ts1.nodes()):
            decoded_metadata = json.loads(node.metadata.decode())
            self.assertEqual(decoded_metadata, metadata[j])
        ts1.dump(self.temp_file)
        ts2 = tskit.load(self.temp_file)
        self.assertEqual(ts1.tables.nodes, ts2.tables.nodes) 

def test_pickle(self):
        ts = msprime.simulate(10, random_seed=1)
        tables = ts.dump_tables()
        # For each node, we create some Python metadata that can be pickled
        metadata = [
            {"one": j, "two": 2 * j, "three": list(range(j))}
            for j in range(ts.num_nodes)
        ]
        encoded, offset = tskit.pack_bytes(list(map(pickle.dumps, metadata)))
        tables.nodes.set_columns(
            flags=tables.nodes.flags,
            time=tables.nodes.time,
            population=tables.nodes.population,
            metadata_offset=offset,
            metadata=encoded,
        )
        self.assertTrue(np.array_equal(tables.nodes.metadata_offset, offset))
        self.assertTrue(np.array_equal(tables.nodes.metadata, encoded))
        ts1 = tables.tree_sequence()
        for j, node in enumerate(ts1.nodes()):
            decoded_metadata = pickle.loads(node.metadata)
            self.assertEqual(decoded_metadata, metadata[j])
        ts1.dump(self.temp_file)
        ts2 = tskit.load(self.temp_file)
        self.assertEqual(ts1.tables.nodes, ts2.tables.nodes) 

def msgpackext_loads(data: bytes) -> Any:
    """Deserializes a msgpack byte representation of known objects into those objects.

    Parameters
    ----------
    data : bytes
        The serialized msgpack byte array.

    Returns
    -------
    Any
        The deserialized Python objects.
    """
    which_import("msgpack", raise_error=True, raise_msg=_msgpack_which_msg)

    return msgpack.loads(data, object_hook=msgpackext_decode, raw=False)


## JSON Ext 

def json_loads(data: str) -> Any:
    """Deserializes a json representation of known objects into those objects.

    Parameters
    ----------
    data : str
        The serialized JSON blob.

    Returns
    -------
    Any
        The deserialized Python objects.
    """

    # Doesn't hurt anything to try to load JSONext as well
    return json.loads(data, object_hook=jsonext_decode)


## Helper functions 

def jsonext_loads(data: Union[str, bytes]) -> Any:
    """Deserializes a json representation of known objects into those objects.

    Parameters
    ----------
    data : str or bytes
        The byte-serialized JSON blob.

    Returns
    -------
    Any
        The deserialized Python objects.
    """

    return json.loads(data, object_hook=jsonext_decode)


## JSON 

def __load_gene(self, jdict):
        gene = Gene(None, logger=self.logger)
        if not isinstance(jdict, dict):
            try:
                jdict = msgpack.loads(jdict, raw=False)
            except TypeError:
                jdict = json.loads(jdict)

        gene.load_dict(jdict, exclude_utr=self.__exclude_utr, protein_coding=self.__protein_coding)
        gene.finalize()
        return gene 

def loads(self, value):
        """
        Deserialize value using ``pickle.loads``.

        :param value: bytes
        :returns: obj
        """
        if value is None:
            return None
        return pickle.loads(value) 

def load_result(self, dbname):

        connection = sqlite3.connect(dbname)
        cursor = connection.cursor()
        done = set()

        if self.printout_tmap is True:
            for tmap_row in cursor.execute("SELECT * from tmap"):
                tmap_row = ResultStorer(state=msgpack.loads(tmap_row[0],
                                                            raw=False,
                                                            use_list=False,
                                                            strict_map_key=False,
                                                            object_hook=msgpack_convert))
                done.add(tmap_row.tid)
                self.print_tmap(tmap_row)

        for gid, gene_match in cursor.execute("SELECT * from gene_matches"):
            gene_match = msgpack.loads(gene_match, raw=False, use_list=False,
                                       strict_map_key=False, object_hook=msgpack_convert)
            for tid in gene_match:
                for match in gene_match[tid]:
                    self.gene_matches[gid][tid].append(match)

        temp_stats = Namespace()
        for attr, stat in cursor.execute("SELECT * from stats"):
            setattr(temp_stats, attr, msgpack.loads(stat, raw=False,
                                                    use_list=False,
                                                    strict_map_key=False,
                                                    object_hook=msgpack_convert))

        self.stat_calculator.merge_into(temp_stats)
        os.remove(dbname)
        self.done += len(done) 

def get(self, idx):
        if idx > NUMBER_OF_IMAGES:
            print("WARNING: Index is to large for number of images.")

        k = self.obj.keys[idx]
        v = self.obj._txn.get(k.encode())
        # v is (img, score) pair packes with msgpack
        v = msgpack.loads(v)
        img, score = v
        return k, img, score 

def loads(self, value):
        """
        Returns value back without transformations
        """
        return value 

def loads(self, value):
        """
        Returns the same value
        """
        return value 

def loads(self, value):
        raise NotImplementedError("loads method must be implemented") 

def loads_msgpack(buf):
    """
    Args:
        buf: the output of `dumps`.
    """
    return msgpack.loads(buf, raw=False) 

def loads(buf):
    #return dill.loads(buf)
    return msgpack.loads(buf) 

def _unpack_json(str):
    return json.loads(str, encoding='utf-8') 

def _unpack_msgpack(str):
    return msgpack.loads(str, encoding='utf-8') 

def _unpack_msgpack_snappy(str):
    if str.startswith(b'S'):
        tmp = snappy.uncompress(str[1:])
        # print "SNAPPY: ", len(str), len(tmp)
        obj = msgpack.loads(tmp, encoding='utf-8')
    elif str.startswith(b'\0'):
        obj = msgpack.loads(str[1:], encoding='utf-8')
    else:
        return None
    
    return obj 

def loads(buf):
    #return dill.loads(buf)
    return msgpack.loads(buf) 

def load(key, default=None):
    result = conn.get(NAMESPACE + key)
    if result is None:
        return default
    try:
        return _serializer.loads(result, encoding='utf-8')
    except Exception as err:
        exc_info = log.level <= logging.DEBUG
        log.warn('cannot load cache (%s): %s', key, err, exc_info=exc_info)
        return default 

def loads(data):
    """ Decodes and processes events received from the bridge """
    #if not data:
    #    raise ValueError("Tried to load empty data!")
    return msgpack.loads(data, use_list=False, raw=False) 

def iter(self):
        c = self.obj._txn.cursor()
        while c.next():
            k, v = c.item()
            if k != b'__keys__':
                v = msgpack.loads(v)
                img, score = v
                yield k, img, score 

def loads(buf):
        """
        Args:
            bytes
        """
        return pickle.loads(buf)


# Define the default serializer to be used that dumps data to bytes