Python cPickle.loads() Examples

def TestCustomFormat():
    OpenClipboard()
    try:
        # Just for the fun of it pickle Python objects through the clipboard
        fmt = RegisterClipboardFormat("Python Pickle Format")
        import cPickle
        pickled_object = Foo(a=1, b=2, Hi=3)
        SetClipboardData(fmt, cPickle.dumps( pickled_object ) )
        # Now read it back.
        data = GetClipboardData(fmt)
        loaded_object = cPickle.loads(data)
        assert cPickle.loads(data) == pickled_object, "Didnt get the correct data!"

        print "Clipboard custom format tests worked correctly"
    finally:
        CloseClipboard() 

def test_float(self):
        for_bin_protos = [4.94e-324, 1e-310]
        neg_for_bin_protos = [-x for x in for_bin_protos]
        test_values = [0.0, 7e-308, 6.626e-34, 0.1, 0.5,
                       3.14, 263.44582062374053, 6.022e23, 1e30]
        test_proto0_values = test_values + [-x for x in test_values]
        test_values = test_proto0_values + for_bin_protos + neg_for_bin_protos

        for value in test_proto0_values:
            pickle = self.dumps(value, 0)
            got = self.loads(pickle)
            self.assertEqual(value, got)

        for proto in pickletester.protocols[1:]:
            for value in test_values:
                pickle = self.dumps(value, proto)
                got = self.loads(pickle)
                self.assertEqual(value, got)

    # Backwards compatibility was explicitly broken in r67934 to fix a bug. 

def load_object(self, value):
        """The reversal of :meth:`dump_object`.  This might be called with
        None.
        """
        if value is None:
            return None
        if value.startswith(b'!'):
            try:
                return pickle.loads(value[1:])
            except pickle.PickleError:
                return None
        try:
            return int(value)
        except ValueError:
            # before 0.8 we did not have serialization.  Still support that.
            return value 

def __init__(self, env, clear, ivfidx=None):
        MemStorage.__init__(self)

        self.env = env
        dbname = 'db%06d' % ivfidx if ivfidx else 'db'
        self.db = LMDBAccessor(self.env, dbname)

        if clear:
            self.clear()

        with self.env.begin() as txn:
            self.num_items = txn.stat(self.db.db)['entries']
            if self.num_items > 0:
                cursor = txn.cursor(self.db.db)
                keys = []
                codes = []
                for key, code in cursor:
                    # print len(key), [ord(c) for c in key], unpack('i', key)[0]
                    keys.append(unpack('i', key)[0])
                    codes.append(pickle.loads(code))
                self.keys = np.array(keys)
                self.codes = np.vstack(tuple(codes)) 

def load_object(self, value):
        """The reversal of :meth:`dump_object`.  This might be called with
        None.
        """
        if value is None:
            return None
        if value.startswith(b'!'):
            try:
                return pickle.loads(value[1:])
            except pickle.PickleError:
                return None
        try:
            return int(value)
        except ValueError:
            # before 0.8 we did not have serialization.  Still support that.
            return value 

def from_metadata(cls, metadata, rng=None):
        if rng is None: rng = gu.gen_rng(0)
        # Unpickle the sklearn forest.
        forest = cPickle.loads(
            base64.b64decode(metadata['params']['forest_binary']))
        metadata['params']['forest'] = forest
        forest = cls(
            outputs=metadata['outputs'],
            inputs=metadata['inputs'],
            hypers=metadata['hypers'],
            params=metadata['params'],
            distargs=metadata['distargs'],
            rng=rng)
        # json keys are strings -- convert back to integers.
        x = ((int(k), v) for k, v in metadata['data']['x'].iteritems())
        Y = ((int(k), v) for k, v in metadata['data']['Y'].iteritems())
        forest.data = Data(x=OrderedDict(x), Y=OrderedDict(Y))
        forest.N = metadata['N']
        forest.counts = metadata['counts']
        return forest 

def from_metadata(cls, metadata, rng=None):
        if rng is None: rng = gu.gen_rng(0)
        # Unpickle the sklearn ols.
        skl_ols = cPickle.loads(
            base64.b64decode(metadata['params']['regressor_binary']))
        metadata['params']['regressor'] = skl_ols
        ols = cls(
            outputs=metadata['outputs'],
            inputs=metadata['inputs'],
            params=metadata['params'],
            distargs=metadata['distargs'],
            rng=rng)
        # json keys are strings -- convert back to integers.
        x = ((int(k), v) for k, v in metadata['data']['x'].iteritems())
        Y = ((int(k), v) for k, v in metadata['data']['Y'].iteritems())
        ols.data = Data(x=OrderedDict(x), Y=OrderedDict(Y))
        ols.N = metadata['N']
        return ols 

def load_object(self, value):
        """The reversal of :meth:`dump_object`.  This might be called with
        None.
        """
        if value is None:
            return None
        if value.startswith(b"!"):
            try:
                return pickle.loads(value[1:])
            except pickle.PickleError:
                return None
        try:
            return int(value)
        except ValueError:
            # before 0.8 we did not have serialization.  Still support that.
            return value 

def _extract(self, rec):
        if rec is None:
            return None
        else:
            key, data = rec
            # Safe in Python 2.x because expresion short circuit
            if sys.version_info[0] < 3 or isinstance(data, bytes) :
                return key, cPickle.loads(data)
            else :
                return key, cPickle.loads(bytes(data, "iso8859-1"))  # 8 bits

    #----------------------------------------------
    # Methods allowed to pass-through to self.dbc
    #
    # close, count, delete, get_recno, join_item


#--------------------------------------------------------------------------- 

def load_object(self, value):
        """The reversal of :meth:`dump_object`.  This might be called with
        None.
        """
        if value is None:
            return None
        if value.startswith(b'!'):
            try:
                return pickle.loads(value[1:])
            except pickle.PickleError:
                return None
        try:
            return int(value)
        except ValueError:
            # before 0.8 we did not have serialization.  Still support that.
            return value 

def testFlow_save():
    dummy_list = [1,2,3]
    flow = _get_default_flow()
    flow[0].dummy_attr = dummy_list
    # test string save
    copy_flow_pic = flow.save(None)
    copy_flow = cPickle.loads(copy_flow_pic)
    assert flow[0].dummy_attr == copy_flow[0].dummy_attr, \
           'Flow save (string) method did not work'
    copy_flow[0].dummy_attr[0] = 10
    assert flow[0].dummy_attr != copy_flow[0].dummy_attr, \
           'Flow save (string) method did not work'
    # test file save
    dummy_file = tempfile.mktemp(prefix='MDP_', suffix=".pic",
                                 dir=py.test.mdp_tempdirname)
    flow.save(dummy_file, protocol=1)
    dummy_file = open(dummy_file, 'rb')
    copy_flow = cPickle.load(dummy_file)
    assert flow[0].dummy_attr == copy_flow[0].dummy_attr, \
           'Flow save (file) method did not work'
    copy_flow[0].dummy_attr[0] = 10
    assert flow[0].dummy_attr != copy_flow[0].dummy_attr, \
           'Flow save (file) method did not work' 

def ListTableColumns(self, table):
        """Return a list of columns in the given table.
        [] if the table doesn't exist.
        """
        assert isinstance(table, str)
        if contains_metastrings(table):
            raise ValueError, "bad table name: contains reserved metastrings"

        columnlist_key = _columns_key(table)
        if not getattr(self.db, "has_key")(columnlist_key):
            return []
        pickledcolumnlist = getattr(self.db, "get_bytes",
                self.db.get)(columnlist_key)
        if pickledcolumnlist:
            return pickle.loads(pickledcolumnlist)
        else:
            return [] 

def test_Node_save():
    test_list = [1,2,3]
    generic_node = mdp.Node()
    generic_node.dummy_attr = test_list
    # test string save
    copy_node_pic = generic_node.save(None)
    copy_node = cPickle.loads(copy_node_pic)
    assert generic_node.dummy_attr == copy_node.dummy_attr,\
           'Node save (string) method did not work'
    copy_node.dummy_attr[0] = 10
    assert generic_node.dummy_attr != copy_node.dummy_attr,\
           'Node save (string) method did not work'
    # test file save
    dummy_file = tempfile.mktemp(prefix='MDP_', suffix=".pic",
                                 dir=py.test.mdp_tempdirname)
    generic_node.save(dummy_file, protocol=1)
    dummy_file = open(dummy_file, 'rb')
    copy_node = cPickle.load(dummy_file)
    assert generic_node.dummy_attr == copy_node.dummy_attr,\
           'Node save (file) method did not work'
    copy_node.dummy_attr[0] = 10
    assert generic_node.dummy_attr != copy_node.dummy_attr,\
           'Node save (file) method did not work' 

def _augment_images_worker(self, augseq, queue_source, queue_result):
        """Worker function that endlessly queries the source queue (input
        batches), augments batches in it and sends the result to the output
        queue."""
        while True:
            # wait for a new batch in the source queue and load it
            batch_str = queue_source.get()
            batch = pickle.loads(batch_str)

            # augment the batch
            if batch.images is not None and batch.keypoints is not None:
                augseq_det = augseq.to_deterministic()
                batch.images_aug = augseq_det.augment_images(batch.images)
                batch.keypoints_aug = augseq_det.augment_keypoints(batch.keypoints)
            elif batch.images is not None:
                batch.images_aug = augseq.augment_images(batch.images)
            elif batch.keypoints is not None:
                batch.keypoints_aug = augseq.augment_keypoints(batch.keypoints)

            # send augmented batch to output queue
            queue_result.put(pickle.dumps(batch, protocol=-1)) 

def unpickle_args(items):
    '''Takes a dict and unpickles values whose keys are found in
    '_pickled' key.

    >>> unpickle_args({'_pickled': ['foo']. 'foo': ['I3%0A.']})
    {'foo': 3}
    '''
    # Technically there can be more than one _pickled value. At this point
    # we'll just use the first one
    pickled= items.pop('_pickled', None)
    if pickled is None:
        return items

    pickled_keys = pickled[0].split(',')
    ret = {}
    for key, vals in items.items():
        if key in pickled_keys:
            ret[key] = [pickle.loads(val) for val in vals]
        else:
            ret[key] = vals
    return ret 

def unpickle_dict(items):
    """un-pickles a dictionary that was pickled with `pickle_dict`.

    Args:
        items (dict): A pickled dictionary.

    Returns:
        dict: An un-pickled dictionary.
    """
    pickled_keys = items.pop('_pickled', '').split(',')
    ret = {}
    for k, v in items.items():
        if k in pickled_keys:
            ret[k] = pickle.loads(v)
        else:
            ret[k] = v
    return ret 

def __getitem__(self, key):
        data = self.db[key]
        return cPickle.loads(data) 

def test_marshal(self):
        import marshal
        self.assertIs(marshal.loads(marshal.dumps(True)), True)
        self.assertIs(marshal.loads(marshal.dumps(False)), False) 

def test_pickle(self):
        for protocol in range(HIGHEST_PROTOCOL + 1):
            a = array.array(self.typecode, self.example)
            b = loads(dumps(a, protocol))
            self.assertNotEqual(id(a), id(b))
            self.assertEqual(a, b)

            a = ArraySubclass(self.typecode, self.example)
            a.x = 10
            b = loads(dumps(a, protocol))
            self.assertNotEqual(id(a), id(b))
            self.assertEqual(a, b)
            self.assertEqual(a.x, b.x)
            self.assertEqual(type(a), type(b)) 

def test_pickle_for_empty_array(self):
        for protocol in range(HIGHEST_PROTOCOL + 1):
            a = array.array(self.typecode)
            b = loads(dumps(a, protocol))
            self.assertNotEqual(id(a), id(b))
            self.assertEqual(a, b)

            a = ArraySubclass(self.typecode)
            a.x = 10
            b = loads(dumps(a, protocol))
            self.assertNotEqual(id(a), id(b))
            self.assertEqual(a, b)
            self.assertEqual(a.x, b.x)
            self.assertEqual(type(a), type(b)) 

def test_pickle(self):
        d = deque(xrange(200))
        for i in range(pickle.HIGHEST_PROTOCOL + 1):
            s = pickle.dumps(d, i)
            e = pickle.loads(s)
            self.assertNotEqual(id(d), id(e))
            self.assertEqual(list(d), list(e))

##    def test_pickle_recursive(self):
##        d = deque('abc')
##        d.append(d)
##        for i in range(pickle.HIGHEST_PROTOCOL + 1):
##            e = pickle.loads(pickle.dumps(d, i))
##            self.assertNotEqual(id(d), id(e))
##            self.assertEqual(id(e), id(e[-1])) 

def loads(self, *args):
        f = self.input(args[0])
        try:
            p = cPickle.Unpickler(f)
            return p.load()
        finally:
            self.close(f) 

def __load_column_info(self, table) :
        """initialize the self.__tablecolumns dict"""
        # check the column names
        try:
            tcolpickles = getattr(self.db, "get_bytes",
                    self.db.get)(_columns_key(table))
        except db.DBNotFoundError:
            raise TableDBError, "unknown table: %r" % (table,)
        if not tcolpickles:
            raise TableDBError, "unknown table: %r" % (table,)
        self.__tablecolumns[table] = pickle.loads(tcolpickles) 

def loads(self, input):
        return self.module.loads(input)

    # These tests are disabled because they require some special setup
    # on the worker that's hard to keep in sync. 

def loads(self, buf):
        # Ignore fast
        return cPickle.loads(buf) 

def test_copy_deepcopy_pickle(self):
        r = F(13, 7)
        dr = DummyFraction(13, 7)
        self.assertEqual(r, loads(dumps(r)))
        self.assertEqual(id(r), id(copy(r)))
        self.assertEqual(id(r), id(deepcopy(r)))
        self.assertNotEqual(id(dr), id(copy(dr)))
        self.assertNotEqual(id(dr), id(deepcopy(dr)))
        self.assertTypedEquals(dr, copy(dr))
        self.assertTypedEquals(dr, deepcopy(dr)) 

def get_both(self, key, value, txn=None, flags=0):
        data = _dumps(value, self.protocol)
        data = self.db.get(key, data, txn, flags)
        return cPickle.loads(data) 

def get(self, *args, **kw):
        # We do it with *args and **kw so if the default value wasn't
        # given nothing is passed to the extension module.  That way
        # an exception can be raised if set_get_returns_none is turned
        # off.
        data = self.db.get(*args, **kw)
        try:
            return cPickle.loads(data)
        except (EOFError, TypeError, cPickle.UnpicklingError):
            return data  # we may be getting the default value, or None,
                         # so it doesn't need unpickled. 

def associate(self, secondaryDB, callback, flags=0):
        def _shelf_callback(priKey, priData, realCallback=callback):
            # Safe in Python 2.x because expresion short circuit
            if sys.version_info[0] < 3 or isinstance(priData, bytes) :
                data = cPickle.loads(priData)
            else :
                data = cPickle.loads(bytes(priData, "iso8859-1"))  # 8 bits
            return realCallback(priKey, data)

        return self.db.associate(secondaryDB, _shelf_callback, flags)


    #def get(self, key, default=None, txn=None, flags=0): 

def items(self, txn=None):
        if txn is not None:
            items = self.db.items(txn)
        else:
            items = self.db.items()
        newitems = []

        for k, v in items:
            newitems.append( (k, cPickle.loads(v)) )
        return newitems