Python _pickle.Pickler() Examples

def test_cpickle(_cache={}):
    import io
    try:
        import _pickle
    except ImportError:
        print("cannot import _pickle, skipped!")
        return
    k, l = None, None
    for n in itertools.count():
        try:
            l = _cache[n]
            continue  # Already tried and it works, let's save some time
        except KeyError:
            for i in range(100):
                l = [k, l]
                k = {i: l}
        _pickle.Pickler(io.BytesIO(), protocol=-1).dump(l)
        _cache[n] = l 

def test_pickler(self):
            basesize = support.calcobjsize('5P2n3i2n3iP')
            p = _pickle.Pickler(io.BytesIO())
            self.assertEqual(object.__sizeof__(p), basesize)
            MT_size = struct.calcsize('3nP0n')
            ME_size = struct.calcsize('Pn0P')
            check = self.check_sizeof
            check(p, basesize +
                MT_size + 8 * ME_size +  # Minimal memo table size.
                sys.getsizeof(b'x'*4096))  # Minimal write buffer size.
            for i in range(6):
                p.dump(chr(i))
            check(p, basesize +
                MT_size + 32 * ME_size +  # Size of memo table required to
                                          # save references to 6 objects.
                0)  # Write buffer is cleared after every dump(). 

def test_signature_on_builtin_class(self):
        self.assertEqual(str(inspect.signature(_pickle.Pickler)),
                         '(file, protocol=None, fix_imports=True)')

        class P(_pickle.Pickler): pass
        class EmptyTrait: pass
        class P2(EmptyTrait, P): pass
        self.assertEqual(str(inspect.signature(P)),
                         '(file, protocol=None, fix_imports=True)')
        self.assertEqual(str(inspect.signature(P2)),
                         '(file, protocol=None, fix_imports=True)')

        class P3(P2):
            def __init__(self, spam):
                pass
        self.assertEqual(str(inspect.signature(P3)), '(spam)')

        class MetaP(type):
            def __call__(cls, foo, bar):
                pass
        class P4(P2, metaclass=MetaP):
            pass
        self.assertEqual(str(inspect.signature(P4)), '(foo, bar)') 

def test_cpickle(_cache={}):
    import io
    try:
        import _pickle
    except ImportError:
        print("cannot import _pickle, skipped!")
        return
    k, l = None, None
    for n in itertools.count():
        try:
            l = _cache[n]
            continue  # Already tried and it works, let's save some time
        except KeyError:
            for i in range(100):
                l = [k, l]
                k = {i: l}
        _pickle.Pickler(io.BytesIO(), protocol=-1).dump(l)
        _cache[n] = l 

def test_cpickle(_cache={}):
    import io
    try:
        import _pickle
    except ImportError:
        print("cannot import _pickle, skipped!")
        return
    k, l = None, None
    for n in itertools.count():
        try:
            l = _cache[n]
            continue  # Already tried and it works, let's save some time
        except KeyError:
            for i in range(100):
                l = [k, l]
                k = {i: l}
        _pickle.Pickler(io.BytesIO(), protocol=-1).dump(l)
        _cache[n] = l 

def test_pickler(self):
            basesize = support.calcobjsize('5P2n3i2n3iP')
            p = _pickle.Pickler(io.BytesIO())
            self.assertEqual(object.__sizeof__(p), basesize)
            MT_size = struct.calcsize('3nP0n')
            ME_size = struct.calcsize('Pn0P')
            check = self.check_sizeof
            check(p, basesize +
                MT_size + 8 * ME_size +  # Minimal memo table size.
                sys.getsizeof(b'x'*4096))  # Minimal write buffer size.
            for i in range(6):
                p.dump(chr(i))
            check(p, basesize +
                MT_size + 32 * ME_size +  # Size of memo table required to
                                          # save references to 6 objects.
                0)  # Write buffer is cleared after every dump(). 

def test_signature_on_builtin_class(self):
        self.assertEqual(str(inspect.signature(_pickle.Pickler)),
                         '(file, protocol=None, fix_imports=True)')

        class P(_pickle.Pickler): pass
        class EmptyTrait: pass
        class P2(EmptyTrait, P): pass
        self.assertEqual(str(inspect.signature(P)),
                         '(file, protocol=None, fix_imports=True)')
        self.assertEqual(str(inspect.signature(P2)),
                         '(file, protocol=None, fix_imports=True)')

        class P3(P2):
            def __init__(self, spam):
                pass
        self.assertEqual(str(inspect.signature(P3)), '(spam)')

        class MetaP(type):
            def __call__(cls, foo, bar):
                pass
        class P4(P2, metaclass=MetaP):
            pass
        self.assertEqual(str(inspect.signature(P4)), '(foo, bar)') 

def test_signature_on_builtin_class(self):
        self.assertEqual(str(inspect.signature(_pickle.Pickler)),
                         '(file, protocol=None, fix_imports=True)')

        class P(_pickle.Pickler): pass
        class EmptyTrait: pass
        class P2(EmptyTrait, P): pass
        self.assertEqual(str(inspect.signature(P)),
                         '(file, protocol=None, fix_imports=True)')
        self.assertEqual(str(inspect.signature(P2)),
                         '(file, protocol=None, fix_imports=True)')

        class P3(P2):
            def __init__(self, spam):
                pass
        self.assertEqual(str(inspect.signature(P3)), '(spam)')

        class MetaP(type):
            def __call__(cls, foo, bar):
                pass
        class P4(P2, metaclass=MetaP):
            pass
        self.assertEqual(str(inspect.signature(P4)), '(foo, bar)') 

def test_pickler(self):
            basesize = support.calcobjsize('5P2n3i2n3iP')
            p = _pickle.Pickler(io.BytesIO())
            self.assertEqual(object.__sizeof__(p), basesize)
            MT_size = struct.calcsize('3nP0n')
            ME_size = struct.calcsize('Pn0P')
            check = self.check_sizeof
            check(p, basesize +
                MT_size + 8 * ME_size +  # Minimal memo table size.
                sys.getsizeof(b'x'*4096))  # Minimal write buffer size.
            for i in range(6):
                p.dump(chr(i))
            check(p, basesize +
                MT_size + 32 * ME_size +  # Size of memo table required to
                                          # save references to 6 objects.
                0)  # Write buffer is cleared after every dump(). 

def _function_reduce(self, obj):
        """Reducer for function objects.

        If obj is a top-level attribute of a file-backed module, this
        reducer returns NotImplemented, making the CloudPickler fallback to
        traditional _pickle.Pickler routines to save obj. Otherwise, it reduces
        obj using a custom cloudpickle reducer designed specifically to handle
        dynamic functions.

        As opposed to cloudpickle.py, There no special handling for builtin
        pypy functions because cloudpickle_fast is CPython-specific.
        """
        if _is_importable_by_name(obj):
            return NotImplemented
        else:
            return self._dynamic_function_reduce(obj) 

def test_unbound_builtin_method(self):
        self.assertEqual(self._get_summary_line(_pickle.Pickler.dump),
            "dump(self, obj, /)")

    # these no longer include "self" 

def test_bound_builtin_method(self):
        s = StringIO()
        p = _pickle.Pickler(s)
        self.assertEqual(self._get_summary_line(p.dump),
            "dump(obj, /) method of _pickle.Pickler instance")

    # this should *never* include self! 

def test_unbound_builtin_method(self):
        self.assertEqual(self._get_summary_line(_pickle.Pickler.dump),
            "dump(self, obj, /)")

    # these no longer include "self" 

def test_getfullargspec_builtin_methods(self):
        self.assertFullArgSpecEquals(_pickle.Pickler.dump,
                                     args_e=['self', 'obj'], formatted='(self, obj)')

        self.assertFullArgSpecEquals(_pickle.Pickler(io.BytesIO()).dump,
                                     args_e=['self', 'obj'], formatted='(self, obj)')

        self.assertFullArgSpecEquals(
             os.stat,
             args_e=['path'],
             kwonlyargs_e=['dir_fd', 'follow_symlinks'],
             kwonlydefaults_e={'dir_fd': None, 'follow_symlinks': True},
             formatted='(path, *, dir_fd=None, follow_symlinks=True)') 

def test_bound_builtin_method(self):
        s = StringIO()
        p = _pickle.Pickler(s)
        self.assertEqual(self._get_summary_line(p.dump),
            "dump(obj, /) method of _pickle.Pickler instance")

    # this should *never* include self! 

def test_unbound_builtin_method(self):
        self.assertEqual(self._get_summary_line(_pickle.Pickler.dump),
            "dump(self, obj, /)")

    # these no longer include "self" 

def dump(self, obj):
        try:
            return Pickler.dump(self, obj)
        except RuntimeError as e:
            if "recursion" in e.args[0]:
                msg = (
                    "Could not pickle object as excessively deep recursion "
                    "required."
                )
                raise pickle.PicklingError(msg)
            else:
                raise 

def __init__(self, file, protocol=None, buffer_callback=None):
        if protocol is None:
            protocol = DEFAULT_PROTOCOL
        Pickler.__init__(self, file, protocol=protocol, buffer_callback=buffer_callback)
        # map functions __globals__ attribute ids, to ensure that functions
        # sharing the same global namespace at pickling time also share their
        # global namespace at unpickling time.
        self.globals_ref = {}

        # Take into account potential custom reducers registered by external
        # modules
        self.dispatch_table = copyreg.dispatch_table.copy()
        self.dispatch_table.update(self.dispatch)
        self.proto = int(protocol) 

def test_getfullargspec_builtin_methods(self):
        self.assertFullArgSpecEquals(_pickle.Pickler.dump,
                                     args_e=['self', 'obj'], formatted='(self, obj)')

        self.assertFullArgSpecEquals(_pickle.Pickler(io.BytesIO()).dump,
                                     args_e=['self', 'obj'], formatted='(self, obj)')

        self.assertFullArgSpecEquals(
             os.stat,
             args_e=['path'],
             kwonlyargs_e=['dir_fd', 'follow_symlinks'],
             kwonlydefaults_e={'dir_fd': None, 'follow_symlinks': True},
             formatted='(path, *, dir_fd=None, follow_symlinks=True)') 

def test_bound_builtin_method(self):
        s = StringIO()
        p = _pickle.Pickler(s)
        self.assertEqual(self._get_summary_line(p.dump),
            "dump(obj, /) method of _pickle.Pickler instance")

    # this should *never* include self! 

def test_unbound_builtin_method(self):
        self.assertEqual(self._get_summary_line(_pickle.Pickler.dump),
            "dump(self, obj, /)")

    # these no longer include "self" 

def test_getfullargspec_builtin_methods(self):
        self.assertFullArgSpecEquals(_pickle.Pickler.dump,
                                     args_e=['self', 'obj'], formatted='(self, obj)')

        self.assertFullArgSpecEquals(_pickle.Pickler(io.BytesIO()).dump,
                                     args_e=['self', 'obj'], formatted='(self, obj)')

        self.assertFullArgSpecEquals(
             os.stat,
             args_e=['path'],
             kwonlyargs_e=['dir_fd', 'follow_symlinks'],
             kwonlydefaults_e={'dir_fd': None, 'follow_symlinks': True},
             formatted='(path, *, dir_fd=None, follow_symlinks=True)') 

def test_bound_builtin_method(self):
        s = StringIO()
        p = _pickle.Pickler(s)
        self.assertEqual(self._get_summary_line(p.dump),
            "dump(obj, /) method of _pickle.Pickler instance")

    # this should *never* include self! 

def reducer_override(self, obj):
        """Type-agnostic reducing callback for function and classes.

        For performance reasons, subclasses of the C _pickle.Pickler class
        cannot register custom reducers for functions and classes in the
        dispatch_table. Reducer for such types must instead implemented in the
        special reducer_override method.

        Note that method will be called for any object except a few
        builtin-types (int, lists, dicts etc.), which differs from reducers in
        the Pickler's dispatch_table, each of them being invoked for objects of
        a specific type only.

        This property comes in handy for classes: although most classes are
        instances of the ``type`` metaclass, some of them can be instances of
        other custom metaclasses (such as enum.EnumMeta for example). In
        particular, the metaclass will likely not be known in advance, and thus
        cannot be special-cased using an entry in the dispatch_table.
        reducer_override, among other things, allows us to register a reducer
        that will be called for any class, independently of its type.


        Notes:

        * reducer_override has the priority over dispatch_table-registered
          reducers.
        * reducer_override can be used to fix other limitations of cloudpickle
          for other types that suffered from type-specific reducers, such as
          Exceptions. See https://github.com/cloudpipe/cloudpickle/issues/248
        """

        # This is a patch for python3.5
        if isinstance(obj, numpy.ndarray):
            if (self.proto < 5 or
                    (not obj.flags.c_contiguous and not obj.flags.f_contiguous) or
                    (issubclass(type(obj), numpy.ndarray) and type(obj) is not numpy.ndarray) or
                    obj.dtype == "O" or obj.itemsize == 0):
                return NotImplemented
            return _numpy_ndarray_reduce(obj)

        t = type(obj)
        try:
            is_anyclass = issubclass(t, type)
        except TypeError:  # t is not a class (old Boost; see SF #502085)
            is_anyclass = False

        if is_anyclass:
            return _class_reduce(obj)
        elif isinstance(obj, types.FunctionType):
            return self._function_reduce(obj)
        else:
            # fallback to save_global, including the Pickler's distpatch_table
            return NotImplemented

    # function reducers are defined as instance methods of CloudPickler
    # objects, as they rely on a CloudPickler attribute (globals_ref)