Python marshal.load() Examples

def test_foreign_code(self):
        py_compile.compile(self.file_name)
        with open(self.compiled_name, "rb") as f:
            header = f.read(8)
            code = marshal.load(f)
        constants = list(code.co_consts)
        foreign_code = test_main.func_code
        pos = constants.index(1)
        constants[pos] = foreign_code
        code = type(code)(code.co_argcount, code.co_nlocals, code.co_stacksize,
                          code.co_flags, code.co_code, tuple(constants),
                          code.co_names, code.co_varnames, code.co_filename,
                          code.co_name, code.co_firstlineno, code.co_lnotab,
                          code.co_freevars, code.co_cellvars)
        with open(self.compiled_name, "wb") as f:
            f.write(header)
            marshal.dump(code, f)
        mod = self.import_module()
        self.assertEqual(mod.constant.co_filename, foreign_code.co_filename) 

def load_bytecode(self, f):
        """Loads bytecode from a file or file like object."""
        # make sure the magic header is correct
        magic = f.read(len(bc_magic))
        if magic != bc_magic:
            self.reset()
            return
        # the source code of the file changed, we need to reload
        checksum = pickle.load(f)
        if self.checksum != checksum:
            self.reset()
            return
        # if marshal_load fails then we need to reload
        try:
            self.code = marshal_load(f)
        except (EOFError, ValueError, TypeError):
            self.reset()
            return 

def load_bytecode(self, f):
        """Loads bytecode from a file or file like object."""
        # make sure the magic header is correct
        magic = f.read(len(bc_magic))
        if magic != bc_magic:
            self.reset()
            return
        # the source code of the file changed, we need to reload
        checksum = pickle.load(f)
        if self.checksum != checksum:
            self.reset()
            return
        # if marshal_load fails then we need to reload
        try:
            self.code = marshal_load(f)
        except (EOFError, ValueError, TypeError):
            self.reset()
            return 

def test_foreign_code(self):
        py_compile.compile(self.file_name)
        with open(self.compiled_name, "rb") as f:
            header = f.read(8)
            code = marshal.load(f)
        constants = list(code.co_consts)
        foreign_code = test_main.func_code
        pos = constants.index(1)
        constants[pos] = foreign_code
        code = type(code)(code.co_argcount, code.co_nlocals, code.co_stacksize,
                          code.co_flags, code.co_code, tuple(constants),
                          code.co_names, code.co_varnames, code.co_filename,
                          code.co_name, code.co_firstlineno, code.co_lnotab,
                          code.co_freevars, code.co_cellvars)
        with open(self.compiled_name, "wb") as f:
            f.write(header)
            marshal.dump(code, f)
        mod = self.import_module()
        self.assertEqual(mod.constant.co_filename, foreign_code.co_filename) 

def load_bytecode(self, f):
        """Loads bytecode from a file or file like object."""
        # make sure the magic header is correct
        magic = f.read(len(bc_magic))
        if magic != bc_magic:
            self.reset()
            return
        # the source code of the file changed, we need to reload
        checksum = pickle.load(f)
        if self.checksum != checksum:
            self.reset()
            return
        # if marshal_load fails then we need to reload
        try:
            self.code = marshal_load(f)
        except (EOFError, ValueError, TypeError):
            self.reset()
            return 

def load_bytecode(self, f):
        """Loads bytecode from a file or file like object."""
        # make sure the magic header is correct
        magic = f.read(len(bc_magic))
        if magic != bc_magic:
            self.reset()
            return
        # the source code of the file changed, we need to reload
        checksum = pickle.load(f)
        if self.checksum != checksum:
            self.reset()
            return
        # if marshal_load fails then we need to reload
        try:
            self.code = marshal_load(f)
        except (EOFError, ValueError, TypeError):
            self.reset()
            return 

def test_imp_module(self):
        # Verify that the imp module can correctly load and find .py files

        # XXX (ncoghlan): It would be nice to use test_support.CleanImport
        # here, but that breaks because the os module registers some
        # handlers in copy_reg on import. Since CleanImport doesn't
        # revert that registration, the module is left in a broken
        # state after reversion. Reinitialising the module contents
        # and just reverting os.environ to its previous state is an OK
        # workaround
        orig_path = os.path
        orig_getenv = os.getenv
        with EnvironmentVarGuard():
            x = imp.find_module("os")
            new_os = imp.load_module("os", *x)
            self.assertIs(os, new_os)
            self.assertIs(orig_path, new_os.path)
            self.assertIsNot(orig_getenv, new_os.getenv) 

def load_stats(self, arg):
        if not arg:  self.stats = {}
        elif isinstance(arg, basestring):
            f = open(arg, 'rb')
            self.stats = marshal.load(f)
            f.close()
            try:
                file_stats = os.stat(arg)
                arg = time.ctime(file_stats.st_mtime) + "    " + arg
            except:  # in case this is not unix
                pass
            self.files = [ arg ]
        elif hasattr(arg, 'create_stats'):
            arg.create_stats()
            self.stats = arg.stats
            arg.stats = {}
        if not self.stats:
            raise TypeError("Cannot create or construct a %r object from %r"
                            % (self.__class__, arg))
        return 

def test_file_multiple_reads(self):
        """calling load w/ a file should only advance the length of the file"""
        l = []
        for i in xrange(10):
            l.append(marshal.dumps({i:i}))

        data = ''.join(l)
        with open('tempfile.txt', 'w') as f:
            f.write(data)

        with open('tempfile.txt') as f:
            for i in xrange(10):
                obj = marshal.load(f)
                self.assertEqual(obj, {i:i})

        self.delete_files('tempfile.txt') 

def load_stats(self, arg):
        if not arg:  self.stats = {}
        elif isinstance(arg, basestring):
            f = open(arg, 'rb')
            self.stats = marshal.load(f)
            f.close()
            try:
                file_stats = os.stat(arg)
                arg = time.ctime(file_stats.st_mtime) + "    " + arg
            except:  # in case this is not unix
                pass
            self.files = [ arg ]
        elif hasattr(arg, 'create_stats'):
            arg.create_stats()
            self.stats = arg.stats
            arg.stats = {}
        if not self.stats:
            raise TypeError,  "Cannot create or construct a %r object from '%r''" % (
                              self.__class__, arg)
        return 

def _import_one(self, parent, modname, fqname):
        "Import a single module."

        # has the module already been imported?
        try:
            return sys.modules[fqname]
        except KeyError:
            pass

        # load the module's code, or fetch the module itself
        result = self.get_code(parent, modname, fqname)
        if result is None:
            return None

        module = self._process_result(result, fqname)

        # insert the module into its parent
        if parent:
            setattr(parent, modname, module)
        return module 

def get_code(self, parent, modname, fqname):
        if parent:
            # these modules definitely do not occur within a package context
            return None

        # look for the module
        if imp.is_builtin(modname):
            type = imp.C_BUILTIN
        elif imp.is_frozen(modname):
            type = imp.PY_FROZEN
        else:
            # not found
            return None

        # got it. now load and return it.
        module = imp.load_module(modname, None, modname, ('', '', type))
        return 0, module, { }


######################################################################
#
# Internal importer used for importing from the filesystem
# 

def py_suffix_importer(filename, finfo, fqname):
    file = filename[:-3] + _suffix
    t_py = long(finfo[8])
    t_pyc = _timestamp(file)

    code = None
    if t_pyc is not None and t_pyc >= t_py:
        f = open(file, 'rb')
        if f.read(4) == imp.get_magic():
            t = struct.unpack('<I', f.read(4))[0]
            if t == t_py:
                code = marshal.load(f)
        f.close()
    if code is None:
        file = filename
        code = _compile(file, t_py)

    return 0, code, { '__file__' : file } 

def get_code(self, parent, modname, fqname):
        if parent:
            # these modules definitely do not occur within a package context
            return None

        # look for the module
        if imp.is_builtin(modname):
            type = imp.C_BUILTIN
        elif imp.is_frozen(modname):
            type = imp.PY_FROZEN
        else:
            # not found
            return None

        # got it. now load and return it.
        module = imp.load_module(modname, None, modname, ('', '', type))
        return 0, module, { }


######################################################################
#
# Internal importer used for importing from the filesystem
# 

def _import_one(self, parent, modname, fqname):
        "Import a single module."

        # has the module already been imported?
        try:
            return sys.modules[fqname]
        except KeyError:
            pass

        # load the module's code, or fetch the module itself
        result = self.get_code(parent, modname, fqname)
        if result is None:
            return None

        module = self._process_result(result, fqname)

        # insert the module into its parent
        if parent:
            setattr(parent, modname, module)
        return module 

def __init__(self, fs_imp=None):
        # we're definitely going to be importing something in the future,
        # so let's just load the OS-related facilities.
        if not _os_stat:
            _os_bootstrap()

        # This is the Importer that we use for grabbing stuff from the
        # filesystem. It defines one more method (import_from_dir) for our use.
        if fs_imp is None:
            cls = self.clsFilesystemImporter or _FilesystemImporter
            fs_imp = cls()
        self.fs_imp = fs_imp

        # Initialize the set of suffixes that we recognize and import.
        # The default will import dynamic-load modules first, followed by
        # .py files (or a .py file's cached bytecode)
        for desc in imp.get_suffixes():
            if desc[2] == imp.C_EXTENSION:
                self.add_suffix(desc[0],
                                DynLoadSuffixImporter(desc).import_file)
        self.add_suffix('.py', py_suffix_importer) 

def test_recursion_limit(self):
        # Create a deeply nested structure.
        head = last = []
        # The max stack depth should match the value in Python/marshal.c.
        if os.name == 'nt' and hasattr(sys, 'gettotalrefcount'):
            MAX_MARSHAL_STACK_DEPTH = 1000
        else:
            MAX_MARSHAL_STACK_DEPTH = 2000
        for i in range(MAX_MARSHAL_STACK_DEPTH - 2):
            last.append([0])
            last = last[-1]

        # Verify we don't blow out the stack with dumps/load.
        data = marshal.dumps(head)
        new_head = marshal.loads(data)
        # Don't use == to compare objects, it can exceed the recursion limit.
        self.assertEqual(len(new_head), len(head))
        self.assertEqual(len(new_head[0]), len(head[0]))
        self.assertEqual(len(new_head[-1]), len(head[-1]))

        last.append([0])
        self.assertRaises(ValueError, marshal.dumps, head) 

def load_stats(self, arg):
        if not arg:  self.stats = {}
        elif isinstance(arg, basestring):
            f = open(arg, 'rb')
            self.stats = marshal.load(f)
            f.close()
            try:
                file_stats = os.stat(arg)
                arg = time.ctime(file_stats.st_mtime) + "    " + arg
            except:  # in case this is not unix
                pass
            self.files = [ arg ]
        elif hasattr(arg, 'create_stats'):
            arg.create_stats()
            self.stats = arg.stats
            arg.stats = {}
        if not self.stats:
            raise TypeError("Cannot create or construct a %r object from %r"
                            % (self.__class__, arg))
        return 

def load_bytecode(self, f):
        """Loads bytecode from a file or file like object."""
        # make sure the magic header is correct
        magic = f.read(len(bc_magic))
        if magic != bc_magic:
            self.reset()
            return
        # the source code of the file changed, we need to reload
        checksum = pickle.load(f)
        if self.checksum != checksum:
            self.reset()
            return
        # if marshal_load fails then we need to reload
        try:
            self.code = marshal_load(f)
        except (EOFError, ValueError, TypeError):
            self.reset()
            return 

def __init__(self, fs_imp=None):
        # we're definitely going to be importing something in the future,
        # so let's just load the OS-related facilities.
        if not _os_stat:
            _os_bootstrap()

        # This is the Importer that we use for grabbing stuff from the
        # filesystem. It defines one more method (import_from_dir) for our use.
        if fs_imp is None:
            cls = self.clsFilesystemImporter or _FilesystemImporter
            fs_imp = cls()
        self.fs_imp = fs_imp

        # Initialize the set of suffixes that we recognize and import.
        # The default will import dynamic-load modules first, followed by
        # .py files (or a .py file's cached bytecode)
        for desc in imp.get_suffixes():
            if desc[2] == imp.C_EXTENSION:
                self.add_suffix(desc[0],
                                DynLoadSuffixImporter(desc).import_file)
        self.add_suffix('.py', py_suffix_importer) 

def test_list(self):
        lst = self.d.items()
        new = marshal.loads(marshal.dumps(lst))
        self.assertEqual(lst, new)
        marshal.dump(lst, file(test_support.TESTFN, "wb"))
        new = marshal.load(file(test_support.TESTFN, "rb"))
        self.assertEqual(lst, new)
        os.unlink(test_support.TESTFN) 

def test_dict(self):
        new = marshal.loads(marshal.dumps(self.d))
        self.assertEqual(self.d, new)
        marshal.dump(self.d, file(test_support.TESTFN, "wb"))
        new = marshal.load(file(test_support.TESTFN, "rb"))
        self.assertEqual(self.d, new)
        os.unlink(test_support.TESTFN) 

def test_tuple(self):
        t = tuple(self.d.keys())
        new = marshal.loads(marshal.dumps(t))
        self.assertEqual(t, new)
        marshal.dump(t, file(test_support.TESTFN, "wb"))
        new = marshal.load(file(test_support.TESTFN, "rb"))
        self.assertEqual(t, new)
        os.unlink(test_support.TESTFN) 

def test_ints(self):
        # Test the full range of Python ints.
        n = sys.maxint
        while n:
            for expected in (-n, n):
                s = marshal.dumps(expected)
                got = marshal.loads(s)
                self.assertEqual(expected, got)
                marshal.dump(expected, file(test_support.TESTFN, "wb"))
                got = marshal.load(file(test_support.TESTFN, "rb"))
                self.assertEqual(expected, got)
            n = n >> 1
        os.unlink(test_support.TESTFN) 

def test_string(self):
        for s in ["", "Andr� Previn", "abc", " "*10000]:
            new = marshal.loads(marshal.dumps(s))
            self.assertEqual(s, new)
            self.assertEqual(type(s), type(new))
            marshal.dump(s, file(test_support.TESTFN, "wb"))
            new = marshal.load(file(test_support.TESTFN, "rb"))
            self.assertEqual(s, new)
            self.assertEqual(type(s), type(new))
        os.unlink(test_support.TESTFN) 

def test_dict(self):
        new = marshal.loads(marshal.dumps(self.d))
        self.assertEqual(self.d, new)
        marshal.dump(self.d, file(test_support.TESTFN, "wb"))
        new = marshal.load(file(test_support.TESTFN, "rb"))
        self.assertEqual(self.d, new)
        os.unlink(test_support.TESTFN) 

def _finish_import(self, top, parts, fromlist):
        # if "a.b.c" was provided, then load the ".b.c" portion down from
        # below the top-level module.
        bottom = self._load_tail(top, parts)

        # if the form is "import a.b.c", then return "a"
        if not fromlist:
            # no fromlist: return the top of the import tree
            return top

        # the top module was imported by self.
        #
        # this means that the bottom module was also imported by self (just
        # now, or in the past and we fetched it from sys.modules).
        #
        # since we imported/handled the bottom module, this means that we can
        # also handle its fromlist (and reliably use __ispkg__).

        # if the bottom node is a package, then (potentially) import some
        # modules.
        #
        # note: if it is not a package, then "fromlist" refers to names in
        #       the bottom module rather than modules.
        # note: for a mix of names and modules in the fromlist, we will
        #       import all modules and insert those into the namespace of
        #       the package module. Python will pick up all fromlist names
        #       from the bottom (package) module; some will be modules that
        #       we imported and stored in the namespace, others are expected
        #       to be present already.
        if bottom.__ispkg__:
            self._import_fromlist(bottom, fromlist)

        # if the form is "from a.b import c, d" then return "b"
        return bottom 

def test_unicode(self):
        for s in [u"", u"Andr� Previn", u"abc", u" "*10000]:
            new = marshal.loads(marshal.dumps(s))
            self.assertEqual(s, new)
            self.assertEqual(type(s), type(new))
            marshal.dump(s, file(test_support.TESTFN, "wb"))
            new = marshal.load(file(test_support.TESTFN, "rb"))
            self.assertEqual(s, new)
            self.assertEqual(type(s), type(new))
        os.unlink(test_support.TESTFN) 

def test_bool(self):
        for b in (True, False):
            new = marshal.loads(marshal.dumps(b))
            self.assertEqual(b, new)
            self.assertEqual(type(b), type(new))
            marshal.dump(b, file(test_support.TESTFN, "wb"))
            new = marshal.load(file(test_support.TESTFN, "rb"))
            self.assertEqual(b, new)
            self.assertEqual(type(b), type(new)) 

def test_buffer(self):
        for s in ["", "Andr� Previn", "abc", " "*10000]:
            with test_support.check_py3k_warnings(("buffer.. not supported",
                                                     DeprecationWarning)):
                b = buffer(s)
            new = marshal.loads(marshal.dumps(b))
            self.assertEqual(s, new)
            marshal.dump(b, file(test_support.TESTFN, "wb"))
            new = marshal.load(file(test_support.TESTFN, "rb"))
            self.assertEqual(s, new)
        os.unlink(test_support.TESTFN)