Python marshal.dumps() Examples

def run_code(self, function, *args, **kwargs):
        log.debug("%s() args:%s kwargs:%s on target", function.func_name, args, kwargs)
        data = {"cmd":self.CODE,
                "code":marshal.dumps(function.func_code),
                "name":function.func_name,
                "args":args,
                "kwargs":kwargs,
                "defaults":function.__defaults__,
                "closure":function.__closure__}
        self.send_data(data)
        log.debug("waiting for code to execute...")
        data = self.recv_data()
        if data["cmd"] == self.EXCEPT:
            log.debug("received exception")
            raise self._process_target_except(data)
        assert data["cmd"] == self.RETURN
        return data["value"] 

def forked_run_report(item):
    # for now, we run setup/teardown in the subprocess
    # XXX optionally allow sharing of setup/teardown
    from _pytest.runner import runtestprotocol
    EXITSTATUS_TESTEXIT = 4
    import marshal

    def runforked():
        try:
            reports = runtestprotocol(item, log=False)
        except KeyboardInterrupt:
            os._exit(EXITSTATUS_TESTEXIT)
        return marshal.dumps([serialize_report(x) for x in reports])

    ff = py.process.ForkedFunc(runforked)
    result = ff.waitfinish()
    if result.retval is not None:
        report_dumps = marshal.loads(result.retval)
        return [runner.TestReport(**x) for x in report_dumps]
    else:
        if result.exitstatus == EXITSTATUS_TESTEXIT:
            pytest.exit("forked test item %s raised Exit" % (item,))
        return [report_process_crash(item, result)] 

def putmessage(self, message):
        self.debug("putmessage:%d:" % message[0])
        try:
            s = pickle.dumps(message)
        except pickle.PicklingError:
            print >>sys.__stderr__, "Cannot pickle:", repr(message)
            raise
        s = struct.pack("<i", len(s)) + s
        while len(s) > 0:
            try:
                r, w, x = select.select([], [self.sock], [])
                n = self.sock.send(s[:BUFSIZE])
            except (AttributeError, TypeError):
                raise IOError, "socket no longer exists"
            except socket.error:
                raise
            else:
                s = s[n:] 

def _write_pyc(state, co, source_stat, pyc):
    # Technically, we don't have to have the same pyc format as
    # (C)Python, since these "pycs" should never be seen by builtin
    # import. However, there's little reason deviate.
    try:
        with atomicwrites.atomic_write(pyc, mode="wb", overwrite=True) as fp:
            fp.write(importlib.util.MAGIC_NUMBER)
            # as of now, bytecode header expects 32-bit numbers for size and mtime (#4903)
            mtime = int(source_stat.st_mtime) & 0xFFFFFFFF
            size = source_stat.st_size & 0xFFFFFFFF
            # "<LL" stands for 2 unsigned longs, little-ending
            fp.write(struct.pack("<LL", mtime, size))
            fp.write(marshal.dumps(co))
    except EnvironmentError as e:
        state.trace("error writing pyc file at {}: errno={}".format(pyc, e.errno))
        # we ignore any failure to write the cache file
        # there are many reasons, permission-denied, __pycache__ being a
        # file etc.
        return False
    return True 

def _write_pyc(state, co, source_stat, pyc):
    # Technically, we don't have to have the same pyc format as
    # (C)Python, since these "pycs" should never be seen by builtin
    # import. However, there's little reason deviate.
    try:
        with atomicwrites.atomic_write(pyc, mode="wb", overwrite=True) as fp:
            fp.write(importlib.util.MAGIC_NUMBER)
            # as of now, bytecode header expects 32-bit numbers for size and mtime (#4903)
            mtime = int(source_stat.st_mtime) & 0xFFFFFFFF
            size = source_stat.st_size & 0xFFFFFFFF
            # "<LL" stands for 2 unsigned longs, little-ending
            fp.write(struct.pack("<LL", mtime, size))
            fp.write(marshal.dumps(co))
    except EnvironmentError as e:
        state.trace("error writing pyc file at {}: errno={}".format(pyc, e.errno))
        # we ignore any failure to write the cache file
        # there are many reasons, permission-denied, __pycache__ being a
        # file etc.
        return False
    return True 

def func_dump(func):
  """Serializes a user defined function.

  Arguments:
      func: the function to serialize.

  Returns:
      A tuple `(code, defaults, closure)`.
  """
  code = marshal.dumps(func.__code__).decode('raw_unicode_escape')
  defaults = func.__defaults__
  if func.__closure__:
    closure = tuple(c.cell_contents for c in func.__closure__)
  else:
    closure = None
  return code, defaults, closure 

def func_dump(func):
    """Serializes a user defined function.

    # Arguments
        func: the function to serialize.

    # Returns
        A tuple `(code, defaults, closure)`.
    """
    raw_code = marshal.dumps(func.__code__)
    code = codecs.encode(raw_code, 'base64').decode('ascii')
    defaults = func.__defaults__
    if func.__closure__:
        closure = tuple(c.cell_contents for c in func.__closure__)
    else:
        closure = None
    return code, defaults, closure 

def func_dump(func):
    """Serializes a user defined function.

    # Arguments
        func: the function to serialize.

    # Returns
        A tuple `(code, defaults, closure)`.
    """
    raw_code = marshal.dumps(func.__code__)
    code = codecs.encode(raw_code, 'base64').decode('ascii')
    defaults = func.__defaults__
    if func.__closure__:
        closure = tuple(c.cell_contents for c in func.__closure__)
    else:
        closure = None
    return code, defaults, closure 

def send_rpc_request(sock, req_obj, unknown):
    marsh = marshal.dumps(req_obj)  # python object

    # build out the header
    header =  "\x78\x01\x01" + struct.pack('<h', len(marsh))
    header += chr(unknown) # not sure exactly what this is
    header += "\xff"

    # add the ADLER32 checksum
    checksum = struct.pack('>i', zlib.adler32(marsh))

    post_data = header + marsh + checksum
    message = build_post(post_data)
    try:
        sock.send(message)
        #print("Sent request.")

        resp = sock.recv(1024)

        if resp is None:
            print("Did not receive a response from server.")
    except Exception as e:
        print("Error with request:")
        print(e) 

def send_rpc_request(sock, req_obj, unknown):
    marsh = marshal.dumps(req_obj)  # python object

    # build out the header
    header =  "\x78\x01\x01" + struct.pack('<h', len(marsh))
    header += chr(unknown) # not sure exactly what this is
    header += "\xff"

    # add the ADLER32 checksum
    checksum = struct.pack('>i', zlib.adler32(marsh))

    post_data = header + marsh + checksum
    message = build_post(post_data)
    try:
        sock.send(message)

        resp = sock.recv(1024)

        if resp is None:
            print("Did not receive a response from server.")
    except Exception as e:
        print("Error with request:")
        print(e) 

def convert_pyc_file(to_opcodes, pyc_file):
    pyc_code_obj_in = unmarshal_pyc_code(pyc_file)
    
    # recursively look for code objects
    # should also rebuild code object
    code_object_out = recurse_convert_code_objects(pyc_code_obj_in, to_opcodes)

    magic = '\x03\xf3\x0d\x0a'  # 62211 version
    time = '\x00'*4
    new_pyc_out = magic + time + marshal.dumps(code_object_out)
    
    filename = os.path.basename(pyc_code_obj_in.co_filename)
    filename = filename.replace(".pyc", "_converted.pyc")

    output_filename = os.path.dirname(pyc_file) + "\\" + filename

    with open(output_filename, 'wb') as f:
        f.write(new_pyc_out)
    
    return output_filename 

def test_file_multiple_reads(self):
        """calling load w/ a file should only advance the length of the file"""
        l = []
        for i in xrange(10):
            l.append(marshal.dumps({i:i}))

        data = ''.join(l)
        with open('tempfile.txt', 'w') as f:
            f.write(data)

        with open('tempfile.txt') as f:
            for i in xrange(10):
                obj = marshal.load(f)
                self.assertEqual(obj, {i:i})

        self.delete_files('tempfile.txt') 

def test_sets(self):
        for constructor in (set, frozenset):
            t = constructor(self.d.keys())
            new = marshal.loads(marshal.dumps(t))
            self.assertEqual(t, new)
            self.assertTrue(isinstance(new, constructor))
            self.assertNotEqual(id(t), id(new))
            marshal.dump(t, file(test_support.TESTFN, "wb"))
            new = marshal.load(file(test_support.TESTFN, "rb"))
            self.assertEqual(t, new)
            os.unlink(test_support.TESTFN) 

def test_tuple(self):
        t = tuple(self.d.keys())
        new = marshal.loads(marshal.dumps(t))
        self.assertEqual(t, new)
        marshal.dump(t, file(test_support.TESTFN, "wb"))
        new = marshal.load(file(test_support.TESTFN, "rb"))
        self.assertEqual(t, new)
        os.unlink(test_support.TESTFN) 

def test_large_marshal(self):
        size = int(1e6)
        testString = 'abc' * size
        marshal.dumps(testString) 

def make_pyc(co, mtime):
    data = marshal.dumps(co)
    if type(mtime) is type(0.0):
        # Mac mtimes need a bit of special casing
        if mtime < 0x7fffffff:
            mtime = int(mtime)
        else:
            mtime = int(-0x100000000L + long(mtime))
    pyc = imp.get_magic() + struct.pack("<i", int(mtime)) + data
    return pyc 

def dumps(obj, protocol=None):
    file = StringIO()
    Pickler(file, protocol).dump(obj)
    return file.getvalue() 

def marshal_dump(code, f):
        if isinstance(f, file):
            marshal.dump(code, f)
        else:
            f.write(marshal.dumps(code)) 

def getPycData(fullname, code, ispkg):
    if ispkg:
        fullname += ".__init__"
    path = fullname.replace(".", os.sep) + PYC_EXT
    return path, MAGIC + '\0\0\0\0' + marshal.dumps(code)

#
# Extension modules can't be in the modules zip archive, so a placeholder
# is added instead, that loads the extension from a specified location.
# 

def dumps(obj, protocol=None):
    file = StringIO()
    Pickler(file, protocol).dump(obj)
    return file.getvalue() 

def dumps(obj, protocol=None):
    file = StringIO()
    Pickler(file, protocol).dump(obj)
    return file.getvalue() 

def make_pyc(co, mtime):
    data = marshal.dumps(co)
    if type(mtime) is type(0.0):
        # Mac mtimes need a bit of special casing
        if mtime < 0x7fffffff:
            mtime = int(mtime)
        else:
            mtime = int(-0x100000000L + long(mtime))
    pyc = imp.get_magic() + struct.pack("<i", int(mtime)) + data
    return pyc 

def test_cp24547(self):
        self.assertEqual(marshal.dumps(2**33), "l\x03\x00\x00\x00\x00\x00\x00\x00\x08\x00") 

def test_binary_floats(self):
        self.assertEqual(marshal.dumps(2.0, 2), 'g\x00\x00\x00\x00\x00\x00\x00@')
        self.assertEqual(marshal.dumps(2.0), 'g\x00\x00\x00\x00\x00\x00\x00@')
        if is_cli: #https://github.com/IronLanguages/main/issues/854
            self.assertEqual(marshal.dumps(2.0, 1), 'f\x032.0')
        else:
            self.assertEqual(marshal.dumps(2.0, 1), 'f\x012')
        self.assertEqual(marshal.loads(marshal.dumps(2.0, 2)), 2.0) 

def test_string_interning(self):
        self.assertEqual(marshal.dumps(['abc', 'abc'], 1), '[\x02\x00\x00\x00t\x03\x00\x00\x00abcR\x00\x00\x00\x00')
        self.assertEqual(marshal.dumps(['abc', 'abc']), '[\x02\x00\x00\x00t\x03\x00\x00\x00abcR\x00\x00\x00\x00')
        self.assertEqual(marshal.dumps(['abc', 'abc'], 0), '[\x02\x00\x00\x00s\x03\x00\x00\x00abcs\x03\x00\x00\x00abc')
        self.assertEqual(marshal.dumps(['abc', 'abc', 'abc', 'def', 'def'], 1), '[\x05\x00\x00\x00t\x03\x00\x00\x00abcR\x00\x00\x00\x00R\x00\x00\x00\x00t\x03\x00\x00\x00defR\x01\x00\x00\x00')
        self.assertEqual(marshal.loads(marshal.dumps(['abc', 'abc'], 1)), ['abc', 'abc'])
        self.assertEqual(marshal.loads(marshal.dumps(['abc', 'abc'], 0)), ['abc', 'abc'])
        self.assertEqual(marshal.loads(marshal.dumps(['abc', 'abc', 'abc', 'def', 'def'], 1)), ['abc', 'abc', 'abc', 'def', 'def']) 

def send_data(self, data):
        is_ack = (data["cmd"] == self.ACK)
        data = pickle.dumps(data, pickle.HIGHEST_PROTOCOL)
        data_len = len(data)
        assert data_len < 0xFFFFFFFF, "Transfer too large!"
        log.debug("-> sending %d bytes", data_len)
        self.conn.sendall(struct.pack("I", data_len))
        self.conn.sendall(data)
        if not is_ack:
            assert self.recv_data()["cmd"] == self.ACK
            log.debug("ACK received") 

def test_negative(self):
        self.assertRaises(TypeError, marshal.dump, 2, None)
        self.assertRaises(TypeError, marshal.load, '-1', None)

        l = [1, 2]
        l.append(l)
        self.assertRaises(ValueError, marshal.dumps, l) ## infinite loop

        class my: pass
        self.assertRaises(ValueError, marshal.dumps, my())  ## unmarshallable object 

def test_read_object_from_file(self):
        obj = ('\u20ac', b'abc', 123, 45.6, 7+8j)
        for v in range(marshal.version + 1):
            data = marshal.dumps(obj, v)
            with open(test_support.TESTFN, 'wb') as f:
                f.write(data + b'xxxx')
            r, p = _testcapi.pymarshal_read_object_from_file(test_support.TESTFN)
            test_support.unlink(test_support.TESTFN)
            self.assertEqual(r, obj)
            self.assertEqual(p, len(data)) 

def test_read_last_object_from_file(self):
        obj = ('\u20ac', b'abc', 123, 45.6, 7+8j)
        for v in range(marshal.version + 1):
            data = marshal.dumps(obj, v)
            with open(test_support.TESTFN, 'wb') as f:
                f.write(data + b'xxxx')
            r, p = _testcapi.pymarshal_read_last_object_from_file(test_support.TESTFN)
            test_support.unlink(test_support.TESTFN)
            self.assertEqual(r, obj) 

def test_bool(self):
        for b in (True, False):
            new = marshal.loads(marshal.dumps(b))
            self.assertEqual(b, new)
            self.assertEqual(type(b), type(new))
            marshal.dump(b, file(test_support.TESTFN, "wb"))
            new = marshal.load(file(test_support.TESTFN, "rb"))
            self.assertEqual(b, new)
            self.assertEqual(type(b), type(new))