Python dill.loads() Examples

def get_function(self, func, name):
        """Returns a precondition/postcondition/execution object.

        Parameters
        ----------
        func : :obj:`str`
            Name of the function group (precondition, postcondition, execution)
        name : :obj:`str`
            Name of the function.

        Returns
        -------
        :obj: `function`
            Function obtained.

        """
        f = bytearray.fromhex(self._functions[func][name])
        return dill.loads(f) 

def test_blue_io(self):
        import dill
        s0 = self.dsp()
        pre_dsp = dill.dumps(self.dsp)
        blue = self.dsp.blue()
        self.assertEqual(pre_dsp, dill.dumps(self.dsp))
        pre = dill.dumps(blue), pre_dsp
        sol = blue()
        post = dill.dumps(blue), dill.dumps(self.dsp)
        self.assertEqual(pre, post)
        s = self.dsp()
        post = dill.dumps(blue), dill.dumps(self.dsp)
        self.assertEqual(pre, post)
        self.assertEqual(s, sol)
        self.assertEqual(s0, sol)
        self.assertLess(*map(len, post))
        self.assertLess(len(post[1]), len(dill.dumps(s)))
        blue, dsp = list(map(dill.loads, post))
        self.assertEqual(dsp.solution, {})
        self.assertEqual(s, dsp())
        self.assertEqual(s, blue()) 

def __init__(self, algo, domain, trials, rstate, asynchronous=None,
                 max_queue_len=1,
                 poll_interval_secs=1.0,
                 max_evals=sys.maxsize,
                 verbose=0,
                 ):
        self.algo = algo
        self.domain = domain
        self.trials = trials
        if asynchronous is None:
            self.asynchronous = trials.asynchronous
        else:
            self.asynchronous = asynchronous
        self.poll_interval_secs = poll_interval_secs
        self.max_queue_len = max_queue_len
        self.max_evals = max_evals
        self.rstate = rstate

        if self.asynchronous:
            if 'FMinIter_Domain' in trials.attachments:
                logger.warn('over-writing old domain trials attachment')
            msg = pickler.dumps(domain)
            # -- sanity check for unpickling
            pickler.loads(msg)
            trials.attachments['FMinIter_Domain'] = msg 

def test_metric():
    def _test(m, values, e):
        for v in values:
            m.update(v)
        assert m.compute() == e

    metric = Accumulation()

    m1 = dill.loads(dill.dumps(metric))

    values = list(range(10))
    expected = sum(values)

    _test(m1, values, expected)

    metric.update(5)

    m2 = dill.loads(dill.dumps(metric))

    _test(m2, values, expected + 5) 

def __init__(self,
                 topic,
                 consumer,
                 callback=None,
                 deserializer=None,
                 logger=None):

        assert is_str(topic), 'topic must be a str'
        assert isinstance(consumer, KafkaConsumer), 'bad consumer instance'
        assert consumer.config['group_id'], 'consumer must have group_id'
        assert is_none_or_func(callback), 'callback must be a callable'
        assert is_none_or_func(deserializer), 'deserializer must be a callable'
        assert is_none_or_logger(logger), 'bad logger instance'

        self._topic = topic
        self._hosts = consumer.config['bootstrap_servers']
        self._group = consumer.config['group_id']
        self._consumer = consumer
        self._callback = callback
        self._deserializer = deserializer or dill.loads
        self._logger = logger or logging.getLogger('kq.worker') 

def dill_helper(payload: Any) -> Any:
    """
    Load single sample from data serialized by dill
    Args:
        payload: data which is loaded with dill

    Returns:
        Any: loaded data

    """
    if not DILL_AVAILABLE:
        raise RuntimeError('dill is not installed. For async loading '
                           'please install it')

    fn, args, kwargs = dill.loads(payload)
    return fn(*args, **kwargs) 

def _handle_loopback(self, message):
        """
        Handle incoming messages in the loopback socket.
        """
        header, data = cloudpickle.loads(message)
        if header == 'EXECUTE_METHOD':
            method, args, kwargs = data
            try:
                response = getattr(self, method)(*args, **kwargs)
            except Exception as error:
                yield format_method_exception(error, method, args, kwargs)
                raise
            yield response or True
        else:
            error = 'Unrecognized loopback message: {} {}'.format(header, data)
            self.log_error(error)
            yield error 

def test_pickle(self):
        print("separator", self.state_cls.separator)
        if sys.version_info < (3, 4):
            import dill as pickle
        else:
            import pickle

        states = ['A', 'B', {'name': 'C', 'children': ['1', '2', {'name': '3', 'children': ['a', 'b', 'c']}]},
                  'D', 'E', 'F']
        transitions = [
            {'trigger': 'walk', 'source': 'A', 'dest': 'B'},
            {'trigger': 'run', 'source': 'B', 'dest': 'C'},
            {'trigger': 'sprint', 'source': 'C', 'dest': 'D'}
        ]
        m = self.stuff.machine_cls(states=states, transitions=transitions, initial='A')
        m.walk()
        dump = pickle.dumps(m)
        self.assertIsNotNone(dump)
        m2 = pickle.loads(dump)
        self.assertEqual(m.state, m2.state)
        m2.run()
        m2.to_C_3_a()
        m2.to_C_3_b() 

def test_excel_model_compile(self):
        xl_model = ExcelModel().loads(self.filename_compile).finish()

        inputs = ["A%d" % i for i in range(2, 5)]
        outputs = ["C%d" % i for i in range(2, 5)]
        func = xl_model.compile(
            ["'[EXCEL.XLSX]DATA'!%s" % i for i in inputs],
            ["'[EXCEL.XLSX]DATA'!%s" % i for i in outputs]
        )
        i = sh.selector(inputs, self.results_compile, output_type='list')
        res = sh.selector(outputs, self.results_compile, output_type='list')
        self.assertEqual([x.value[0, 0] for x in func(*i)], res)
        self.assertIsNot(xl_model, copy.deepcopy(xl_model))
        self.assertIsNot(func, copy.deepcopy(func))

        xl_model = ExcelModel().loads(self.filename_circular).finish(circular=1)
        func = xl_model.compile(
            ["'[CIRCULAR.XLSX]DATA'!A10"], ["'[CIRCULAR.XLSX]DATA'!E10"]
        )
        self.assertEqual(func(False).value[0, 0], 2.0)
        self.assertIs(func(True).value[0, 0], ERR_CIRCULAR)
        self.assertIsNot(xl_model, copy.deepcopy(xl_model))
        self.assertIsNot(func, copy.deepcopy(func)) 

def test_pickle(self):
        import sys
        if sys.version_info < (3, 4):
            import dill as pickle
        else:
            import pickle

        states = ['A', 'B', 'C', 'D']
        # Define with list of dictionaries
        transitions = [
            {'trigger': 'walk', 'source': 'A', 'dest': 'B'},
            {'trigger': 'run', 'source': 'B', 'dest': 'C'},
            {'trigger': 'sprint', 'source': 'C', 'dest': 'D'}
        ]
        m = Machine(states=states, transitions=transitions, initial='A')
        m.walk()
        dump = pickle.dumps(m)
        self.assertIsNotNone(dump)
        m2 = pickle.loads(dump)
        self.assertEqual(m.state, m2.state)
        m2.run() 

def getRepresentation(name, process):
    obj_class = getClass(name, process)
    converters = pythonwhat.State.State.root_state.converters
    if obj_class in converters:
        repres = convert(name, dill.dumps(converters[obj_class]), process)
        if errored(repres):
            return ReprFail("manual conversion failed: {}".format(repres))
        else:
            return repres
    else:
        # first try to pickle
        try:
            stream = getStreamPickle(name, process)
            if not errored(stream):
                return pickle.loads(stream)
        except:
            pass

        # if it failed, try to dill
        try:
            stream = getStreamDill(name, process)
            if not errored(stream):
                return dill.loads(stream)
            return ReprFail(
                "dilling inside process failed for %s - write manual converter"
                % obj_class
            )
        except PicklingError:
            return ReprFail(
                "undilling of bytestream failed with PicklingError - write manual converter"
            )
        except Exception as e:
            return ReprFail(
                "undilling of bytestream failed for class %s - write manual converter."
                "Error: %s - %s" % (obj_class, type(e), e)
            ) 

def func_wrapper(args):
    func = dill.loads(args[0])
    return func(mp.current_process()._identity, *args[1:]) 

def testPickle_old(self):
        import pickle
        out1 = pickle.dumps(self.mModel);
        lModel2 = pickle.loads(out1);
        out2 = pickle.dumps(lModel2);
        print(sorted(self.mModel.__dict__))
        print(sorted(lModel2.__dict__))
        for (k , v) in self.mModel.__dict__.items():
            print(k , self.mModel.__dict__[k])
            print(k , lModel2.__dict__[k])
        assert(out1 == out2)
        print("TEST_PICKLE_OLD_OK") 

def testPickle(self):
        import dill
        dill.settings['recurse'] = False
        out1 = dill.dumps(self.mModel);
        lModel2 = dill.loads(out1);
        out2 = dill.dumps(lModel2);
        print(sorted(self.mModel.__dict__))
        print(sorted(lModel2.__dict__))
        for (k , v) in self.mModel.__dict__.items():
            print(k , self.mModel.__dict__[k])
            print(k , lModel2.__dict__[k])
        assert(out1 == out2)
        print("TEST_PICKLE_OK") 

def test_pickle(self):
        import sys
        if sys.version_info < (3, 4):
            import dill as pickle
        else:
            import pickle

        # go to non initial state B
        self.stuff.to_B()
        # pickle Stuff model
        dump = pickle.dumps(self.stuff)
        self.assertIsNotNone(dump)
        stuff2 = pickle.loads(dump)
        self.assertTrue(stuff2.is_B())
        # check if machines of stuff and stuff2 are truly separated
        stuff2.to_A()
        self.stuff.to_C()
        self.assertTrue(stuff2.is_A())
        thread = Thread(target=stuff2.forward)
        thread.start()
        # give thread some time to start
        time.sleep(0.01)
        # both objects should be in different states
        # and also not share locks
        begin = time.time()
        # stuff should not be locked and execute fast
        self.assertTrue(self.stuff.is_C())
        fast = time.time()
        # stuff2 should be locked and take about 1 second
        # to be executed
        self.assertTrue(stuff2.is_B())
        blocked = time.time()
        self.assertAlmostEqual(fast - begin, 0, delta=0.1)
        self.assertAlmostEqual(blocked - begin, 1, delta=0.1) 

def test_pickle_model(self):
        import sys
        if sys.version_info < (3, 4):
            import dill as pickle
        else:
            import pickle

        self.stuff.to_B()
        dump = pickle.dumps(self.stuff)
        self.assertIsNotNone(dump)
        model2 = pickle.loads(dump)
        self.assertEqual(self.stuff.state, model2.state)
        model2.to_F() 

def import_object(obj):
    import dill as pickle
    import base64
#    if obj is None:
#        obj = sys.stdin.read().strip().encode('utf-8')
    if obj is str:
        obj = obj.strip().encode('utf-8')
    return pickle.loads(gzip.zlib.decompress(base64.b64decode(obj))) 

def __init__(self, map_=map, mapper_pickles=False):
        super().__init__()
        self.map_ = map_
        self.pickle, self.unpickle = ((identity, identity)
                                      if mapper_pickles
                                      else (pickle.dumps, pickle.loads)) 

def import_object(obj):
    import dill as pickle
    import base64
#    if obj is None:
#        obj = sys.stdin.read().strip().encode('utf-8')
    if obj is str:
        obj = obj.strip().encode('utf-8')
    return pickle.loads(gzip.zlib.decompress(base64.b64decode(obj))) 

def _get_function_and_execute(f_dill, *args):
    f = dill.loads(f_dill)
    return f(*args) 

def get_obj(f, argfile=None, argstr=None, args=(), kwargs=None):
    """
    XXX: document me
    """
    if kwargs is None:
        kwargs = {}
    if argfile is not None:
        argstr = open(argfile).read()
    if argstr is not None:
        argd = pickler.loads(argstr)
    else:
        argd = {}
    args = args + argd.get('args', ())
    kwargs.update(argd.get('kwargs', {}))
    return json_call(f, args=args, kwargs=kwargs) 

def getValue(self):
        """
        Returns PromisedRequirement value
        """
        func = dill.loads(self._func)
        return func(*self._args) 

def convert(name, converter, process, shell):
    return dill.loads(converter)(get_env(shell.user_ns)[name]) 

def get(self, key, shared=False):
        """Retrieve data by key

        :param key: key for the data to retrieve
        :param shared: ``True/False`` wether to retrieve data from the shared (global) namespace.
        :return: the data, or ``None`` if the key cannot be found/has expired
        """
        namespaced_key = self._namespace_key(key, shared)
        value = Storage.get_instance().get(namespaced_key)
        if value:
            return dill.loads(value)
        else:
            return None 

def test_pickling_unpickling():

    import dill

    mg = ModelGetter()
    m1 = mg.model

    pick = dill.dumps(m1)

    m_reloaded = dill.loads(pick) 

def pf(request) -> ProfanityFilter:
    config: Config = request.param
    result = ProfanityFilter(
        languages=config.languages,
        analyses=config.analyses,
        censor_whole_words=config.censor_whole_words,
    )
    for analysis in config.analyses:
        if analysis not in result.analyses:
            pytest.skip(f"Couldn't initialize {analysis.value} analysis")
    if config.deep_copy:
        result = deepcopy(result)
    if config.dill:
        result = dill.loads(dill.dumps(result))
    return result 

def invoke(self):
        """
        Invoke the captured function with the captured arguments.
        """
        logger.debug('Running deferred function %s.', self)
        self.module.makeLoadable()
        function, args, kwargs = list(map(dill.loads, (self.function, self.args, self.kwargs)))
        return function(*args, **kwargs) 

def test_serializarion(self):
    import pickle, dill
    model = LinearStudentT(ndim_x=5, mu=5, random_seed=22)
    X, Y = model.simulate(200)

    pkl_str = dill.dumps(model)
    model_loaded = dill.loads(pkl_str) 

def run(self):
        """
        Begin execution of the agent process and start the main loop.
        """
        # Capture SIGINT
        signal.signal(signal.SIGINT, self._sigint_handler)

        try:
            ns = NSProxy(self.nsaddr)
            self._daemon = Pyro4.Daemon(self._host, self.port)
            self.base = cloudpickle.loads(self.base)
            self.agent = self.base(
                name=self.name,
                host=self._host,
                serializer=self._serializer,
                transport=self._transport,
                attributes=self.attributes,
            )
        except Exception:
            self._queue.put(format_exception())
            return

        self.name = self.agent.name
        uri = self._daemon.register(self.agent)
        try:
            ns.register(self.name, uri, safe=True)
        except Pyro4.errors.NamingError:
            self._queue.put(format_exception())
            return
        finally:
            ns.release()

        self._queue.put('STARTED:' + self.name)

        self._daemon.requestLoop(lambda: not self._shutdown_event.is_set())
        self._daemon.unregister(self.agent)

        self._teardown() 

def _handle_loopback_safe(self, data):
        """
        Handle incoming messages in the _loopback_safe socket.
        """
        method, args, kwargs = cloudpickle.loads(data)
        try:
            response = getattr(self, method)(*args, **kwargs)
        except Exception as error:
            yield format_method_exception(error, method, args, kwargs)
            raise
        yield response