Python cloudpickle.loads() Examples

def _set_mesh(self, value):
        """setter of mesh"""
        try:  # Check the type
            check_var("mesh", value, "dict")
        except CheckTypeError:
            check_var("mesh", value, "pyvista.core.pointset.UnstructuredGrid")
            # property can be set from a list to handle loads
        if (
            type(value) == dict
        ):  # Load type from saved dict {"type":type(value),"str": str(value),"serialized": serialized(value)]
            self._mesh = loads(value["serialized"].encode("ISO-8859-2"))
        else:
            self._mesh = value

    # Pyvista object of the mesh (optional)
    # Type : pyvista.core.pointset.UnstructuredGrid 

def install_custom_package(self, model_blob, model_id, model_version, delete_previous):
        """

        Args:
            model_blob:
            model_id:
            model_version:
            delete_previous:

        Returns:

        """
        model_obj = cloudpickle.loads(model_blob)

        if isinstance(model_obj, dict) and 'custom_package_blob' in model_obj.keys():
            (custom_package_name, custom_package_version, tar_file_content) = self.unpack_pkg(model_obj)
            self.check_package_version(custom_package_version, model_id, model_version)

            self.install_custom_module(custom_package_name, custom_package_version, model_id, model_version, tar_file_content, delete_previous=delete_previous) 

def _set_Br(self, value):
        """setter of Br"""
        try:  # Check the type
            check_var("Br", value, "dict")
        except CheckTypeError:
            check_var("Br", value, "SciDataTool.Classes.DataND.DataND")
            # property can be set from a list to handle loads
        if (
            type(value) == dict
        ):  # Load type from saved dict {"type":type(value),"str": str(value),"serialized": serialized(value)]
            self._Br = loads(value["serialized"].encode("ISO-8859-2"))
        else:
            self._Br = value

    # Radial airgap flux density
    # Type : SciDataTool.Classes.DataND.DataND 

def _set_selector(self, value):
        """setter of selector"""
        try:
            check_var("selector", value, "list")
        except CheckTypeError:
            check_var("selector", value, "function")
        if isinstance(value, list):  # Load function from saved dict
            self._selector = [loads(value[0].encode("ISO-8859-2")), value[1]]
        elif value is None:
            self._selector = [None, None]
        elif callable(value):
            self._selector = [value, getsource(value)]
        else:
            raise TypeError(
                "Expected function or list from a saved file, got: " + str(type(value))
            )

    # Selector of the genetic algorithm
    # Type : function 

def _set_crossover(self, value):
        """setter of crossover"""
        try:
            check_var("crossover", value, "list")
        except CheckTypeError:
            check_var("crossover", value, "function")
        if isinstance(value, list):  # Load function from saved dict
            self._crossover = [loads(value[0].encode("ISO-8859-2")), value[1]]
        elif value is None:
            self._crossover = [None, None]
        elif callable(value):
            self._crossover = [value, getsource(value)]
        else:
            raise TypeError(
                "Expected function or list from a saved file, got: " + str(type(value))
            )

    # Crossover of the genetic algorithm
    # Type : function 

def _set_mutator(self, value):
        """setter of mutator"""
        try:
            check_var("mutator", value, "list")
        except CheckTypeError:
            check_var("mutator", value, "function")
        if isinstance(value, list):  # Load function from saved dict
            self._mutator = [loads(value[0].encode("ISO-8859-2")), value[1]]
        elif value is None:
            self._mutator = [None, None]
        elif callable(value):
            self._mutator = [value, getsource(value)]
        else:
            raise TypeError(
                "Expected function or list from a saved file, got: " + str(type(value))
            )

    # Mutator of the genetic algorithm
    # Type : function 

def _set_Bt(self, value):
        """setter of Bt"""
        try:  # Check the type
            check_var("Bt", value, "dict")
        except CheckTypeError:
            check_var("Bt", value, "SciDataTool.Classes.DataND.DataND")
            # property can be set from a list to handle loads
        if (
            type(value) == dict
        ):  # Load type from saved dict {"type":type(value),"str": str(value),"serialized": serialized(value)]
            self._Bt = loads(value["serialized"].encode("ISO-8859-2"))
        else:
            self._Bt = value

    # Tangential airgap flux density
    # Type : SciDataTool.Classes.DataND.DataND 

def _set_function(self, value):
        """setter of function"""
        try:
            check_var("function", value, "list")
        except CheckTypeError:
            check_var("function", value, "function")
        if isinstance(value, list):  # Load function from saved dict
            self._function = [loads(value[0].encode("ISO-8859-2")), value[1]]
        elif value is None:
            self._function = [None, None]
        elif callable(value):
            self._function = [value, getsource(value)]
        else:
            raise TypeError(
                "Expected function or list from a saved file, got: " + str(type(value))
            )

    # Function of the space to initiate the variable
    # Type : function 

def _set_surf(self, value):
        """setter of surf"""
        try:  # Check the type
            check_var("surf", value, "dict")
        except CheckTypeError:
            check_var("surf", value, "pyvista.core.pointset.PolyData")
            # property can be set from a list to handle loads
        if (
            type(value) == dict
        ):  # Load type from saved dict {"type":type(value),"str": str(value),"serialized": serialized(value)]
            self._surf = loads(value["serialized"].encode("ISO-8859-2"))
        else:
            self._surf = value

    # Pyvista object of the outer surface
    # Type : pyvista.core.pointset.PolyData 

def test_ephemeral_persisting(zk, dsk2):
    with Persist(zk, name="dsk2", ns="/test/dags", ephemeral=True):
        with Ran() as r:
            assert get(dsk2, 'e') == 10
            assert r.steps == ['e']
        with Ran() as r:
            assert get(dsk2, 's') == 0.4
            assert r.steps == ['f', 's']
        with Ran() as r:
            assert get(dsk2, 's') == 0.4
            assert r.steps == []

        assert loads(zk.get("/test/dags/dsk2/e")[0]) == 10

    with pytest.raises(NoNodeError):
        zk.get("/test/dags/dsk2/e") 

def test_pickle_objective():
    """Test serializing objectives (needed for MultiThreadEngine)."""
    petab_importer = pypesto.petab.PetabImporter.from_yaml(
        folder_base + "Zheng_PNAS2012/Zheng_PNAS2012.yaml")
    objective = petab_importer.create_objective()

    objective.amici_solver.setSensitivityMethod(
        amici.SensitivityMethod_adjoint)

    objective2 = pickle.loads(pickle.dumps(objective))

    # test some properties
    assert objective.amici_model.getParameterIds() \
        == objective2.amici_model.getParameterIds()
    assert objective.amici_solver.getSensitivityOrder() \
        == objective2.amici_solver.getSensitivityOrder()
    assert objective.amici_solver.getSensitivityMethod() \
        == objective2.amici_solver.getSensitivityMethod()
    assert len(objective.edatas) == len(objective2.edatas) 

def get_args(key=None, default=None):
    args = __get_arg_config()

    if args.args_data:
        if args.use_cloudpickle:
            import cloudpickle
            assert args.cloudpickle_version == cloudpickle.__version__, "Cloudpickle versions do not match! (host) %s vs (remote) %s" % (args.cloudpickle_version, cloudpickle.__version__)
            data = cloudpickle.loads(base64.b64decode(args.args_data))
        else:
            data = pickle.loads(base64.b64decode(args.args_data))
    else:
        data = {}

    if key is not None:
        return data.get(key, default)
    return data 

def _handle_loopback(self, message):
        """
        Handle incoming messages in the loopback socket.
        """
        header, data = cloudpickle.loads(message)
        if header == 'EXECUTE_METHOD':
            method, args, kwargs = data
            try:
                response = getattr(self, method)(*args, **kwargs)
            except Exception as error:
                yield format_method_exception(error, method, args, kwargs)
                raise
            yield response or True
        else:
            error = 'Unrecognized loopback message: {} {}'.format(header, data)
            self.log_error(error)
            yield error 

def _set_toolbox(self, value):
        """setter of toolbox"""
        try:  # Check the type
            check_var("toolbox", value, "dict")
        except CheckTypeError:
            check_var("toolbox", value, "deap.base.Toolbox")
            # property can be set from a list to handle loads
        if (
            type(value) == dict
        ):  # Load type from saved dict {"type":type(value),"str": str(value),"serialized": serialized(value)]
            self._toolbox = loads(value["serialized"].encode("ISO-8859-2"))
        else:
            self._toolbox = value

    # DEAP toolbox
    # Type : deap.base.Toolbox 

def _set_eval_func(self, value):
        """setter of eval_func"""
        try:
            check_var("eval_func", value, "list")
        except CheckTypeError:
            check_var("eval_func", value, "function")
        if isinstance(value, list):  # Load function from saved dict
            self._eval_func = [loads(value[0].encode("ISO-8859-2")), value[1]]
        elif value is None:
            self._eval_func = [None, None]
        elif callable(value):
            self._eval_func = [value, getsource(value)]
        else:
            raise TypeError(
                "Expected function or list from a saved file, got: " + str(type(value))
            )

    # Function to evaluate before computing obj function and constraints
    # Type : function 

def _set_Yr(self, value):
        """setter of Yr"""
        try:  # Check the type
            check_var("Yr", value, "dict")
        except CheckTypeError:
            check_var("Yr", value, "SciDataTool.Classes.DataND.DataND")
            # property can be set from a list to handle loads
        if (
            type(value) == dict
        ):  # Load type from saved dict {"type":type(value),"str": str(value),"serialized": serialized(value)]
            self._Yr = loads(value["serialized"].encode("ISO-8859-2"))
        else:
            self._Yr = value

    # Displacement output
    # Type : SciDataTool.Classes.DataND.DataND 

def _set_Vr(self, value):
        """setter of Vr"""
        try:  # Check the type
            check_var("Vr", value, "dict")
        except CheckTypeError:
            check_var("Vr", value, "SciDataTool.Classes.DataND.DataND")
            # property can be set from a list to handle loads
        if (
            type(value) == dict
        ):  # Load type from saved dict {"type":type(value),"str": str(value),"serialized": serialized(value)]
            self._Vr = loads(value["serialized"].encode("ISO-8859-2"))
        else:
            self._Vr = value

    # Velocity output
    # Type : SciDataTool.Classes.DataND.DataND 

def _set_Ar(self, value):
        """setter of Ar"""
        try:  # Check the type
            check_var("Ar", value, "dict")
        except CheckTypeError:
            check_var("Ar", value, "SciDataTool.Classes.DataND.DataND")
            # property can be set from a list to handle loads
        if (
            type(value) == dict
        ):  # Load type from saved dict {"type":type(value),"str": str(value),"serialized": serialized(value)]
            self._Ar = loads(value["serialized"].encode("ISO-8859-2"))
        else:
            self._Ar = value

    # Acceleration output
    # Type : SciDataTool.Classes.DataND.DataND 

def _set_Prad(self, value):
        """setter of Prad"""
        try:  # Check the type
            check_var("Prad", value, "dict")
        except CheckTypeError:
            check_var("Prad", value, "SciDataTool.Classes.DataND.DataND")
            # property can be set from a list to handle loads
        if (
            type(value) == dict
        ):  # Load type from saved dict {"type":type(value),"str": str(value),"serialized": serialized(value)]
            self._Prad = loads(value["serialized"].encode("ISO-8859-2"))
        else:
            self._Prad = value

    # Radial magnetic air-gap surface force
    # Type : SciDataTool.Classes.DataND.DataND 

def _set_get_variable(self, value):
        """setter of get_variable"""
        try:
            check_var("get_variable", value, "list")
        except CheckTypeError:
            check_var("get_variable", value, "function")
        if isinstance(value, list):  # Load function from saved dict
            self._get_variable = [loads(value[0].encode("ISO-8859-2")), value[1]]
        elif value is None:
            self._get_variable = [None, None]
        elif callable(value):
            self._get_variable = [value, getsource(value)]
        else:
            raise TypeError(
                "Expected function or list from a saved file, got: " + str(type(value))
            )

    # Function to get the variable to compare
    # Type : function 

def _set_func(self, value):
        """setter of func"""
        try:
            check_var("func", value, "list")
        except CheckTypeError:
            check_var("func", value, "function")
        if isinstance(value, list):  # Load function from saved dict
            self._func = [loads(value[0].encode("ISO-8859-2")), value[1]]
        elif value is None:
            self._func = [None, None]
        elif callable(value):
            self._func = [value, getsource(value)]
        else:
            raise TypeError(
                "Expected function or list from a saved file, got: " + str(type(value))
            )

    # Function to minimize
    # Type : function 

def deploy_func(df, other, apply_func, call_queue_df=None, call_queue_other=None):
    if call_queue_df is not None and len(call_queue_df) > 0:
        for call, kwargs in call_queue_df:
            if isinstance(call, bytes):
                call = pkl.loads(call)
            if isinstance(kwargs, bytes):
                kwargs = pkl.loads(kwargs)
            df = call(df, **kwargs)
    if call_queue_other is not None and len(call_queue_other) > 0:
        for call, kwargs in call_queue_other:
            if isinstance(call, bytes):
                call = pkl.loads(call)
            if isinstance(kwargs, bytes):
                kwargs = pkl.loads(kwargs)
            other = call(other, **kwargs)
    if isinstance(apply_func, bytes):
        apply_func = pkl.loads(apply_func)
    return apply_func(df, other) 

def run_function(self, function_packed, data_item):
        """
        Receives a serialized function unpack it and run it
        Passes the model communicator if ther is more than one process per model
        """
        func = cloudpickle.loads(function_packed)
        res = None
        try:
            if(self.mpimanager.get_model_size() > 1):
                npc = NestedParallelizationControllerMPI(self.mpimanager.get_model_communicator())
                if self.mpimanager.get_model_communicator().Get_rank() == 0:
                    self.logger.debug("Executing map function on master rank 0.")
                    res = func(data_item, npc=npc)
                del(npc)
            else:
                res = func(data_item)
        except Exception as e:
            msg = "Exception occured while calling the map function {}: ".format(func.__name__)
            res = type(e)(msg + str(e))
        return res 

def tcp_recv_object(sock, chunk_size=1048576):
    """Receives a python object sent by 'tcp_send_object' over TCP. Returns the
    received object, or None if connection closed"""
    metadata = sock.recv(8)
    if len(metadata) != 8:  # Connection closed
        return None
    sz, compressed = struct.unpack("II", metadata)

    buffer = bytearray(sz)
    buffer_view = memoryview(buffer)
    offset = 0
    while sz > 0:
        amount_get = min(chunk_size, sz)
        amount_read = sock.recv_into(buffer_view[offset:], amount_get)
        if not amount_read:
            return None  # Connection closed

        sz -= amount_read
        offset += amount_read

    if compressed:
        import lz4.frame
        buffer = lz4.frame.decompress(buffer)
    return cloudpickle.loads(buffer) 

def test_pickle_meta_evaluator():
    set_seed(100)
    tasks = SetTaskSampler(lambda: GarageEnv(PointEnv()))
    max_path_length = 200
    env = GarageEnv(PointEnv())
    n_traj = 3
    with tempfile.TemporaryDirectory() as log_dir_name:
        runner = LocalRunner(
            SnapshotConfig(snapshot_dir=log_dir_name,
                           snapshot_mode='last',
                           snapshot_gap=1))
        meta_eval = MetaEvaluator(test_task_sampler=tasks,
                                  max_path_length=max_path_length,
                                  n_test_tasks=10,
                                  n_exploration_traj=n_traj)
        policy = RandomPolicy(env.spec.action_space)
        algo = MockAlgo(env, policy, max_path_length, n_traj, meta_eval)
        runner.setup(algo, env)
        log_file = tempfile.NamedTemporaryFile()
        csv_output = CsvOutput(log_file.name)
        logger.add_output(csv_output)
        meta_eval.evaluate(algo)
        meta_eval_pickle = cloudpickle.dumps(meta_eval)
        meta_eval2 = cloudpickle.loads(meta_eval_pickle)
        meta_eval2.evaluate(algo) 

def scatter_from_master(self, key):
        """
        master: set `key` to DB, then set trigger and wait sampler completion
        sampler: wait trigger, then get `key` and reset trigger
        """

        if self.rank < 0:
            obj = getattr(self, key)
            self.r.set(key, cloudpickle.dumps(obj))
            trigger = ['{}_trigger_{}'.format(
                key, rank) for rank in range(self.world_size)]
            self.wait_trigger_completion(trigger)
        else:
            trigger = '{}_trigger_{}'.format(key, self.rank)
            self.wait_trigger(trigger)
            obj = cloudpickle.loads(self.r.get(key))
            setattr(self, key, obj)
            self.reset_trigger(trigger) 

def __init__(
                self,
                env,
                policy,
                exploration_policy,
                max_path_length,
                train_rollout_function,
                eval_rollout_function,
        ):
            torch.set_num_threads(1)
            self._env = env
            self._policy = policy
            self._exploration_policy = exploration_policy
            self._max_path_length = max_path_length
            self.train_rollout_function = cloudpickle.loads(train_rollout_function)
            self.eval_rollout_function = cloudpickle.loads(eval_rollout_function) 

def deserialize_model(self, blob, model_id, model_version):
        """
        Deserializes the given blob to Model object which can be used for predictions
        :param blob:
        :param model_id:
        :param model_version:
        :return:
        """
        model_obj = cloudpickle.loads(blob)

        if not isinstance(model_obj, dict):  # Is a plain cloud-pickled model
            return model_obj

        if isinstance(model_obj, dict) and 'custom_package_blob' in model_obj.keys():
            self.__custom_package_deployer.install_custom_package(blob, model_id, model_version, delete_previous = True)

        if 'serialization_mechanism' in model_obj and model_obj['serialization_mechanism'] == 'asm':  # Is an ASM model
            self.__extract_model_resources(model_obj, model_id, model_version)
            self.__extract_prediction_module(model_obj, model_id, model_version)
            return self.__deserialize_asm_model(model_id, model_version)

        # tar_file_content = model_obj['custom_package_blob']
        # custom_package_name = model_obj['custom_package_name']
        # custom_package_version = model_obj['custom_package_version']
        return cloudpickle.loads(model_obj['model_blob'])  # Is a cloud-pickled model with custom code 

def _set_mmf_unit(self, value):
        """setter of mmf_unit"""
        try:  # Check the type
            check_var("mmf_unit", value, "dict")
        except CheckTypeError:
            check_var("mmf_unit", value, "SciDataTool.Classes.DataND.DataND")
            # property can be set from a list to handle loads
        if (
            type(value) == dict
        ):  # Load type from saved dict {"type":type(value),"str": str(value),"serialized": serialized(value)]
            self._mmf_unit = loads(value["serialized"].encode("ISO-8859-2"))
        else:
            self._mmf_unit = value

    # Unit magnetomotive force
    # Type : SciDataTool.Classes.DataND.DataND 

def apply_list_of_funcs(funcs, df):
    for func, kwargs in funcs:
        if isinstance(func, bytes):
            func = pkl.loads(func)
        df = func(df, **kwargs)
    return df