Python multiprocessing.RawValue() Examples

def test_value(self, raw=False):
        if raw:
            values = [self.RawValue(code, value)
                      for code, value, _ in self.codes_values]
        else:
            values = [self.Value(code, value)
                      for code, value, _ in self.codes_values]

        for sv, cv in zip(values, self.codes_values):
            self.assertEqual(sv.value, cv[1])

        proc = self.Process(target=self._test, args=(values,))
        proc.daemon = True
        proc.start()
        proc.join()

        for sv, cv in zip(values, self.codes_values):
            self.assertEqual(sv.value, cv[2]) 

def test_getobj_getlock(self):
        val1 = self.Value('i', 5)
        lock1 = val1.get_lock()
        obj1 = val1.get_obj()

        val2 = self.Value('i', 5, lock=None)
        lock2 = val2.get_lock()
        obj2 = val2.get_obj()

        lock = self.Lock()
        val3 = self.Value('i', 5, lock=lock)
        lock3 = val3.get_lock()
        obj3 = val3.get_obj()
        self.assertEqual(lock, lock3)

        arr4 = self.Value('i', 5, lock=False)
        self.assertFalse(hasattr(arr4, 'get_lock'))
        self.assertFalse(hasattr(arr4, 'get_obj'))

        self.assertRaises(AttributeError, self.Value, 'i', 5, lock='navalue')

        arr5 = self.RawValue('i', 5)
        self.assertFalse(hasattr(arr5, 'get_lock'))
        self.assertFalse(hasattr(arr5, 'get_obj')) 

def __init__(self, ndata, nprocs, chunk=None, schedule='guided'):
        if not schedule in ['guided', 'dynamic', 'static']:
            raise ValueError('unknown scheduling strategy')
        self._ndata = mp.RawValue(ctypes.c_int, ndata)
        self._start = mp.RawValue(ctypes.c_int, 0)
        self._lock = mp.Lock()
        self._schedule = schedule
        self._nprocs = nprocs
        if schedule == 'guided' or schedule == 'dynamic':
            min_chunk = ndata // (10 * nprocs)
            if chunk:
                min_chunk = chunk
            min_chunk = max(min_chunk, 1)
            self._chunk = min_chunk
        elif schedule == 'static':
            min_chunk = ndata // nprocs
            if chunk:
                min_chunk = max(chunk, min_chunk)
            min_chunk = max(min_chunk, 1)
            self._chunk = min_chunk 

def __init__(self, ModelCls=None, model_kwargs=None, initial_model_state_dict=None):
        """
        Arguments are saved but no model initialization occurs.

        Args:
            ModelCls: The model class to be used.
            model_kwargs (optional): Any keyword arguments to pass when instantiating the model.
            initial_model_state_dict (optional): Initial model parameter values.
        """

        save__init__args(locals())
        self.model = None  # type: torch.nn.Module
        self.shared_model = None
        self.distribution = None
        self.device = torch.device("cpu")
        self._mode = None
        if self.model_kwargs is None:
            self.model_kwargs = dict()
        # The rest only for async operations:
        self._rw_lock = RWLock()
        self._send_count = mp.RawValue("l", 0)
        self._recv_count = 0 

def build_ctrl(self, world_size):
        """
        Builds several parallel communication mechanisms for controlling the
        workflow across processes.
        """
        opt_throttle = (mp.Barrier(world_size) if world_size > 1 else
            None)
        return AttrDict(
            quit=mp.Value('b', lock=True),
            quit_opt=mp.RawValue('b'),
            sample_ready=[mp.Semaphore(0) for _ in range(2)],  # Double buffer.
            sample_copied=[mp.Semaphore(1) for _ in range(2)],
            sampler_itr=mp.Value('l', lock=True),
            opt_throttle=opt_throttle,
            eval_time=mp.Value('d', lock=True),
        ) 

def __init__(self, params, opt_type=None, lr=0, step=0):
        self.var_shapes = [
            var.get_shape().as_list()
            for var in params]
        self.size = sum([np.prod(shape) for shape in self.var_shapes])
        self.step = RawValue(ctypes.c_int, step)

        if opt_type == 'adam':
            self.ms = self.malloc_contiguous(self.size)
            self.vs = self.malloc_contiguous(self.size)
            self.lr = RawValue(ctypes.c_float, lr)
        elif opt_type == 'adamax':
            self.ms = self.malloc_contiguous(self.size)
            self.vs = self.malloc_contiguous(self.size)
            self.lr = RawValue(ctypes.c_float, lr)
        elif opt_type == 'rmsprop':
            self.vars = self.malloc_contiguous(self.size, np.ones(self.size, dtype=np.float))
        elif opt_type == 'momentum':
            self.vars = self.malloc_contiguous(self.size)
        else:
            self.vars = self.malloc_contiguous(self.size) 

def test_value(self, raw=False):
        if raw:
            values = [self.RawValue(code, value)
                      for code, value, _ in self.codes_values]
        else:
            values = [self.Value(code, value)
                      for code, value, _ in self.codes_values]

        for sv, cv in zip(values, self.codes_values):
            self.assertEqual(sv.value, cv[1])

        proc = self.Process(target=self._test, args=(values,))
        proc.daemon = True
        proc.start()
        proc.join()

        for sv, cv in zip(values, self.codes_values):
            self.assertEqual(sv.value, cv[2]) 

def test_getobj_getlock(self):
        val1 = self.Value('i', 5)
        lock1 = val1.get_lock()
        obj1 = val1.get_obj()

        val2 = self.Value('i', 5, lock=None)
        lock2 = val2.get_lock()
        obj2 = val2.get_obj()

        lock = self.Lock()
        val3 = self.Value('i', 5, lock=lock)
        lock3 = val3.get_lock()
        obj3 = val3.get_obj()
        self.assertEqual(lock, lock3)

        arr4 = self.Value('i', 5, lock=False)
        self.assertFalse(hasattr(arr4, 'get_lock'))
        self.assertFalse(hasattr(arr4, 'get_obj'))

        self.assertRaises(AttributeError, self.Value, 'i', 5, lock='navalue')

        arr5 = self.RawValue('i', 5)
        self.assertFalse(hasattr(arr5, 'get_lock'))
        self.assertFalse(hasattr(arr5, 'get_obj')) 

def __init__(self, delta_list_queue, comp_delta_queue,
                 overlay_mode,
                 block_size=1024*1024*2):
        """
        comparisons of compression algorithm
        http://pokecraft.first-world.info/wiki/Quick_Benchmark:_Gzip_vs_Bzip2_vs_LZMA_vs_XZ_vs_LZ4_vs_LZO
        """
        self.delta_list_queue = delta_list_queue
        self.comp_delta_queue = comp_delta_queue
        self.overlay_mode = overlay_mode
        self.num_proc = VMOverlayCreationMode.MAX_THREAD_NUM
        self.comp_type = overlay_mode.COMPRESSION_ALGORITHM_TYPE
        self.comp_level = overlay_mode.COMPRESSION_ALGORITHM_SPEED
        self.block_size = block_size
        self.proc_list = list()

        # monitor value specific to compression
        self.monitor_time_first_input_recved = multiprocessing.RawValue(
            ctypes.c_double, 0)

        super(CompressProc, self).__init__(target=self.compress_stream) 

def __init__(self, command_queue, task_queue, mode_queue,
                 output_queue, comp_type, comp_level):
        self.command_queue = command_queue
        self.task_queue = task_queue
        self.mode_queue = mode_queue
        self.output_queue = output_queue
        self.comp_type = comp_type
        self.comp_level = comp_level

        # shared variables between processes
        self.child_process_time_total = multiprocessing.RawValue(
            ctypes.c_double, 0)
        self.child_process_block_total = multiprocessing.RawValue(
            ctypes.c_double, 0)
        self.child_input_size_total = multiprocessing.RawValue(
            ctypes.c_ulong, 0)
        self.child_output_size_total = multiprocessing.RawValue(
            ctypes.c_ulong, 0)

        super(CompChildProc, self).__init__(target=self._comp) 

def __init__(self, remote_addr, remote_port, metadata, compdata_queue, process_controller):
        self.remote_addr = remote_addr
        self.remote_port = remote_port
        self.metadata = metadata
        self.compdata_queue = compdata_queue
        self.process_controller = process_controller

        # measurement
        self.monitor_network_bw = multiprocessing.RawValue(ctypes.c_double, 0)
        self.monitor_network_bw.value = 0.0
        self.vm_resume_time_at_dest = multiprocessing.RawValue(ctypes.c_double, 0)
        self.time_finish_transmission = multiprocessing.RawValue(ctypes.c_double, 0)

        self.is_first_recv = False
        self.time_first_recv = 0

        super(StreamSynthesisClient, self).__init__(target=self.transfer) 

def __init__(self, modified_mem_queue, deltalist_queue,
                 basemem_meta, basemem_path, overlay_mode,
                 apply_free_memory=True,
                 free_memory_info=None):
        self.modified_mem_queue = modified_mem_queue
        self.deltalist_queue = deltalist_queue
        self.basemem_meta = basemem_meta
        self.memory_hashlist = Memory.import_hashlist(basemem_meta)
        self.apply_free_memory = apply_free_memory
        self.free_memory_info = free_memory_info
        self.basemem_path = basemem_path
        self.proc_list = list()
        self.overlay_mode = overlay_mode
        self.num_proc = VMOverlayCreationMode.MAX_THREAD_NUM
        self.diff_algorithm = overlay_mode.MEMORY_DIFF_ALGORITHM

        self.monitor_current_iteration = multiprocessing.RawValue(
            ctypes.c_ulong, 0)

        super(CreateMemoryDeltalist, self).__init__(
            target=self.create_memory_deltalist) 

def test_getobj_getlock(self):
        val1 = self.Value('i', 5)
        lock1 = val1.get_lock()
        obj1 = val1.get_obj()

        val2 = self.Value('i', 5, lock=None)
        lock2 = val2.get_lock()
        obj2 = val2.get_obj()

        lock = self.Lock()
        val3 = self.Value('i', 5, lock=lock)
        lock3 = val3.get_lock()
        obj3 = val3.get_obj()
        self.assertEqual(lock, lock3)

        arr4 = self.Value('i', 5, lock=False)
        self.assertFalse(hasattr(arr4, 'get_lock'))
        self.assertFalse(hasattr(arr4, 'get_obj'))

        self.assertRaises(AttributeError, self.Value, 'i', 5, lock='navalue')

        arr5 = self.RawValue('i', 5)
        self.assertFalse(hasattr(arr5, 'get_lock'))
        self.assertFalse(hasattr(arr5, 'get_obj')) 

def test_value(self, raw=False):
        if raw:
            values = [self.RawValue(code, value)
                      for code, value, _ in self.codes_values]
        else:
            values = [self.Value(code, value)
                      for code, value, _ in self.codes_values]

        for sv, cv in zip(values, self.codes_values):
            self.assertEqual(sv.value, cv[1])

        proc = self.Process(target=self._test, args=(values,))
        proc.daemon = True
        proc.start()
        proc.join()

        for sv, cv in zip(values, self.codes_values):
            self.assertEqual(sv.value, cv[2]) 

def __init__(self, command_queue, task_queue, mode_queue, deltalist_queue,
                 diff_algorithm, basedisk_path, modified_disk, chunk_size):
        self.command_queue = command_queue
        self.task_queue = task_queue
        self.mode_queue = mode_queue
        self.deltalist_queue = deltalist_queue
        self.diff_algorithm = diff_algorithm
        self.basedisk_path = basedisk_path
        self.modified_disk = modified_disk
        self.chunk_size = chunk_size

        # shared variables between processes
        self.child_process_time_total = multiprocessing.RawValue(
            ctypes.c_double, 0)
        self.child_process_block_total = multiprocessing.RawValue(
            ctypes.c_double, 0)
        self.child_input_size_total = multiprocessing.RawValue(
            ctypes.c_ulong, 0)
        self.child_output_size_total = multiprocessing.RawValue(
            ctypes.c_ulong, 0)

        super(DiskDiffProc, self).__init__(target=self.process_diff) 

def __init__(self, *, slot_bytes, slot_count):
        """Initializer.

        Args:
            slot_bytes: The maximum size of slots in the buffer.
            slot_count: How many slots should be in the buffer.
        """
        self.slot_count = slot_count
        self.array = SlotArray(slot_bytes=slot_bytes, slot_count=slot_count)
        self.lock = ReadersWriterLock()
        # Each reading process may modify its own Pointer while the read
        # lock is being held. Each reading process can also load the position
        # of the writer, but not load any other readers. Each reading process
        # can also load the value of the 'active' count.
        self.readers = []
        # The writer can load and store the Pointer of all the reader Pointers
        # or the writer Pointer while the write lock is held. It can also load
        # and store the value of the 'active' acount.
        self.writer = Pointer(self.slot_count)
        self.active = multiprocessing.RawValue(ctypes.c_uint, 0) 

def __init__(self):
        # RawValue because we don't need it to create a Lock:
        self.val = RawValue('d', 0)
        self.num = RawValue('i', 0)
        self.lock = Lock() 

def __init__(self, *args, **kwargs):
        # measurement
        self.monitor_total_time_block = multiprocessing.RawValue(
            ctypes.c_double, 0)
        self.monitor_total_ratio_block = multiprocessing.RawValue(
            ctypes.c_double, 0)
        self.monitor_total_time_block_cur = multiprocessing.RawValue(
            ctypes.c_double, 0)
        self.monitor_total_ratio_block_cur = multiprocessing.RawValue(
            ctypes.c_double, 0)
        self.monitor_total_input_size = multiprocessing.RawValue(
            ctypes.c_ulong, 0)
        self.monitor_total_output_size = multiprocessing.RawValue(
            ctypes.c_ulong, 0)
        self.monitor_total_input_size_cur = multiprocessing.RawValue(
            ctypes.c_ulong, 0)
        self.monitor_total_output_size_cur = multiprocessing.RawValue(
            ctypes.c_ulong, 0)
        self.in_size = 0
        self.out_size = 0
        self.is_processing_alive = multiprocessing.RawValue(ctypes.c_bool)
        self.finish_processing_input = multiprocessing.RawValue(ctypes.c_bool)
        self.is_processing_alive.value = True
        self.finish_processing_input.value = False

        self.worker_name = str(
            kwargs.pop('worker_name',
                       self.__class__.__name__))
        process_manager = get_instance()
        (self.control_queue, self.response_queue) = \
            process_manager.register(self)  # shared dictionary

        # not used
        self.monitor_current_bw = float(0)
        self.monitor_current_inqueue_length = multiprocessing.Value('d', -1.0)
        self.monitor_current_outqueue_length = multiprocessing.Value('d', -1.0)
        self.monitor_current_get_time = multiprocessing.Value('d', -1.0)
        self.monitor_current_put_time = multiprocessing.Value('d', -1.0)
        super(ProcWorker, self).__init__(*args, **kwargs) 

def __init__(self, value=0):
        # RawValue because we don't need it to create a Lock:
        self.val = RawValue('i', value)
        self.lock = Lock() 

def __init__(self, conf={}):
        self.conf = DEFAULT_CONF
        self.conf.update(conf)

        # Init one-to-one mapped variables
        self.net_man = None
        self.state_man = None
        self.traffic_gen = None
        self.bw_ctrl = None
        self.sampler = None
        self.input_file = None
        self.terminated = False
        self.reward = RawValue('d', 0)

        # set the id of this environment
        self.short_id = dc_utils.generate_id()
        if self.conf["parallel_envs"]:
            self.conf["topo_conf"]["id"] = self.short_id
        # initialize the topology
        self.topo = TopoFactory.create(self.conf["topo"],
                                       self.conf["topo_conf"])
        # Save the configuration we have, id does not matter here
        dc_utils.dump_json(path=self.conf["output_dir"],
                           name="env_config", data=self.conf)
        dc_utils.dump_json(path=self.conf["output_dir"],
                           name="topo_config", data=self.topo.conf)
        # set the dimensions of the state matrix
        self._set_gym_matrices()
        # Set the active traffic matrix
        self._set_traffic_matrix(
            self.conf["tf_index"], self.conf["input_dir"], self.topo)

        # each unique id has its own sub folder
        if self.conf["parallel_envs"]:
            self.conf["output_dir"] += f"/{self.short_id}"
        # check if the directory we are going to work with exists
        dc_utils.check_dir(self.conf["output_dir"])

        # handle unexpected exits scenarios gracefully
        atexit.register(self.close) 

def __init__(self, initval=0):
        self.val = multiprocessing.RawValue('i', initval)
        self.lock = multiprocessing.Lock() 

def __init__(self, slot_count, *, start=None):
        default = start if start is not None else 0
        self.counter = multiprocessing.RawValue(ctypes.c_longlong, default)
        self.position = Position(slot_count) 

def __init__(self, initval=0):
        self.val = RawValue('i', initval)
        self.last_step_update_target = RawValue('i', initval)
        self.lock = Lock() 

def __init__(self, *args, **kwargs):
        self.async_t = mp.RawValue("l", 0)
        super().__init__(*args, **kwargs)
        # Wrap guard behavior should be fine without async--each will catch it. 

def __init__(self):
        self.write_lock = mp.Lock()
        self._read_lock = mp.Lock()
        self._read_count = mp.RawValue("i") 

def launch_workers(self, double_buffer_slice, affinity, seed, n_envs_list):
        self.n_worker = n_worker = len(n_envs_list)
        # A little slight-of-hand to make 2-level signal:
        self.ctrl.stop_eval = self.sync.stop_eval
        self.sync = AttrDict(
            obs_ready=[mp.Semaphore(0) for _ in range(n_worker)],
            act_ready=[mp.Semaphore(0) for _ in range(n_worker)],
            stop_eval=mp.RawValue(ctypes.c_bool, False),  # Overwrite.
            # stop_eval=self.ctrl.stop_eval,  # No, make 2-level signal.
            db_idx=self.ctrl.db_idx,  # Copy into sync which passes to Collector.
        )
        self.step_buffer_pyt, self.step_buffer_np = build_step_buffer(
            self.examples, sum(n_envs_list))
        self.agent_inputs = AgentInputs(self.step_buffer_pyt.observation,
            self.step_buffer_pyt.action, self.step_buffer_pyt.reward)

        if self.eval_n_envs > 0:
            eval_n_envs = self.eval_n_envs_per * n_worker
            eval_step_buffer_pyt, eval_step_buffer_np = build_step_buffer(
                self.examples, eval_n_envs)
            self.eval_step_buffer_pyt = eval_step_buffer_pyt
            self.eval_step_buffer_np = eval_step_buffer_np
            self.eval_agent_inputs = AgentInputs(
                self.eval_step_buffer_pyt.observation,
                self.eval_step_buffer_pyt.action,
                self.eval_step_buffer_pyt.reward,
            )
            # eval_max_T already made in earlier initialize.

        self.double_buffer = double_buffer_slice  # Now only see my part.
        common_kwargs = self._assemble_common_kwargs(affinity)
        common_kwargs["agent"] = None  # Remove.
        workers_kwargs = self._assemble_workers_kwargs(affinity, seed,
            n_envs_list)

        # Yes, fork again.
        self.workers = [mp.Process(target=sampling_process,
            kwargs=dict(common_kwargs=common_kwargs, worker_kwargs=w_kwargs))
            for w_kwargs in workers_kwargs]
        for w in self.workers:
            w.start() 

def _build_parallel_ctrl(self, *args, **kwargs):
        super()._build_parallel_ctrl(*args, **kwargs)
        self.ctrl.db_idx = mp.RawValue("i", 0)
        self.sync.db_idx = self.ctrl.db_idx  # Pass along to collectors.
        # CPU maybe only needs it in sync? 

def _build_parallel_ctrl(self, n_worker):
        self.ctrl = AttrDict(
            quit=mp.RawValue(ctypes.c_bool, False),
            barrier_in=mp.Barrier(n_worker + 1),
            barrier_out=mp.Barrier(n_worker + 1),
            do_eval=mp.RawValue(ctypes.c_bool, False),
            itr=mp.RawValue(ctypes.c_long, 0),
        )
        self.traj_infos_queue = mp.Queue()
        self.eval_traj_infos_queue = mp.Queue()
        self.sync = AttrDict(stop_eval=mp.RawValue(ctypes.c_bool, False)) 

def __init__(self, ndata, nprocs):
        self._ndata = mp.RawValue(ctypes.c_int, ndata)
        self._start = mp.RawValue(ctypes.c_int, 0)
        self._lock = mp.Lock()
        min_chunk = ndata // nprocs
        min_chunk = ndata if min_chunk <= 2 else min_chunk
        self._chunk = min_chunk 

def __init__(self, context_size):
        # use raw values because both indices have
        # to manually be locked together
        self._doc_id = multiprocessing.RawValue('i', 0)
        self._in_doc_pos = multiprocessing.RawValue('i', context_size)
        self._lock = multiprocessing.Lock()