Python multiprocessing.managers() Examples

def __stop__(self):
        self._running = False
        try:  # try/except needed only for python2 compatibility
            # end daemon threads which distribute the work load
            for th, q1, q2 in self.threads:
                q1.put(None)
                th.join()
            delattr(self, 'threads')
            # end managers which handle the convolvers
            delattr(self, 'conv_handlers')
            for m in self.managers:
                m.shutdown()
            delattr(self, 'managers')
        except AttributeError:
            pass
        atexit.unregister(self.__stop__) 

def test_manager_initializer(self):
        m = multiprocessing.managers.SyncManager()
        self.assertRaises(TypeError, m.start, 1)
        m.start(initializer, (self.ns,))
        self.assertEqual(self.ns.test, 1)
        m.shutdown() 

def process_manager(self):
        if self._process_manager is None:
            self._process_manager = multiprocessing.managers.SyncManager()
            self._process_manager.start(_manager_initializer)
        return self._process_manager 

def __init__(self):
        self._memories = {}
        self._manager = multiprocessing.managers.SyncManager()

        self._locks = set()
        self._locks_queues = {} 

def test_manager_initializer(self):
        m = multiprocessing.managers.SyncManager()
        self.assertRaises(TypeError, m.start, 1)
        m.start(initializer, (self.ns,))
        self.assertEqual(self.ns.test, 1)
        m.shutdown()
        m.join() 

def start(self, initializer=None, initargs=()):
            '''
            Spawn a server process for this manager object
            '''
            assert self._state.value == multiprocessing.managers.State.INITIAL

            if initializer is not None and not hasattr(initializer,
                                                       '__call__'):
                raise TypeError('initializer must be a callable')

            # pipe over which we will retrieve address of server
            reader, writer = multiprocessing.Pipe(duplex=False)

            # spawn process which runs a server
            self._process = multiprocessing.Process(
                target=type(self)._run_server,
                args=(self._registry, self._address, self._authkey,
                      self._serializer, writer, initializer, initargs),
            )
            ident = ':'.join(str(i) for i in self._process._identity)
            self._process.name = type(self).__name__ + '-' + ident
            self._process.start()

            # get address of server
            writer.close()
            self._address = reader.recv()
            reader.close()

            # register a finalizer
            self._state.value = multiprocessing.managers.State.STARTED
            self.shutdown = multiprocessing.util.Finalize(
                self, type(self)._finalize_manager,
                args=(self._process, self._address, self._authkey,
                      self._state, self._Client),
                exitpriority=0
            ) 

def test_manager_initializer(self):
        m = multiprocessing.managers.SyncManager()
        self.assertRaises(TypeError, m.start, 1)
        m.start(initializer, (self.ns,))
        self.assertEqual(self.ns.test, 1)
        m.shutdown()
        m.join() 

def test_manager_initializer(self):
        m = multiprocessing.managers.SyncManager()
        self.assertRaises(TypeError, m.start, 1)
        m.start(initializer, (self.ns,))
        self.assertEqual(self.ns.test, 1)
        m.shutdown()
        m.join() 

def _init_multiprocessing(self):
        """
        initialize multiprocessing for powder pattern calculation
        """
        # The structure of the multiprocessing code is as follows:
        # There are nproc "manager"s which handle the actual convolver code.
        # Additionally there are 4 daemon threads which listen for work to be
        # distributed to the managers.
        np = config.NTHREADS
        self.nproc = np if np != 0 else multiprocessing.cpu_count()
        self.chunks = chunkify(list(self.data), self.nproc)
        self.next_proc = len(self.data) % self.nproc
        manager.register("conv", convolver_handler)
        self.managers = [manager() for idx in range(self.nproc)]
        self.conv_handlers = []
        self.threads = []
        self.output_queue = queue.Queue()
        for idx, mg in enumerate(self.managers):
            mg.start()
            m = mg.conv()
            for h in self.chunks[idx]:
                m.add_convolver(self.data[h]['conv'])
            self.conv_handlers.append(m)
            self.threads.append((
                threading.Thread(target=self._send_work, args=(idx, )),
                queue.Queue(), self.output_queue))
        self._running = True
        for th, q1, q2 in self.threads:
            th.daemon = True
            th.start()
        atexit.register(self.__stop__) 

def testException(self):
    manager = multiprocessing.managers.SyncManager()
    manager.start()
    lock = manager.Lock()
    counter = manager.Value('i', 0)
    calls = [(_IncrementCounter, (lock, counter), {}),
             (_RaiseValueError, (), {}),
             (_IncrementCounter, (lock, counter), {})]
    with self.assertRaises(errors.VmUtil.CalledProcessException):
      background_tasks.RunParallelProcesses(calls, max_concurrency=1)
    self.assertEqual(counter.value, 2) 

def start(self, initializer=None, initargs=()):
            '''
            Spawn a server process for this manager object
            '''
            assert self._state.value == multiprocessing.managers.State.INITIAL

            if initializer is not None and not hasattr(initializer,
                                                       '__call__'):
                raise TypeError('initializer must be a callable')

            # pipe over which we will retrieve address of server
            reader, writer = multiprocessing.Pipe(duplex=False)

            # spawn process which runs a server
            self._process = multiprocessing.Process(
                target=type(self)._run_server,
                args=(self._registry, self._address, self._authkey,
                      self._serializer, writer, initializer, initargs),
            )
            ident = ':'.join(str(i) for i in self._process._identity)
            self._process.name = type(self).__name__ + '-' + ident
            self._process.start()

            # get address of server
            writer.close()
            self._address = reader.recv()
            reader.close()

            # register a finalizer
            self._state.value = multiprocessing.managers.State.STARTED
            self.shutdown = multiprocessing.util.Finalize(
                self, type(self)._finalize_manager,
                args=(self._process, self._address, self._authkey,
                      self._state, self._Client),
                exitpriority=0
            ) 

def test_import(self):
        modules = [
            'multiprocessing', 'multiprocessing.connection',
            'multiprocessing.heap', 'multiprocessing.managers',
            'multiprocessing.pool', 'multiprocessing.process',
            'multiprocessing.synchronize', 'multiprocessing.util'
            ]

        if HAS_REDUCTION:
            modules.append('multiprocessing.reduction')

        if c_int is not None:
            # This module requires _ctypes
            modules.append('multiprocessing.sharedctypes')

        for name in modules:
            __import__(name)
            mod = sys.modules[name]

            for attr in getattr(mod, '__all__', ()):
                self.assertTrue(
                    hasattr(mod, attr),
                    '%r does not have attribute %r' % (mod, attr)
                    )

#
# Quick test that logging works -- does not test logging output
#