Python multiprocessing.active_children() Examples

def test_skip(self):
        autosave_file = BytesIONonClosing()
        btcrpass.parse_arguments(self.E2E_ARGS + [tstr("--skip=103763")],
                                 tokenlist            = StringIO(self.E2E_TOKENLIST),
                                 exclude_passwordlist = StringIO(self.E2E_EXCLUDELIST),
                                 data_extract         = self.E2E_DATA_EXTRACT,
                                 autosave             = autosave_file)
        self.assertIn("Password search exhausted", btcrpass.main()[1])
        for process in multiprocessing.active_children():
            process.join()  # wait for any remaining child processes to exit cleanly

        # Verify the password number where the search started
        autosave_file.seek(0)
        savestate = cPickle.load(autosave_file)
        self.assertEqual(savestate.get(b"skip"), 103763)

        # Verify the total count of passwords
        autosave_file.seek(SAVESLOT_SIZE)
        savestate = cPickle.load(autosave_file)
        self.assertEqual(savestate.get(b"skip"), 139652)


# QuickTests: all of Test01Basics, Test02Anchors, Test03WildCards, and Test04Typos,
# all of Test05CommandLine except the "large" tests, and select quick tests from
# Test08KeyDecryption 

def test_end_to_end(self):
        autosave_file = self.autosave_file
        btcrpass.parse_arguments(self.E2E_ARGS,
                                 tokenlist            = StringIO(self.E2E_TOKENLIST),
                                 exclude_passwordlist = StringIO(self.E2E_EXCLUDELIST),
                                 data_extract         = self.E2E_DATA_EXTRACT,
                                 autosave             = autosave_file)
        self.assertEqual("btcr-test-password", btcrpass.main()[0])
        for process in multiprocessing.active_children():
            process.join()  # wait for any remaining child processes to exit cleanly

        # Verify the exact password number where it was found to ensure password ordering hasn't changed
        autosave_file.seek(SAVESLOT_SIZE)
        savestate = cPickle.load(autosave_file)
        self.assertEqual(savestate.get(b"skip"), 103762)

    # Repeat the test above using the same autosave file, starting off just before the password was found 

def test_terminate(self):
        if self.TYPE == 'threads':
            self.skipTest('test not appropriate for {}'.format(self.TYPE))

        p = self.Process(target=self._test_terminate)
        p.daemon = True
        p.start()

        self.assertEqual(p.is_alive(), True)
        self.assertIn(p, self.active_children())
        self.assertEqual(p.exitcode, None)

        p.terminate()

        join = TimingWrapper(p.join)
        self.assertEqual(join(), None)
        self.assertTimingAlmostEqual(join.elapsed, 0.0)

        self.assertEqual(p.is_alive(), False)
        self.assertNotIn(p, self.active_children())

        p.join()

        # XXX sometimes get p.exitcode == 0 on Windows ...
        #self.assertEqual(p.exitcode, -signal.SIGTERM) 

def tearDownClass(cls):
        # only the manager process should be returned by active_children()
        # but this can take a bit on slow machines, so wait a few seconds
        # if there are other children too (see #17395)
        t = 0.01
        while len(multiprocessing.active_children()) > 1 and t < 5:
            time.sleep(t)
            t *= 2
        gc.collect()                       # do garbage collection
        if cls.manager._number_of_objects() != 0:
            # This is not really an error since some tests do not
            # ensure that all processes which hold a reference to a
            # managed object have been joined.
            print('Shared objects which still exist at manager shutdown:')
            print(cls.manager._debug_info())
        cls.manager.shutdown()
        cls.manager.join()
        cls.manager = None 

def test_server_multiproc_custom_stop_signals(
        mocker, set_timeout, restore_signal, start_method):

    mpctx = mp.get_context(start_method)
    mocker.patch('aiotools.server.mp', mpctx)

    started = mpctx.Value('i', 0)
    terminated = mpctx.Value('i', 0)
    received_signals = mpctx.Array('i', 2)
    proc_idxs = mpctx.Array('i', 2)

    set_timeout(0.2, interrupt_usr1)
    aiotools.start_server(myserver_multiproc_custom_stop_signals,
                          num_workers=2,
                          stop_signals={signal.SIGUSR1},
                          args=(started, terminated, received_signals, proc_idxs))

    assert started.value == 2
    assert terminated.value == 2
    assert list(received_signals) == [signal.SIGUSR1, signal.SIGUSR1]
    assert list(proc_idxs) == [0, 1]
    assert len(mpctx.active_children()) == 0 

def test_server_multiproc(mocker, set_timeout, restore_signal, start_method):

    mpctx = mp.get_context(start_method)
    mocker.patch('aiotools.server.mp', mpctx)

    started = mpctx.Value('i', 0)
    terminated = mpctx.Value('i', 0)
    proc_idxs = mpctx.Array('i', 3)

    set_timeout(0.2, interrupt)
    aiotools.start_server(myserver_multiproc, num_workers=3,
                          args=(started, terminated, proc_idxs))

    assert started.value == 3
    assert terminated.value == 3
    assert list(proc_idxs) == [0, 1, 2]
    assert len(mp.active_children()) == 0 

def tearDownClass(cls):
        # only the manager process should be returned by active_children()
        # but this can take a bit on slow machines, so wait a few seconds
        # if there are other children too (see #17395)
        t = 0.01
        while len(multiprocessing.active_children()) > 1 and t < 5:
            time.sleep(t)
            t *= 2
        gc.collect()                       # do garbage collection
        if cls.manager._number_of_objects() != 0:
            # This is not really an error since some tests do not
            # ensure that all processes which hold a reference to a
            # managed object have been joined.
            print('Shared objects which still exist at manager shutdown:')
            print(cls.manager._debug_info())
        cls.manager.shutdown()
        cls.manager.join()
        cls.manager = None 

def teardown_slow(runtimes, request_handler, hostname):
    request_handler.set_credentials({"user": "user0", "password": "pass0"})
    for i in range(1, len(runtimes)):
        _log.info("kill runtime {}".format(i))
        try:
            request_handler.quit(runtimes[i]["RT"])
        except Exception:
            _log.error("Failed quit for node {}".format(i))
    # Kill Auth/Authz node last since the other nodes need it for authorization
    # of the kill requests
    time.sleep(2)
    try:
        request_handler.quit(runtimes[0]["RT"])
    except Exception:
        _log.error("Failed quit for node 0")
    time.sleep(0.2)
    for p in multiprocessing.active_children():
        p.terminate()
    # They will die eventually (about 5 seconds) in most cases, but this makes sure without wasting time
    for i in range(len(runtimes)):
        os.system("pkill -9 -f 'csruntime -n {} -p 500{}'" .format(hostname,i))
    time.sleep(0.2) 

def teardown_test_type(test_type, runtimes, request_handler):
    from functools import partial
    def wait_for_it(peer):
        while True:
            try:
                request_handler.get_node_id(peer)
            except Exception:
                return True
        return False

    if test_type == "local":
        for peer in runtimes:
            request_handler.quit(peer)
        for peer in runtimes:
            retry(10, partial(wait_for_it, peer), lambda r: r, "Failed to stop peer %r" % (peer,))
        for p in multiprocessing.active_children():
            p.terminate()
            time.sleep(1) 

def tvm_wrap_excepthook(exception_hook):
    """Wrap given excepthook with TVM additional work."""

    def wrapper(exctype, value, trbk):
        """Clean subprocesses when TVM is interrupted."""
        exception_hook(exctype, value, trbk)
        if hasattr(multiprocessing, 'active_children'):
            # pylint: disable=not-callable
            for p in multiprocessing.active_children():
                p.terminate()

    return wrapper 

def close(self):
        """Closes the producer, flushing all buffered messages into Kafka.
        Calling this method directly is not recommended, instead, use the
        producer as a context manager::

            with Producer() as producer:
                producer.publish(message)
                ...
                producer.publish(message)
        """
        self.registrar.stop()
        self.monitor.close()
        self._kafka_producer.close()
        assert len(multiprocessing.active_children()) == 0 

def shutdown(self, c):
        '''
        Shutdown this process
        '''
        try:
            try:
                util.debug('manager received shutdown message')
                c.send(('#RETURN', None))

                if sys.stdout != sys.__stdout__:
                    util.debug('resetting stdout, stderr')
                    sys.stdout = sys.__stdout__
                    sys.stderr = sys.__stderr__

                util._run_finalizers(0)

                for p in active_children():
                    util.debug('terminating a child process of manager')
                    p.terminate()

                for p in active_children():
                    util.debug('terminating a child process of manager')
                    p.join()

                util._run_finalizers()
                util.info('manager exiting with exitcode 0')
            except:
                import traceback
                traceback.print_exc()
        finally:
            exit(0) 

def shutdown(self, c):
        '''
        Shutdown this process
        '''
        try:
            try:
                util.debug('manager received shutdown message')
                c.send(('#RETURN', None))

                if sys.stdout != sys.__stdout__:
                    util.debug('resetting stdout, stderr')
                    sys.stdout = sys.__stdout__
                    sys.stderr = sys.__stderr__

                util._run_finalizers(0)

                for p in active_children():
                    util.debug('terminating a child process of manager')
                    p.terminate()

                for p in active_children():
                    util.debug('terminating a child process of manager')
                    p.join()

                util._run_finalizers()
                util.info('manager exiting with exitcode 0')
            except:
                import traceback
                traceback.print_exc()
        finally:
            exit(0) 

def _stop_pool(self):
        if self.pool is None:
            return
        self.pool.close()
        self.pool.terminate()
        while len(multiprocessing.active_children()) > 0:
            time.sleep(1) 

def tearDown(self):
        for r in self.rt:
            request_handler.quit(r)
        time.sleep(0.2)
        for p in multiprocessing.active_children():
            p.terminate()
        time.sleep(0.2) 

def tearDown(self):
        request_handler.quit(self.rt1)
        request_handler.quit(self.rt2)
        request_handler.quit(self.rt3)
        time.sleep(0.2)
        for p in multiprocessing.active_children():
            p.terminate()
        time.sleep(0.2) 

def tearDown(self):
        request_handler.quit(self.rt1)
        request_handler.quit(self.rt2)
        request_handler.quit(self.rt3)
        time.sleep(0.2)
        for p in multiprocessing.active_children():
            p.terminate()
        time.sleep(0.2) 

def test_active_children(self):
        assert type(self.active_children()) == list

        p = self.Process(target=time.sleep, args=(DELTA,))
        assert p not in self.active_children()

        p.daemon = True
        p.start()
        assert p in self.active_children()

        p.join()
        assert p not in self.active_children()
        assert p.exitcode == 0 

def wait_for_children():
    """Wait for child processes to exit

    The testing system launches and terminates child processes, but
    doesn't wait for them to actually die. So in a few places we need
    this extra call"""
    wait(lambda: len(multiprocessing.active_children()) == 0) 

def slay(plist):
    import signal
    for p in plist:
        if p.is_alive():
            p.terminate()
            p.join(timeout=.2)
            if p.is_alive():
                print "Warning: process %s still alive slay!!" % p._name
                os.kill(p.pid, signal.SIGKILL)
    time.sleep(.1)
    if len(multiprocessing.active_children()) > 1:
        print "Error: children is still alive", multiprocessing.active_children()
        for a in multiprocessing.active_children():
            a.terminate() 

def teardown(self):
        global rt1
        global rt2
        global rt3
        request_handler.quit(rt1)
        request_handler.quit(rt2)
        request_handler.quit(rt3)
        time.sleep(0.2)
        for p in multiprocessing.active_children():
            p.terminate()
        # They will die eventually (about 5 seconds) in most cases, but this makes sure without wasting time
        os.system("pkill -9 -f 'csruntime -n %s -p 5000'" % (ip_addr,))
        os.system("pkill -9 -f 'csruntime -n %s -p 5001'" % (ip_addr,))
        os.system("pkill -9 -f 'csruntime -n %s -p 5002'" % (ip_addr,))
        time.sleep(0.2) 

def test_messages_published_without_flush(self, message, producer_instance):
        with capture_new_messages(
            message.topic
        ) as get_messages, producer_instance as producer:
            producer.publish(message)

        assert len(multiprocessing.active_children()) == 0
        assert len(get_messages()) == 1 

def test_messages_not_duplicated(self, message, producer_instance):
        with capture_new_messages(
            message.topic
        ) as get_messages, producer_instance as producer:
            producer.publish(message)
            producer.flush()

        assert len(multiprocessing.active_children()) == 0
        assert len(get_messages()) == 1 

def producer(self, producer_instance):
        with producer_instance as producer:
            yield producer
        assert len(multiprocessing.active_children()) == 0 

def test_publish_pii_payload_data_message(
        self, pii_schema, example_payload_data, producer_instance
    ):
        with reconfigure(
            encryption_type='AES_MODE_CBC-1',
            skip_messages_with_pii=False
        ), producer_instance as producer:
            pii_message = CreateMessage(
                schema_id=pii_schema.schema_id,
                payload_data=example_payload_data
            )
            self._publish_and_assert_pii_message(pii_message, producer)
        assert len(multiprocessing.active_children()) == 0 

def saveStrategyContext2File(self):
        self.logger.debug("save strategy context to file")
        jsonFile = open('config/StrategyContext.json', 'w', encoding='utf-8')
        result = {}
        result["StrategyConfig"] = self._strategyMgr.getStrategyConfig()
        result["MaxStrategyId"] = self._maxStrategyId
        #result["StrategyOrder"] = self._engineOrderModel.getData()
        json.dump(result, jsonFile, ensure_ascii=False, indent=4)
        for child in multiprocessing.active_children():
            try:
                child.terminate()
                child.join(timeout=0.5)
            except Exception as e:
                pass
        self.logger.debug("saveStrategyContext2File exit") 

def monitor_progress(self):
    # loop until SyncManager, LogSvc and Producer are the only children left alive
    while len(multiprocessing.active_children()) > 3 and not self.ns.quit_now:
      self.report_progress()
      self.monitor_interaction() 

def test_active_children(self):
        self.assertEqual(type(self.active_children()), list)

        p = self.Process(target=time.sleep, args=(DELTA,))
        self.assertNotIn(p, self.active_children())

        p.daemon = True
        p.start()
        self.assertIn(p, self.active_children())

        p.join()
        self.assertNotIn(p, self.active_children()) 

def test_process(self):
        q = self.Queue(1)
        e = self.Event()
        args = (q, 1, 2)
        kwargs = {'hello':23, 'bye':2.54}
        name = 'SomeProcess'
        p = self.Process(
            target=self._test, args=args, kwargs=kwargs, name=name
            )
        p.daemon = True
        current = self.current_process()

        if self.TYPE != 'threads':
            self.assertEqual(p.authkey, current.authkey)
        self.assertEqual(p.is_alive(), False)
        self.assertEqual(p.daemon, True)
        self.assertNotIn(p, self.active_children())
        self.assertTrue(type(self.active_children()) is list)
        self.assertEqual(p.exitcode, None)

        p.start()

        self.assertEqual(p.exitcode, None)
        self.assertEqual(p.is_alive(), True)
        self.assertIn(p, self.active_children())

        self.assertEqual(q.get(), args[1:])
        self.assertEqual(q.get(), kwargs)
        self.assertEqual(q.get(), p.name)
        if self.TYPE != 'threads':
            self.assertEqual(q.get(), current.authkey)
            self.assertEqual(q.get(), p.pid)

        p.join()

        self.assertEqual(p.exitcode, 0)
        self.assertEqual(p.is_alive(), False)
        self.assertNotIn(p, self.active_children()) 

def get_result(self):
        res = None

        print 'process count', multiprocessing.active_children()
        if self.lock.acquire():
            if self.topics:
                res = self.topics.pop(0)
            self.lock.release()
        # self.topic_trends.terminate()
        return res