Python multiprocessing.process() Examples

def _validate(self):
        """Raise exception if this handle is closed or not registered to be
        used in the current process.

        Intended to be called before every operation on `self._fd`.
        Reveals wrong usage of this module in the context of multiple
        processes. Might prevent tedious debugging sessions. Has little
        performance impact.
        """
        if self._closed:
            raise GIPCClosed(
                "GIPCHandle has been closed before.")
        if os.getpid() != self._legit_pid:
            raise GIPCError(
                "GIPCHandle %s not registered for current process %s." % (
                    self, os.getpid())) 

def run_trial(self, trial_num, param):
        '''
        algo step 2, construct and run Trial with the next param
        args trial_num, param must be provided externally,
        otherwise they will not progress within mp.process
        '''
        experiment_spec = self.compose_experiment_spec(param)
        trial = self.Trial(
            experiment_spec, trial_num=trial_num,
            times=self.times,
            num_of_trials=self.num_of_trials,
            run_timestamp=self.run_timestamp,
            experiment_id_override=self.experiment_id_override)
        trial_data = trial.run()
        del trial
        import gc
        gc.collect()
        debug_mem_usage()
        return trial_data

    # retrieve the trial_num, param, fitness_score from trial_data 

def _winapi_childhandle_after_createprocess_child(self):
        """Called on Windows in the child process after the CreateProcess()
        system call. This is required for making the handle usable in the child.
        """

        if WINAPI_HANDLE_TRANSFER_STEAL:
            # In this case the handle has not been inherited by the child
            # process during CreateProcess(). Steal it from the parent.
            new_winapihandle = multiprocessing.reduction.steal_handle(
                self._parent_pid, self._parent_winapihandle)
            del self._parent_winapihandle
            del self._parent_pid
            # Restore C file descriptor with (read/write)only flag.
            self._fd = msvcrt.open_osfhandle(new_winapihandle, self._fd_flag)
            return

        # In this case the handle has been inherited by the child process during
        # the CreateProcess() system call. Get C file descriptor from Windows
        # file handle.
        self._fd = msvcrt.open_osfhandle(
            self._inheritable_winapihandle, self._fd_flag)

        del self._inheritable_winapihandle 

def startASLogger(loggerAddr, logDefs, transport, capabilities,
                  aggregatorAddress=None,
                  concurrency_context = None):
    endpointPrep = transport.prepEndpoint(loggerAddr, capabilities)
    multiprocessing.process._current_process._daemonic = False
    NewProc = concurrency_context.Process if concurrency_context else multiprocessing.Process
    logProc = NewProc(target=startupASLogger,
                      args = (transport.myAddress, endpointPrep,
                              logDefs,
                              transport.__class__, aggregatorAddress))
    logProc.daemon = True
    logProc.start()
    transport.connectEndpoint(endpointPrep)
    # When the caller that owns the transport starts their run(), it
    # will receive the LoggerConnected from the child to complete the
    # handshake and the sender will be the actual address of the
    # logger.
    return ChildInfo(loggerAddr, 'logger', logProc, endpointPrep.addrInst) 

def __init__(self):
        global _all_handles
        # Generate label of text/unicode type from three random bytes.
        self._id = codecs.encode(os.urandom(3), "hex_codec").decode("ascii")
        self._legit_pid = os.getpid()
        self._make_nonblocking()

        # Define lock for synchronizing access to this handle within the current
        # process. Note that a `gevent.lock.Semaphore` instance lives on the
        # heap of the current process and cannot be used to synchronize access
        # across multiple processes. That is, this lock is only meaningful in
        # the current process. This is especially important to consider when the
        # platform supports fork()ing.
        self._lock = gevent.lock.Semaphore(value=1)
        self._closed = False
        _all_handles.append(self) 

def temp_umask(umask):
        """Context manager that temporarily sets the process umask."""
        oldmask = os.umask(umask)
        try:
            yield
        finally:
            os.umask(oldmask) 

def temp_umask(umask):
        """Context manager that temporarily sets the process umask."""
        oldmask = os.umask(umask)
        try:
            yield
        finally:
            os.umask(oldmask) 

def strip_python_stderr(stderr):
    """Strip the stderr of a Python process from potential debug output
    emitted by the interpreter.

    This will typically be run on the result of the communicate() method
    of a subprocess.Popen object.
    """
    stderr = re.sub(br"\[\d+ refs\]\r?\n?", b"", stderr).strip()
    return stderr 

def detach_child(childref):
    if hasattr(multiprocessing.process, '_children'):
        # Python 3.4
        multiprocessing.process._children.remove(childref)
    if hasattr(multiprocessing.process, '_current_process'):
        if hasattr(multiprocessing.process._current_process, '_children'):
            # Python 2.6
            multiprocessing.process._current_process._children.remove(childref) 

def strip_python_stderr(stderr):
    """Strip the stderr of a Python process from potential debug output
    emitted by the interpreter.

    This will typically be run on the result of the communicate() method
    of a subprocess.Popen object.
    """
    stderr = re.sub(br"\[\d+ refs\]\r?\n?", b"", stderr).strip()
    return stderr 

def temp_umask(umask):
        """Context manager that temporarily sets the process umask."""
        oldmask = os.umask(umask)
        try:
            yield
        finally:
            os.umask(oldmask) 

def bind_port(sock, host=HOST):
    """Bind the socket to a free port and return the port number.  Relies on
    ephemeral ports in order to ensure we are using an unbound port.  This is
    important as many tests may be running simultaneously, especially in a
    buildbot environment.  This method raises an exception if the sock.family
    is AF_INET and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR
    or SO_REUSEPORT set on it.  Tests should *never* set these socket options
    for TCP/IP sockets.  The only case for setting these options is testing
    multicasting via multiple UDP sockets.

    Additionally, if the SO_EXCLUSIVEADDRUSE socket option is available (i.e.
    on Windows), it will be set on the socket.  This will prevent anyone else
    from bind()'ing to our host/port for the duration of the test.
    """

    if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM:
        if hasattr(socket, 'SO_REUSEADDR'):
            if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1:
                raise TestFailed("tests should never set the SO_REUSEADDR "   \
                                 "socket option on TCP/IP sockets!")
        if hasattr(socket, 'SO_REUSEPORT'):
            try:
                if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1:
                    raise TestFailed("tests should never set the SO_REUSEPORT "   \
                                     "socket option on TCP/IP sockets!")
            except socket.error:
                # Python's socket module was compiled using modern headers
                # thus defining SO_REUSEPORT but this process is running
                # under an older kernel that does not support SO_REUSEPORT.
                pass
        if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'):
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1)

    sock.bind((host, 0))
    port = sock.getsockname()[1]
    return port 

def signal_detector(name, addr, am):
    def signal_detected(signum, frame):
        if signum == getattr(signal, 'SIGUSR1', 'no-sigusr1-avail'):
            am.thesplogStatus()
        else:
            thesplog('Actor %s @ %s got signal: %s', name, addr, signum,
                     level = logging.WARNING)
        # Simply exit; just by catching the signal the atexit handlers are enabled
        # if this signal is going to cause a process exit.
    return signal_detected 

def _handleReplicatorMessages(self, envelope):
        # This method handles any messages related to multi-process
        # management operations.  This also ensures that any messages
        # received before initialization is completed are held to be
        # handled after initialization finishes.
        if isinstance(envelope.message, EndpointConnected):
            return True, self.h_EndpointConnected(envelope)
        if self.asLogger is None and not isinstance(envelope.message, LoggerConnected):
            self._pre_init_msgs.append(envelope)
            return True, True
        if isinstance(envelope.message, logging.LogRecord):
            return True, self.h_LogRecord(envelope)
        if isinstance(envelope.message, ChildMayHaveDied):
            return True, self.childDied()
        return False, True 

def h_EndpointConnected(self, envelope):
        for C in getattr(self, '_child_procs', []):
            if envelope.message.childInstance == C.childNum:
                C.childRealAddr = envelope.sender
                break
        else:
            thesplog('Unknown child process endpoint connected: %s', envelope, level=logging.WARNING)
        self._pendingActorReady(envelope.message.childInstance, envelope.sender)
        return True 

def _startAdmin(self, adminAddr, addrOfStarter, capabilities, logDefs):
        mp = self.mpcontext if self.mpcontext else multiprocessing
        endpointPrep = self.transport.prepEndpoint(adminAddr, capabilities)

        multiprocessing.process._current_process._daemonic = False
        admin = mp.Process(target=startAdmin,
                                        args=(MultiProcAdmin,
                                              addrOfStarter,
                                              endpointPrep,
                                              self.transport.__class__,
                                              adminAddr,
                                              capabilities,
                                              logDefs,
                                              self.mpcontext),
                                        name='ThespianAdmin')
        admin.start()
        # admin must be explicitly shutdown and is not automatically
        # stopped when this current process exits.
        detach_child(admin)

        self.transport.connectEndpoint(endpointPrep)

        response = self.transport.run(None, MAX_ADMIN_STARTUP_DELAY)
        if not isinstance(response, ReceiveEnvelope) or \
           not isinstance(response.message, EndpointConnected):
            raise InvalidActorAddress(adminAddr,
                                      'not a valid ActorSystem admin') 

def restore_multiprocessing_process__dangling(self, saved):
        if not multiprocessing:
            return
        multiprocessing.process._dangling.clear()
        multiprocessing.process._dangling.update(saved) 

def bind_port(sock, host=HOST):
    """Bind the socket to a free port and return the port number.  Relies on
    ephemeral ports in order to ensure we are using an unbound port.  This is
    important as many tests may be running simultaneously, especially in a
    buildbot environment.  This method raises an exception if the sock.family
    is AF_INET and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR
    or SO_REUSEPORT set on it.  Tests should *never* set these socket options
    for TCP/IP sockets.  The only case for setting these options is testing
    multicasting via multiple UDP sockets.

    Additionally, if the SO_EXCLUSIVEADDRUSE socket option is available (i.e.
    on Windows), it will be set on the socket.  This will prevent anyone else
    from bind()'ing to our host/port for the duration of the test.
    """

    if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM:
        if hasattr(socket, 'SO_REUSEADDR'):
            if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1:
                raise TestFailed("tests should never set the SO_REUSEADDR "   \
                                 "socket option on TCP/IP sockets!")
        if hasattr(socket, 'SO_REUSEPORT'):
            try:
                if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1:
                    raise TestFailed("tests should never set the SO_REUSEPORT "   \
                                     "socket option on TCP/IP sockets!")
            except socket.error:
                # Python's socket module was compiled using modern headers
                # thus defining SO_REUSEPORT but this process is running
                # under an older kernel that does not support SO_REUSEPORT.
                pass
        if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'):
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1)

    sock.bind((host, 0))
    port = sock.getsockname()[1]
    return port 

def strip_python_stderr(stderr):
    """Strip the stderr of a Python process from potential debug output
    emitted by the interpreter.

    This will typically be run on the result of the communicate() method
    of a subprocess.Popen object.
    """
    stderr = re.sub(br"\[\d+ refs\]\r?\n?", b"", stderr).strip()
    return stderr 

def restore_multiprocessing_process__dangling(self, saved):
        if not multiprocessing:
            return
        multiprocessing.process._dangling.clear()
        multiprocessing.process._dangling.update(saved) 

def bind_port(sock, host=HOST):
    """Bind the socket to a free port and return the port number.  Relies on
    ephemeral ports in order to ensure we are using an unbound port.  This is
    important as many tests may be running simultaneously, especially in a
    buildbot environment.  This method raises an exception if the sock.family
    is AF_INET and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR
    or SO_REUSEPORT set on it.  Tests should *never* set these socket options
    for TCP/IP sockets.  The only case for setting these options is testing
    multicasting via multiple UDP sockets.

    Additionally, if the SO_EXCLUSIVEADDRUSE socket option is available (i.e.
    on Windows), it will be set on the socket.  This will prevent anyone else
    from bind()'ing to our host/port for the duration of the test.
    """

    if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM:
        if hasattr(socket, 'SO_REUSEADDR'):
            if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1:
                raise TestFailed("tests should never set the SO_REUSEADDR "   \
                                 "socket option on TCP/IP sockets!")
        if hasattr(socket, 'SO_REUSEPORT'):
            try:
                if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1:
                    raise TestFailed("tests should never set the SO_REUSEPORT "   \
                                     "socket option on TCP/IP sockets!")
            except socket.error:
                # Python's socket module was compiled using modern headers
                # thus defining SO_REUSEPORT but this process is running
                # under an older kernel that does not support SO_REUSEPORT.
                pass
        if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'):
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1)

    sock.bind((host, 0))
    port = sock.getsockname()[1]
    return port 

def strip_python_stderr(stderr):
    """Strip the stderr of a Python process from potential debug output
    emitted by the interpreter.

    This will typically be run on the result of the communicate() method
    of a subprocess.Popen object.
    """
    stderr = re.sub(br"\[\d+ refs\]\r?\n?", b"", stderr).strip()
    return stderr 

def temp_umask(umask):
        """Context manager that temporarily sets the process umask."""
        oldmask = os.umask(umask)
        try:
            yield
        finally:
            os.umask(oldmask) 

def bind_port(sock, host=HOST):
    """Bind the socket to a free port and return the port number.  Relies on
    ephemeral ports in order to ensure we are using an unbound port.  This is
    important as many tests may be running simultaneously, especially in a
    buildbot environment.  This method raises an exception if the sock.family
    is AF_INET and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR
    or SO_REUSEPORT set on it.  Tests should *never* set these socket options
    for TCP/IP sockets.  The only case for setting these options is testing
    multicasting via multiple UDP sockets.

    Additionally, if the SO_EXCLUSIVEADDRUSE socket option is available (i.e.
    on Windows), it will be set on the socket.  This will prevent anyone else
    from bind()'ing to our host/port for the duration of the test.
    """

    if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM:
        if hasattr(socket, 'SO_REUSEADDR'):
            if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1:
                raise TestFailed("tests should never set the SO_REUSEADDR "   \
                                 "socket option on TCP/IP sockets!")
        if hasattr(socket, 'SO_REUSEPORT'):
            try:
                if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1:
                    raise TestFailed("tests should never set the SO_REUSEPORT "   \
                                     "socket option on TCP/IP sockets!")
            except socket.error:
                # Python's socket module was compiled using modern headers
                # thus defining SO_REUSEPORT but this process is running
                # under an older kernel that does not support SO_REUSEPORT.
                pass
        if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'):
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1)

    sock.bind((host, 0))
    port = sock.getsockname()[1]
    return port 

def temp_umask(umask):
        """Context manager that temporarily sets the process umask."""
        oldmask = os.umask(umask)
        try:
            yield
        finally:
            os.umask(oldmask)

# TEST_HOME_DIR refers to the top level directory of the "test" package
# that contains Python's regression test suite 

def bind_port(sock, host=HOST):
    """Bind the socket to a free port and return the port number.  Relies on
    ephemeral ports in order to ensure we are using an unbound port.  This is
    important as many tests may be running simultaneously, especially in a
    buildbot environment.  This method raises an exception if the sock.family
    is AF_INET and sock.type is SOCK_STREAM, *and* the socket has SO_REUSEADDR
    or SO_REUSEPORT set on it.  Tests should *never* set these socket options
    for TCP/IP sockets.  The only case for setting these options is testing
    multicasting via multiple UDP sockets.

    Additionally, if the SO_EXCLUSIVEADDRUSE socket option is available (i.e.
    on Windows), it will be set on the socket.  This will prevent anyone else
    from bind()'ing to our host/port for the duration of the test.
    """

    if sock.family == socket.AF_INET and sock.type == socket.SOCK_STREAM:
        if hasattr(socket, 'SO_REUSEADDR'):
            if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR) == 1:
                raise TestFailed("tests should never set the SO_REUSEADDR "   \
                                 "socket option on TCP/IP sockets!")
        if hasattr(socket, 'SO_REUSEPORT'):
            try:
                if sock.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT) == 1:
                    raise TestFailed("tests should never set the SO_REUSEPORT "   \
                                     "socket option on TCP/IP sockets!")
            except OSError:
                # Python's socket module was compiled using modern headers
                # thus defining SO_REUSEPORT but this process is running
                # under an older kernel that does not support SO_REUSEPORT.
                pass
        if hasattr(socket, 'SO_EXCLUSIVEADDRUSE'):
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1)

    sock.bind((host, 0))
    port = sock.getsockname()[1]
    return port 

def restore_multiprocessing_process__dangling(self, saved):
        if not multiprocessing:
            return
        multiprocessing.process._dangling.clear()
        multiprocessing.process._dangling.update(saved) 

def get_multiprocessing_process__dangling(self):
        if not multiprocessing:
            return None
        # Unjoined process objects can survive after process exits
        multiprocessing.process._cleanup()
        # This copies the weakrefs without making any strong reference
        return multiprocessing.process._dangling.copy() 

def strip_python_stderr(stderr):
    """Strip the stderr of a Python process from potential debug output
    emitted by the interpreter.

    This will typically be run on the result of the communicate() method
    of a subprocess.Popen object.
    """
    stderr = re.sub(br"\[\d+ refs\]\r?\n?", b"", stderr).strip()
    return stderr 

def temp_umask(umask):
        """Context manager that temporarily sets the process umask."""
        oldmask = os.umask(umask)
        try:
            yield
        finally:
            os.umask(oldmask)