Python twisted.internet.reactor.spawnProcess() Examples

def test_twisted(self):
        """Invoking python -m twisted should execute twist."""
        cmd = sys.executable
        p = Accumulator()
        d = p.endedDeferred = defer.Deferred()
        reactor.spawnProcess(p, cmd, [cmd, '-m', 'twisted', '--help'], env=None)
        p.transport.closeStdin()

        def processEnded(ign):
            f = p.outF
            output = f.getvalue().replace(b'\r\n', b'\n')

            options = TwistOptions()
            message = '{}\n'.format(options).encode('utf-8')
            self.assertEqual(output, message)
        return d.addCallback(processEnded) 

def test_openingTTY(self):
        scriptPath = b"twisted.test.process_tty"
        p = Accumulator()
        d = p.endedDeferred = defer.Deferred()
        reactor.spawnProcess(p, pyExe, [pyExe, b"-u", b"-m", scriptPath],
                             env=properEnv, usePTY=self.usePTY)
        p.transport.write(b"hello world!\n")

        def processEnded(ign):
            self.assertRaises(
                error.ProcessExitedAlready, p.transport.signalProcess, 'HUP')
            self.assertEqual(
                p.outF.getvalue(),
                b"hello world!\r\nhello world!\r\n",
                ("Error message from process_tty "
                 "follows:\n\n%s\n\n" % (p.outF.getvalue(),)))
        return d.addCallback(processEnded) 

def test_useReactorArgument(self):
        """
        L{twcgi.FilteredScript.runProcess} uses the reactor passed as an
        argument to the constructor.
        """
        class FakeReactor:
            """
            A fake reactor recording whether spawnProcess is called.
            """
            called = False
            def spawnProcess(self, *args, **kwargs):
                """
                Set the C{called} flag to C{True} if C{spawnProcess} is called.

                @param args: Positional arguments.
                @param kwargs: Keyword arguments.
                """
                self.called = True

        fakeReactor = FakeReactor()
        request = DummyRequest(['a', 'b'])
        resource = twcgi.FilteredScript("dummy-file", reactor=fakeReactor)
        _render(resource, request)

        self.assertTrue(fakeReactor.called) 

def test_mockErrorInPipe(self):
        """
        If C{os.pipe} raises an exception after some pipes where created, the
        created pipes are closed and don't leak.
        """
        pipes = [-1, -2, -3, -4]
        def pipe():
            try:
                return pipes.pop(0), pipes.pop(0)
            except IndexError:
                raise OSError()
        self.mockos.pipe = pipe
        protocol = TrivialProcessProtocol(None)
        self.assertRaises(OSError, reactor.spawnProcess, protocol, None)
        self.assertEqual(self.mockos.actions, [])
        self.assertEqual(set(self.mockos.closed), set([-4, -3, -2, -1])) 

def test_process(self):
        cmd = self.getCommand('gzip')
        s = b"there's no place like home!\n" * 3
        p = Accumulator()
        d = p.endedDeferred = defer.Deferred()
        reactor.spawnProcess(p, cmd, [cmd, b"-c"], env=None, path="/tmp",
                             usePTY=self.usePTY)
        p.transport.write(s)
        p.transport.closeStdin()

        def processEnded(ign):
            f = p.outF
            f.seek(0, 0)
            with gzip.GzipFile(fileobj=f) as gf:
                self.assertEqual(gf.read(), s)
        return d.addCallback(processEnded) 

def test_mockErrorECHILDInReapProcess(self):
        """
        Test that reapProcess doesn't log anything when waitpid raises a
        C{OSError} with errno C{ECHILD}.
        """
        self.mockos.child = False
        cmd = b'/mock/ouch'
        self.mockos.waitChild = (0, 0)

        d = defer.Deferred()
        p = TrivialProcessProtocol(d)
        proc = reactor.spawnProcess(p, cmd, [b'ouch'], env=None,
                                    usePTY=False)
        self.assertEqual(self.mockos.actions, [("fork", False), "waitpid"])

        self.mockos.raiseWaitPid = OSError()
        self.mockos.raiseWaitPid.errno = errno.ECHILD
        # This should not produce any errors
        proc.reapProcess() 

def launchWorkerProcesses(self, spawner, protocols, arguments):
        """
        Spawn processes from a list of process protocols.

        @param spawner: A C{IReactorProcess.spawnProcess} implementation.

        @param protocols: An iterable of C{ProcessProtocol} instances.

        @param arguments: Extra arguments passed to the processes.
        """
        workertrialPath = theSystemPath[
            'twisted.trial._dist.workertrial'].filePath.path
        childFDs = {0: 'w', 1: 'r', 2: 'r', _WORKER_AMP_STDIN: 'w',
                    _WORKER_AMP_STDOUT: 'r'}
        environ = os.environ.copy()
        # Add an environment variable containing the raw sys.path, to be used by
        # subprocesses to make sure it's identical to the parent. See
        # workertrial._setupPath.
        environ['TRIAL_PYTHONPATH'] = os.pathsep.join(sys.path)
        for worker in protocols:
            args = [sys.executable, workertrialPath]
            args.extend(arguments)
            spawner(worker, sys.executable, args=args, childFDs=childFDs,
                    env=environ) 

def _spawn(script, outputFD):
    """
    Start a script that is a peer of this test as a subprocess.

    @param script: the module name of the script in this directory (no
        package prefix, no '.py')
    @type script: C{str}

    @rtype: L{StartStopProcessProtocol}
    """
    pyExe = FilePath(sys.executable).asBytesMode().path
    env = bytesEnviron()
    env[b"PYTHONPATH"] = FilePath(
        pathsep.join(sys.path)).asBytesMode().path
    sspp = StartStopProcessProtocol()
    reactor.spawnProcess(
        sspp, pyExe, [
            pyExe,
            FilePath(__file__).sibling(script + ".py").asBytesMode().path,
            intToBytes(outputFD),
        ],
        env=env,
        childFDs={0: "w", 1: "r", 2: "r", outputFD: outputFD}
    )
    return sspp 

def test_mockWithWaitError(self):
        """
        Test that reapProcess logs errors raised.
        """
        self.mockos.child = False
        cmd = b'/mock/ouch'
        self.mockos.waitChild = (0, 0)

        d = defer.Deferred()
        p = TrivialProcessProtocol(d)
        proc = reactor.spawnProcess(p, cmd, [b'ouch'], env=None,
                             usePTY=False)
        self.assertEqual(self.mockos.actions, [("fork", False), "waitpid"])

        self.mockos.raiseWaitPid = OSError()
        proc.reapProcess()
        errors = self.flushLoggedErrors()
        self.assertEqual(len(errors), 1)
        errors[0].trap(OSError) 

def test_mockPTYSetUidInParent(self):
        """
        When spawning a child process with PTY and a UID different from the UID
        of the current process, the current process does not have its UID
        changed.
        """
        self.mockos.child = False
        cmd = b'/mock/ouch'

        d = defer.Deferred()
        p = TrivialProcessProtocol(d)
        oldPTYProcess = process.PTYProcess
        try:
            process.PTYProcess = DumbPTYProcess
            reactor.spawnProcess(p, cmd, [b'ouch'], env=None,
                                 usePTY=True, uid=8080)
        finally:
            process.PTYProcess = oldPTYProcess
        self.assertEqual(self.mockos.actions, [('fork', False), 'waitpid']) 

def test_unsetPid(self):
        """
        Test if pid is None/non-None before/after process termination.  This
        reuses process_echoer.py to get a process that blocks on stdin.
        """
        finished = defer.Deferred()
        p = TrivialProcessProtocol(finished)
        scriptPath = b"twisted.test.process_echoer"
        procTrans = reactor.spawnProcess(p, pyExe,
                                         [pyExe, b'-u', b"-m", scriptPath],
                                         env=properEnv)
        self.assertTrue(procTrans.pid)

        def afterProcessEnd(ignored):
            self.assertIsNone(procTrans.pid)

        p.transport.closeStdin()
        return finished.addCallback(afterProcessEnd) 

def test_stdio(self):
        """
        L{twisted.internet.stdio} test.
        """
        scriptPath = b"twisted.test.process_twisted"
        p = Accumulator()
        d = p.endedDeferred = defer.Deferred()
        reactor.spawnProcess(p, pyExe, [pyExe, b'-u', b"-m", scriptPath],
                             env=properEnv,
                             path=None, usePTY=self.usePTY)
        p.transport.write(b"hello, world")
        p.transport.write(b"abc")
        p.transport.write(b"123")
        p.transport.closeStdin()

        def processEnded(ign):
            self.assertEqual(p.outF.getvalue(), b"hello, worldabc123",
                              "Output follows:\n"
                              "%s\n"
                              "Error message from process_twisted follows:\n"
                              "%s\n" % (p.outF.getvalue(), p.errF.getvalue()))
        return d.addCallback(processEnded) 

def test_stdio(self):
        """
        L{twisted.internet.stdio} test.
        """
        scriptPath = b"twisted.test.process_twisted"
        p = Accumulator()
        d = p.endedDeferred = defer.Deferred()
        reactor.spawnProcess(p, pyExe, [pyExe, b'-u', b"-m", scriptPath],
                             env=properEnv,
                             path=None, usePTY=self.usePTY)
        p.transport.write(b"hello, world")
        p.transport.write(b"abc")
        p.transport.write(b"123")
        p.transport.closeStdin()

        def processEnded(ign):
            self.assertEqual(p.outF.getvalue(), b"hello, worldabc123",
                              "Output follows:\n"
                              "%s\n"
                              "Error message from process_twisted follows:\n"
                              "%s\n" % (p.outF.getvalue(), p.errF.getvalue()))
        return d.addCallback(processEnded) 

def test_stderr(self):
        """
        Bytes written to stderr by the spawned process are passed to the
        C{errReceived} callback on the C{ProcessProtocol} passed to
        C{spawnProcess}.
        """
        value = "42"

        p = Accumulator()
        d = p.endedDeferred = defer.Deferred()
        reactor.spawnProcess(p, pyExe,
                             [pyExe, b"-c",
                              networkString("import sys; sys.stderr.write"
                                            "('{0}')".format(value))],
                             env=None, path="/tmp",
                             usePTY=self.usePTY)

        def processEnded(ign):
            self.assertEqual(b"42", p.errF.getvalue())
        return d.addCallback(processEnded) 

def _mockForkInParentTest(self):
        """
        Assert that in the main process, spawnProcess disables the garbage
        collector, calls fork, closes the pipe file descriptors it created for
        the child process, and calls waitpid.
        """
        self.mockos.child = False
        cmd = b'/mock/ouch'

        d = defer.Deferred()
        p = TrivialProcessProtocol(d)
        reactor.spawnProcess(p, cmd, [b'ouch'], env=None,
                             usePTY=False)
        # It should close the first read pipe, and the 2 last write pipes
        self.assertEqual(set(self.mockos.closed), set([-1, -4, -6]))
        self.assertEqual(self.mockos.actions, [("fork", False), "waitpid"]) 

def test_mockFork(self):
        """
        Test a classic spawnProcess. Check the path of the client code:
        fork, exec, exit.
        """
        gc.enable()

        cmd = b'/mock/ouch'

        d = defer.Deferred()
        p = TrivialProcessProtocol(d)
        try:
            reactor.spawnProcess(p, cmd, [b'ouch'], env=None,
                                 usePTY=False)
        except SystemError:
            self.assertTrue(self.mockos.exited)
            self.assertEqual(
                self.mockos.actions, [("fork", False), "exec", ("exit", 1)])
        else:
            self.fail("Should not be here")

        # It should leave the garbage collector disabled.
        self.assertFalse(gc.isenabled()) 

def test_executionError(self):
        """
        Raise an error during execvpe to check error management.
        """
        cmd = self.getCommand('false')

        d = defer.Deferred()
        p = TrivialProcessProtocol(d)
        def buggyexecvpe(command, args, environment):
            raise RuntimeError("Ouch")
        oldexecvpe = os.execvpe
        os.execvpe = buggyexecvpe
        try:
            reactor.spawnProcess(p, cmd, [b'false'], env=None,
                                 usePTY=self.usePTY)

            def check(ignored):
                errData = b"".join(p.errData + p.outData)
                self.assertIn(b"Upon execvpe", errData)
                self.assertIn(b"Ouch", errData)
            d.addCallback(check)
        finally:
            os.execvpe = oldexecvpe
        return d 

def test_encodableUnicodeEnvironment(self):
        """
        Test C{os.environ} (inherited by every subprocess on Windows) that
        contains an ascii-encodable Unicode string. This is different from
        passing Unicode environment explicitly to spawnProcess (which is not
        supported on Python 2).
        """
        os.environ[self.goodKey] = self.goodValue
        self.addCleanup(operator.delitem, os.environ, self.goodKey)

        p = GetEnvironmentDictionary.run(reactor, [], properEnv)
        def gotEnvironment(environ):
            self.assertEqual(
                environ[self.goodKey.encode('ascii')],
                self.goodValue.encode('ascii'))
        return p.getResult().addCallback(gotEnvironment) 

def test_mockSetUid(self):
        """
        Try creating a process with setting its uid: it's almost the same path
        as the standard path, but with a C{switchUID} call before the exec.
        """
        cmd = b'/mock/ouch'

        d = defer.Deferred()
        p = TrivialProcessProtocol(d)
        try:
            reactor.spawnProcess(p, cmd, [b'ouch'], env=None,
                                 usePTY=False, uid=8080)
        except SystemError:
            self.assertTrue(self.mockos.exited)
            self.assertEqual(
                self.mockos.actions,
                [('fork', False), ('setuid', 0), ('setgid', 0),
                 ('switchuid', 8080, 1234), 'exec', ('exit', 1)])
        else:
            self.fail("Should not be here") 

def test_stdin(self):
        """
        Making sure getPassword accepts a password from standard input by
        running a child process which uses getPassword to read in a string
        which it then writes it out again.  Write a string to the child
        process and then read one and make sure it is the right string.
        """
        p = PasswordTestingProcessProtocol()
        p.finished = Deferred()
        reactor.spawnProcess(
            p, pyExe,
            [pyExe,
             b'-c',
             (b'import sys\n'
              b'from twisted.python.util import getPassword\n'
              b'sys.stdout.write(getPassword())\n'
              b'sys.stdout.flush()\n')],
            env={b'PYTHONPATH': os.pathsep.join(sys.path).encode("utf8")})

        def processFinished(result):
            (reason, output) = result
            reason.trap(ProcessDone)
            self.assertIn((1, b'secret'), output)

        return p.finished.addCallback(processFinished) 

def test_echo(self):
        """
        A spawning a subprocess which echoes its stdin to its stdout via
        L{IReactorProcess.spawnProcess} will result in that echoed output being
        delivered to outReceived.
        """
        finished = defer.Deferred()
        p = EchoProtocol(finished)

        scriptPath = b"twisted.test.process_echoer"
        reactor.spawnProcess(p, pyExe, [pyExe, b'-u', b"-m", scriptPath],
                             env=properEnv)

        def asserts(ignored):
            self.assertFalse(p.failure, p.failure)
            self.assertTrue(hasattr(p, 'buffer'))
            self.assertEqual(len(p.buffer), len(p.s * p.n))

        def takedownProcess(err):
            p.transport.closeStdin()
            return err

        return finished.addCallback(asserts).addErrback(takedownProcess) 

def _spawn(script, outputFD):
    """
    Start a script that is a peer of this test as a subprocess.

    @param script: the module name of the script in this directory (no
        package prefix, no '.py')
    @type script: C{str}

    @rtype: L{StartStopProcessProtocol}
    """
    pyExe = FilePath(sys.executable).asBytesMode().path
    env = bytesEnviron()
    env[b"PYTHONPATH"] = FilePath(
        pathsep.join(sys.path)).asBytesMode().path
    sspp = StartStopProcessProtocol()
    reactor.spawnProcess(
        sspp, pyExe, [
            pyExe,
            FilePath(__file__).sibling(script + ".py").asBytesMode().path,
            intToBytes(outputFD),
        ],
        env=env,
        childFDs={0: "w", 1: "r", 2: "r", outputFD: outputFD}
    )
    return sspp 

def test_unsetPid(self):
        """
        Test if pid is None/non-None before/after process termination.  This
        reuses process_echoer.py to get a process that blocks on stdin.
        """
        finished = defer.Deferred()
        p = TrivialProcessProtocol(finished)
        scriptPath = b"twisted.test.process_echoer"
        procTrans = reactor.spawnProcess(p, pyExe,
                                         [pyExe, b'-u', b"-m", scriptPath],
                                         env=properEnv)
        self.assertTrue(procTrans.pid)

        def afterProcessEnd(ignored):
            self.assertIsNone(procTrans.pid)

        p.transport.closeStdin()
        return finished.addCallback(afterProcessEnd) 

def _spawnProcess(self, proto, sibling, *args, **kw):
        """
        Launch a child Python process and communicate with it using the
        given ProcessProtocol.

        @param proto: A L{ProcessProtocol} instance which will be connected
        to the child process.

        @param sibling: The basename of a file containing the Python program
        to run in the child process.

        @param *args: strings which will be passed to the child process on
        the command line as C{argv[2:]}.

        @param **kw: additional arguments to pass to L{reactor.spawnProcess}.

        @return: The L{IProcessTransport} provider for the spawned process.
        """
        args = [sys.executable,
                b"-m", b"twisted.test." + sibling,
                reactor.__class__.__module__] + list(args)
        return reactor.spawnProcess(
            proto,
            sys.executable,
            args,
            env=properEnv,
            **kw) 

def test_useReactorArgument(self):
        """
        L{twcgi.FilteredScript.runProcess} uses the reactor passed as an
        argument to the constructor.
        """
        class FakeReactor:
            """
            A fake reactor recording whether spawnProcess is called.
            """
            called = False
            def spawnProcess(self, *args, **kwargs):
                """
                Set the C{called} flag to C{True} if C{spawnProcess} is called.

                @param args: Positional arguments.
                @param kwargs: Keyword arguments.
                """
                self.called = True

        fakeReactor = FakeReactor()
        request = DummyRequest(['a', 'b'])
        request.client = address.IPv4Address('TCP', '127.0.0.1', 12345)
        resource = twcgi.FilteredScript("dummy-file", reactor=fakeReactor)
        _render(resource, request)

        self.assertTrue(fakeReactor.called) 

def _testSignal(self, sig):
        scriptPath = b"twisted.test.process_signal"
        d = defer.Deferred()
        p = Win32SignalProtocol(d, sig)
        reactor.spawnProcess(p, pyExe, [pyExe, b"-u", b"-m", scriptPath],
                             env=properEnv)
        return d 

def test_stdinReader(self):
        scriptPath = b"twisted.test.process_stdinreader"
        p = Accumulator()
        d = p.endedDeferred = defer.Deferred()
        reactor.spawnProcess(p, pyExe, [pyExe, b"-u", b"-m", scriptPath],
                             env=properEnv)
        p.transport.write(b"hello, world")
        p.transport.closeStdin()

        def processEnded(ign):
            self.assertEqual(p.errF.getvalue(), b"err\nerr\n")
            self.assertEqual(p.outF.getvalue(), b"out\nhello, world\nout\n")
        return d.addCallback(processEnded) 

def test_badArgs(self):
        pyArgs = [pyExe, b"-u", b"-c", b"print('hello')"]
        p = Accumulator()
        self.assertRaises(ValueError,
                          reactor.spawnProcess, p, pyExe, pyArgs, uid=1)
        self.assertRaises(ValueError,
                          reactor.spawnProcess, p, pyExe, pyArgs, gid=1)
        self.assertRaises(ValueError,
                          reactor.spawnProcess, p, pyExe, pyArgs, usePTY=1)
        self.assertRaises(ValueError,
                          reactor.spawnProcess, p, pyExe, pyArgs, childFDs={1:'r'}) 

def testCallBeforeStartupUnexecuted(self):
        progname = self.mktemp()
        with open(progname, 'w') as progfile:
            progfile.write(_callBeforeStartupProgram % {'reactor': reactor.__module__})

        def programFinished(result):
            (out, err, reason) = result
            if reason.check(error.ProcessTerminated):
                self.fail("Process did not exit cleanly (out: %s err: %s)" % (out, err))

            if err:
                log.msg("Unexpected output on standard error: %s" % (err,))
            self.assertFalse(
                out,
                "Expected no output, instead received:\n%s" % (out,))

        def programTimeout(err):
            err.trap(error.TimeoutError)
            proto.signalProcess('KILL')
            return err

        env = os.environ.copy()
        env['PYTHONPATH'] = os.pathsep.join(sys.path)
        d = defer.Deferred().addCallbacks(programFinished, programTimeout)
        proto = ThreadStartupProcessProtocol(d)
        reactor.spawnProcess(proto, sys.executable, ('python', progname), env)
        return d 

def testChildResolve(self):
        # I've seen problems with reactor.run under gtk2reactor. Spawn a
        # child which just does reactor.resolve after the reactor has
        # started, fail if it does not complete in a timely fashion.
        helperPath = os.path.abspath(self.mktemp())
        with open(helperPath, 'w') as helperFile:

            # Eeueuuggg
            reactorName = reactor.__module__

            helperFile.write(resolve_helper % {'reactor': reactorName})

        env = os.environ.copy()
        env['PYTHONPATH'] = os.pathsep.join(sys.path)

        helperDeferred = Deferred()
        helperProto = ChildResolveProtocol(helperDeferred)

        reactor.spawnProcess(helperProto, sys.executable, ("python", "-u", helperPath), env)

        def cbFinished(result):
            (reason, output, error) = result
            # If the output is "done 127.0.0.1\n" we don't really care what
            # else happened.
            output = b''.join(output)
            if _PY3:
                expected_output = (b'done 127.0.0.1' +
                                   os.linesep.encode("ascii"))
            else:
                expected_output = b'done 127.0.0.1\n'
            if output != expected_output:
                self.fail((
                    "The child process failed to produce the desired results:\n"
                    "   Reason for termination was: %r\n"
                    "   Output stream was: %r\n"
                    "   Error stream was: %r\n") % (reason.getErrorMessage(), output, b''.join(error)))

        helperDeferred.addCallback(cbFinished)
        return helperDeferred