Python asyncio.create_subprocess_exec() Examples

def run(args, universal_newlines=False, check=False, shell=False, **kwargs):
    """Provide the subprocess.run() function as asyncio corouting.

    This takes care of the missing 'universal_newlines' and 'check' options.
    Everything else is passed through. Will also raise the same exceptions as
    subprocess.run() to act as a drop-in replacement.
    """
    import asyncio
    import io
    import locale
    import subprocess

    if shell:
        proc = await asyncio.create_subprocess_shell(args, **kwargs)
    else:
        proc = await asyncio.create_subprocess_exec(*args, **kwargs)
    stdout, stderr = await proc.communicate()

    if universal_newlines and (stdout is not None):
        stdout = io.TextIOWrapper(io.BytesIO(stdout)).read()
    if universal_newlines and (stderr is not None):
        stderr = io.TextIOWrapper(io.BytesIO(stderr)).read()

    if check and (proc.returncode != 0):
        raise subprocess.CalledProcessError(proc.returncode, args,
            stdout, stderr)

    return subprocess.CompletedProcess(args, proc.returncode, stdout,
        stderr) 

def _start_remote_processors_ssh(self, host, port, num_workers):
        # establish a reverse SSH tunnel from remote unix socket to
        # the local unix socket that our Unix server is listening on
        port_arg = ('-p', port) if port else ()
        remote_unix_path = util.get_temp_path()
        proc = await asyncio.create_subprocess_exec(
            'ssh',
            '-T', host,
            *port_arg,
            '-R', f'{remote_unix_path}:{self._unix_path}',
            stdin=asyncio.subprocess.PIPE,
            stderr=asyncio.subprocess.PIPE)
        # spawn processors that will connect to the tunneled Unix server
        cmd = (
            f'distex_proc '
            f'-u {remote_unix_path} '
            f'-l {self._worker_loop} '
            f'-f {self._func_pickle} '
            f'-d {self._data_pickle} '
            f'& \n'.encode()) * num_workers
        proc.stdin.write(cmd)
        await proc.stdin.drain()
        self._ssh_tunnels.append(proc)
        self._total_workers += num_workers 

def init_cache_coro(log=logging.getLogger('qubes.storage.lvm')):
    cmd = _init_cache_cmd
    if os.getuid() != 0:
        cmd = ['sudo'] + cmd
    environ = os.environ.copy()
    environ['LC_ALL'] = 'C.utf8'
    p = yield from asyncio.create_subprocess_exec(*cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        close_fds=True, env=environ)
    out, err = yield from p.communicate()
    return_code = p.returncode
    if return_code == 0 and err:
        log.warning(err)
    elif return_code != 0:
        raise qubes.storage.StoragePoolException(err)

    return _parse_lvm_cache(out) 

def run_npm_audit(pkg: str) -> Dict[str, Any]:
    """
    CLI usage: dffml service dev run -log debug shouldi.npm_audit:run_npm_audit -pkg .
    """
    proc = await asyncio.create_subprocess_exec(
        "npm",
        "audit",
        "--json",
        cwd=pkg,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )

    stdout, stderr = await proc.communicate()
    if proc.returncode != 0 and stderr:
        raise NPMAuditError(stderr.decode())

    npm_audit_op = stdout.decode()
    npm_audit_op = json.loads(npm_audit_op)
    result = npm_audit_op["metadata"]["vulnerabilities"]
    return {"report": result} 

def test_cancel_make_subprocess_transport_exec(self):
        @asyncio.coroutine
        def cancel_make_transport():
            coro = asyncio.create_subprocess_exec(*PROGRAM_BLOCKED,
                                                  loop=self.loop)
            task = self.loop.create_task(coro)

            self.loop.call_soon(task.cancel)
            try:
                yield from task
            except asyncio.CancelledError:
                pass

        # ignore the log:
        # "Exception during subprocess creation, kill the subprocess"
        with test_utils.disable_logger():
            self.loop.run_until_complete(cancel_make_transport()) 

def test_cancel_process_wait(self):
        # Issue #23140: cancel Process.wait()

        @asyncio.coroutine
        def cancel_wait():
            proc = yield from asyncio.create_subprocess_exec(
                                          *PROGRAM_BLOCKED,
                                          loop=self.loop)

            # Create an internal future waiting on the process exit
            task = self.loop.create_task(proc.wait())
            self.loop.call_soon(task.cancel)
            try:
                yield from task
            except asyncio.CancelledError:
                pass

            # Cancel the future
            task.cancel()

            # Kill the process and wait until it is done
            proc.kill()
            yield from proc.wait()

        self.loop.run_until_complete(cancel_wait()) 

def _convert_fb_sticker(data: bytes, frames_per_row: int, frames_per_col: int
                                  ) -> Tuple[bytes, int, int]:
        ntf = NamedTemporaryFile
        with ntf(suffix=".png") as input_file, ntf(suffix=".gif") as output_file:
            input_file.write(data)
            with Image.open(input_file) as img:
                width, height = img.size
            width /= frames_per_row
            height /= frames_per_col
            proc = await asyncio.create_subprocess_exec(convert_cmd,
                                                        "-dispose", "Background",
                                                        input_file.name,
                                                        "-crop", f"{width}x{height}",
                                                        "+adjoin", "+repage", "-adjoin",
                                                        "-loop", "0",
                                                        output_file.name)
            await proc.wait()
            return output_file.read(), width, height 

def test_stdin_not_inheritable(self):
        # asyncio issue #209: stdin must not be inheritable, otherwise
        # the Process.communicate() hangs
        @asyncio.coroutine
        def len_message(message):
            code = 'import sys; data = sys.stdin.read(); print(len(data))'
            proc = yield from asyncio.create_subprocess_exec(
                                          sys.executable, '-c', code,
                                          stdin=asyncio.subprocess.PIPE,
                                          stdout=asyncio.subprocess.PIPE,
                                          stderr=asyncio.subprocess.PIPE,
                                          close_fds=False,
                                          loop=self.loop)
            stdout, stderr = yield from proc.communicate(message)
            exitcode = yield from proc.wait()
            return (stdout, exitcode)

        output, exitcode = self.loop.run_until_complete(len_message(b'abc'))
        self.assertEqual(output.rstrip(), b'3')
        self.assertEqual(exitcode, 0) 

def safety_check(package: str, version: str) -> int:
    pinned = f"{package}=={version}"

    proc = await asyncio.create_subprocess_exec(
        sys.executable,
        "-m",
        "safety",
        "check",
        "--stdin",
        "--json",
        stdin=asyncio.subprocess.PIPE,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )

    stdout, _stderr = await proc.communicate(pinned.encode() + b"\n")

    issues = json.loads(stdout)

    return len(issues) 

def __init__(
            self,
            pt_exec: Union[List[str], List[bytes]],
            state: Union[str, bytes, os.PathLike],
            *,
            exit_on_stdin_close: bool = True,
    ) -> None:
        """Create the adapter.

        Args:
            pt_exec: The pluggable transport command line to execute. This has
                to be a list of str / bytes, since
                :func:`asyncio.create_subprocess_exec` does not accept an
                entire command line as a string. On non-Windows platforms
                :func:`shlex.split` can be used to split a command line string
                into a list, while on Windows it's a bit more complicated.
            state: The state directory. This is a directory where the PT is
                allowed to store state. Either specify a path (which
                is not required to exist, in which case the PT will create
                the directory), or specify ``None`` to use a temporary
                directory created using :mod:`tempfile`.
            exit_on_stdin_close: Whether closing the PT's STDIN indicates the
                PT should gracefully exit.
        """
        if isinstance(pt_exec, (str, bytes)):
            self._pt_args = [pt_exec]
        else:
            self._pt_args = list(pt_exec)
        if state is not None:
            self._state = os.path.abspath(state)
        else:
            self._state = None
        self._exit_on_stdin_close = exit_on_stdin_close

        self._process: asyncio.subprocess.Process = None
        self._stdout_task: asyncio.Task = None
        self._ready = asyncio.Future()
        self._accepted_version: str = None
        self._transports: Dict[str, asyncio.Future] = {}
        self._stopping = False
        self._stack = contextlib.AsyncExitStack() 

def start(self) -> None:
        """(async) Start the PT executable and wait until it's ready.

        "Ready" means that all transports have finished initializing.
        """
        self._check_not_started()
        await self._pre_start()
        env = self._build_env()
        self._logger.debug('PT environment variables: %r', env)
        self._process = await asyncio.create_subprocess_exec(
            *self._pt_args,
            env=env,
            stdin=asyncio.subprocess.PIPE,
            stdout=asyncio.subprocess.PIPE,
            stderr=None,
        )
        self._logger.debug('Started PT subprocess: %r', self._process)
        self._stdout_task = asyncio.create_task(self._process_stdout())
        try:
            await self._ready
        except Exception:
            await self.stop()
            raise 

def test_communicate(self):
        args = PROGRAM_CAT

        @asyncio.coroutine
        def run(data):
            proc = yield from asyncio.create_subprocess_exec(
                                          *args,
                                          stdin=subprocess.PIPE,
                                          stdout=subprocess.PIPE,
                                          loop=self.loop)
            stdout, stderr = yield from proc.communicate(data)
            return proc.returncode, stdout

        task = run(b'some data')
        task = asyncio.wait_for(task, 60.0, loop=self.loop)
        exitcode, stdout = self.loop.run_until_complete(task)
        self.assertEqual(exitcode, 0)
        self.assertEqual(stdout, b'some data') 

def test_read_all_from_pipe_reader(self):
        # See asyncio issue 168.  This test is derived from the example
        # subprocess_attach_read_pipe.py, but we configure the
        # StreamReader's limit so that twice it is less than the size
        # of the data writter.  Also we must explicitly attach a child
        # watcher to the event loop.

        code = """\
import os, sys
fd = int(sys.argv[1])
os.write(fd, b'data')
os.close(fd)
"""
        rfd, wfd = os.pipe()
        args = [sys.executable, '-c', code, str(wfd)]

        pipe = open(rfd, 'rb', 0)
        reader = asyncio.StreamReader(loop=self.loop, limit=1)
        protocol = asyncio.StreamReaderProtocol(reader, loop=self.loop)
        transport, _ = self.loop.run_until_complete(
            self.loop.connect_read_pipe(lambda: protocol, pipe))

        watcher = asyncio.SafeChildWatcher()
        watcher.attach_loop(self.loop)
        try:
            asyncio.set_child_watcher(watcher)
            create = asyncio.create_subprocess_exec(*args,
                                                    pass_fds={wfd},
                                                    loop=self.loop)
            proc = self.loop.run_until_complete(create)
            self.loop.run_until_complete(proc.wait())
        finally:
            asyncio.set_child_watcher(None)

        os.close(wfd)
        data = self.loop.run_until_complete(reader.read(-1))
        self.assertEqual(data, b'data') 

def _scp(self, source, destination, scp_opts):
        """ Execute an scp command. Requires a fully qualified source and
        destination.
        """
        cmd = [
            'scp',
            '-i', os.path.expanduser('~/.local/share/juju/ssh/juju_id_rsa'),
            '-o', 'StrictHostKeyChecking=no',
            '-q',
            '-B'
        ]
        cmd.extend(scp_opts.split() if isinstance(scp_opts, str) else scp_opts)
        cmd.extend([source, destination])
        loop = self.model.loop
        process = await asyncio.create_subprocess_exec(*cmd, loop=loop)
        await process.wait()
        if process.returncode != 0:
            raise JujuError("command failed: %s" % cmd) 

def execute_process(*cmd, log=None, loop=None):
    '''
    Wrapper around asyncio.create_subprocess_exec.

    '''
    p = await asyncio.create_subprocess_exec(
        *cmd,
        stdin=asyncio.subprocess.PIPE,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
        loop=loop)
    stdout, stderr = await p.communicate()
    if log:
        log.debug("Exec %s -> %d", cmd, p.returncode)
        if stdout:
            log.debug(stdout.decode('utf-8'))
        if stderr:
            log.debug(stderr.decode('utf-8'))
    return p.returncode == 0 

def test_user_admin_add():
    """
    User is made an admin, logs in and we check if they are in admin group
    """
    # This *must* be localhost, not an IP
    # aiohttp throws away cookies if we are connecting to an IP!
    hub_url = 'http://localhost'
    username = secrets.token_hex(8)

    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'add-item', 'users.admin', username)).wait()
    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()

    async with User(username, hub_url, partial(login_dummy, password='')) as u:
            await u.login()
            await u.ensure_server()

            # Assert that the user exists
            assert pwd.getpwnam(f'jupyter-{username}') is not None

            # Assert that the user has admin rights
            assert f'jupyter-{username}' in grp.getgrnam('jupyterhub-admins').gr_mem


# FIXME: Make this test pass 

def test_user_code_execute():
    """
    User logs in, starts a server & executes code
    """
    # This *must* be localhost, not an IP
    # aiohttp throws away cookies if we are connecting to an IP!
    hub_url = 'http://localhost'
    username = secrets.token_hex(8)

    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'set', 'auth.type', 'dummyauthenticator.DummyAuthenticator')).wait()
    assert 0 == await (await asyncio.create_subprocess_exec(*TLJH_CONFIG_PATH, 'reload')).wait()

    async with User(username, hub_url, partial(login_dummy, password='')) as u:
            await u.login()
            await u.ensure_server()
            await u.start_kernel()
            await u.assert_code_output("5 * 4", "20", 5, 5)

            # Assert that the user exists
            assert pwd.getpwnam(f'jupyter-{username}') is not None 

def handle_request(self, reader, writer):
        req_host, req_port = writer.get_extra_info('peername')
        peername = f'{req_host}:{req_port}'
        self._logger.info(f'Connection from {peername}')
        data = await reader.readline()
        nw, port, worker_loop, func_pickle, data_pickle = data.split()
        num_workers = int(nw) or os.cpu_count()
        self._logger.info(
            f'Starting up {num_workers} processors for {peername}')

        # start processors that will connect back to the remote server
        asyncio.gather(
            *[asyncio.create_subprocess_exec(
                'distex_proc',
                '-H', req_host,
                '-p', port,
                '-l', worker_loop,
                '-f', func_pickle,
                '-d', data_pickle,
                stdout=None, stderr=None)
                for _ in range(num_workers)])

        writer.close() 

def qubes_lvm_coro(cmd, log=logging.getLogger('qubes.storage.lvm')):
    ''' Call :program:`lvm` to execute an LVM operation

    Coroutine version of :py:func:`qubes_lvm`'''
    environ = os.environ.copy()
    environ['LC_ALL'] = 'C.utf8'
    if cmd[0] == "remove":
        pre_cmd = ['blkdiscard', '/dev/'+cmd[1]]
        p = yield from asyncio.create_subprocess_exec(*pre_cmd,
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL,
            close_fds=True, env=environ)
        _, _ = yield from p.communicate()
    cmd = _get_lvm_cmdline(cmd)
    p = yield from asyncio.create_subprocess_exec(*cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        close_fds=True, env=environ)
    out, err = yield from p.communicate()
    return _process_lvm_output(p.returncode, out, err, log) 

def test_kill(self):
        args = PROGRAM_BLOCKED
        create = asyncio.create_subprocess_exec(*args, loop=self.loop)
        proc = self.loop.run_until_complete(create)
        proc.kill()
        returncode = self.loop.run_until_complete(proc.wait())
        if sys.platform == 'win32':
            self.assertIsInstance(returncode, int)
            # expect 1 but sometimes get 0
        else:
            self.assertEqual(-signal.SIGKILL, returncode) 

def run_bandit(pkg: str) -> dict:
    """
    CLI usage: dffml service dev run -log debug shouldi.bandit:run_bandit -pkg .
    """
    proc = await asyncio.create_subprocess_exec(
        sys.executable,
        "-m",
        "bandit",
        "-r",
        "-f",
        "json",
        pkg,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )

    stdout, _stderr = await proc.communicate()
    if len(stdout) == 0:
        raise Exception
    bandit_op = stdout.decode()
    bandit_op = json.loads(bandit_op)
    t_results = bandit_op["results"]
    final_result = bandit_op["metrics"]["_totals"]
    # Count put number of issues that are high confidence
    for level in ["LOW", "MEDIUM", "HIGH"]:
        high_conf = 0
        for item in t_results:
            if (
                item["issue_confidence"] == "HIGH"
                and item["issue_severity"] == level
            ):
                high_conf += 1
        final_result["CONFIDENCE.HIGH_AND_SEVERITY." + level] = high_conf
    return final_result 

def async_new_subprocess(self,
                                   args: Union[str, Sequence[str]],
                                   log: str,
                                   env: Optional[Dict[str, str]] = None,
                                   cwd: Optional[str] = None) -> Optional[int]:
        """A coroutine which starts a subprocess.

        If this coroutine is cancelled, it will shut down the subprocess gracefully using
        SIGTERM/SIGKILL, then raise CancelledError.

        Parameters
        ----------
        args : Union[str, Sequence[str]]
            command to run, as string or sequence of strings.
        log : str
            the log file name.
        env : Optional[Dict[str, str]]
            an optional dictionary of environment variables.  None to inherit from parent.
        cwd : Optional[str]
            the working directory.  None to inherit from parent.

        Returns
        -------
        retcode : Optional[int]
            the return code of the subprocess.
        """
        if isinstance(args, str):
            args = [args]

        # get log file name, make directory if necessary
        log = os.path.abspath(log)
        if os.path.isdir(log):
            raise ValueError('log file %s is a directory.' % log)
        os.makedirs(os.path.dirname(log), exist_ok=True)

        async with self._semaphore:
            proc = None
            with open(log, 'w') as logf:
                logf.write('command: %s\n' % (' '.join(args)))
                logf.flush()
                try:
                    proc = await asyncio.create_subprocess_exec(*args, stdout=logf,
                                                                stderr=subprocess.STDOUT,
                                                                env=env, cwd=cwd)
                    retcode = await proc.wait()
                    return retcode
                except CancelledError as err:
                    await self._kill_subprocess(proc)
                    raise err 

def start(self):
        env = dict(PYTHONPATH=":".join(sys.path), PYTHONHOME=sys.prefix)
        args = ["-u", "-m", "fontgoggles.compile.workServer"]
        self.process = await asyncio.create_subprocess_exec(
            sys.executable, *args,
            env=env,
            stdin=asyncio.subprocess.PIPE,
            stdout=asyncio.subprocess.PIPE,
            stderr=asyncio.subprocess.STDOUT) 

def test_can_terminate_subprocess(loop):
    """Verify that a subprocess can be terminated."""
    # Start a never-ending process
    @asyncio.coroutine
    def mycoro():
        process = yield from asyncio.create_subprocess_exec(
            sys.executable or 'python', '-c', 'import time\nwhile True: time.sleep(1)')
        process.terminate()
        yield from process.wait()
        assert process.returncode != 0
    loop.run_until_complete(mycoro()) 

def test_can_communicate_subprocess(loop):
    """Verify that a subprocess's data can be passed in/out via stdin/stdout."""
    @asyncio.coroutine
    def mycoro():
        process = yield from asyncio.create_subprocess_exec(
            sys.executable or 'python', '-c', 'print(input())', stdout=subprocess.PIPE, stdin=subprocess.PIPE)
        received_stdout, received_stderr = yield from process.communicate(b'Hello async world!\n')
        yield from process.wait()
        assert process.returncode == 0
        assert received_stdout.strip() == b'Hello async world!'
    loop.run_until_complete(asyncio.wait_for(mycoro(), timeout=3)) 

def test_can_read_subprocess(loop):
    """Verify that a subprocess's data can be read from stdout."""
    @asyncio.coroutine
    def mycoro():
        process = yield from asyncio.create_subprocess_exec(
            sys.executable or 'python', '-c', 'print("Hello async world!")', stdout=subprocess.PIPE)
        received_stdout = yield from process.stdout.readexactly(len(b'Hello async world!\n'))
        yield from process.wait()
        assert process.returncode == 0
        assert received_stdout.strip() == b'Hello async world!'

    loop.run_until_complete(asyncio.wait_for(mycoro(), timeout=3)) 

def test_can_execute_subprocess(loop):
    """Verify that a subprocess can be executed."""
    @asyncio.coroutine
    def mycoro():
        process = yield from asyncio.create_subprocess_exec(
            sys.executable or 'python', '-c', 'import sys; sys.exit(5)')
        yield from process.wait()
        assert process.returncode == 5
    loop.run_until_complete(asyncio.wait_for(mycoro(), timeout=3)) 

def cleanup_test_volumes(self):
        p = self.loop.run_until_complete(asyncio.create_subprocess_exec(
            'sudo', 'lvs', '--noheadings', '-o', 'lv_name', self.pool.volume_group,
            stdout=subprocess.PIPE
        ))
        volumes, _ = self.loop.run_until_complete(p.communicate())
        for volume in volumes.decode().splitlines():
            volume = volume.strip()
            if not volume.startswith('vm-' + qubes.tests.VMPREFIX):
                continue
            p = self.loop.run_until_complete(asyncio.create_subprocess_exec(
                'sudo', 'lvremove', '-f', '/'.join([self.pool.volume_group, volume])
            ))
            self.loop.run_until_complete(p.wait()) 

def test_034_import_data_empty(self):
        config = {
            'name': 'root',
            'pool': self.pool.name,
            'save_on_stop': True,
            'rw': True,
            'size': 1024 * 1024,
        }
        vm = qubes.tests.storage.TestVM(self)
        volume = self.app.get_pool(self.pool.name).init_volume(vm, config)
        with unittest.mock.patch('time.time') as mock_time:
            mock_time.side_effect = [1521065905]
            self.loop.run_until_complete(volume.create())
        p = self.loop.run_until_complete(asyncio.create_subprocess_exec(
            'sudo', 'dd', 'if=/dev/urandom', 'of=' + volume.path, 'count=1', 'bs=1M'
        ))
        self.loop.run_until_complete(p.wait())
        import_path = self.loop.run_until_complete(
            volume.import_data(volume.size))
        self.assertNotEqual(volume.path, import_path)
        p = self.loop.run_until_complete(asyncio.create_subprocess_exec(
            'sudo', 'touch', import_path))
        self.loop.run_until_complete(p.wait())
        self.loop.run_until_complete(volume.import_data_end(True))
        self.assertFalse(os.path.exists(import_path), import_path)
        p = self.loop.run_until_complete(asyncio.create_subprocess_exec(
            'sudo', 'cat', volume.path,
            stdout=subprocess.PIPE
        ))
        volume_data, _ = self.loop.run_until_complete(p.communicate())
        self.assertEqual(volume_data.strip(b'\0'), b'') 

def git_clone(git_url, path, verbose=False):
    """Clone git repository at $git_url to $path. Return True if succesful,
    otherwise False."""
    if verbose:
        print("Cloning {}...".format(git_url))
    if os.path.exists(os.path.join(path, ".git")):
        # get rid of local repo if it already exists
        shutil.rmtree(path)

    os.makedirs(path, exist_ok=True)

    proc_env = os.environ.copy()
    proc_env["GIT_TERMINAL_PROMPT"] = "0"
    git_proc = await asyncio.create_subprocess_exec(
        "git", "clone", git_url, path, stderr=asyncio.subprocess.PIPE, env=proc_env
    )
    stdout, stderr = await git_proc.communicate()

    if git_proc.returncode != 0:
        # remove created directory if it's empty
        try:
            os.rmdir(path)
        except OSError:
            pass

        verb_msg("{}:\n{}".format(git_url, stderr.decode("utf-8")))
        return False
    elif verbose:
        print("Cloned {}".format(git_url))
    return True