Python subprocess.CompletedProcess() Examples

def _run(*args, env=None, check=False, timeout=None):
    encoded_args = [a.encode('utf-8') for a in args] if sys.platform != 'win32' else args
    with subprocess.Popen(encoded_args, env=env, stdout=PIPE, stderr=PIPE) as process:
        try:
            stdout, stderr = process.communicate(input, timeout=timeout)
        except TimeoutExpired:
            process.kill()
            stdout, stderr = process.communicate()
            raise TimeoutExpired(
                process.args, timeout, output=stdout, stderr=stderr,
            )
        except Exception:
            process.kill()
            process.wait()
            raise
        retcode = process.poll()
        if check and retcode:
            raise subprocess.CalledProcessError(
                retcode, process.args, output=stdout, stderr=stderr,
            )
        return subprocess.CompletedProcess(process.args, retcode, stdout, stderr) 

def run(args, universal_newlines=False, check=False, shell=False, **kwargs):
    """Provide the subprocess.run() function as asyncio corouting.

    This takes care of the missing 'universal_newlines' and 'check' options.
    Everything else is passed through. Will also raise the same exceptions as
    subprocess.run() to act as a drop-in replacement.
    """
    import asyncio
    import io
    import locale
    import subprocess

    if shell:
        proc = await asyncio.create_subprocess_shell(args, **kwargs)
    else:
        proc = await asyncio.create_subprocess_exec(*args, **kwargs)
    stdout, stderr = await proc.communicate()

    if universal_newlines and (stdout is not None):
        stdout = io.TextIOWrapper(io.BytesIO(stdout)).read()
    if universal_newlines and (stderr is not None):
        stderr = io.TextIOWrapper(io.BytesIO(stderr)).read()

    if check and (proc.returncode != 0):
        raise subprocess.CalledProcessError(proc.returncode, args,
            stdout, stderr)

    return subprocess.CompletedProcess(args, proc.returncode, stdout,
        stderr) 

def run_command(cmd, check=False, timeout=None, shell=False, log=True):
    try:
        proc = run_command_async(cmd, shell=shell, start_new_session=True,
                                 log=log)
        proc_stdout, proc_stderr = proc.communicate(timeout=timeout)
    except subprocess.TimeoutExpired as e:
        os.killpg(proc.pid, signal.SIGKILL)
        raise SpawnedProcessTimeout(e.cmd,
                                    proc.stdout.read(),
                                    proc.stderr.read(), timeout) from None

    completed = subprocess.CompletedProcess(args=shlex.split(cmd),
                                            returncode=proc.returncode,
                                            stdout=proc_stdout,
                                            stderr=proc_stderr)

    if check and proc.returncode != 0:
        raise SpawnedProcessError(completed.args,
                                  completed.stdout, completed.stderr,
                                  completed.returncode)

    return completed 

def check_output(
    args: Sequence[str],
    *,
    cwd: str = None,
    env: Mapping[str, str] = None,
    shell: bool = False
) -> subprocess.CompletedProcess:
    """
    Get the output of an invoked command.

    :param args: args should be a sequence of program arguments
    :param cwd: the working directory for the subprocess
    :param env: a dictionary with environment variables
    :param shell: whether to use the shell as the program to execute
    :returns: The `stdout` output of the command
    :rtype: str
    """
    rc, stdout_data, stderr_data = await _async_check_call(
        args, subprocess.PIPE, subprocess.PIPE,
        cwd=cwd, env=env, shell=shell, use_pty=False)
    if rc:
        stderr_data = stderr_data.decode(errors='replace')
    assert not rc, \
        'Expected {args} to pass: {stderr_data}'.format_map(locals())
    return stdout_data 

def run_command(cmd: str,
                verbose: bool = False,
                env: dict={}) -> subprocess.CompletedProcess:
    """ Run a command in a subprocess.

    Args:
        verbose: Show the output.

    Raises:
        subprocess.CalledProcessError: The given cmd exits with a non-0 exit
                                       code.
    """
    stdout = None if verbose else subprocess.PIPE
    stderr = None if verbose else subprocess.STDOUT
    p = subprocess.run(
        args=shlex.split(cmd),
        stdout=stdout,
        stderr=stderr,
        encoding='utf-8',
        check=True,
        env=env,
    )
    return p 

def execute(
            self,
            args: str,
            user_master_endpoint: bool=True,
            use_v3: bool = False,
    ) -> subprocess.CompletedProcess:
        cmd = self.etcdctl_path
        endpoint = self.get_endpoint(self.scheme, user_master_endpoint)
        cmd = "{} --endpoints={}".format(cmd, endpoint)
        cmd = "{} --cacert={} --cert={} --key={}".format(
            cmd, self.ca_cert_file, self.cert_file, self.key_file)
        cmd = "{} {}".format(cmd, args)
        env = {} if not use_v3 else {"ETCDCTL_API": "3"}
        result = run_command(cmd, env=env)

        self.log_cmd_result(args, result)

        return result 

def _app_deploy_with_retry(gcloud_path: str, project: str,
                               app_yaml_path: str, env_vars: Dict[str, Any]
                              ) -> subprocess.CompletedProcess:
        """Run 'gcloud app deploy' with retries.

        Args:
            gcloud_path: The path of your gcloud cli tool.
            project: GCP project id.
            app_yaml_path: Absolute path of your app.yaml.
            env_vars: A dictionary of the environment variables.

        Returns:
            The result of the subprocess run.
        """
        gcloud_result = subprocess.run(
            [gcloud_path, '-q', project, 'app', 'deploy', app_yaml_path],
            stdout=subprocess.DEVNULL,
            stderr=subprocess.PIPE,
            universal_newlines=True,
            env=env_vars)
        return gcloud_result 

def _execute(
    file: str, args: Sequence[str], cwd: Optional[Union[Path, str]], env: dict
) -> CompletedProcess:

    proc = Popen(args, executable=file, cwd=cwd, env=env)

    while True:
        try:
            returncode = proc.wait()
        except KeyboardInterrupt:
            pass
        else:
            break

    if returncode != 0:
        raise CalledProcessError(
            returncode=returncode, cmd=" ".join(shlex.quote(arg) for arg in args),
        )

    return CompletedProcess(args=args, returncode=returncode) 

def mulled_upload(image: str, quay_target: str) -> sp.CompletedProcess:
    """
    Upload the build Docker images to quay.io with ``mulled-build push``.

    Calls ``mulled-build push <image> -n <quay_target>``

    Args:
      image: name of image to push
      quary_target: name of image on quay
    """
    cmd = ['mulled-build', 'push', image, '-n', quay_target]
    mask = []
    if os.environ.get('QUAY_OAUTH_TOKEN', False):
        token = os.environ['QUAY_OAUTH_TOKEN']
        cmd.extend(['--oauth-token', token])
        mask = [token]
    return utils.run(cmd, mask=mask) 

def test__lvm_report_returns_specific_reports(self, m_run):
        report_key = 'report-1'
        report_data = [{random_string(): random_string()}]
        extra_data1 = [list(range(0, 10))]
        extra_data2 = ""
        cmd_out = json.dumps({
            "report": [
                {report_key: report_data},
                {random_string(): extra_data1},
                {random_string(): extra_data2},
             ]
        })
        cp = subprocess.CompletedProcess(args=[random_string()], returncode=0,
                                         stdout=cmd_out.encode('utf-8'),
                                         stderr="")
        m_run.return_value = cp
        expected_result = report_data
        result = lvm._lvm_report(random_string(), report_key)
        self.assertEqual(expected_result, result) 

def check(self):
        """
        Check if the process has exited, and if an error occured.

        Returns None if process hasn't exited, or a CompletedProcess
        object if it has exited without an error. If the process
        has exited with an error, raises a CalledProcessError.

        If process has exited, all available stdout and stderr
        are captured into the returned object or raised exception.
        """
        retcode = self.poll()

        if retcode is not None:
            stdout, stderr = self.communicate(timeout=0)
            completed = subprocess.CompletedProcess(
                args=self.args,
                returncode=retcode,
                stdout=stdout,
                stderr=stderr,
            )
            completed.check_returncode()
            return completed 

def custom(
    path: Union[PurePath, str], context: Optional[dict] = None
) -> CompletedProcess:
    """
    Calls the pretf_workflow() function from the specified Python file.
    This is useful for having a custom workflow that is used by multiple
    pretf.workflow.py files in different directories.

    """

    with import_file(path) as module:

        if not hasattr(module, "pretf_workflow"):
            raise log.bad(f"workflow: {path} does not have a 'pretf_workflow' function")

        # Call the pretf_workflow() function,
        # passing in "path" and "terraform" if required.
        result = call_pretf_function(func=module.pretf_workflow, context=context)  # type: ignore

    if isinstance(result, int):
        result = CompletedProcess(args=[str(path)], returncode=result)

    return result 

def test_dasdview_returns_stdout(self, m_run, m_exists):
        devname = random_string()
        dasdview_out = random_string()
        cp = subprocess.CompletedProcess(args=['foo'], returncode=0,
                                         stdout=dasdview_out.encode('utf-8'),
                                         stderr="")
        m_run.return_value = cp
        m_exists.return_value = True
        result = dasd.dasdview(devname)
        self.assertEqual(dasdview_out, result)
        m_run.assert_called_with(['dasdview', '--extended', devname],
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.DEVNULL) 

def test_multipath_show_paths_serial_with_spaces(self, m_run):
        mp_out = MP_SEP.join(['sda', 'IPR-0 1234567890'] +
                             [random_string() for x in range(0, 6)])
        cp = subprocess.CompletedProcess(args=['foo'], returncode=0,
                                         stdout=mp_out.encode('utf-8'),
                                         stderr="")
        m_run.return_value = cp
        expected_result = [multipath.MPath(*mp_out.split(MP_SEP))._asdict()]
        result = multipath.multipath_show_paths()
        self.assertEqual(expected_result, result) 

def test_multipath_show_maps_skips_unparsable_output(self, m_run):
        lines = [
            MP_SEP.join([random_string() for x in range(0, 3)]),
            "",
            MP_SEP.join([random_string() for x in range(0, 3)]),
        ]
        mp_out = "\n".join(lines)
        cp = subprocess.CompletedProcess(args=['foo'], returncode=0,
                                         stdout=mp_out.encode('utf-8'),
                                         stderr="")
        m_run.return_value = cp
        expected_result = [multipath.MMap(*lines[0].split(MP_SEP))._asdict(),
                           multipath.MMap(*lines[2].split(MP_SEP))._asdict()]
        result = multipath.multipath_show_maps()
        self.assertEqual(expected_result, result) 

def test_multipath_show_paths_skips_unparsable_output(self, m_run):
        lines = [
            MP_SEP.join([random_string() for x in range(0, 8)]),
            "",
            MP_SEP.join([random_string() for x in range(0, 8)]),
        ]
        mp_out = "\n".join(lines)
        cp = subprocess.CompletedProcess(args=['foo'], returncode=0,
                                         stdout=mp_out.encode('utf-8'),
                                         stderr="")
        m_run.return_value = cp
        expected_result = [multipath.MPath(*lines[0].split(MP_SEP))._asdict(),
                           multipath.MPath(*lines[2].split(MP_SEP))._asdict()]
        result = multipath.multipath_show_paths()
        self.assertEqual(expected_result, result) 

def test_multipath_show_maps(self, m_run):
        mp_out = MP_SEP.join([random_string() for x in range(0, 3)])
        cp = subprocess.CompletedProcess(args=['foo'], returncode=0,
                                         stdout=mp_out.encode('utf-8'),
                                         stderr="")
        m_run.return_value = cp
        expected_result = [multipath.MMap(*mp_out.split(MP_SEP))._asdict()]
        result = multipath.multipath_show_maps()
        self.assertEqual(expected_result, result) 

def execute(
            self,
            args: str,
            user_master_endpoint: bool=True,
            use_v3: bool = False,
    ) -> subprocess.CompletedProcess:
        cmd = self.etcdctl_path
        endpoint = self.get_endpoint(self.scheme, user_master_endpoint)
        cmd = "{} --endpoints={}".format(cmd, endpoint)
        cmd = "{} {}".format(cmd, args)
        env = {} if not use_v3 else {"ETCDCTL_API": "3"}
        result = run_command(cmd, env=env)
        self.log_cmd_result(args, result)

        return result 

def exec_cmd(cmd: str, check=False) -> subprocess.CompletedProcess:
    print("Executing: {}".format(cmd))
    process = subprocess.run(
        shlex.split(cmd),
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        encoding='utf-8',
        check=check)
    return process 

def _postprocess_result(result: subprocess.CompletedProcess) -> None:
    if isinstance(result.stdout, bytes):
        result.stdout = result.stdout.decode('utf-8', 'ignore')
    if isinstance(result.stderr, bytes):
        result.stderr = result.stderr.decode('utf-8', 'ignore')
    if len(result.stderr) > 10240:
        result.stderr = result.stderr[:10240] 

def run(
    args: Sequence[str],
    stdout_callback: Callable[[bytes], None],
    stderr_callback: Callable[[bytes], None],
    *,
    cwd: str = None,
    env: Mapping[str, str] = None,
    shell: bool = False,
    use_pty: Optional[bool] = None
) -> subprocess.CompletedProcess:
    """
    Run the command described by args.

    Invokes the callbacks for every line read from the subprocess pipes.

    :param args: args should be a sequence of program arguments
    :param stdout_callback: the callable is invoked for every line read from
      the stdout pipe of the process
    :param stderr_callback: the callable is invoked for every line read from
      the stderr pipe of the process
    :param cwd: the working directory for the subprocess
    :param env: a dictionary with environment variables
    :param shell: whether to use the shell as the program to execute
    :param use_pty: whether to use a pseudo terminal
    :returns: the result of the completed process
    :rtype subprocess.CompletedProcess
    """
    assert callable(stdout_callback) or stdout_callback is None
    assert callable(stderr_callback) or stderr_callback is None

    # if use_pty is neither True nor False choose based on isatty of stdout
    if use_pty is None:
        use_pty = sys.stdout.isatty()
    # the pty module is only supported on Windows
    if use_pty and platform.system() != 'Linux':
        use_pty = False

    rc, _, _ = await _async_check_call(
        args, stdout_callback, stderr_callback,
        cwd=cwd, env=env, shell=shell, use_pty=use_pty)
    return subprocess.CompletedProcess(args, rc) 

def run(
        self,
        args: List[str],
        user: str,
        log_output_live: bool,
        env: Dict[str, Any],
        tty: bool,
        ssh_key_path: Path,
        public_ip_address: IPv4Address,
        capture_output: bool,
    ) -> subprocess.CompletedProcess:
        """
        Run a command on this node the given user.

        Args:
            args: The command to run on the node.
            user: The username to communicate as.
            log_output_live: If ``True``, log output live. If ``True``, stderr
                is merged into stdout in the return value.
            env: Environment variables to be set on the node before running
                the command. A mapping of environment variable names to
                values.
            tty: If ``True``, allocate a pseudo-tty. This means that the users
                terminal is attached to the streams of the process.
            ssh_key_path: The path to an SSH key which can be used to SSH to
                the node as the ``user`` user.
            public_ip_address: The public IP address of the node.
            capture_output: Whether to capture output in the result.

        Returns:
            The representation of the finished process.

        Raises:
            subprocess.CalledProcessError: The process exited with a non-zero
                code.
        """ 

def run(self, cmd: List[str], check: bool = True,
            env: dict = {}) -> subprocess.CompletedProcess:
        process = subprocess.run(
            self._base_args.copy() + cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            env=env,
            check=check)

        return process 

def run(self, cmd: List[str], check: bool = True,
            env: dict = {}) -> subprocess.CompletedProcess:
        process = subprocess.run(
            self._base_args.copy() + cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            env=env,
            check=check)

        return process 

def test_cmdstep_runstep_cmd_is_dict_save_true_shell_true_cwd_set():
    """Dict command with save false and shell true with cwd set."""
    context = Context({'cmd': {'run': 'blah -blah1 --blah2',
                               'save': True,
                               'cwd': 'pathhere'}})
    with patch_logger('blahname', logging.DEBUG) as mock_logger_debug:
        obj = CmdStep('blahname', context)

    assert obj.is_save
    assert obj.logger.name == 'blahname'
    assert obj.context == Context({'cmd': {'run': 'blah -blah1 --blah2',
                                           'save': True,
                                           'cwd': 'pathhere'}})
    assert obj.cmd_text == 'blah -blah1 --blah2'
    mock_logger_debug.assert_called_once_with(
        'Processing command string in dir pathhere: blah -blah1 --blah2')

    with patch('subprocess.run') as mock_run:
        mock_run.return_value = subprocess.CompletedProcess(None,
                                                            0,
                                                            'std',
                                                            None)
        with patch_logger('blahname', logging.INFO) as mock_logger_info:
            obj.run_step(is_shell=True)

    mock_logger_info.assert_called_once_with('stdout: std')

    # blah is in a list because shell == false
    mock_run.assert_called_once_with('blah -blah1 --blah2',
                                     cwd='pathhere',
                                     shell=True,
                                     # capture_output=True,
                                     stderr=subprocess.PIPE,
                                     stdout=subprocess.PIPE,
                                     # text=True,
                                     universal_newlines=True)

    assert context['cmdOut']['returncode'] == 0
    assert context['cmdOut']['stdout'] == 'std'
    assert context['cmdOut']['stderr'] is None 

def test_cmdstep_runstep_cmd_is_dict_save_true_shell_true():
    """Dict command with save false and shell true."""
    context = Context({'cmd': {'run': 'blah -blah1 --blah2',
                               'save': True}})
    with patch_logger('blahname', logging.DEBUG) as mock_logger_debug:
        obj = CmdStep('blahname', context)

    assert obj.is_save
    assert obj.logger.name == 'blahname'
    assert obj.context == Context({'cmd': {'run': 'blah -blah1 --blah2',
                                           'save': True}})
    assert obj.cmd_text == 'blah -blah1 --blah2'
    mock_logger_debug.assert_called_once_with(
        'Processing command string: blah -blah1 --blah2')

    with patch('subprocess.run') as mock_run:
        mock_run.return_value = subprocess.CompletedProcess(None,
                                                            0,
                                                            'std',
                                                            None)
        with patch_logger('blahname', logging.INFO) as mock_logger_info:
            obj.run_step(is_shell=True)

    mock_logger_info.assert_called_once_with('stdout: std')

    # blah is in a list because shell == false
    mock_run.assert_called_once_with('blah -blah1 --blah2',
                                     cwd=None,
                                     shell=True,
                                     # capture_output=True,
                                     stderr=subprocess.PIPE,
                                     stdout=subprocess.PIPE,
                                     # text=True,
                                     universal_newlines=True)

    assert context['cmdOut']['returncode'] == 0
    assert context['cmdOut']['stdout'] == 'std'
    assert context['cmdOut']['stderr'] is None 

def test_cmdstep_runstep_cmd_is_dict_save_true_shell_false():
    """Dict command with save false and shell false."""
    context = Context({'cmd': {'run': 'blah -blah1 --blah2',
                               'save': True}})
    with patch_logger('blahname', logging.DEBUG) as mock_logger_debug:
        obj = CmdStep('blahname', context)

    assert obj.is_save
    assert obj.logger.name == 'blahname'
    assert obj.context == Context({'cmd': {'run': 'blah -blah1 --blah2',
                                           'save': True}})
    assert obj.cmd_text == 'blah -blah1 --blah2'
    mock_logger_debug.assert_called_once_with(
        'Processing command string: blah -blah1 --blah2')

    with patch('subprocess.run') as mock_run:
        mock_run.return_value = subprocess.CompletedProcess(None,
                                                            0,
                                                            'std',
                                                            'err')
        with patch_logger('blahname', logging.ERROR) as mock_logger_error:
            obj.run_step(is_shell=False)

    mock_logger_error.assert_called_once_with('stderr: err')

    # blah is in a list because shell == false
    mock_run.assert_called_once_with(['blah', '-blah1', '--blah2'],
                                     cwd=None,
                                     shell=False,
                                     stderr=subprocess.PIPE,
                                     stdout=subprocess.PIPE,
                                     # text=True
                                     universal_newlines=True)

    assert context['cmdOut']['returncode'] == 0
    assert context['cmdOut']['stdout'] == 'std'
    assert context['cmdOut']['stderr'] == 'err' 

def test_runner_git_version(self):
        """ProcessRunner should return subprocess.CompletedProcess"""
        runner = ProcessRunner(['git'], workdir=str(self.tempdir))
        proc = runner('version')
        self.assertIsInstance(proc, subprocess.CompletedProcess)
        self.assertIn('git version', proc.stdout) 

def extract_contigs(fname: Path, ename: Path) -> CompletedProcess:
    """Extract contents of fname to ename using gunzip.

    :param fname:  str, path to input compressed file
    :param ename:  str, path to output uncompressed file

    Returns status of subprocess call
    """
    cmd = ["gunzip", "-c", shlex.quote(str(fname))]
    with open(ename, "w") as efh:
        return subprocess.run(cmd, stdout=efh, check=True, shell=False)


# Using a genomes UID, create class and label text files 

def run(cmd: List[str], **kwargs) -> subprocess.CompletedProcess:
    print("$ " + " ".join(cmd))
    return subprocess.run(cmd, **kwargs)