Python subprocess32.Popen() Examples

def exec_command(cmd, args, cwd, timeout=None):
    """
    Executes shell command inside @repo_path
    returns exec code
    """
    pipe = None
    try:
        env = os.environ
        env["PATH"] = env["NDK_TOOLCHAIN"] + "/bin:" + env["PATH"]
        pipe = Popen(args, stdin=PIPE, stdout=PIPE, cwd=cwd, env=env)
        stdout, error = pipe.communicate(timeout=timeout) if (timeout and timeout > 0) else pipe.communicate()
        logger.debug("stdout: %s", stdout)
        return pipe.returncode

    except TimeoutExpired as te:
        pipe.terminate()
        logger.error("%s timed out: %s", cmd, str(te))
        return 0

    except Exception as e:
        logger.error("%s subprocess failed: %s", cmd, str(e))
        return -1 

def command(cmd, *arguments, **options):

    close_fds = options.get('close_fds', True)
    cwd = options.get('cwd', None)
    shell = options.get('shell', False)
    env = options.get('env', None)
    if env is not None:
        env = dict(os.environ, **env)
    stdin = options.get('stdin', None)

    subproc = subprocess32.Popen([cmd] + list(arguments),
                                 close_fds=close_fds,
                                 shell=shell,
                                 cwd=cwd,
                                 env=env,
                                 stdin=subprocess32.PIPE,
                                 stdout=subprocess32.PIPE,
                                 stderr=subprocess32.PIPE, )

    out, err = subproc.communicate(input=stdin)

    subproc.wait()

    return (subproc.returncode, out, err) 

def start_worker(node_ip_address, worker_path, scheduler_address, objstore_address=None, cleanup=True):
  """This method starts a worker process.

  Args:
    node_ip_address (str): The IP address of the node that the worker runs on.
    worker_path (str): The path of the source code which the worker process will
      run.
    scheduler_address (str): The ip address and port of the scheduler to connect
      to.
    objstore_address (Optional[str]): The ip address and port of the object
      store to connect to.
    cleanup (Optional[bool]): True if using Ray in local mode. If cleanup is
      true, then this process will be killed by serices.cleanup() when the
      Python process that imported services exits. This is True by default.
  """
  command = ["python",
             worker_path,
             "--node-ip-address=" + node_ip_address,
             "--scheduler-address=" + scheduler_address]
  if objstore_address is not None:
    command.append("--objstore-address=" + objstore_address)
  p = subprocess.Popen(command)
  if cleanup:
    all_processes.append(p) 

def xvfb_supports_listen():
    """Determine whether the '-listen' option is supported by Xvfb."""
    p = subprocess.Popen(
        ['Xvfb', '-listen', 'TCP', '-__sentinel_parameter__'],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    p.wait()
    _, stderr = p.communicate()

    match = re.search(
        br'^Unrecognized option: (?P<option>.*)$',
        stderr,
        flags=re.MULTILINE,
    ).groupdict()
    unrecognized_option = match['option']
    return unrecognized_option != b'-listen' 

def check_output(*popenargs, **kwargs):
    """Run command with arguments and return its output (both stdout and stderr) as a byte string.

    If the exit code was non-zero it raises a CalledProcessWithOutputError.
    """
    if 'stdout' in kwargs:
        raise ValueError('stdout argument not allowed, it will be overridden.')

    if 'stderr' in kwargs:
        raise ValueError('stderr argument not allowed, it will be overridden.')

    process = subprocess.Popen(
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        *popenargs, **kwargs)
    output, err = process.communicate()
    retcode = process.poll()
    if retcode:
        cmd = kwargs.get("args")
        if cmd is None:
            cmd = popenargs[0]
        raise CalledProcessWithOutputError(retcode, cmd, output, err)

    return output, err 

def open_process(command, cwd=None, shell=True, _popen_lock=threading.Lock()):
    kwargs = {
        "shell": shell,
        "stdout": subprocess.PIPE,
        "stderr": subprocess.STDOUT,
        "stdin": subprocess.PIPE,
        "bufsize": 1,  # Line buffered
        "universal_newlines": True,
    }

    if cwd is not None:
        kwargs["cwd"] = cwd

    # Prevent signal propagation from parent process
    try:
        # Windows
        kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
    except AttributeError:
        # Unix
        kwargs["preexec_fn"] = os.setpgrp

    with _popen_lock:  # Work around Python 2 Popen race condition
        return subprocess.Popen(command, **kwargs) 

def ensure_terminated(self, status=""):
        if self._popen:
            try:
                os.kill(self._popen.pid, signal.SIGINT)
            except OSError:
                print "Process does not exist"
                return
            time.sleep(0.5)
            if self._popen:
                self._popen.poll()

                if self._popen.returncode is None:
                    self._popen.terminate()
                    time.sleep(0.2)
                    self._popen.poll()

                while self._popen.returncode is None:
                    time.sleep(1)
                    if status:
                        print(status)
                    self._popen.poll()
        else:
            print "ERROR Popen is NONE" 

def subproc(script, cwd=None):

    subproc = subprocess32.Popen(['sh'],
                               close_fds=True,
                               cwd=cwd,
                               stdin=subprocess32.PIPE,
                               stdout=subprocess32.PIPE,
                               stderr=subprocess32.PIPE)

    out, err = subproc.communicate(script)

    subproc.wait()

    if subproc.returncode != 0:
        print out
        print err

    return (subproc.returncode, out, err) 

def execute(cls, cmd, stdin_payload=None, **kwargs):
    """Execute a command via subprocess.Popen and returns the stdio.

    :param string|list cmd: A list or string representing the command to run.
    :param string stdin_payload: A string representing the stdin payload, if any, to send.
    :param **kwargs: Additional kwargs to pass through to subprocess.Popen.
    :return: A tuple of strings representing (stdout, stderr), pre-decoded for utf-8.
    :raises: `Executor.ExecutableNotFound` when the executable requested to run does not exist.
             `Executor.NonZeroExit` when the execution fails with a non-zero exit code.
    """
    process = cls.open_process(cmd=cmd,
                               stdin=subprocess.PIPE,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               **kwargs)
    stdout_raw, stderr_raw = process.communicate(input=stdin_payload)
    # N.B. In cases where `stdout` or `stderr` is passed as parameters, these can be None.
    stdout = stdout_raw.decode('utf-8') if stdout_raw is not None else stdout_raw
    stderr = stderr_raw.decode('utf-8') if stderr_raw is not None else stderr_raw

    if process.returncode != 0:
      raise cls.NonZeroExit(cmd, process.returncode, stdout, stderr)

    return stdout, stderr 

def open_process(cls, cmd, **kwargs):
    """Opens a process object via subprocess.Popen().

    :param string|list cmd: A list or string representing the command to run.
    :param **kwargs: Additional kwargs to pass through to subprocess.Popen.
    :return: A `subprocess.Popen` object.
    :raises: `Executor.ExecutableNotFound` when the executable requested to run does not exist.
    """
    assert len(cmd) > 0, 'cannot execute an empty command!'

    try:
      return subprocess.Popen(cmd, **kwargs)
    except (IOError, OSError) as e:
      if e.errno == errno.ENOENT:
        raise cls.ExecutableNotFound(cmd, e)
      else:
        raise cls.ExecutionError(repr(e), cmd, e) 

def run_cmd(cmd):
    c = cmd[:-1]
    timeout = cmd[-1]
    display("Executing command: %s" % " ".join(c))

    current_time = time.time()

    if timeout:
        process = Popen(c)
        while time.time() < current_time + timeout and process.poll() is None:
            time.sleep(5)
        if process.poll() is None:

            display_error(
                "Timeout of %s reached. Aborting thread for command: %s"
                % (timeout, " ".join(c))
            )
            process.terminate()

    else:
        Popen(c).wait()

    return cmd 

def _work(args_and_log):
    args, log_file = args_and_log
    print " ".join(args)
    if log_file:
        with open(log_file, 'w') as the_log:
            p = subprocess32.Popen(args, stdout=the_log, stderr=DEVNULL)
            p.wait()
    else:
        p = subprocess32.Popen(args, stdout=DEVNULL)
        p.wait() 

def long_task_streaming(config, api):
    import time

    for i in range(10 * 6):
        echo_cmd = ["echo", "Echoed by an external process.\n"]
        subprocess.Popen(echo_cmd)
        print("Print by an internal statement. %f \n" % (time.time()))
        sys.stderr.write("This goes to stderr \n")
        time.sleep(10) 

def run(self):
        self._popen = subprocess.Popen(self._cmd, env=self._env, stdout = DEVNULL)
        # print "popen: ", self._popen, " id", self._popen.pid 

def test_rejects_client_that_does_not_specifiy_a_supported_protocol(server):

    node_script = '''
        module.paths.push('{0}')
        WebSocket = require('ws')
        const client = new WebSocket('ws://localhost:{1}/socket')
        client.on('close', (code) => {{
            console.log(JSON.stringify(code))
          }}
        );
    '''.format(
        os.path.join(os.path.dirname(__file__), 'node_modules'), TEST_PORT)

    p = subprocess.Popen(
        ['node', '-e', node_script],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    q = queue.Queue()
    t = threading.Thread(target=enqueue_output, args=(p.stdout, q))
    t.daemon = True
    t.start()
    time.sleep(.2)
    ret_values = []
    while True:
        try:
            _line = q.get_nowait()
            if isinstance(_line, bytes):
                line = _line.decode()
            line = json.loads(line)
            ret_values.append(line)
        except ValueError:
            pass
        except queue.Empty:
            break
    assert ret_values[0] == 1002 or 1006 

def run_in_subprocess(cmd, check_output=False, **kwargs):
    """
    Execute command using default subprocess configuration.

    Parameters
    ----------
    cmd : string
        Command to be executed in subprocess.
    kwargs : keywords
        Options to pass to subprocess.Popen.

    Returns
    -------
    proc : Popen subprocess
        Subprocess used to run command.
    """

    logger.debug('Executing command: {0}'.format(cmd))
    config = DEFAULT_SUBPROCESS_CONFIG.copy()
    config.update(kwargs)
    if not check_output:
        if omniduct_config.logging_level < 20:
            config['stdout'] = None
            config['stderr'] = None
        else:
            config['stdout'] = open(os.devnull, 'w')
            config['stderr'] = open(os.devnull, 'w')
    timeout = config.pop('timeout', None)

    process = subprocess.Popen(cmd, **config)
    try:
        stdout, stderr = process.communicate(None, timeout=timeout)
    except subprocess.TimeoutExpired:
        os.killpg(os.getpgid(process.pid), signal.SIGINT)  # send signal to the process group, recursively killing all children
        output, unused_err = process.communicate()
        raise subprocess.TimeoutExpired(process.args, timeout, output=output)
    return SubprocessResults(returncode=process.returncode, stdout=stdout or b'', stderr=stderr or b'') 

def spawn(self, command):
        """ Spawns a new process and adds it to the pool """

        # process_name
        # output
        # time before starting (wait for port?)
        # start_new_session=True : avoid sending parent signals to child

        env = dict(os.environ)
        env["MRQ_IS_SUBPROCESS"] = "1"
        env.update(self.extra_env or {})

        # Extract env variables from shell commands.
        parts = shlex.split(command)
        for p in list(parts):
            if "=" in p:
                env[p.split("=")[0]] = p[len(p.split("=")[0]) + 1:]
                parts.pop(0)
            else:
                break

        p = subprocess.Popen(parts, shell=False, close_fds=True, env=env, cwd=os.getcwd())

        self.processes.append({
            "subprocess": p,
            "pid": p.pid,
            "command": command,
            "psutil": psutil.Process(pid=p.pid)
        }) 

def raw_cmd(self, *args):
        '''adb command. return the subprocess.Popen object.'''
        cmd_line = [self.adb()] + self.adb_host_port_options + list(args)
        if os.name != "nt":
            cmd_line = [" ".join(cmd_line)]
        return subprocess.Popen(cmd_line, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 

def popen(cmdline, environ=(), stdin=None, stdout=None, stderr=None):
    """Open a subprocess wrapping subprocess.Popen.

    :param cmdline:
        Command to run
    :type cmdline:
        ``list``
    :param environ:
        *optional* Environ variable to set prior to running the command
    :type environ:
        ``dict``
    :param stdin:
        *optional* File object to hook to the subprocess' stdin
    :type stdin:
        ``file object``
    :param stdout:
        *optional* File object to hook to the subprocess' stdout
    :type stdout:
        ``file object``
    :param stderr:
        *optional* File object to hook to the subprocess' stderr
    :type stderr:
        ``file object``
    """
    _LOGGER.debug('popen: %r', cmdline)
    args = _alias_command(cmdline)

    # Setup a copy of the environ with the provided overrides
    cmd_environ = dict(os.environ.items())
    cmd_environ.update(environ)

    return subprocess.Popen(
        args,
        close_fds=_CLOSE_FDS, shell=False,
        stdin=stdin or subprocess.PIPE,
        stdout=stdout or subprocess.PIPE,
        stderr=stderr or subprocess.PIPE,
        env=cmd_environ
    ) 

def background_test(self):
        self.running = True
        proc = subprocess.Popen('echo hello', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        while True:
            line = proc.stdout.readline()
            if line == '':
                break
            print line
            for client in ProgressHandler.clients:
                client.write_message(line)
            self.output = self.output + line
        self.running = False 

def run(self, capture=False):
        if capture:
            self._captured = True
            self._popen = subprocess.Popen(self._cmd,
                                           env=self._env,
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE, cwd=self._cwd)
        else:
            self._popen = subprocess.Popen(self._cmd, env=self._env, stdout=DEVNULL, cwd=self._cwd) 

def setUpClass(cls):
        # In Python 3 Click makes a call to locale to determine how the
        # terminal wants to handle unicode. Because we mock Popen to avoid
        # calling the real verifier, we need to get the actual result of
        # locale to provide it to Click during the test run.
        if os.name == 'nt':
            cls.locale = ''  # pragma: no cover
        else:
            cls.locale = Popen(
                ['locale', '-a'], stdout=PIPE, stderr=PIPE).communicate()[0] 

def setUp(self):
        super(mainTestCase, self).setUp()
        self.addCleanup(patch.stopall)
        self.mock_Popen = patch.object(
            verify.subprocess, 'Popen', spec=verify.subprocess.Popen,
            stdout=['6 interactions, 0 failures']).start()

        self.mock_Popen.return_value.communicate.return_value = self.locale

        self.mock_isfile = patch.object(
            verify, 'isfile', autospec=True).start()

        self.mock_rerun_command = patch.object(
            verify, 'rerun_command', autospec=True).start()

        self.runner = CliRunner()
        self.default_call = [
            './pacts/consumer-provider.json',
            './pacts/consumer-provider2.json',
            './pacts/consumer-provider3.json',
            '--provider-base-url=http://localhost']

        self.default_opts = [
            '--provider-base-url=http://localhost',
            '--pact-url=./pacts/consumer-provider.json',
            '--pact-urls=./pacts/consumer-provider2.json,'
            './pacts/consumer-provider3.json'] 

def setUp(self):
        self.addCleanup(patch.stopall)
        self.mock_write = patch.object(
            verify.sys.stdout, 'write', autospec=True).start()

        self.process = Mock(Popen, stdout=[
            'Actual: {"username":123,"id":"100","groups":["users","admins"]}',
            '# /Users/matthewbalvanz/dev/pact-python/pact/bin/pact/lib/vendor'
            '/ruby/2.2.0/gems/pact-provider-verifier-1.6.0/lib/pact/provider'
            '_verifier/cli/custom_thor.rb:17:in `start\'',
            '# /Users/matthewbalvanz/dev/pact-python/pact/bin/pact/lib/app'
            '/pact-provider-verifier.rb:2:in `<main>\''
        ]) 

def Popen(*args, **kwargs):
        raise OSError("subprocess.Popen is not supported") 

def run_background_command(command, pid_filename, env=None):
    """Run `command` in the background, writing its PID in `pid_filename`."""
    if ON_WINDOWS:
        process = subprocess.Popen(command, env=env, creationflags=subprocess.CREATE_NEW_PROCESS_GROUP)
    else:
        process = subprocess.Popen(command, env=env, start_new_session=True)
    with open(pid_filename, 'w') as pid_file:
        pid_file.write(str(process.pid)) 

def start_app(self, bundle_id):
        '''
        Start app by bundle_id
        Args:
            - bundle_id(string): ex com.netease.my
        Returns:
            idevicedebug subprocess instance
        '''
        idevicedebug = must_look_exec('idevicedebug')

        # run in background
        kwargs = {'stdout': subprocess.PIPE, 'stderr': subprocess.PIPE}
        if sys.platform != 'darwin':
            kwargs['close_fds'] = True
        return subprocess.Popen([idevicedebug, "--udid", self.udid, 'run', bundle_id], **kwargs) 

def run_cmd(cmd_data):
    cmd = cmd_data[0]
    output = cmd_data[1]
    c = cmd[:-1]
    timeout = cmd[-1]
    display("Executing command: %s" % " ".join(c))

    current_time = time.time()
    f = open(output, 'w')
    if timeout:
        
        process = Popen(c, stdout=f)
        while time.time() < current_time + timeout and process.poll() is None:
            time.sleep(5)
        if process.poll() is None:

            display_error(
                "Timeout of %s reached. Aborting thread for command: %s"
                % (timeout, " ".join(c))
            )
            process.terminate()

    else:
        Popen(c, stdout=f).wait()
    f.close()
    return cmd 

def run_cmd_noout(cmd_data):
    cmd = cmd_data[0]
    output = cmd_data[1]
    c = cmd[:-1]
    timeout = cmd[-1]
    display("Executing command: %s" % " ".join(c))

    current_time = time.time()
    f = open(output, 'w')
    if timeout:
        
        process = Popen(c, stdout=f, stderr=STDOUT)
        while time.time() < current_time + timeout and process.poll() is None:
            time.sleep(5)
        if process.poll() is None:

            display_error(
                "Timeout of %s reached. Aborting thread for command: %s"
                % (timeout, " ".join(c))
            )
            process.terminate()

    else:
        Popen(c, stdout=f, stderr=STDOUT).wait()
    f.close()
    return cmd_data 

def run_proc_with_quit(proc_id, quit_dict, args, logfile=None, append=False, env=None, cwd=None):
    if logfile is None:
        logfile = os.devnull

    mode = 'ab' if append else 'wb'
    with open(logfile, mode) as logf:
        if proc_id in quit_dict:
            return None
        proc = subprocess.Popen(args, stdout=logf, stderr=subprocess.STDOUT,
                                env=env, cwd=cwd)
        retcode = None
        num_kill = 0
        timeout = 0.05
        while retcode is None and num_kill <= 2:
            try:
                retcode = proc.wait(timeout=timeout)
            except subprocess.TimeoutExpired:
                if proc_id in quit_dict:
                    if num_kill == 0:
                        proc.terminate()
                        timeout = quit_dict[proc_id]
                    elif num_kill == 1:
                        proc.kill()
                    num_kill += 1

        return proc.returncode