Python subprocess.check_output() Examples

def call(self, cmd, **kwargs):
        print('Running "{}"'.format(cmd), file=sys.stderr)
        expect = kwargs.pop("expect", [dict(return_codes=[os.EX_OK], stdout=None, stderr=None)])
        process = subprocess.Popen(cmd, stdin=kwargs.get("stdin", subprocess.PIPE), stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE, **kwargs)
        out, err = process.communicate()
        return_code = process.poll()
        out = out.decode(sys.stdin.encoding)
        err = err.decode(sys.stdin.encoding)

        def match(return_code, out, err, expected):
            exit_ok = return_code in expected["return_codes"]
            stdout_ok = re.search(expected.get("stdout") or "", out)
            stderr_ok = re.search(expected.get("stderr") or "", err)
            return exit_ok and stdout_ok and stderr_ok
        if not any(match(return_code, out, err, exp) for exp in expect):
            print(err)
            e = subprocess.CalledProcessError(return_code, cmd, output=out)
            e.stdout, e.stderr = out, err
            raise e
        return self.SubprocessResult(out, err, return_code) 

def get_num_triggering_samples(signature, samples):
    """
        Get number of samples triggering ClamAV signature _signature_.
        :param signature: A dictionary with keys 'type' for the signature type
            and 'signature' for the signature string.
        :param samples: A list of sample paths to scan.
        :returns: The number of samples triggering this signature.
    """
    handle, temp_sig = tempfile.mkstemp(suffix = "." + signature["type"])
    try:
        with os.fdopen(handle, "w") as f:
            f.write(signature["signature"])
        proc_clamscan = subprocess.Popen(["clamscan", 
                                          "-d", temp_sig,
                                          "--no-summary", "--infected"] + samples, 
                                         stdout = subprocess.PIPE,
                                         stderr = subprocess.PIPE)
        stdout, stderr = proc_clamscan.communicate()
        if not stdout:
            return 0
        else:
            return len(stdout.strip().split("\n"))
    finally:
        os.unlink(temp_sig) 

def pipe_fopen(command, mode, background=True):
    if mode not in ["rb", "r"]:
        raise RuntimeError("Now only support input from pipe")

    p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)

    def background_command_waiter(command, p):
        p.wait()
        if p.returncode != 0:
            warnings.warn("Command \"{0}\" exited with status {1}".format(
                command, p.returncode))
            _thread.interrupt_main()

    if background:
        thread = threading.Thread(target=background_command_waiter,
                                  args=(command, p))
        # exits abnormally if main thread is terminated .
        thread.daemon = True
        thread.start()
    else:
        background_command_waiter(command, p)
    return p.stdout 

def alignProcWrapper(ref, seq):
    cmd = "python {} {} {}".format(
        os.path.realpath(__file__),
        ref,
        seq)
    
    proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = proc.communicate()

    if proc.returncode != 0:
        return None

    fields = out.split()
    strand = fields[0].decode()
    
    aln = Aln(int(fields[1]), int(fields[2]), fields[3].decode(), int(fields[4]), int(fields[5]))

    return strand, aln 

def disas_objdump(b):
    with open("/dev/shm/shifter", "w") as f:
        f.write(b)
    if arch == "64":
        dis, errors = subprocess.Popen("objdump -D --insn-width=256 -b binary \
                -mi386 -Mx86-64 /dev/shm/shifter | head -8 | tail -1",
                stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True).communicate()
    else:
        dis, errors = subprocess.Popen("objdump -D --insn-width=256 -b binary \
                -mi386 /dev/shm/shifter | head -8 | tail -1",
                stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True).communicate()
    dis = dis[6:] # address
    raw = dis[:256*3].replace(" ","")
    dis = dis[256*3:].strip().split(None, 2)
    mnemonic = dis[0]
    if len(dis) > 1:
        op_str = dis[1]
    else:
        op_str = ""
    if mnemonic == "(bad)":
        mnemonic = "(unk)"
        insn = ""
        op_str = ""
    size = len(raw)/2
    return (mnemonic, op_str, size) 

def decompress(fileName, settings, type):
    """
        Decompresses a tar.xz file.
    """
    srcFilePath = os.path.join(settings.getLocalDst(type), fileName)

    if srcFilePath.split('.')[-1] != 'xz':
        raise("Trying to decompress a non-xz file: " + srcFilePath)

    cmd = str('tar -xJf ' + srcFilePath)
    print("Decompressing file: %s" % srcFilePath)
    try:
        cmdProc = subprocess.Popen(cmd, shell=True, bufsize=-1, cwd=os.path.dirname(srcFilePath), stdout=None)
        cmdProc.wait()
    except Exception as e:
        print("Can't run command: %s" % cmd)
        raise e

    if cmdProc.returncode != 0:
        print("Command returned with a non-zero status: %s: %s" % (cmdProc.returncode, cmd))
        raise Exception("Can't decompress file: %s" % srcFilePath)

    print("File '%s' was successfully decompressed." % fileName) 

def main():
    env = os.environ.copy()

    # in case the PYTHONHASHSEED was not set, set to 0 to denote
    # that hash randomization should be disabled and
    # restart python for the changes to take effect
    if 'PYTHONHASHSEED' not in env:
        env['PYTHONHASHSEED'] = "0"
        proc = subprocess.Popen([sys.executable] + sys.argv,
                                env=env)
        proc.communicate()
        exit(proc.returncode)

    # check if hash has been properly de-randomized in python 3
    # by comparing hash of magic tuple
    h = hash(eden.__magic__)
    assert h == eden.__magic_py2hash__ or h == eden.__magic_py3hash__, 'Unexpected hash value: "{}". Please check if python 3 hash normalization is disabled by setting shell variable PYTHONHASHSEED=0.'.format(h)

    # run program and exit
    print("This is the magic python hash restart script.")
    exit(0) 

def update_version_py():
    if not os.path.isdir(".git"):
        print("This does not appear to be a Git repository.")
        return
    try:
        # p = subprocess.Popen(["git", "describe","--tags", "--always"],
        #        stdout=subprocess.PIPE)
        p = subprocess.Popen("git rev-list HEAD --count".split(),
                             stdout=subprocess.PIPE)

    except EnvironmentError:
        print("unable to run git, leaving eden/_version.py alone")
        return
    stdout = p.communicate()[0]
    if p.returncode != 0:
        print("unable to run git, leaving eden/_version.py alone")
        return
    ver = "0.3."+stdout.strip()
    # ver = str(int(ver,16)) # pypi doesnt like base 16
    f = open("eden/_version.py", "w")
    f.write(VERSION_PY % ver)
    f.close()
    print("set eden/_version.py to '%s'" % ver) 

def dd_iso_image(self, input_, output, gui_update, status_update):
        ''' Implementation for OS that use dd to write the iso image. 
        '''
        in_file_size = os.path.getsize(input_)
        cmd = [self.dd_exe, 'if=' + input_,
               'of=' + self.physical_disk(output), 'bs=1M']
        self.dd_iso_image_add_args(cmd, input_, output)
        kw_args = {
            'stdout' : subprocess.PIPE,
            'stderr' : subprocess.PIPE,
            'shell'  : False,
            }
        self.add_dd_iso_image_popen_args(kw_args)
        self.dd_iso_image_prepare(input, output, status_update)
        log('Executing => ' + str(cmd))
        dd_process = subprocess.Popen(cmd, **kw_args)
        output_q = queue.Queue()
        while dd_process.poll() is None:
            self.dd_iso_image_readoutput(dd_process, gui_update, in_file_size,
                                         output_q)
        output_lines = [output_q.get() for i in range(output_q.qsize())]
        for l in output_lines:
            log('dd: ' + l)
        return self.dd_iso_image_interpret_result(
            dd_process.returncode, output_lines) 

def sortSeqDesc(usearch5, usearch6, mergedDir, sortedDir, mapFilePathList):
    """
        Sort sequences in the descending order.
    """
    taxonIdSet = getAllTaxonIdSet(mapFilePathList)
    inFilePathList = []
    sortedFilePathList = []
    for taxonId in taxonIdSet:
        inFilePathList.append(os.path.join(mergedDir, str(str(taxonId) + '.fna')))
        sortedFilePathList.append(os.path.join(sortedDir, str(str(taxonId) + '.fna')))

    # sort sequences, longer first
    for inFile, outFile in zip(inFilePathList, sortedFilePathList):
        sortCmd = str(usearch5 + ' -sort ' + inFile + ' -output ' + outFile + ' --maxlen ' + str(int(getMaxLen(inFile) + 1000)) + ' --minlen 10')
        #sortCmd = str(usearch6 + ' -sortbylength ' + inFile + ' -output ' + outFile) # + ' -maxqt ' + str(int(getMaxLen(inFile))))
        print sortCmd
        sortProc = subprocess.Popen(sortCmd, shell=True, cwd=os.path.dirname(usearch6), bufsize=-1)
        sortProc.wait()
        if sortProc.returncode != 0:
            sys.stderr.write(str('Cmd: ' + sortCmd + ' ended with return code: ' + str(sortProc.returncode) + '\n'))
            return
    print 'sequences sorted' 

def start(self):
        self.command = "%s %s -%c -R %s -s %d" % \
                (
                    INJECTOR,
                    " ".join(self.settings.args),
                    self.settings.synth_mode,
                    "-0" if self.settings.root else "",
                    self.settings.seed
                )
        self.process = subprocess.Popen(
            "exec %s" % self.command,
            shell=True,
            stdout=subprocess.PIPE,
            stdin=subprocess.PIPE,
            preexec_fn=os.setsid
            ) 

def popen(fullcmd):
        p = subprocess.Popen(fullcmd, shell=True,
                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                             close_fds=True)
        return p.stdout 

def calculate(self):
        """
        Calculate the vacancy formation given Potential class.
        """
        with ScratchDir('.'):
            input_file, energy_per_atom, num_atoms = self._setup()
            p = subprocess.Popen([self.LMP_EXE, '-in', input_file], stdout=subprocess.PIPE)
            stdout = p.communicate()[0]

            rc = p.returncode
            if rc != 0:
                error_msg = 'LAMMPS exited with return code %d' % rc
                msg = stdout.decode("utf-8").split('\n')[:-1]
                try:
                    error_line = [i for i, m in enumerate(msg)
                                  if m.startswith('ERROR')][0]
                    error_msg += ', '.join([e for e in msg[error_line:]])
                except Exception:
                    error_msg += msg[-1]
                raise RuntimeError(error_msg)
            defect_energy, _, _ = self._parse()
        defect_formation_energy = defect_energy - energy_per_atom * num_atoms

        return defect_formation_energy 

def get_git_hash():

    def _minimal_ext_cmd(cmd):
        # construct minimal environment
        env = {}
        for k in ['SYSTEMROOT', 'PATH', 'HOME']:
            v = os.environ.get(k)
            if v is not None:
                env[k] = v
        # LANGUAGE is used on win32
        env['LANGUAGE'] = 'C'
        env['LANG'] = 'C'
        env['LC_ALL'] = 'C'
        out = subprocess.Popen(
            cmd, stdout=subprocess.PIPE, env=env).communicate()[0]
        return out

    try:
        out = _minimal_ext_cmd(['git', 'rev-parse', 'HEAD'])
        sha = out.strip().decode('ascii')
    except OSError:
        sha = 'unknown'

    return sha 

def checkProgramIsInstalled(self, program, args, where_to_download,
                                affected_tools):
        try:
            subprocess.Popen([program, args], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
            return True
        except EnvironmentError:
            # handle file not found error.
            # the config file is installed in:
            msg = "\n**{0} not found. This " \
                  "program is needed for the following "\
                  "tools to work properly:\n"\
                  " {1}\n"\
                  "{0} can be downloaded from here:\n " \
                  " {2}\n".format(program, affected_tools,
                                  where_to_download)
            sys.stderr.write(msg)

        except Exception as e:
            sys.stderr.write("Error: {}".format(e)) 

def calculate(self):
        """
        Calculate the NEB barrier given Potential class.
        """
        with ScratchDir('.'):
            input_file = self._setup()
            p = subprocess.Popen(['mpirun', '-n', str(self.num_replicas),
                                  'lmp_mpi', '-partition', '{}x1'.format(self.num_replicas),
                                  '-in', input_file],
                                 stdout=subprocess.PIPE)
            stdout = p.communicate()[0]
            rc = p.returncode
            if rc != 0:
                error_msg = 'LAMMPS exited with return code %d' % rc
                msg = stdout.decode("utf-8").split('\n')[:-1]
                try:
                    error_line = [i for i, m in enumerate(msg)
                                  if m.startswith('ERROR')][0]
                    error_msg += ', '.join([e for e in msg[error_line:]])
                except Exception:
                    error_msg += msg[-1]
                raise RuntimeError(error_msg)
            result = self._parse()
        return result 

def calculate(self):
        """
        Calculate the elastic constant given Potential class.
        """
        with ScratchDir('.'):
            input_file = self._setup()
            p = subprocess.Popen([self.LMP_EXE, '-in', input_file],
                                 stdout=subprocess.PIPE)
            stdout = p.communicate()[0]
            rc = p.returncode
            if rc != 0:
                error_msg = 'LAMMPS exited with return code %d' % rc
                msg = stdout.decode("utf-8").split('\n')[:-1]
                try:
                    error_line = [i for i, m in enumerate(msg)
                                  if m.startswith('ERROR')][0]
                    error_msg += ', '.join([e for e in msg[error_line:]])
                except Exception:
                    error_msg += msg[-1]
                raise RuntimeError(error_msg)
            result = self._parse()
        return result 

def get_perf(filename):
    ''' run conlleval.pl perl script to obtain
    precision/recall and F1 score '''
    _conlleval = PREFIX + 'conlleval'
    if not isfile(_conlleval):
        #download('http://www-etud.iro.umontreal.ca/~mesnilgr/atis/conlleval.pl') 
        os.system('wget https://www.comp.nus.edu.sg/%7Ekanmy/courses/practicalNLP_2008/packages/conlleval.pl')
        chmod('conlleval.pl', stat.S_IRWXU) # give the execute permissions
    
    out = []
    proc = subprocess.Popen(["perl", _conlleval], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    stdout, _ = proc.communicate(open(filename).read())
    for line in stdout.split('\n'):
        if 'accuracy' in line:
            out = line.split()
            break
    
    # out = ['accuracy:', '16.26%;', 'precision:', '0.00%;', 'recall:', '0.00%;', 'FB1:', '0.00']
    precision = float(out[3][:-2])
    recall    = float(out[5][:-2])
    f1score   = float(out[7])

    return {'p':precision, 'r':recall, 'f1':f1score} 

def gpt_device(self, dev_name):
        disk_dev = self.physical_disk(dev_name)
        cmd = ['parted', disk_dev, '-s', 'print']
        with open(os.devnull) as devnull:
            p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE, stdin=devnull)
            _cmd_out, _err_out = p.communicate()
            p.wait()
        if p.returncode != 0:
            lang = os.getenv('LANG')
            encoding = lang.rsplit('.')[-1] if lang else 'utf-8'
            raise RuntimeError(str(_err_out, encoding))
        subprocess.check_call(['partprobe', disk_dev])
        if b'msdos' in _cmd_out:
            return False
        if b'gpt' in _cmd_out:
            return True
        raise RuntimeError("Disk '%s' is uninitialized and not usable." %
                           disk_dev) 

def predict_on_batch(self, inputs):
        # write test fasta file
        temp_input = tempfile.NamedTemporaryFile(suffix = ".txt")
        test_fname = temp_input.name
        encode_sequence_into_fasta_file(ofname = test_fname, seq = inputs.tolist())
        # test gkmsvm
        temp_ofp = tempfile.NamedTemporaryFile(suffix = ".txt")
        threads_option = '-T %s' % (str(self.threads))
        verbosity_option = '-v 0'
        command = ' '.join(['gkmpredict',
                            test_fname,
                            self.model_file,
                            temp_ofp.name,
                            threads_option,
                            verbosity_option])
        #process = subprocess.Popen(command, shell=True)
        #process.wait()  # wait for it to finish
        exit_code = os.system(command)
        temp_input.close()
        assert exit_code == 0
        # get classification results
        temp_ofp.seek(0)
        y = np.array([line.split()[-1] for line in temp_ofp], dtype=float)
        temp_ofp.close()
        return np.expand_dims(y, 1) 

def detect_missing_tools(distro):
    tools_dir = os.path.join('data', 'tools')
    if platform.system() == 'Windows':
        _7zip_exe = gen.resource_path(
            os.path.join(tools_dir, '7zip', '7z.exe'))
        e2fsck_exe = gen.resource_path(os.path.join(tools_dir, 'cygwin', 'e2fsck.exe'))
        resize2fs_exe = gen.resource_path(os.path.join(tools_dir, 'cygwin', 'resize2fs.exe'))
    else:
        _7zip_exe = '7z'
        e2fsck_exe = 'e2fsck'
        resize2fs_exe = 'resize2fs'
    if distro not in creator_dict or \
       creator_dict[distro][0] is not create_persistence_using_resize2fs:
        return None
    try:
        with open(os.devnull) as devnull:
            for tool in [e2fsck_exe, resize2fs_exe]:
                p = subprocess.Popen([tool], stdout=devnull, stderr=devnull)
                p.communicate()
    except FileNotFoundError:  # Windows
        return "'%s.exe' is not installed or not available for use." % tool
    except OSError:            # Linux
        return "'%s' is not installed or not available for use." % tool
    return None 

def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
                env=None):
    """Call the given command(s)."""
    assert isinstance(commands, list)
    p = None
    for c in commands:
        try:
            dispcmd = str([c] + args)
            # remember shell=False, so use git.cmd on windows, not just git
            p = subprocess.Popen([c] + args, cwd=cwd, env=env,
                                 stdout=subprocess.PIPE,
                                 stderr=(subprocess.PIPE if hide_stderr
                                         else None))
            break
        except EnvironmentError:
            e = sys.exc_info()[1]
            if e.errno == errno.ENOENT:
                continue
            if verbose:
                print("unable to run %s" % dispcmd)
                print(e)
            return None, None
    else:
        if verbose:
            print("unable to find command, tried %s" % (commands,))
        return None, None
    stdout = p.communicate()[0].strip()
    if sys.version_info[0] >= 3:
        stdout = stdout.decode()
    if p.returncode != 0:
        if verbose:
            print("unable to run %s (error)" % dispcmd)
            print("stdout was %s" % stdout)
        return None, p.returncode
    return stdout, p.returncode 

def check_disassembler(name):
    result, errors = \
        subprocess.Popen(
                ['which', name],
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE
                ).communicate()
    return result.strip() != "" 

def check_vfat_filesystem(usb_disk, result=None):
    p = subprocess.Popen(['fsck.vfat', '-n', usb_disk],
                         stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                         stdin=subprocess.PIPE)
    output = p.communicate()
    gen.log("fsck.vfat -n returned %d" % p.returncode)
    gen.log(b"fsck.vfat -n said:" + b'\n---\n'.join(f for f in output if f))
    if result is not None:
        result.append((p.returncode, output, 'fsck.vfat -n'))
    return len(output[0].split(b'\n'))==3 and output[1]==b'' \
           and p.returncode==0 

def onedit_syslinux(self):
        """
        Function to edit main syslinux.cfg file.
        :return:
        """
        # Function to edit syslinux.cfg file on editors like gedit, notepad etc.
        # Suggest me more editor which can be included in to this function.
        sys_cfg_file = os.path.join(config.usb_mount, "multibootusb", "syslinux.cfg")
        log("Locating " + sys_cfg_file)
        editor = ''
        if not os.path.exists(sys_cfg_file):
            log("syslinux.cfg file not found...")
            QtWidgets.QMessageBox.information(self, 'File not found...', 'Sorry. Unable to locate syslinux.cfg file.\n'
                                                                         'You can only edit syslinux.cfg file generated by multibootusb.')
        else:
            if platform.system() == "Linux":
                for e in config.editors_linux:
                    if subprocess.call('which ' + e, shell=True) == 0:
                        log("Editor found is " + e)
                        editor = e
                        break
            elif platform.system() == "Windows":
                for e in config.editors_win:
                    if not shutil.which(e) is None:
                        log("Editor found is " + e)
                        editor = e
                        break
            if not editor:
                QtWidgets.QMessageBox.information(self, 'Editor not found...',
                                                  'Sorry. Installed editor is not supported by multibootusb\n'
                                                  'Edit ' + sys_cfg_file + ' manually.\n')
            else:
                try:
                    subprocess.Popen(editor + " '" + sys_cfg_file + "'", shell=True).pid
                except OSError:
                    QtWidgets.QMessageBox.warning(self, 'Error...',
                                                  'Failed to open syslinux.cfg file.\n'
                                                  'Edit syslinux.cfg file manually.\n') 

def disassemble(disassembler, bitness, data):
    if supported[disassembler] and disassemblers[disassembler][bitness]:
        temp_file = tempfile.NamedTemporaryFile()
        temp_file.write(data)

        # disassemble
        result, errors = \
            subprocess.Popen(
                    disassemblers[disassembler][bitness][0].format(temp_file.name),
                    shell=True,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE
                    ).communicate()

        disas = cleanup(result)

        # raw
        result, errors = \
            subprocess.Popen(
                    disassemblers[disassembler][bitness][1].format(temp_file.name),
                    shell=True,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE
                    ).communicate()

        raw = cleanup(result)

        temp_file.close()

        return (disas, raw)
    else:
        return (None, None) 

def _format_files(self, paths):
        """Helper method to start a seaparate subprocess."""
        cmd = [sys.executable, os.path.join(HERE, 'format_task.py')]
        env = os.environ.copy()
        env['CIOCHECK_PROJECT_ROOT'] = self.cmd_root
        env['CIOCHECK_CHECK'] = str(self.check)
        proc = subprocess.Popen(
            cmd + paths,
            env=env,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        return proc 

def run_command(args, cwd=None):
    """Run command."""
    process = subprocess.Popen(
        args,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        cwd=cwd, )
    output, error = process.communicate()

    if isinstance(output, bytes):
        output = output.decode()
    if isinstance(error, bytes):
        error = error.decode()

    return output, error 

def check_output(command):
    """
    Compatibility with :func:`subprocess.check_output` from Python 2.7 and
    upwards.
    """
    proc = Popen(command, stdout=PIPE)
    output = proc.communicate()[0]
    if proc.returncode != 0:
        raise CalledProcessError(proc.returncode, command)
    return output 

def dd_iso_image_readoutput(self, dd_process, gui_update, in_file_size,
                                output_q):
        # If this time delay is not given, the Popen does not execute
        # the actual command
        time.sleep(0.1)
        dd_process.send_signal(signal.SIGUSR1)
        dd_process.stderr.flush()
        while True:
            time.sleep(0.1)
            out_error = dd_process.stderr.readline().decode()
            if out_error:
                if 'bytes' in out_error:
                    bytes_copied = float(out_error.split(' ', 1)[0])
                    gui_update( bytes_copied / in_file_size * 100. )
                    break
                if 15 < output_q.qsize():
                    output_q.get()
                output_q.put(out_error.rstrip())
            else:
                # stderr is closed
                break