Python subprocess.call() Examples

def install_update():
    """If a newer release is available, download and install it.

    :returns: ``True`` if an update is installed, else ``False``

    """
    update_data = wf().cached_data('__workflow_update_status', max_age=0)

    if not update_data or not update_data.get('available'):
        wf().logger.info('no update available')
        return False

    local_file = download_workflow(update_data['download_url'])

    wf().logger.info('installing updated workflow ...')
    subprocess.call(['open', local_file])

    update_data['available'] = False
    wf().cache_data('__workflow_update_status', update_data)
    return True 

def convert_image(inpath, outpath, size):
    """Convert an image file using ``sips``.

    Args:
        inpath (str): Path of source file.
        outpath (str): Path to destination file.
        size (int): Width and height of destination image in pixels.

    Raises:
        RuntimeError: Raised if ``sips`` exits with non-zero status.
    """
    cmd = [
        b'sips',
        b'-z', str(size), str(size),
        inpath,
        b'--out', outpath]
    # log().debug(cmd)
    with open(os.devnull, 'w') as pipe:
        retcode = subprocess.call(cmd, stdout=pipe, stderr=subprocess.STDOUT)

    if retcode != 0:
        raise RuntimeError('sips exited with %d' % retcode) 

def copy_iso(src, dst):
    """
    A simple wrapper for copying larger files. This is necessary as
    shutil copy files is much slower under Windows platform
    :param src: Path to source file
    :param dst: Destination directory
    :return:
    """
    if platform.system() == "Windows":
        # Note that xcopy asks if the target is a file or a directory when
        # source filename (or dest filename) contains space(s) and the target
        # does not exist.
        assert os.path.exists(dst)
        subprocess.call(['xcopy', '/Y', src, dst], shell=True)
    elif platform.system() == "Linux":
        shutil.copy(src, dst) 

def run_test_trials(args):
    test_path = args.test_path + ":" + args.test_name
    logging.info("Testing: %s", test_path)

    new_env = os.environ.copy()
    new_env["MXNET_TEST_COUNT"] = str(args.num_trials)
    
    if args.seed is None:
        logging.info("No test seed provided, using random seed")
    else:
        new_env["MXNET_TEST_SEED"] = str(args.seed)

    verbosity = "--verbosity=" + str(args.verbosity)

    code = subprocess.call(["nosetests", verbosity, test_path], 
                           env = new_env)
    
    logging.info("Nosetests terminated with exit code %d", code) 

def launchFile(filepath):
    if sys.platform.startswith('darwin'):
        subprocess.call(('open', filepath))
    elif os.name == 'nt':
        os.startfile(filepath)
    elif os.name == 'posix':
        subprocess.call(('xdg-open', filepath))


############################ String utilities ############################ 

def main(args):
  print_in_box('Validating submission ' + args.submission_filename)
  random.seed()
  temp_dir = args.temp_dir
  delete_temp_dir = False
  if not temp_dir:
    temp_dir = tempfile.mkdtemp()
    logging.info('Created temporary directory: %s', temp_dir)
    delete_temp_dir = True
  validator = validate_submission_lib.SubmissionValidator(temp_dir,
                                                          args.use_gpu)
  if validator.validate_submission(args.submission_filename,
                                   args.submission_type):
    print_in_box('Submission is VALID!')
  else:
    print_in_box('Submission is INVALID, see log messages for details')
  if delete_temp_dir:
    logging.info('Deleting temporary directory: %s', temp_dir)
    subprocess.call(['rm', '-rf', temp_dir]) 

def shell_call(command, **kwargs):
  """Calls shell command with parameter substitution.

  Args:
    command: command to run as a list of tokens
    **kwargs: dirctionary with substitutions

  Returns:
    whether command was successful, i.e. returned 0 status code

  Example of usage:
    shell_call(['cp', '${A}', '${B}'], A='src_file', B='dst_file')
  will call shell command:
    cp src_file dst_file
  """
  command = list(command)
  for i in range(len(command)):
    m = CMD_VARIABLE_RE.match(command[i])
    if m:
      var_id = m.group(1)
      if var_id in kwargs:
        command[i] = kwargs[var_id]
  return subprocess.call(command) == 0 

def run(self, input_dir, output_dir, epsilon):
    """Runs attack inside Docker.

    Args:
      input_dir: directory with input (dataset).
      output_dir: directory where output (adversarial images) should be written.
      epsilon: maximum allowed size of adversarial perturbation,
        should be in range [0, 255].
    """
    print('Running attack ', self.name)
    cmd = [self.docker_binary(), 'run',
           '-v', '{0}:/input_images'.format(input_dir),
           '-v', '{0}:/output_images'.format(output_dir),
           '-v', '{0}:/code'.format(self.directory),
           '-w', '/code',
           self.container,
           './' + self.entry_point,
           '/input_images',
           '/output_images',
           str(epsilon)]
    print(' '.join(cmd))
    subprocess.call(cmd) 

def run(self, input_dir, output_dir):
    """Runs defense inside Docker.

    Args:
      input_dir: directory with input (adversarial images).
      output_dir: directory to write output (classification result).
    """
    print('Running defense ', self.name)
    cmd = [self.docker_binary(), 'run',
           '-v', '{0}:/input_images'.format(input_dir),
           '-v', '{0}:/output_data'.format(output_dir),
           '-v', '{0}:/code'.format(self.directory),
           '-w', '/code',
           self.container,
           './' + self.entry_point,
           '/input_images',
           '/output_data/result.csv']
    print(' '.join(cmd))
    subprocess.call(cmd) 

def save_id_to_path_mapping(self):
    """Saves mapping from submission IDs to original filenames.

    This mapping is saved as CSV file into target directory.
    """
    if not self.id_to_path_mapping:
      return
    with open(self.local_id_to_path_mapping_file, 'w') as f:
      writer = csv.writer(f)
      writer.writerow(['id', 'path'])
      for k, v in sorted(iteritems(self.id_to_path_mapping)):
        writer.writerow([k, v])
    cmd = ['gsutil', 'cp', self.local_id_to_path_mapping_file,
           os.path.join(self.target_dir, 'id_to_path_mapping.csv')]
    if subprocess.call(cmd) != 0:
      logging.error('Can\'t copy id_to_path_mapping.csv to target directory') 

def run_without_time_limit(self, cmd):
    """Runs docker command without time limit.

    Args:
      cmd: list with the command line arguments which are passed to docker
        binary

    Returns:
      how long it took to run submission in seconds

    Raises:
      WorkerError: if error occurred during execution of the submission
    """
    cmd = [DOCKER_BINARY, 'run', DOCKER_NVIDIA_RUNTIME] + cmd
    logging.info('Docker command: %s', ' '.join(cmd))
    start_time = time.time()
    retval = subprocess.call(cmd)
    elapsed_time_sec = long(time.time() - start_time)
    logging.info('Elapsed time of attack: %d', elapsed_time_sec)
    logging.info('Docker retval: %d', retval)
    if retval != 0:
      logging.warning('Docker returned non-zero retval: %d', retval)
      raise WorkerError('Docker returned non-zero retval ' + str(retval))
    return elapsed_time_sec 

def fetch_attacks_data(self):
    """Initializes data necessary to execute attacks.

    This method could be called multiple times, only first call does
    initialization, subsequent calls are noop.
    """
    if self.attacks_data_initialized:
      return
    # init data from datastore
    self.submissions.init_from_datastore()
    self.dataset_batches.init_from_datastore()
    self.adv_batches.init_from_datastore()
    # copy dataset locally
    if not os.path.exists(LOCAL_DATASET_DIR):
      os.makedirs(LOCAL_DATASET_DIR)
    eval_lib.download_dataset(self.storage_client, self.dataset_batches,
                              LOCAL_DATASET_DIR,
                              os.path.join(LOCAL_DATASET_COPY,
                                           self.dataset_name, 'images'))
    # download dataset metadata
    self.read_dataset_metadata()
    # mark as initialized
    self.attacks_data_initialized = True 

def find_qemu():
        """
        Check if QEMU is available on host system and return path of the binary
        :return: path to QEMU program or None otherwise.
        """
        if platform.system() == "Linux":
            if subprocess.call('which qemu-system-x86_64', shell=True) == 0:
                qemu = "qemu-system-x86_64"
            elif subprocess.call('which qemu', shell=True) == 0:
                qemu = "qemu"
            else:
                qemu = ""

        elif platform.system() == "Windows":
            qemu = find_qemu_exe()

        if qemu:
            log("QEMU: using " + qemu)
        else:
            log("QEMU: ERROR: not found!")

        return qemu 

def install_dependency_package():
        if subprocess.call("which pacman", shell=True) == 0:
            subprocess.call("pacman -Sy --noconfirm", shell=True)
            # Thank you Neitsab for "--needed"  argument.
            if subprocess.call("pacman -S --needed --noconfirm p7zip python-pyqt5 mtools python3-six parted util-linux python-dbus") == 0:
                result = True
        elif subprocess.call("which yum", shell=True) == 0:
            subprocess.call("yum check-update", shell=True)
            if subprocess.call("dnf install mtools python3-PyQt5 util-linux python3-six parted p7zip p7zip-plugins python3-pyudev python3-dbus -y", shell=True) == 0:
                result = True
        elif subprocess.call("which apt-get", shell=True) == 0:
            subprocess.call("apt-get -q update", shell=True)
            if subprocess.call("apt-get -q -y install python3-pyqt5 p7zip-full parted util-linux python3-pyudev mtools python3-dbus", shell=True) == 0:
                result = True
        elif subprocess.call("which zypper", shell=True) == 0:
            subprocess.call("zypper refresh", shell=True)
            if subprocess.call("zypper install -y mtools python3-qt5 p7zip python3-pyudev python3-six util-linux parted", shell=True) == 0:
                result = True
        elif subprocess.call("which urpmi", shell=True) == 0:
            subprocess.call("urpmi.update -a", shell=True)
            if subprocess.call("urpmi install -auto mtools util-linux p7zip python3-pyudev python3-six parted python3-qt5", shell=True) == 0:
                result = True

        return bool(result) 

def bamToGold(bamtogold, merged, out, metadata, threads):
    """
    Calls the bamToGold script for all of the merged bam files, creating the gold standard
    """
    out_name = os.path.join(out, "anonymous_gsa.fasta")
    all_files = os.listdir(merged)
    bams = []
    for f in all_files:
        if f.endswith(".bam"):
            bams.append(f)
    for bam in bams:
        genome = bam.rstrip(".bam")
        otu, ncbi, novelty, path = metadata[genome]
        cmd = "{bamToGold} -r {path} -b {bam} -l 1 -c 1 >> {gsa}".format(
            bamToGold = bamtogold,
            path = path,
            bam = os.path.join(out,"bam",bam),
            gsa = out_name
        )
        subprocess.call([cmd],shell=True) 

def merge_bam_files(bams_per_genome, out, threads):
    """
    Merges (+sort +index)  all given bam files per genome (exact paths, single sample/multiple runs or multiple samples)
    """
    out_path = os.path.join(out,"bam")
    os.mkdir(out_path)
    for genome in bams_per_genome:
        list_of_bam = " ".join(bams_per_genome[genome]) # can be used as input to samtools immediately
        header = fix_headers(genome, bams_per_genome[genome], out_path)
        if header is not None:
            for bam in bams_per_genome[genome]: # add new header to all bam files
                cmd = "samtools reheader {header} {bam} >> {out}/out.bam; mv {out}/out.bam {bam}".format(
                    header = header,
                    out = out_path,
                    bam = bam
                )
                subprocess.call([cmd],shell=True)
        cmd = "samtools merge -@ {threads} - {bam_files} | samtools sort -@ {threads} - {path}/{genome}; samtools index {path}/{genome}.bam".format(
            threads = threads,
            bam_files = list_of_bam,
            path = out_path,
            genome = genome
        )
        subprocess.call([cmd],shell=True) # this runs a single command at a time (but that one multi threaded)
    return out_path 

def shell_call(command, **kwargs):
  """Calls shell command with parameter substitution.

  Args:
    command: command to run as a list of tokens
    **kwargs: dirctionary with substitutions

  Returns:
    whether command was successful, i.e. returned 0 status code

  Example of usage:
    shell_call(['cp', '${A}', '${B}'], A='src_file', B='dst_file')
  will call shell command:
    cp src_file dst_file
  """
  command = list(command)
  for i in range(len(command)):
    m = CMD_VARIABLE_RE.match(command[i])
    if m:
      var_id = m.group(1)
      if var_id in kwargs:
        command[i] = kwargs[var_id]
  return subprocess.call(command) == 0 

def call(cls, cmd, shell=False):
        '''
            Run command with arguments, wait to complete and return ``True`` on success.

        :param cls: The class as implicit first argument.
        :param cmd: Command string to be executed.
        :returns  : ``True`` on success, otherwise ``None``.
        :rtype    : ``bool``
        '''
        logger = logging.getLogger('SPOT.INGEST.COMMON.UTIL')
        logger.debug('Execute command: {0}'.format(cmd))

        try:
            subprocess.call(cmd, shell=shell)
            return True

        except Exception as exc:
            logger.error('[{0}] {1}'.format(exc.__class__.__name__, exc.message)) 

def execute_cmd(cls,command,logger):

        try:
            logger.info("SPOT.Utils: Executing: {0}".format(command))
            subprocess.call(command,shell=True)

        except subprocess.CalledProcessError as e:
            logger.error("SPOT.Utils: There was an error executing: {0}".format(e.cmd))
            sys.exit(1) 

def linux_install_default_bootsector(usb_disk, mbr_install_cmd):

    with usb.UnmountedContext(usb_disk, config.update_usb_mount):
        syslinux_cmd = [syslinux_path, '-i', '-d', 'multibootusb', usb_disk]
        if os.access(syslinux_path, os.X_OK) is False:
            subprocess.call('chmod +x ' + syslinux_path, shell=True)
        log("\nExecuting ==> %s\n" % syslinux_cmd)
        config.status_text = 'Installing default syslinux version 4...'
        if subprocess.call(syslinux_cmd) == 0:

            # On my system, it takes hours long to complete a single check
            # So not included as of now
            # usb.repair_vfat_filesystem(usb_disk)

            log("\nDefault syslinux install is success...\n")
            config.status_text = 'Default syslinux successfully installed...'
            log('\nExecuting ==> ' + mbr_install_cmd)
            if subprocess.call(mbr_install_cmd, shell=True) == 0:
                config.status_text = 'mbr install is success...'
                log("\nmbr install is success...\n")
                if set_boot_flag(usb_disk) is True:
                    return True
                else:
                    log("\nFailed to install default syslinux...\n")
                    config.status_text = 'Failed to install default syslinux...'
                    return False
    return None 

def main(vfl, run=False, version=True, threads=1, time=True):
    """Compiles and optionally runs vex code. Arguments:
            vfl:     the path to compile '/some/code.vlf'
            run:     if this is true, we call execute() and thus vexexec
            version: print the houdini version called for compiling and running
            threads: number of threads when run is true
            time:    print the execution time when running
    """
    hfs, version = find_houdini()
    if version:
        print('Using houdini path:\n\t%s' % hfs)

    # compile
    print('Compiling:\n\t%s' % vfl)
    bin = os.path.join(hfs, 'bin')
    vex = build(bin, vfl)

    # execute
    if run:
        print('Executing:\n\t%s' % vex)
        execute(bin, vex, threads, time) 

def onComboChange(self):
        """
        Detects and updates GUI with populated USB device details.
        :return:
        """
        self.ui.installed_distros.clear()
        config.usb_disk = osdriver.listbox_entry_to_device(
            self.ui.combo_drives.currentText())
        if config.usb_disk == 0 or config.usb_disk:
            # Get the GPT status of the disk and store it on a variable
            try:
                usb.gpt_device(config.usb_disk)
                config.imager_usb_disk \
                    = self.ui.combo_drives.currentText()
                config.usb_details \
                    = usb.details(config.usb_disk)
            except Exception as e:
                o = io.StringIO()
                traceback.print_exc(None, o)
                log(o.getvalue())
                QtWidgets.QMessageBox.critical(
                    self, "The disk/partition is not usable.",
                    str(e))
                self.ui.combo_drives.setCurrentIndex(0)
                # Above statement triggers call to this method.
                return
            log("Selected device " +
                            osdriver.usb_disk_desc(config.usb_disk))
            self.update_target_info()
            self.update_list_box(config.usb_disk)
            self.ui_update_persistence()
        else:
            self.ui.usb_dev.clear()
            self.ui.usb_vendor.clear()
            self.ui.usb_model.clear()
            self.ui.usb_size.clear()
            self.ui.usb_mount.clear()
            self.ui.usb_type.clear()
            self.ui.usb_fs.clear()
            log("No USB disk found...") 

def build_distro_bootsector_impl(usb_disk, options,
                                 distro_syslinux_install_dir):
    syslinux_path = os.path.join(
        multibootusb_host_dir(), "syslinux", "bin", "syslinux") \
        + config.syslinux_version

    if os.access(syslinux_path, os.X_OK) is False:
        subprocess.call('chmod +x ' + syslinux_path, shell=True)
    sys_cmd = [syslinux_path] + options + [
        distro_syslinux_install_dir, usb_disk]
    log("Executing ==> %s" % sys_cmd)
    if subprocess.call(sys_cmd) == 0:
        config.status_text = \
            'Syslinux install on distro directory is successful...'
        log("\nSyslinux install on distro directory is successful...\n")

        # On my system, it takes hours long to complete a single check
        # So not included as of now
        # usb.repair_vfat_filesystem(usb_disk)

        tmp_bs_file = '/tmp/mbusb_temp.bs'
        dd_cmd = ['dd', 'if=' + usb_disk, 'of=' + tmp_bs_file, 'count=1']
        log('Executing ==> %s' % dd_cmd + '\n')
        config.status_text = 'Copying boot sector...'
        config.status_text = 'Installing distro specific syslinux...'
        if subprocess.call(dd_cmd) == 0:
            config.status_text = 'Bootsector copy is successful...'
            log("\nBootsector copy is successful...\n")
        else:
            config.status_text = 'Failed to copy boot sector...'
            log("\nFailed to copy boot sector...\n")
        return tmp_bs_file
    else:
        config.status_text = 'Failed to install syslinux on distro directory...'
        log("\nFailed to install syslinux on distro directory...\n")
        return None 

def mbusb(self):
        try:
#             from PyQt5 import QtGui
            if subprocess.call("python3 setup.py install --record ./.install_files.txt", shell=True) == 0:
                    print("Installation finished.")
                    print("Find multibootusb under system menu or run from terminal  using the following command...")
                    print("\nmultibootusb\n")
                    print("You can uninstall multibootusb at any time using follwing command (with root/sudo previlage)")
                    print("\n./uninstall.sh\n")
        except:
            print("Installing missing package.")
            if self.supported_pac_manager() is not True:
                print("Unsupported package manager.")
                print("Please install parted, util-linux and python3-pyqt5/PyQt5, mtools and python3-dbus\n"
                      "Whatever the package name is applicable to your distro and rerun this script.")
                sys.exit(0)
            elif self.internet_on() is False:
                print("Unable to connect to internet.")
                print("Please install parted, util-linux and python3-pyqt5/PyQt5, pkexec, mtools and python3-dbus \n"
                      "Whatever the package name is applicable to your distro and rerun this script.")
                sys.exit(0)
            elif self.internet_on() is True:
                if self.install_dependency_package() is not True:
                    print("Error installing dependency packages.")
                else:
                    if subprocess.call("python3 setup.py install --record ./.install_files.txt", shell=True) == 0:
                        print("Installation finished.")
                        print("Find multibootusb under system menu or run from terminal  using the following command...")
                        print("\nmultibootusb\n")
                        print("You can uninstall multibootusb at any time using follwing command (with root/sudo previlage)")
                        print("\nsudo ./uninstall.sh\n") 

def supported_pac_manager():
        pac_managers = ["pacman", "yum", "apt-get", "zypper", "urpmi"]
        result = "0"
        for pac_man in pac_managers:
            if subprocess.call("which " + pac_man, shell=True) == 0:
                result = "1"
                return True

        if not result == "1":
            return False 

def load_to_hdfs(cls,file_local_path,file_hdfs_path,logger):
        # move file to hdfs.
        load_to_hadoop_script = "hadoop fs -moveFromLocal {0} {1}".format(file_local_path,file_hdfs_path)
        logger.info("SPOT.Utils: Loading file to hdfs: {0}".format(load_to_hadoop_script))
        subprocess.call(load_to_hadoop_script,shell=True) 

def test_model(model_name, caplog):
    """kipoi test ...
    """
    caplog.set_level(logging.INFO)

    source_name = "kipoi"
    assert source_name == "kipoi"

    env_name = conda_env_name(model_name, model_name, source_name)
    env_name = "test-" + env_name  # prepend "test-"

    # if environment already exists, remove it
    if env_exists(env_name):
        print("Removing the environment: {0}".format(env_name))
        remove_env(env_name)

    # create the model test environment
    args = ["kipoi", "env", "create",
            "--source", source_name,
            "--env", env_name,
            model_name]
    returncode = subprocess.call(args=args)
    assert returncode == 0

    if model_name == "basenji":
        batch_size = str(2)
    else:
        batch_size = str(4)

    # run the tests in the environment
    args = [get_kipoi_bin(env_name), "test",
            "--batch_size", batch_size,
            "--source", source_name,
            model_name]
    returncode = subprocess.call(args=args)
    assert returncode == 0

    for record in caplog.records:
        # there shoudn't be any warning
        assert record.levelname not in ['WARN', 'WARNING', 'ERROR', 'CRITICAL'] 

def creat_hdfs_folder(cls,hdfs_path,logger):
        hadoop_create_folder="hadoop fs -mkdir -p {0}".format(hdfs_path)
        logger.info("SPOT.Utils: Creating hdfs folder: {0}".format(hadoop_create_folder))
        subprocess.call(hadoop_create_folder,shell=True) 

def fix_headers(genome, bams, out):
    """
    Sometimes short sequences are not present in one or the other bam file since no reads were created for them, the header of the merged file needs to have the union of all sequences in all bam files
    """
    sequences = set()
    header = ""
    header_name = os.path.join(out, genome + "_header.sam")
    for bam in bams:
        cmd = "samtools view -H {bam} >> {hname}".format(
            bam = bam,
            hname = header_name
        )
        subprocess.call([cmd],shell=True)
        with open(header_name, 'r') as header_file:
            for line in header_file:
                if line.startswith("@SQ"): # sequences 
                    sq, sn, ln = line.strip().split('\t')
                    sequence_name = sn.split(":",1)[1]
                    if sequence_name not in sequences:
                        sequences.add(sequence_name)
                        header += line
                elif line.startswith("@HD") and header == "": #primary header, use arbitrary one
                    header += line 
    with open(header_name,'w+') as header_file:
        header_file.write(header)
    return header_name 

def adminCmd(cmd, fork=False, gui=False):
    """
    This simple function checks for a sudo command and runs a command using it.
    This function tries to launch given script with root access using pkexec/gksu/gksudo/kdesu/kdesudo,
    if one of them is already installed.
    PyQt4 is used as GUI.
    Author : sundar
    """
    sudo_cmd = ''
    if os.getuid() == 0:
        sudo_cmd = cmd
    else:
        if os.system('which pkexec') == 0:
            if gui:
                # By default, pkexec disallows X11 apps. Restore DISPLAY & XAUTHORITY
                # to allow it. man 1 pkexec/"SECURITY NOTES" section
                cmd = ['export DISPLAY=$DISPLAY; export XAUTHORITY=$XAUTHORITY; '] + cmd
            sudo_cmd = ['pkexec', '/bin/sh', '-c']
        elif os.system('which gksudo') == 0:
            sudo_cmd = ["gksudo", "--", "/bin/sh", "-c"]
        elif os.system('which gksu') == 0:
            sudo_cmd = ["gksu"]
        elif os.system('which kdesudo') == 0:
            sudo_cmd = ["kdesudo", "-t", "-c"]    # http://www.unix.com/man-page/debian/1/kdesudo/
        elif os.system('which kdesu') == 0:
            sudo_cmd = ["kdesu", "-t", "-c"]      # http://linux.die.net/man/1/kdesu
        else:
            QtWidgets.QMessageBox.information('No root...',
                                          'Could not find any of: pkexec, sudo, gksu, kdesu, gksudo, or kdesudo.\n'
                                          'Please install one then restart multibootusb.')
            sys.exit(0)
    final_cmd = ' '.join(sudo_cmd + ['"' + ' '.join(cmd).replace('"', '\\"') + '"'])
    gen.log("Executing ==>  " + final_cmd)
    if fork:
        return subprocess.Popen(final_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1, shell=True)
    else:
        ret = subprocess.call(final_cmd, shell=True)
        gen.log("Process returned ==>   " + str(ret))
        return ret