Python subprocess32.STDOUT Examples

def run_cmd(cmd):
    out = []

    fh = NamedTemporaryFile(delete=False)
    es = subprocess.call(cmd, stdin=None,
            stdout=fh, stderr=subprocess.STDOUT, shell=True)
    fh.close()
    with open(fh.name, 'r') as f:
        for line in f:
            out.append(line.rstrip('\n'))
    os.unlink(fh.name)

    if (es != 0):
        print "[-] Non-zero exit status '%d' for CMD: '%s'" % (es, cmd)
        for line in out:
            print line

    return es, out 

def parent_identical_or_crashes(self, crash, parent):

        # Base names
        cbasename = os.path.basename(crash)
        pbasename = os.path.basename(parent)

        ## Filter queue filenames with sig info
        if self.find_crash_parent_regex.match(pbasename):
            self.logr("Parent ({}) looks like crashing input!".format(pbasename))
            return True

        try:
            diff_out = subprocess.check_output("diff -q {} {}".format(crash, parent),
                                               stderr=subprocess.STDOUT, shell=True)
        except Exception, e:
            diff_out = e.output 

def parent_identical_or_crashes(self, crash, parent):

        # Base names
        cbasename = os.path.basename(crash)
        pbasename = os.path.basename(parent)

        ## Filter queue filenames with sig info
        if self.find_crash_parent_regex.match(pbasename):
            self.logr("Parent ({}) looks like crashing input!".format(pbasename))
            return True

        try:
            diff_out = subprocess.check_output("diff -q {} {}".format(crash, parent),
                                               stderr=subprocess.STDOUT, shell=True)
        except Exception, e:
            diff_out = e.output 

def parent_identical_or_crashes(self, crash, parent):

        # Base names
        cbasename = os.path.basename(crash)
        pbasename = os.path.basename(parent)

        ## Filter queue filenames with sig info
        if self.find_crash_parent_regex.match(pbasename):
            self.logr("Parent ({}) looks like crashing input!".format(pbasename))
            return True

        try:
            diff_out = subprocess.check_output("diff -q {} {}".format(crash, parent),
                                               stderr=subprocess.STDOUT, shell=True)
        except Exception, e:
            diff_out = e.output 

def open_process(command, cwd=None, shell=True, _popen_lock=threading.Lock()):
    kwargs = {
        "shell": shell,
        "stdout": subprocess.PIPE,
        "stderr": subprocess.STDOUT,
        "stdin": subprocess.PIPE,
        "bufsize": 1,  # Line buffered
        "universal_newlines": True,
    }

    if cwd is not None:
        kwargs["cwd"] = cwd

    # Prevent signal propagation from parent process
    try:
        # Windows
        kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
    except AttributeError:
        # Unix
        kwargs["preexec_fn"] = os.setpgrp

    with _popen_lock:  # Work around Python 2 Popen race condition
        return subprocess.Popen(command, **kwargs) 

def test_should_trigger_on_connect_if_client_connect_valid(server_with_mocks):
    node_script = '''
        module.paths.push('{0}')
        WebSocket = require('ws')
        const SubscriptionClient =
        require('subscriptions-transport-ws').SubscriptionClient
        new SubscriptionClient('ws://localhost:{1}/socket')
    '''.format(
        os.path.join(os.path.dirname(__file__), 'node_modules'), TEST_PORT)
    try:
        subprocess.check_output(
            ['node', '-e', node_script], stderr=subprocess.STDOUT, timeout=.2)
    except:
        mock = server_with_mocks.get_nowait()
        assert mock.name == 'on_connect'
        mock.assert_called_once() 

def test_should_trigger_on_connect_with_correct_cxn_params(server_with_mocks):
    node_script = '''
        module.paths.push('{0}')
        WebSocket = require('ws')
        const SubscriptionClient =
        require('subscriptions-transport-ws').SubscriptionClient
        const connectionParams = {{test: true}}
        new SubscriptionClient('ws://localhost:{1}/socket', {{
        connectionParams,
        }})
    '''.format(
        os.path.join(os.path.dirname(__file__), 'node_modules'), TEST_PORT)
    try:
        subprocess.check_output(
            ['node', '-e', node_script], stderr=subprocess.STDOUT, timeout=.2)
    except:
        mock = server_with_mocks.get_nowait()
        assert mock.name == 'on_connect'
        mock.assert_called_once()
        mock.assert_called_with({'test': True}) 

def CheckRG(samtools,bam,out_name,result):
    file = out_name+'.bam.header'
    command = [samtools,'view','-SH',bam,'-o',file]
    print (' '.join(command))
    subprocess.check_output(' '.join(command),stderr=subprocess.STDOUT,shell=True)

    result=[]

    with open(file,'r') as f:
        header = f.readlines()
    for l in range(len(header)):
        i = 0
        if header[l][0:3] == "@RG":
          RG = {x.split(':')[0]:x.split(':')[-1].replace('\n','') for x in header[l].split('@RG')[-1].split('\t')[1:]}
          result.append(RG)

    clean = ['rm','-f',file]
    subprocess.check_output(' '.join(clean),stderr=subprocess.STDOUT,shell=True)
    return result 

def fastq_bwa_mem_piped(fastqs,i,j,t,rg,out_dir,ref_path):
    output = ''
    bam = out_dir+'/'+fastqs[0].rsplit('/')[-1].rsplit('.fq')[0].rsplit('_')[0]+'.%s.bam'%j
    piped_mem = [bwa_mem,'-M','-t %s'%t,'-R',r"'%s'"%rg,ref_path,
                 '<(%s -f %s -i %s -j %s)'%(route,fastqs[0],i,j),
                 '<(%s -f %s -i %s -j %s)'%(route,fastqs[1],i,j),
                 '|',samtools_view,'-Sb','-','-@ %s'%t,
                 '|',sambamba_sort,'-t %s'%t,'--tmpdir=%s/temp'%out_dir,'-o',bam,'/dev/stdin']  #-@ for threads here
    try:#bwa mem call here-------------------------------------------
        output += subprocess.check_output(' '.join(piped_mem),
                                          stderr=subprocess.STDOUT,
                                          executable='/bin/bash',
                                          shell=True)
        output += subprocess.check_output(['rm','-rf','%s/temp'%out_dir])
    except Exception as E:
        output += str(E) #error on the call-------------------------
    return bam #list of bam files to merge into next step

    
#|| by number of fastq files presented 

def subproc_call(cmd, timeout=None):
    """
    Execute a command with timeout, and return both STDOUT/STDERR.

    Args:
        cmd(str): the command to execute.
        timeout(float): timeout in seconds.

    Returns:
        output(bytes), retcode(int). If timeout, retcode is -1.
    """
    try:
        output = subprocess.check_output(
            cmd, stderr=subprocess.STDOUT,
            shell=True, timeout=timeout)
        return output, 0
    except subprocess.TimeoutExpired as e:
        logger.warn("Command '{}' timeout!".format(cmd))
        logger.warn(e.output.decode('utf-8'))
        return e.output, -1
    except subprocess.CalledProcessError as e:
        logger.warn("Command '{}' failed, return code={}".format(cmd, e.returncode))
        logger.warn(e.output.decode('utf-8'))
        return e.output, e.returncode
    except Exception:
        logger.warn("Command '{}' failed to run.".format(cmd))
        return "", -2 

def check_output(cmds, shell=False):
    try:
        output = subprocess.check_output(cmds, stderr=subprocess.STDOUT, shell=shell)
        return output
    except subprocess.CalledProcessError:
        # logger.warn('Failed to run command: %s', ' '.join(cmds))
        # logger.warn('Error output:\n%s', e.output)
        raise 

def __init__(self, device_url, bundle_id=None):
        DeviceMixin.__init__(self)
        self.__device_url = device_url
        self.__scale = None
        
        self._wda = wda.Client(device_url)
        self._session = None
        self._bundle_id = None

        if bundle_id:
            self.start_app(bundle_id)
            
        # ioskit.Device.__init__(self, udid)

        # # xcodebuild -project  -scheme WebDriverAgentRunner -destination "id=1002c0174e481a651d71e3d9a89bd6f90d253446" test
        # # Test Case '-[UITestingUITests testRunner]' started.
        # xproj_dir = os.getenv('WEBDRIVERAGENT_DIR')
        # if not xproj_dir:
        #     raise RuntimeError("env-var WEBDRIVERAGENT_DIR need to be set")

        # proc = self._xcproc = subprocess.Popen(['/usr/bin/xcodebuild',
        #     '-project', 'WebDriverAgent.xcodeproj',
        #     '-scheme', 'WebDriverAgentRunner',
        #     '-destination', 'id='+self.udid, 'test'],
        #     stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=xproj_dir, bufsize=1, universal_newlines=True)
        # for line in iter(proc.stdout.readline, b""):
        #     print 'STDOUT:', line.strip()
        #     if 'TEST FAILED' in line:
        #         raise RuntimeError("webdriver start test failed, maybe need to unlock the keychain, try\n" + 
        #             '$ security unlock-keychain ~/Library/Keychains/login.keychain')
        #     elif "Successfully wrote Manifest cache" in line:
        #         print 'GOOD ^_^, wait 5s'
        #         time.sleep(5.0)
        #         break 

def run_cmd_noout(cmd_data):
    cmd = cmd_data[0]
    output = cmd_data[1]
    c = cmd[:-1]
    timeout = cmd[-1]
    display("Executing command: %s" % " ".join(c))

    current_time = time.time()
    f = open(output, 'w')
    if timeout:
        
        process = Popen(c, stdout=f, stderr=STDOUT)
        while time.time() < current_time + timeout and process.poll() is None:
            time.sleep(5)
        if process.poll() is None:

            display_error(
                "Timeout of %s reached. Aborting thread for command: %s"
                % (timeout, " ".join(c))
            )
            process.terminate()

    else:
        Popen(c, stdout=f, stderr=STDOUT).wait()
    f.close()
    return cmd_data 

def idevice(name, *args):
    exec_name = 'idevice' + name
    exec_path = look_exec(exec_name)
    if not exec_path:
        raise EnvironmentError('Necessary binary ("%s") not found.' % exec_name)

    cmds = [exec_path] + list(args)
    try:
        output = subprocess.check_output(cmds, stderr=subprocess.STDOUT, shell=False)
        return output
    except subprocess.CalledProcessError:
        raise 

def background_test(self):
        self.running = True
        proc = subprocess.Popen('echo hello', shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        while True:
            line = proc.stdout.readline()
            if line == '':
                break
            print line
            for client in ProgressHandler.clients:
                client.write_message(line)
            self.output = self.output + line
        self.running = False 

def test_should_call_unsubscribe_when_client_closes_cxn(server_with_mocks):
    node_script = '''
        module.paths.push('{0}')
        WebSocket = require('ws')
        const SubscriptionClient =
        require('subscriptions-transport-ws').SubscriptionClient
        const client = new SubscriptionClient('ws://localhost:{1}/socket')
        client.subscribe({{
            query: `subscription useInfo($id: String) {{
            user(id: $id) {{
              id
              name
            }}
          }}`,
            operationName: 'useInfo',
            variables: {{
              id: 3,
            }},
          }}, function (error, result) {{
            // nothing
          }}
        )
        setTimeout(() => {{
            client.client.close()
        }}, 500)
    '''.format(
        os.path.join(os.path.dirname(__file__), 'node_modules'), TEST_PORT)
    try:
        subprocess.check_output(
            ['node', '-e', node_script], stderr=subprocess.STDOUT, timeout=1)
    except:
        while True:
            mock = server_with_mocks.get_nowait()
            if mock.name == 'on_unsubscribe':
                mock.assert_called_once()
                break 

def test_should_trigger_on_subscribe_when_client_subscribes(server_with_mocks):
    node_script = '''
        module.paths.push('{0}')
        WebSocket = require('ws')
        const SubscriptionClient =
        require('subscriptions-transport-ws').SubscriptionClient
        const client = new SubscriptionClient('ws://localhost:{1}/socket')
        client.subscribe({{
            query: `subscription useInfo($id: String) {{
            user(id: $id) {{
              id
              name
            }}
          }}`,
            operationName: 'useInfo',
            variables: {{
              id: 3,
            }},
          }}, function (error, result) {{
            // nothing
          }})
    '''.format(
        os.path.join(os.path.dirname(__file__), 'node_modules'), TEST_PORT)
    try:
        subprocess.check_output(
            ['node', '-e', node_script], stderr=subprocess.STDOUT, timeout=.2)
    except:
        while True:
            mock = server_with_mocks.get_nowait()
            if mock.name == 'on_subscribe':
                mock.assert_called_once()
                break 

def test_should_trigger_on_unsubscribe_when_client_unsubscribes(
        server_with_mocks):
    node_script = '''
        module.paths.push('{0}')
        WebSocket = require('ws')
        const SubscriptionClient =
        require('subscriptions-transport-ws').SubscriptionClient
        const client = new SubscriptionClient('ws://localhost:{1}/socket')
        const subId = client.subscribe({{
            query: `subscription useInfo($id: String) {{
            user(id: $id) {{
              id
              name
            }}
          }}`,
            operationName: 'useInfo',
            variables: {{
              id: 3,
            }},
          }}, function (error, result) {{
            // nothing
          }})
        client.unsubscribe(subId)
    '''.format(
        os.path.join(os.path.dirname(__file__), 'node_modules'), TEST_PORT)
    try:
        subprocess.check_output(
            ['node', '-e', node_script], stderr=subprocess.STDOUT, timeout=.2)
    except:
        while True:
            mock = server_with_mocks.get_nowait()
            if mock.name == 'on_unsubscribe':
                mock.assert_called_once()
                break 

def test_passes_through_websocket_request_to_on_subscribe(server):
    node_script = '''
        module.paths.push('{0}')
        WebSocket = require('ws')
        const SubscriptionClient =
        require('subscriptions-transport-ws').SubscriptionClient
        const client = new SubscriptionClient('ws://localhost:{1}/socket')
        client.subscribe({{
            query: `subscription context {{
                context
            }}`,
            variables: {{}},
            }}, (error, result) => {{
                if (error) {{
                  console.log(JSON.stringify(error));
                }}
            }}
        );
    '''.format(
        os.path.join(os.path.dirname(__file__), 'node_modules'), TEST_PORT)

    try:
        subprocess.check_output(
            ['node', '-e', node_script], stderr=subprocess.STDOUT, timeout=.2)
    except:
        while True:
            mock = server.get_nowait()
            if mock.name == 'on_subscribe':
                mock.assert_called_once()
                mock.assert_called_with_contains('websocket')
                break 

def test_rejects_client_that_does_not_specifiy_a_supported_protocol(server):

    node_script = '''
        module.paths.push('{0}')
        WebSocket = require('ws')
        const client = new WebSocket('ws://localhost:{1}/socket')
        client.on('close', (code) => {{
            console.log(JSON.stringify(code))
          }}
        );
    '''.format(
        os.path.join(os.path.dirname(__file__), 'node_modules'), TEST_PORT)

    p = subprocess.Popen(
        ['node', '-e', node_script],
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    q = queue.Queue()
    t = threading.Thread(target=enqueue_output, args=(p.stdout, q))
    t.daemon = True
    t.start()
    time.sleep(.2)
    ret_values = []
    while True:
        try:
            _line = q.get_nowait()
            if isinstance(_line, bytes):
                line = _line.decode()
            line = json.loads(line)
            ret_values.append(line)
        except ValueError:
            pass
        except queue.Empty:
            break
    assert ret_values[0] == 1002 or 1006 

def subproc_call(cmd, timeout=None):
    try:
        output = subprocess.check_output(
                cmd, stderr=subprocess.STDOUT,
                shell=True, timeout=timeout)
        return output
    except subprocess.TimeoutExpired as e:
        logger.warn("Command timeout!")
        logger.warn(e.output)
    except subprocess.CalledProcessError as e:
        logger.warn("Commnad failed: {}".format(e.returncode))
        logger.warn(e.output) 

def subproc_call(cmd, timeout=None):
    try:
        output = subprocess.check_output(
                cmd, stderr=subprocess.STDOUT,
                shell=True, timeout=timeout)
        return output
    except subprocess.TimeoutExpired as e:
        logger.warn("Command timeout!")
        logger.warn(e.output)
    except subprocess.CalledProcessError as e:
        logger.warn("Commnad failed: {}".format(e.returncode))
        logger.warn(e.output) 

def does_dry_run_throw_error(self, cmd):

        try:
            out = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
        except Exception, e:
            return (e.returncode > 128) 

def run_cmd(self, cmd, collect, env=None):

        out = []

        if self.args.verbose:
            self.logr("    CMD: %s" % cmd)

        fh = None
        if self.args.disable_cmd_redirection or collect == self.Want_Output:
            fh = open(self.cov_paths['tmp_out'], 'w')
        else:
            fh = open(os.devnull, 'w')

        if env is None:
            subprocess.call(cmd, stdin=None,
                            stdout=fh, stderr=subprocess.STDOUT, shell=True, executable='/bin/bash')
        else:
            subprocess.call(cmd, stdin=None,
                            stdout=fh, stderr=subprocess.STDOUT, shell=True, env=env, executable='/bin/bash')

        fh.close()

        if self.args.disable_cmd_redirection or collect == self.Want_Output:
            with open(self.cov_paths['tmp_out'], 'r') as f:
                for line in f:
                    out.append(line.rstrip('\n'))

        return out 

def does_dry_run_throw_error(self, cmd):

        try:
            out = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
        except Exception, e:
            return (e.returncode > 128) 

def run_cmd(self, cmd, collect, env=None):

        out = []

        if self.args.verbose:
            self.logr("    CMD: %s" % cmd)

        fh = None
        if self.args.disable_cmd_redirection or collect == self.Want_Output:
            fh = open(self.cov_paths['tmp_out'], 'w')
        else:
            fh = open(os.devnull, 'w')

        if env is None:
            subprocess.call(cmd, stdin=None,
                            stdout=fh, stderr=subprocess.STDOUT, shell=True, executable='/bin/bash')
        else:
            subprocess.call(cmd, stdin=None,
                            stdout=fh, stderr=subprocess.STDOUT, shell=True, env=env, executable='/bin/bash')

        fh.close()

        if self.args.disable_cmd_redirection or collect == self.Want_Output:
            with open(self.cov_paths['tmp_out'], 'r') as f:
                for line in f:
                    out.append(line.rstrip('\n'))

        return out 

def subproc_call(cmd, timeout=None):
    try:
        output = subprocess.check_output(
                cmd, stderr=subprocess.STDOUT,
                shell=True, timeout=timeout)
        return output
    except subprocess.TimeoutExpired as e:
        logger.warn("Command timeout!")
        logger.warn(e.output)
    except subprocess.CalledProcessError as e:
        logger.warn("Commnad failed: {}".format(e.returncode))
        logger.warn(e.output) 

def run_proc_with_quit(proc_id, quit_dict, args, logfile=None, append=False, env=None, cwd=None):
    if logfile is None:
        logfile = os.devnull

    mode = 'ab' if append else 'wb'
    with open(logfile, mode) as logf:
        if proc_id in quit_dict:
            return None
        proc = subprocess.Popen(args, stdout=logf, stderr=subprocess.STDOUT,
                                env=env, cwd=cwd)
        retcode = None
        num_kill = 0
        timeout = 0.05
        while retcode is None and num_kill <= 2:
            try:
                retcode = proc.wait(timeout=timeout)
            except subprocess.TimeoutExpired:
                if proc_id in quit_dict:
                    if num_kill == 0:
                        proc.terminate()
                        timeout = quit_dict[proc_id]
                    elif num_kill == 1:
                        proc.kill()
                    num_kill += 1

        return proc.returncode 

def run(self,run_id,inputs):
        #workflow is to run through the stage correctly and then check for error handles
        #[1b]get some metadata for I/O names
        
        #[2]build command args
        samtools = self.tools['SAMTOOLS']
        view = [samtools, 'view', '-Sh', inputs['.bam']]

        path = self.tools['BWA-POSTALT']
        alt_fix = [self.tools['BWA-POSTALT'], '-p', self.files['GRCH38-EXTRA'], self.files['GRCH38-ALT']]

        out_file = self.strip_in_ext(inputs['.bam'],'.bam') + '.alt.bam'
        if ('out_file' in inputs) and (inputs['out_file'] != ''):
            out_file = inputs['out_file']
        view2 = [samtools, 'view', '-1', '-', '-o', out_file]
        
        #[1a]make start entry which is a new staged_run row
        
        #[3a]execute the command here----------------------------------------------------
        print ("<<<<<<<<<<<<<SVE command>>>>>>>>>>>>>>>\n")
        print (' '.join(view + ['|'] + alt_fix + ['|'] + view2))
        subprocess.check_output(' '.join(view + ['|'] + alt_fix + ['|'] + view2),stderr=subprocess.STDOUT, shell=True)
        
        #[3b]check results--------------------------------------------------
        if err == {}:
            #self.db_stop(run_id,{'output':output},'',True)
            results = [out_file]
            #for i in results: print i
            if all([os.path.exists(r) for r in results]):
                print("<<<<<<<<<<<<<bwa index sucessfull>>>>>>>>>>>>>>>\n")
                return results
            else:
                print("<<<<<<<<<<<<<bwa index failure>>>>>>>>>>>>>>>\n")
                return None
        else:
            #self.db_stop(run_id,{'output':output},err['message'],False)
            return None 

def run(self,run_id,inputs={}):
        in_file = ''
        
        #retrieve all the parameters and map them to the series of calls to be executed...
        command = ['ls','-als']
        self.db_start(run_id,in_file,self.params)
        
        #workflow is to run through the stage correctly and then check for error handles
        #[3a]execute the command here----------------------------------------------------
        output,err = '',{}
        try:
            output = subprocess.check_output(command,stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError as E:
            print('ouput: '+E.output)             #what you would see in the term
            err['output'] = E.output
            #the python exception issues (shouldn't have any...
            print('message: '+E.message)          #?? empty
            err['message'] = E.message
            #return codes used for failure....
            print('code: '+str(E.returncode))     #return 1 for a fail in art?
            err['code'] = E.returncode
        except OSError as E:
            print('ouput: '+E.strerror)             #what you would see in the term
            err['output'] = E.strerror
            #the python exception issues (shouldn't have any...
            print('message: '+E.message)          #?? empty
            err['message'] = E.message
            #the error num
            print('code: '+str(E.errno))
            err['code'] = E.errno
        print('output:\n'+output)
        #[3a]execute the command here----------------------------------------------------
        #[3b]do a os directory/data check or a ls type command to check the size
        #[3b]of the produced data to ensure that everything is fine...        
        if err == {}:
            self.db_stop(run_id,{'output':output},'',True)
            return output
        else:
            self.db_stop(run_id,{'output':output},err['message'],False)
            return None