Python shlex.split() Examples

def _get_terminal_size_tput(self):
        # get terminal width
        # src: http://stackoverflow.com/questions/263890/how-do-i-find-the-width-height-of-a-terminal-window
        try:
            cols = int(
                subprocess.check_output(
                    shlex.split("tput cols"), stderr=subprocess.STDOUT
                )
            )
            rows = int(
                subprocess.check_output(
                    shlex.split("tput lines"), stderr=subprocess.STDOUT
                )
            )

            return (cols, rows)
        except:
            pass 

def parse_json_message(messages):
    if isinstance(messages, (list, set, tuple)):
        for message in messages:
            parse_json_message(message)
    else:
        if len(agent_config_vars['json_top_level']) == 0:
            parse_json_message_single(messages)
        else:
            top_level = _get_json_field_helper(
                    messages,
                    agent_config_vars['json_top_level'].split(JSON_LEVEL_DELIM),
                    allow_list=True)
            if isinstance(top_level, (list, set, tuple)):
                for message in top_level:
                    parse_json_message_single(message)
            else:
                parse_json_message_single(top_level) 

def update_state(setting, value, append=False):
    # update in-mem
    if append:
        current = ','.join(agent_config_vars['state'][setting])
        value = '{},{}'.format(current, value) if current else value
        agent_config_vars['state'][setting] = value.split(',')
    else:
        agent_config_vars['state'][setting] = value
    logger.debug('setting {} to {}'.format(setting, value))
    # update config file
    if 'TAIL' in agent_config_vars['data_format']:
        config_ini = config_ini_path()
        if os.path.exists(config_ini):
            config_parser = ConfigParser.SafeConfigParser()
            config_parser.read(config_ini)
            config_parser.set('state', setting, str(value))
            with open(config_ini, 'w') as config_file:
                config_parser.write(config_file)
    # return new value (if append)
    return value 

def parse_messages_kafka(consumer):
    logger.info('Reading messages')
    for message in consumer:
        try:
            logger.debug('Message received')
            logger.debug(message.value)
            if 'JSON' in agent_config_vars['data_format']:
                parse_json_message(json.loads(str(message.value)))
            elif 'CSV' in agent_config_vars['data_format']:
                parse_json_message(label_message(message.value.split(',')))
            else:
                parse_raw_message(message.value)
        except Exception as e:
            logger.warn('Error when parsing message')
            logger.warn(e)
            continue 

def parse_json_message(messages):
    if len(agent_config_vars['json_top_level']) != 0:
        if agent_config_vars['json_top_level'] == '[]' and isinstance(messages, (list, set, tuple)):
            for message in messages:
                parse_json_message_single(message)
        else:
            top_level = _get_json_field_helper(
                messages,
                agent_config_vars['json_top_level'].split(JSON_LEVEL_DELIM),
                allow_list=True)
            if isinstance(top_level, (list, set, tuple)):
                for message in top_level:
                    parse_json_message_single(message)
            else:
                parse_json_message_single(top_level)
    elif isinstance(messages, (list, set, tuple)):
        for message_single in messages:
            parse_json_message_single(message_single)
    else:
        parse_json_message_single(messages) 

def parse_json_message(messages):
    if len(agent_config_vars['json_top_level']) != 0:
        if agent_config_vars['json_top_level'] == '[]' and isinstance(messages, (list, set, tuple)):
            for message in messages:
                parse_json_message_single(message)
        else:
            top_level = _get_json_field_helper(
                    messages,
                    agent_config_vars['json_top_level'].split(JSON_LEVEL_DELIM),
                    allow_list=True)
            if isinstance(top_level, (list, set, tuple)):
                for message in top_level:
                    parse_json_message_single(message)
            else:
                parse_json_message_single(top_level)
    elif isinstance(messages, (list, set, tuple)):
        for message_single in messages:
            parse_json_message_single(message_single)
    else:
        parse_json_message_single(messages) 

def get_one_hot_C(sequence, region_dict  ):
    #also replace non excisting nucleos with 0
    repl='TGAN'
    seq_Cmeth_new=""
    seq_C_new=""

    for nucl in repl:
        sequence=sequence.replace(nucl, '0')
    # split the Cs in C meth and C unmeth
    for dict_key in sorted(region_dict):
        meth_true=region_dict[dict_key][2]
        start_meth=int(region_dict[dict_key][0])
        end_meth=int(region_dict[dict_key][1])
        if meth_true==1:
            seqsnip_Cmeth=sequence[start_meth:end_meth+1].replace("C", '1')
            seqsnip_C=sequence[start_meth:end_meth+1].replace("C", '0')
        else:
            seqsnip_Cmeth=sequence[start_meth:end_meth+1].replace("C", '0')
            seqsnip_C=sequence[start_meth:end_meth+1].replace("C", '1')
        seq_C_new=seq_C_new + seqsnip_C
        seq_Cmeth_new=seq_Cmeth_new+seqsnip_Cmeth

    return seq_C_new, seq_Cmeth_new 

def parse_json_message(messages):
    if len(agent_config_vars['json_top_level']) != 0:
        if agent_config_vars['json_top_level'] == '[]' and isinstance(messages, (list, set, tuple)):
            for message in messages:
                parse_json_message_single(message)
        else:
            top_level = _get_json_field_helper(
                    messages,
                    agent_config_vars['json_top_level'].split(JSON_LEVEL_DELIM),
                    allow_list=True)
            if isinstance(top_level, (list, set, tuple)):
                for message in top_level:
                    parse_json_message_single(message)
            else:
                parse_json_message_single(top_level)
    else:
        parse_json_message_single(messages) 

def update_state(setting, value, append=False, write=False):
    # update in-mem
    if append:
        current = ','.join(agent_config_vars['state'][setting])
        value = '{},{}'.format(current, value) if current else value
        agent_config_vars['state'][setting] = value.split(',')
    else:
        agent_config_vars['state'][setting] = value
    logger.debug('setting {} to {}'.format(setting, value))
    # update config file
    if write and not cli_config_vars['testing']:
        config_ini = config_ini_path()
        if os.path.exists(config_ini):
            config_parser = ConfigParser.SafeConfigParser()
            config_parser.read(config_ini)
            config_parser.set('state', setting, str(value))
            with open(config_ini, 'w') as config_file:
                config_parser.write(config_file)
    # return new value (if append)
    return value 

def sendFile(clientSocket, parameters):
    request = clientSocket.recv(1024)
    logger.debug("Request: " + str(request))
    requestParts = shlex.split(request)
    if len(requestParts) >= 4:
        if requestParts[0] == 'CUSTOM':
            action = " ".join(requestParts[1:-3])
            userName = requestParts[len(requestParts) - 3]
            licenseKey = requestParts[len(requestParts) - 2]
            projectName = requestParts[len(requestParts) - 1]
        else:
            action = requestParts[0]
            userName = requestParts[1]
            licenseKey = requestParts[2]
            projectName = requestParts[3]
            if str(action).lower() == "cleandisk":
                action = "clean_disk.sh"
        command = os.path.join(parameters['homepath'], "script_runner", action)
        if verifyUser(userName, licenseKey, projectName):
            runCommand(command, clientSocket)
        else:
            clientSocket.send("Status: 500")
    else:
        clientSocket.send("Status: 500")
    clientSocket.close() 

def update_state(setting, value, append=False, write=False):
    # update in-mem
    if append:
        current = ','.join(agent_config_vars['state'][setting])
        value = '{},{}'.format(current, value) if current else value
        agent_config_vars['state'][setting] = value.split(',')
    else:
        agent_config_vars['state'][setting] = value
    logger.debug('setting {} to {}'.format(setting, value))
    # update config file
    if write and not cli_config_vars['testing']:
        config_ini = config_ini_path()
        if os.path.exists(config_ini):
            config_parser = ConfigParser.SafeConfigParser()
            config_parser.read(config_ini)
            config_parser.set('state', setting, str(value))
            with open(config_ini, 'w') as config_file:
                config_parser.write(config_file)
    # return new value (if append)
    return value 

def find_executable(name) -> str:
    is_windows = os.name == 'nt'
    windows_exts = os.environ['PATHEXT'].split(ENV_PATH_SEP) if is_windows else None
    path_dirs = os.environ['PATH'].split(ENV_PATH_SEP)

    search_dirs = path_dirs + [os.getcwd()] # cwd is last in the list

    for dir in search_dirs:
        path = os.path.join(dir, name)

        if is_windows:
            for extension in windows_exts:
                path_with_ext = path + extension

                if os.path.isfile(path_with_ext) and os.access(path_with_ext, os.X_OK):
                    return path_with_ext
        else:
            if os.path.isfile(path) and os.access(path, os.X_OK):
                return path

    return '' 

def parse_csv_data(csv_data, instance, device=''):
    """
    parses CSV data, assuming the format is given as:
        header row:  timestamp,field_1,field_2,...,field_n
        n data rows: TIMESTAMP,value_1,value_2,...,value_n
    """

    # get field names from header row
    field_names = csv_data.pop(0).split(CSV_DELIM)[1:]

    # go through each row
    for row in csv_data:
        if len(row) > 0:
            parse_csv_row(row.split(CSV_DELIM), field_names, instance, device) 

def get_json_field(message, config_setting, default='', allow_list=False, remove=False):
    if config_setting not in agent_config_vars or len(agent_config_vars[config_setting]) == 0:
        return default
    setting_value = agent_config_vars[config_setting]
    field_val = json_format_field_value(
                    _get_json_field_helper(
                        message,
                        setting_value.split(JSON_LEVEL_DELIM),
                        allow_list=allow_list,
                        remove=remove))
    if len(field_val) == 0:
        field_val = default
    return field_val 

def get_datetime_from_date_string(date_string):
    timestamp_datetime = date_string.partition('.')[0]
    if 'strip_tz' in agent_config_vars and agent_config_vars['strip_tz']:
        date_string = ''.join(agent_config_vars['strip_tz_fmt'].split(date_string))
    if 'timestamp_format' in agent_config_vars:
        for timestamp_format in agent_config_vars['timestamp_format']:
            try:
                if timestamp_format == 'epoch':
                    timestamp_datetime = get_datetime_from_unix_epoch(date_string)
                else:
                    timestamp_datetime = datetime.strptime(date_string,
                                                           timestamp_format)
                break
            except Exception as e:
                logger.info('timestamp {} does not match {}'.format(
                    date_string,
                    timestamp_format))
                return -1
    else:
        try:
            timestamp_datetime = dateutil.parse.parse(date_string)
        except:
            timestamp_datetime = get_datetime_from_unix_epoch(date_string)
            agent_config_vars['timestamp_format'] = ['epoch']

    return timestamp_datetime 

def get_sentence_segment(sentence, start, end=None):
    segment = sentence.split(' ')[start:end]
    return ' '.join(segment) 

def run_subproc_once(command, **passthrough):
    command = format_command(command)
    output = subprocess.check_output(command,
                                     universal_newlines=True,
                                     **passthrough).split('\n')
    for line in output:
        yield line 

def merge_data(field, value, data={}):
    fields = field.split(JSON_LEVEL_DELIM)
    for i in range(len(fields) - 1):
        field = fields[i]
        if field not in data:
            data[field] = dict()
        data = data[field]
    data[fields[-1]] = value
    return data 

def get_single_value(message, setting_value, default='', allow_list=False, remove=False):
    if isinstance(setting_value, (set, list, tuple)):
        setting_value_single = setting_value[0]
    else:
        setting_value_single = setting_value
        setting_value = [setting_value]
    if is_complex(setting_value_single):
        this_field, metadata, that_field = setting_value_single.split('!!')
        return get_complex_value(message,
                                 this_field,
                                 metadata,
                                 that_field,
                                 default=default,
                                 allow_list=allow_list,
                                 remove=remove)
    elif is_formatted(setting_value_single):
        return parse_formatted(message,
                               setting_value_single,
                               default=default,
                               allow_list=False,
                               remove=remove)
    else:
        return get_json_field_by_pri(message,
                                    [i for i in setting_value],
                                    default=default,
                                    allow_list=allow_list,
                                    remove=remove) 

def transpose_metrics():
    """ flatten data up to the timestamp"""
    for timestamp, kvs in track['current_dict'].itemss():
        track['line_count'] += 1
        new_row = dict()
        new_row['timestamp'] = timestamp
        for key, value in kvs.items():
            if '|' in value:
                value = statistics.median(
                    map(lambda v: float(v), value.split('|')))
            new_row[key] = str(value)
        track['current_row'].append(new_row) 

def get_sentence_segment(sentence, start, end=None):
    segment = sentence.split(' ')[start:end]
    return ' '.join(segment) 

def reader(_format, _file, st_ino):
    if _format in {'XLS', 'XLSX'}:
        for line in read_xls(_file):
            yield line
    else:
        mode = 'r' if _format != 'AVRO' else 'rb'
        with open(_file, mode) as data:
            # get field names from header
            if 'IFEXPORT' in _format:
                agent_config_vars['csv_field_names'] = data.readline().strip().split(',')
            # preformatting on all data
            if 'TAIL' in _format:
                update_state('current_file', json.dumps({st_ino: _file}))
                data.seek(int(agent_config_vars['state']['current_file_offset'])) # read from state
            elif _format == 'AVRO':
                data = avro.datafile.DataFileReader(data, avro.io.DatumReader())
            # read data
            if _format == 'XML':
                data = xml2dict.parse(data)
                yield data
            else:
                # read each line
                logger.debug('reading each line')
                for line in data:
                    yield reader_next_line(_format, data, line)
                if 'TAIL' in _format:
                    if 'TAILF' in _format:
                        logger.debug('tailing file')
                        # keep reading file
                        for line2 in tail_file(_file, data):
                            yield reader_next_line(_format, data, line2)
                        # move from current file to completed, reset position
                        update_state('completed_files_st_ino', st_ino, append=True)
                        update_state('current_file', '')
                        update_state('current_file_offset', 0) 

def format_sentence(sentence):
    '''
    Takes a sentence and chops it into an array by word
    Implementation-specifc
    '''
    words = sentence.strip(':')
    words = COLONS.sub('/', words)
    words = UNDERSCORE.sub('/', words)
    words = words.split('/')
    return words 

def transpose_metrics():
    """ flatten data up to the timestamp"""
    for timestamp, kvs in track['current_dict'].items():
        track['line_count'] += 1
        new_row = dict()
        new_row['timestamp'] = timestamp
        for key, value in kvs.items():
            if '|' in value:
                value = statistics.median(
                    map(lambda v: float(v), value.split('|')))
            new_row[key] = str(value)
        track['current_row'].append(new_row) 

def format_command(cmd):
    if not isinstance(cmd, (list, tuple)): # no sets, as order matters
        cmd = shlex.split(cmd)
    return list(cmd) 

def run_subproc_once(command, **passthrough):
    command = format_command(command)
    output = subprocess.check_output(command,
                                     universal_newlines=True,
                                     **passthrough).split('\n')
    for line in output:
        yield line 

def get_datetime_from_date_string(date_string):
    timestamp_datetime = date_string.partition('.')[0]
    if 'strip_tz' in agent_config_vars and agent_config_vars['strip_tz']:
        date_string = ''.join(agent_config_vars['strip_tz_fmt'].split(date_string))
    if 'timestamp_format' in agent_config_vars:
        for timestamp_format in agent_config_vars['timestamp_format']:
            try:
                if timestamp_format == 'epoch':
                    timestamp_datetime = get_datetime_from_unix_epoch(date_string)
                else:
                    timestamp_datetime = datetime.strptime(date_string,
                                                           timestamp_format)
                break
            except Exception as e:
                logger.info('timestamp {} does not match {}'.format(
                    date_string,
                    timestamp_format))
                continue
    else:
        try:
            timestamp_datetime = dateutil.parse.parse(date_string)
        except:
            timestamp_datetime = get_datetime_from_unix_epoch(date_string)
            agent_config_vars['timestamp_format'] = ['epoch']

    return timestamp_datetime 

def label_message(message, fields=[]):
    """ turns unlabeled, split data into labeled data """
    if agent_config_vars['data_format'] in {'CSV', 'CSVTAIL', 'IFEXPORT'}:
        fields = agent_config_vars['csv_field_names']
    json = dict()
    for i in range(minlen(fields, message)):
        json[fields[i]] = message[i]
    return json 

def get_json_field(message, setting_value, default='', allow_list=False, remove=False):
    field_val = json_format_field_value(
                    _get_json_field_helper(
                        message,
                        setting_value.split(JSON_LEVEL_DELIM),
                        allow_list=allow_list,
                        remove=remove))
    if len(field_val) == 0:
        field_val = default
    return field_val 

def merge_data(field, value, data=None):
    if data is None:
        data = {}
    fields = field.split(JSON_LEVEL_DELIM)
    for i in range(len(fields) - 1):
        field = fields[i]
        if field not in data:
            data[field] = dict()
        data = data[field]
    data[fields[-1]] = value
    return data