Python re.compile() Examples

def interpolate_url(self):
        pattern = r'\{([a-z_]+)\}'
        regex = re.compile(pattern)
        start = 0
        new_url = self.resource_url

        while (start + 1) < len(self.resource_url):
            match = regex.search(self.resource_url, start)
            if match is None:
                return new_url

            param = match.group(1)
            val = getattr(self, param, None)
            if val is None:
                raise ValueError("Missing variable value")
            new_url = new_url.replace('{' + param + '}', str(val))
            start = match.end(1) + 1

        return new_url 

def validate_hostname(hostname):
    if len(hostname) > 255:
        raise Exception("Hostname {} is longer than 255 characters".format(hostname))
    if hostname[-1] == ".":
        hostname = hostname[:-1]
    allowed = re.compile(r"(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)
    if not all(allowed.match(x) for x in hostname.split(".")):
        raise Exception("Hostname {} is not RFC 1123 compliant".format(hostname)) 

def test_invalid_partition_mountpoints(self, deckhand_ingester,
                                           drydock_state, input_files,
                                           mock_get_build_data):

        input_file = input_files.join("invalid_validation.yaml")
        design_ref = "file://%s" % str(input_file)

        orch = Orchestrator(
            state_manager=drydock_state, ingester=deckhand_ingester)

        status, site_design = Orchestrator.get_effective_site(orch, design_ref)

        validator = StorageMountpoints()
        message_list = validator.execute(site_design, orchestrator=orch)

        regex = re.compile('Mountpoint .+ already exists')

        for msg in message_list:
            msg = msg.to_dict()
            LOG.debug(msg)
            assert regex.search(msg.get('message')) is not None
            assert msg.get('error') is True 

def test_invalid_storage_partitioning(self, deckhand_ingester,
                                          drydock_state, input_files,
                                          mock_get_build_data):
        input_file = input_files.join("invalid_validation.yaml")

        design_ref = "file://%s" % str(input_file)

        orch = Orchestrator(
            state_manager=drydock_state, ingester=deckhand_ingester)

        status, site_design = Orchestrator.get_effective_site(orch, design_ref)

        validator = StoragePartitioning()
        message_list = validator.execute(site_design)

        regex = re.compile('Volume group .+ not assigned any physical volumes')

        for msg in message_list:
            msg = msg.to_dict()
            LOG.debug(msg)
            assert len(msg.get('documents')) > 0
            assert msg.get('error')
            assert regex.search(msg.get('message')) is not None

        assert len(message_list) == 2 

def tweak(self, content):
        tweaked = self.legacy_tweak(content)
        if tweaked:
            return tweaked
        apply_persistence_to_all_lines = \
            0 < self.setup_params.persistence_size and \
            not self.config_is_persistence_aware(content)
        matching_re = r'^(\s*(%s)\s*)(.*)$' % self.BOOT_PARAMS_STARTER
        kernel_parameter_line_pattern = re.compile(
            matching_re,
            flags = re.I | re.MULTILINE)
        out = self.tweak_first_match(
            content,
            kernel_parameter_line_pattern,
            apply_persistence_to_all_lines,
            self.param_operations(),
            self.param_operations_for_persistence())
        
        return self.post_process(out) 

def test_invalid_network_cidr(self, mocker, deckhand_ingester,
                                  drydock_state, input_files):

        input_file = input_files.join("invalid_network_cidr.yaml")
        design_ref = "file://%s" % str(input_file)

        orch = Orchestrator(
            state_manager=drydock_state, ingester=deckhand_ingester)

        status, site_design = Orchestrator.get_effective_site(orch, design_ref)

        validator = CidrValidity()
        message_list = validator.execute(site_design, orchestrator=orch)
        msg = message_list[0].to_dict()

        regex_diagnostic = re.compile('Provide a CIDR acceptable by MAAS: .+')
        regex_message = re.compile('The provided CIDR .+ has host bits set')

        assert len(message_list) >= 1
        assert msg.get('error') is True
        assert any([
            regex_diagnostic.search(msg.get('diagnostic')),
            regex_message.search(msg.get('message'))
                  ]) 

def test_invalid_boot_storage_small(self, deckhand_ingester, drydock_state,
                                        input_files, mock_get_build_data):
        input_file = input_files.join("invalid_boot_storage_small.yaml")
        design_ref = "file://%s" % str(input_file)

        orch = Orchestrator(
            state_manager=drydock_state, ingester=deckhand_ingester)

        status, site_design = Orchestrator.get_effective_site(orch, design_ref)

        validator = BootStorageRational()
        message_list = validator.execute(site_design, orchestrator=orch)

        regex = re.compile('.+ volume must be > .+GB')

        for msg in message_list:
            msg = msg.to_dict()
            LOG.debug(msg)
            assert len(msg.get('documents')) > 0
            assert regex.search(msg.get('message')) is not None
            assert msg.get('error')

        assert len(message_list) == 4 

def test_invalid_vg_mountpoints(self, deckhand_ingester, drydock_state,
                                    input_files, mock_get_build_data):

        input_file = input_files.join("invalid_mountpoint.yaml")
        design_ref = "file://%s" % str(input_file)

        orch = Orchestrator(
            state_manager=drydock_state, ingester=deckhand_ingester)

        status, site_design = Orchestrator.get_effective_site(orch, design_ref)

        validator = StorageMountpoints()
        message_list = validator.execute(site_design, orchestrator=orch)

        regex = re.compile('Mountpoint .+ already exists')

        for msg in message_list:
            msg = msg.to_dict()
            LOG.debug(msg)
            assert regex.search(msg.get('message')) is not None
            assert msg.get('error') is True 

def escape_wikitext(wikitext):
    """Escape wikitext for use in file description."""
    rep = OrderedDict([
        ('{|', '{{(}}|'),
        ('|}', '|{{)}}'),
        ('||', '||'),
        ('|', '|'),
        ('[[', '{{!((}}'),
        (']]', '{{))!}}'),
        ('{{', '{{((}}'),
        ('}}', '{{))}}'),
        ('{', '{{(}}'),
        ('}', '{{)}}'),
    ])
    rep = dict((re.escape(k), v) for k, v in rep.iteritems())
    pattern = re.compile("|".join(rep.keys()))
    return pattern.sub(lambda m: rep[re.escape(m.group(0))], wikitext)


# Source: mediawiki.Title.js@9df363d 

def __init__(self, interval, stat_func=None, pattern='.*', sort=False):
        if stat_func is None:
            def asum_stat(x):
                """returns |x|/size(x), async execution."""
                return ndarray.norm(x)/sqrt(x.size)
            stat_func = asum_stat
        self.stat_func = stat_func
        self.interval = interval
        self.activated = False
        self.queue = []
        self.step = 0
        self.exes = []
        self.re_prog = re.compile(pattern)
        self.sort = sort
        def stat_helper(name, array):
            """wrapper for executor callback"""
            array = ctypes.cast(array, NDArrayHandle)
            array = NDArray(array, writable=False)
            if not self.activated or not self.re_prog.match(py_str(name)):
                return
            self.queue.append((self.step, py_str(name), self.stat_func(array)))
        self.stat_helper = stat_helper 

def test_invalid_no_duplicate_IPs(self, input_files, drydock_state,
                                      deckhand_ingester):
        input_file = input_files.join("invalid_validation.yaml")
        design_ref = "file://%s" % str(input_file)

        orch = Orchestrator(
            state_manager=drydock_state, ingester=deckhand_ingester)

        status, site_design = Orchestrator.get_effective_site(orch, design_ref)

        validator = NoDuplicateIpsCheck()
        message_list = validator.execute(site_design)

        regex = re.compile('Duplicate IP Address Found: [0-9.]+')

        for msg in message_list:
            msg = msg.to_dict()
            LOG.debug(msg)
            assert len(msg.get('documents')) > 0
            assert msg.get('error') is True
            assert regex.search(msg.get('message')) is not None 

def _collect_params_with_prefix(self, prefix=''):
        if prefix:
            prefix += '.'
        pattern = re.compile(r'(l|r)(\d)_(i2h|h2h)_(weight|bias)\Z')
        def convert_key(m, bidirectional): # for compatibility with old parameter format
            d, l, g, t = [m.group(i) for i in range(1, 5)]
            if bidirectional:
                return '_unfused.{}.{}_cell.{}_{}'.format(l, d, g, t)
            else:
                return '_unfused.{}.{}_{}'.format(l, g, t)
        bidirectional = any(pattern.match(k).group(1) == 'r' for k in self._reg_params)

        ret = {prefix + convert_key(pattern.match(key), bidirectional) : val
               for key, val in self._reg_params.items()}
        for name, child in self._children.items():
            ret.update(child._collect_params_with_prefix(prefix + name))
        return ret 

def convert_param(method, param):
    # remove notation, split by upper, convert to lowercase
    param_sanitized = param.replace('*', '')
    substr = param_sanitized
    try:
        substr = re.search('([A-Z]\w+)', param_sanitized).group(1)
    except:
        pass
    case_re = re.compile(r'((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))')
    converted_param = case_re.sub(r'_\1', substr).lower()
    if converted_param in keyword.kwlist or converted_param in dir(__builtins__):
        converted_param += '_param'
    # check for duplicates. if seen, append number to end
    if 'params' in method and len([param for param in method['params'] if param['name'] == converted_param]):
        param_names = [param['name'] for param in method['params']]
        for x in range(2, 10):
            count_name = '{:s}{:d}'.format(converted_param, x)
            if count_name not in param_names:
                converted_param = count_name
                break
    return converted_param 

def _match_major_minor(cls, value):
        """
        Match the number under the assumption that it is a major,minor pair.

        :param str value: value to match
        :returns: the device number or None
        :rtype: int or NoneType
        """
        major_minor_re = re.compile(
           r'^(?P<major>\d+)(\D+)(?P<minor>\d+)$'
        )
        match = major_minor_re.match(value)
        return match and os.makedev(
           int(match.group('major')),
           int(match.group('minor'))
        ) 

def _parse_summary(self):
        """Grab signature (if given) and summary"""
        summary = self._doc.read_to_next_empty_line()
        summary_str = "\n".join([s.strip() for s in summary])
        if re.compile('^([\\w. ]+=)?[\\w\\.]+\\(.*\\)$').match(summary_str):
            self['Signature'] = summary_str
            if not self._is_at_section():
                self['Summary'] = self._doc.read_to_next_empty_line()
        elif re.compile('^[\\w]+\n[-]+').match(summary_str):
            self['Summary'] = ''
            self._doc.reset()
        else:
            self['Summary'] = summary

        if not self._is_at_section():
            self['Extended Summary'] = self._read_to_next_section() 

def AddDropout(self, prev_layer, index):
    """Adds a dropout layer.

    Args:
      prev_layer: Input tensor.
      index:      Position in model_str to start parsing

    Returns:
      Output tensor, end index in model_str.
    """
    pattern = re.compile(R'(Do)({\w+})?')
    m = pattern.match(self.model_str, index)
    if m is None:
      return None, None
    name = self._GetLayerName(m.group(0), index, m.group(2))
    layer = slim.dropout(
        prev_layer, 0.5, is_training=self.is_training, scope=name)
    return layer, m.end() 

def AddMaxPool(self, prev_layer, index):
    """Add a maxpool layer.

    Args:
      prev_layer: Input tensor.
      index:      Position in model_str to start parsing

    Returns:
      Output tensor, end index in model_str.
    """
    pattern = re.compile(R'(Mp)({\w+})?(\d+),(\d+)(?:,(\d+),(\d+))?')
    m = pattern.match(self.model_str, index)
    if m is None:
      return None, None
    name = self._GetLayerName(m.group(0), index, m.group(2))
    height = int(m.group(3))
    width = int(m.group(4))
    y_stride = height if m.group(5) is None else m.group(5)
    x_stride = width if m.group(6) is None else m.group(6)
    self.reduction_factors[1] *= y_stride
    self.reduction_factors[2] *= x_stride
    return slim.max_pool2d(
        prev_layer, [height, width], [y_stride, x_stride],
        padding='SAME',
        scope=name), m.end() 

def format_pixiv_illust_original_url(id_url, mode=1):
    tag = 'Format_Pixiv_Illust_Original_Url'
    if mode == 1:
        contents = get_text_from_url(id_url)
        try:
            img_src_re = re.compile(r'\"urls\":{.*?}')
            img_src = img_src_re.findall(contents)
            final_dict = json.loads("{" + img_src[0] + "}")
            return final_dict['urls']['original']
        except Exception as e:
            print_with_tag(tag, "An error occurred when parsing the json file.")
            print_with_tag(tag, e)
    elif mode == 2:
        data_list = []
        json_datas = get_text_from_url(id_url)
        json_datas_format = json.loads(json_datas)['body']
        for urls in json_datas_format:
            data_list.append(urls['urls']['original'])
        return data_list 

def main(unused_argv):
  logging.set_verbosity(logging.INFO)
  with tf.Session() as sess:
    src = gen_parser_ops.document_source(batch_size=32,
                                         corpus_name=FLAGS.corpus_name,
                                         task_context=FLAGS.task_context)
    sentence = sentence_pb2.Sentence()
    while True:
      documents, finished = sess.run(src)
      logging.info('Read %d documents', len(documents))
      for d in documents:
        sentence.ParseFromString(d)
        tr = asciitree.LeftAligned()
        d = to_dict(sentence)
        print 'Input: %s' % sentence.text
        print 'Parse:'
        tr_str = tr(d)
        pat = re.compile(r'\s*@\d+$')
        for tr_ln in tr_str.splitlines():
          print pat.sub('', tr_ln)

      if finished:
        break 

def load_qa(self):
    data_source = os.path.join(self.data_folder, self.data_name)
    f = tf.gfile.GFile(data_source, "r")
    id_regex = re.compile("\(id ([^\)]*)\)")
    for line in f:
      id_match = id_regex.search(line)
      id = id_match.group(1)
      self.examples.append(id) 

def read_charset(filename, null_character=u'\u2591'):
  """Reads a charset definition from a tab separated text file.

  charset file has to have format compatible with the FSNS dataset.

  Args:
    filename: a path to the charset file.
    null_character: a unicode character used to replace '<null>' character. the
      default value is a light shade block '░'.

  Returns:
    a dictionary with keys equal to character codes and values - unicode
    characters.
  """
  pattern = re.compile(r'(\d+)\t(.+)')
  charset = {}
  with tf.gfile.GFile(filename) as f:
    for i, line in enumerate(f):
      m = pattern.match(line)
      if m is None:
        logging.warning('incorrect charset file. line #%d: %s', i, line)
        continue
      code = int(m.group(1))
      char = m.group(2).decode('utf-8')
      if char == '<nul>':
        char = null_character
      charset[code] = char
  return charset 

def patch_uboot(self):
        port = self._port

        log.info("Patching U-Boot")
        port.reset_input_buffer()
        sleep(0.1)
        port.write("printenv\n")
        port.flush()
        add_misc_match = re.compile(r'^addmisc=(.+)$')
        add_misc_val = None

        sleep(0.5)

        lines = port.read_until("U-Boot>")
        log.debug(lines)
        for line in lines.split('\n'):
            line = line.strip()
            log.debug(line)
            match = add_misc_match.match(line)
            if match:
                add_misc_val = match.group(1)

        if add_misc_val is None:
            log.error("Could not find value for addmisc environment variable")
            return

        cmd = "setenv addmisc " + re.sub(r'([\$;])',r'\\\1', add_misc_val + " init=/bin/sh")
        port.write(cmd + "\n")
        port.flush()
        log.debug(port.read_until("U-Boot>"))
        port.write("run boot_nand\n")
        port.flush() 

def extract_distroinfo_from_fname(self, which_dir, regex, distro_group,
                                      version_group):
        p = re.compile(regex, re.I)
        for fname in os.listdir(self.fullpath(which_dir)):
            m = p.match(fname)
            if m:
                return (m.group(distro_group),
                        [int(x) for x in m.group(version_group).split('.')])
        return None 

def waitingTradeSystemStateEntered(self):
        # 장시작 전에 조건이 시작하도록 함 
        self.sigSelectCondition.emit()       

        # 반환값 : 조건인덱스1^조건명1;조건인덱스2^조건명2;…;
        # result = '조건인덱스1^조건명1;조건인덱스2^조건명2;'
        result = self.getConditionNameList()
        searchPattern = r'(?P<index>[^\/:*?"<>|;]+)\^(?P<name>[^\/:*?"<>|;]+);'
        fileSearchObj = re.compile(searchPattern, re.IGNORECASE)
        findList = fileSearchObj.findall(result)
        
        tempDict = dict(findList)
        print(tempDict)
        

        condition_name_screenNo_dict = {}
        for number, condition in tempDict.items():
            condition_name_screenNo_dict[condition] = [kw_util.sendConditionScreenNo + '{}'.format(int (number)), number]
        
        start_info = []
        start_name = None

        for name, info in condition_name_screenNo_dict.items():

            if name == self.current_condition_name:
                start_info = info
                start_name = name
            else: 
                print("stop condition " + name + ", screen_no: " + info[0] + ", nIndex " + '{}'.format(int(info[1]) ) )
                self.sendConditionStop( info[0], name, int(info[1])  )
                pass

        print("start condition " + start_name + ", screen_no: " + start_info[0] + ", nIndex " + '{}'.format(int(start_info[1])) )
        self.sendCondition( start_info[0], start_name, int(start_info[1]) , 1) 

        pass 

def __init__(self, *args, **kw):
        persistence_awareness_checking_re = re.compile(
            r'^\s*(%s).*?\s%s(\s.*|)$' % \
            (self.BOOT_PARAMS_STARTER, self.PERSISTENCY_TOKEN),
            flags=re.I|re.MULTILINE)
        super(ConfigTweakerWithDebianStylePersistenceParam,
              self).__init__(persistence_awareness_checking_re, *args, **kw) 

def __init__(self, pattern, flags=0):
        self._re = re.compile(pattern, flags) 

def _match_number(cls, value):
        """
        Match the number under the assumption that it is a single number.

        :param str value: value to match
        :returns: the device number or None
        :rtype: int or NoneType
        """
        number_re = re.compile(r'^(?P<number>\d+)$')
        match = number_re.match(value)
        return match and int(match.group('number')) 

def from_kernel_device(cls, context, kernel_device):
        """
        Locate a device based on the kernel device.

        :param `Context` context: the libudev context
        :param str kernel_device: the kernel device
        :returns: the device corresponding to ``kernel_device``
        :rtype: `Device`
        """
        switch_char = kernel_device[0]
        rest = kernel_device[1:]
        if switch_char in ('b', 'c'):
            number_re = re.compile(r'^(?P<major>\d+):(?P<minor>\d+)$')
            match = number_re.match(rest)
            if match:
                number = os.makedev(
                   int(match.group('major')),
                   int(match.group('minor'))
                )
                return cls.from_device_number(context, switch_char, number)
            else:
                raise DeviceNotFoundByKernelDeviceError(kernel_device)
        elif switch_char == 'n':
            return cls.from_interface_index(context, rest)
        elif switch_char == '+':
            (subsystem, _, kernel_device_name) = rest.partition(':')
            if kernel_device_name and subsystem:
                return cls.from_name(context, subsystem, kernel_device_name)
            else:
                raise DeviceNotFoundByKernelDeviceError(kernel_device)
        else:
            raise DeviceNotFoundByKernelDeviceError(kernel_device) 

def get_enumeration_sparql(rq, v, endpoint, auth=None):
    """
    Returns a list of enumerated values for variable 'v' in query 'rq'
    """
    glogger.info('Retrieving enumeration for variable {}'.format(v))
    vcodes = []
    # tpattern_matcher = re.compile(".*(FROM\s+)?(?P<gnames>.*)\s+WHERE.*[\.\{][\n\t\s]*(?P<tpattern>.*\?" + re.escape(v) + ".*?\.).*", flags=re.DOTALL)
    # tpattern_matcher = re.compile(".*?((FROM\s*)(?P<gnames>(\<.*\>)+))?\s*WHERE\s*\{(?P<tpattern>.*)\}.*", flags=re.DOTALL)

    # WHERE is optional too!!
    tpattern_matcher = re.compile(".*?(FROM\s*(?P<gnames>\<.*\>+))?\s*(WHERE\s*)?\{(?P<tpattern>.*)\}.*",
                                  flags=re.DOTALL)

    glogger.debug(rq)
    tp_match = tpattern_matcher.match(rq)
    if tp_match:
        vtpattern = tp_match.group('tpattern')
        gnames = tp_match.group('gnames')
        glogger.debug("Detected graph names: {}".format(gnames))
        glogger.debug("Detected BGP: {}".format(vtpattern))
        glogger.debug("Matched triple pattern with parameter")
        if gnames:
            codes_subquery = re.sub("SELECT.*\{.*\}.*",
                                    "SELECT DISTINCT ?" + v + " FROM " + gnames + " WHERE { " + vtpattern + " }", rq,
                                    flags=re.DOTALL)
        else:
            codes_subquery = re.sub("SELECT.*\{.*\}.*",
                                    "SELECT DISTINCT ?" + v + " WHERE { " + vtpattern + " }", rq,
                                    flags=re.DOTALL)
        glogger.debug("Codes subquery: {}".format(codes_subquery))
        glogger.debug(endpoint)
        codes_json = requests.get(endpoint, params={'query': codes_subquery},
                                  headers={'Accept': static.mimetypes['json'],
                                           'Authorization': 'token {}'.format(static.ACCESS_TOKEN)}, auth=auth).json()
        for code in codes_json['results']['bindings']:
            vcodes.append(list(code.values())[0]["value"])
    else:
        glogger.debug("No match between variable name and query.")

    return vcodes 

def writeDir(self, path, output, pattern="", r=True, all_type=False):
        """ Extract a directory
        Return 0 means success otherwise failure.
        """
        d = self.searchDir(path)
        if d != None:
            if output.endswith("/"):
                output = output[0:-1]
            # Try to make target directory.
            if not os.path.exists(output):
                try:
                    os.makedirs(output)
                except(OSError):
                    sys.stderr.write("can't make dirs({0})\n".format(output))
                    return E_FAILURE
            pp = None
            if pattern != "":
                p = r'{0}'.format(pattern)
                pp = re.compile(p)
            #gen.log "writeDir: flag(%x)"%(d.fFlag)
            if d.fFlag & 0x02 == 0x02:
                # Check if a clean directory.
                #try:
                #    if len(os.listdir(output)) > 0:
                #        sys.stderr.write("The target directory is not empty\n")
                #        return E_FAILURE
                #except(OSError):
                #    sys.stderr.write("can't access dirs({0})\n".format(p))
                #    return E_FAILURE
                self.writeDir_r(output, d, pp, r, all_type)
                return E_SUCCESS
            else:
                return self.writeFile(d, output+path, all_type)
        else:
            return E_FAILURE