Python re.S Examples

def get_module_source_metadata(cls, module_source, full_line_map=False):
        source_map = re.search(
            r"__M_BEGIN_METADATA(.+?)__M_END_METADATA", module_source, re.S
        ).group(1)
        source_map = json.loads(source_map)
        source_map["line_map"] = dict(
            (int(k), int(v)) for k, v in source_map["line_map"].items()
        )
        if full_line_map:
            f_line_map = source_map["full_line_map"] = []
            line_map = source_map["line_map"]

            curr_templ_line = 1
            for mod_line in range(1, max(line_map)):
                if mod_line in line_map:
                    curr_templ_line = line_map[mod_line]
                f_line_map.append(curr_templ_line)
        return source_map 

def __init__(self, code, **exception_kwargs):
        self.codeargs = []
        self.args = []
        self.declared_identifiers = set()
        self.undeclared_identifiers = set()
        if isinstance(code, compat.string_types):
            if re.match(r"\S", code) and not re.match(r",\s*$", code):
                # if theres text and no trailing comma, insure its parsed
                # as a tuple by adding a trailing comma
                code += ","
            expr = pyparser.parse(code, "exec", **exception_kwargs)
        else:
            expr = code

        f = pyparser.FindTuple(self, PythonCode, **exception_kwargs)
        f.visit(expr) 

def ipinfo(host):
    out = []
    if not re.search(r'\d+\.\d+\.\d+\.\d+', host):
        req = Requests()
        # noinspection PyBroadException
        try:
            r = req.get('https://viewdns.info/iphistory/?domain={}'.format(host))
            result = re.findall(r'(?<=<tr><td>)\d+\.\d+\.\d+\.\d+(?=</td><td>)', r.text, re.S | re.I)
            if result:
                for i in result:
                    if iscdn(i):
                        out.append(i)
        except Exception:
            pass

    return out 

def get_azlyrics(url):
    az_html = get_az_html(url)
    if isinstance(az_html, tuple):
        return az_html[0]

    az_regex = re.compile(r'<!-- Usage of azlyrics.com content by any third-party lyrics provider is prohibited by our licensing agreement. Sorry about that. -->(.*)<!-- MxM banner -->', re.S)

    ly = az_regex.search(az_html)
    if ly == None:
        # Az lyrics not found
        return 'Azlyrics missing...'

    rep = {'&quot;': '\"', '&amp;': '&', '\r' : ''}

    ly = re.sub(r'<[/]?\w*?>', '', ly.group(1)).strip()  
    # ly = ly.replace('&quot;', '\"').replace('&amp;', '&')
    # regex = re.compile('|'.join(substrings))
    ly = re.sub('|'.join(rep.keys()), lambda match: rep[match.group(0)], ly)
    lyrics_lines = ly.split('\n')

    return lyrics_lines 

def compile_rules(environment):
    """Compiles all the rules from the environment into a list of rules."""
    e = re.escape
    rules = [
        (len(environment.comment_start_string), 'comment',
         e(environment.comment_start_string)),
        (len(environment.block_start_string), 'block',
         e(environment.block_start_string)),
        (len(environment.variable_start_string), 'variable',
         e(environment.variable_start_string))
    ]

    if environment.line_statement_prefix is not None:
        rules.append((len(environment.line_statement_prefix), 'linestatement',
                      r'^[ \t\v]*' + e(environment.line_statement_prefix)))
    if environment.line_comment_prefix is not None:
        rules.append((len(environment.line_comment_prefix), 'linecomment',
                      r'(?:^|(?<=\S))[^\S\r\n]*' +
                      e(environment.line_comment_prefix)))

    return [x[1:] for x in sorted(rules, reverse=True)] 

def split_arg_string(string):
    """Given an argument string this attempts to split it into small parts."""
    rv = []
    for match in re.finditer(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
                             r'|"([^"\\]*(?:\\.[^"\\]*)*)"'
                             r'|\S+)\s*', string, re.S):
        arg = match.group().strip()
        if arg[:1] == arg[-1:] and arg[:1] in '"\'':
            arg = arg[1:-1].encode('ascii', 'backslashreplace') \
                .decode('unicode-escape')
        try:
            arg = type(string)(arg)
        except UnicodeError:
            pass
        rv.append(arg)
    return rv 

def compile_rules(environment):
    """Compiles all the rules from the environment into a list of rules."""
    e = re.escape
    rules = [
        (len(environment.comment_start_string), 'comment',
         e(environment.comment_start_string)),
        (len(environment.block_start_string), 'block',
         e(environment.block_start_string)),
        (len(environment.variable_start_string), 'variable',
         e(environment.variable_start_string))
    ]

    if environment.line_statement_prefix is not None:
        rules.append((len(environment.line_statement_prefix), 'linestatement',
                      r'^[ \t\v]*' + e(environment.line_statement_prefix)))
    if environment.line_comment_prefix is not None:
        rules.append((len(environment.line_comment_prefix), 'linecomment',
                      r'(?:^|(?<=\S))[^\S\r\n]*' +
                      e(environment.line_comment_prefix)))

    return [x[1:] for x in sorted(rules, reverse=True)] 

def test_simple_usage(self):
        test_loop = asyncio.get_event_loop()
        console = SharedConsole(interval=0.0)

        async def add_lines():
            console.print("one")
            console.print("two")
            console.print("3")
            try:
                1 + "e"
            except Exception as e:
                console.print_error(e)
                console.print_error(e, sys.exc_info()[2])
            await asyncio.sleep(0.2)
            await console.stop()

        with catch_output() as (stdout, stderr):
            adder = asyncio.ensure_future(add_lines())
            displayer = asyncio.ensure_future(console.display())
            test_loop.run_until_complete(asyncio.gather(adder, displayer))

        output = stdout.read()
        test_loop.close()
        self.assertTrue(re.match(OUTPUT, output, re.S | re.M) is not None, output) 

def versions_from_file(filename):
    """Try to determine the version from _version.py if present."""
    try:
        with open(filename) as f:
            contents = f.read()
    except EnvironmentError:
        raise NotThisMethod("unable to read _version.py")
    mo = re.search(
        r"version_json = '''\n(.*)'''  # END VERSION_JSON", contents, re.M | re.S
    )
    if not mo:
        mo = re.search(
            r"version_json = '''\r\n(.*)'''  # END VERSION_JSON", contents, re.M | re.S
        )
    if not mo:
        raise NotThisMethod("no version_json in _version.py")
    return json.loads(mo.group(1)) 

def getmatch(self, haystack):
        if not isinstance(haystack, basestring):
            return None
        flags = 0
        if self.flags is not None:
            if "i" in self.flags or "I" in self.flags:
                flags |= re.I
            if "l" in self.flags or "L" in self.flags:
                flags |= re.L
            if "m" in self.flags or "M" in self.flags:
                flags |= re.M
            if "s" in self.flags or "S" in self.flags:
                flags |= re.S
            if "u" in self.flags or "U" in self.flags:
                flags |= re.U
            if "x" in self.flags or "X" in self.flags:
                flags |= re.X
        if re.match(self.pattern, haystack, flags=flags) is None:
            return None
        elif self.to is None:
            return Match(haystack, haystack)
        else:
            return Match(haystack, re.sub(self.pattern, self.to, haystack, flags=flags)) 

def _getCrashDetails(self):
        ret = self.gdbOutput
        logging.info("get crash details, res: " + str(len(ret)))
        p = re.compile('#.*\(gdb\)', re.S)
        backtrace = re.search(p, ret, flags=0).group()
        #self.queue_stdout.put(backtrace)
        backtraceFrames = backtrace.split('\n')
        i = 0
        res = []
        while(i < len(backtraceFrames)):
            if backtraceFrames[i].startswith("#"):
                res.append(backtraceFrames[i].rstrip("\n\r"))
            i += 1

        serverCrashData = ServerCrashData(
            backtrace=res,
            analyzerOutput=ret,
            analyzerType="gdb"
        )
        gdbOutput = targetutils.getAsanOutput(self.config, self.pid)
        if gdbOutput is not None:
            serverCrashData.setAsan(gdbOutput)

        return serverCrashData 

def versions_from_file(filename):
    """Try to determine the version from _version.py if present."""
    try:
        with open(filename) as f:
            contents = f.read()
    except EnvironmentError:
        raise NotThisMethod("unable to read _version.py")
    mo = re.search(
        r"version_json = '''\n(.*)'''  # END VERSION_JSON", contents, re.M | re.S
    )
    if not mo:
        mo = re.search(
            r"version_json = '''\r\n(.*)'''  # END VERSION_JSON", contents, re.M | re.S
        )
    if not mo:
        raise NotThisMethod("no version_json in _version.py")
    return json.loads(mo.group(1)) 

def parse_page(self, response):
        pattern = re.compile('gp.insertPrx\((.*?)\)', re.S)
        items = re.findall(pattern, response.body.decode())
        for item in items:
            data = json.loads(item)
            #端口用的是十六进制
            port = data.get('PROXY_PORT')
            port = str(int(port, 16))

            proxy = Proxy()
            proxy.set_value(
                    ip = data.get('PROXY_IP'),
                    port = port,
                    country = data.get('PROXY_COUNTRY'),
                    anonymity = data.get('PROXY_TYPE'),
                    source = self.name,
            )

            self.add_proxy(proxy = proxy) 

def __init__(self, code, **exception_kwargs):
        m = re.match(r'^(\w+)(?:\s+(.*?))?:\s*(#|$)', code.strip(), re.S)
        if not m:
            raise exceptions.CompileException(
                "Fragment '%s' is not a partial control statement" %
                code, **exception_kwargs)
        if m.group(3):
            code = code[:m.start(3)]
        (keyword, expr) = m.group(1, 2)
        if keyword in ['for', 'if', 'while']:
            code = code + "pass"
        elif keyword == 'try':
            code = code + "pass\nexcept:pass"
        elif keyword == 'elif' or keyword == 'else':
            code = "if False:pass\n" + code + "pass"
        elif keyword == 'except':
            code = "try:pass\n" + code + "pass"
        elif keyword == 'with':
            code = code + "pass"
        else:
            raise exceptions.CompileException(
                "Unsupported control keyword: '%s'" %
                keyword, **exception_kwargs)
        super(PythonFragment, self).__init__(code, **exception_kwargs) 

def __init__(self, code, **exception_kwargs):
        self.codeargs = []
        self.args = []
        self.declared_identifiers = set()
        self.undeclared_identifiers = set()
        if isinstance(code, compat.string_types):
            if re.match(r"\S", code) and not re.match(r",\s*$", code):
                # if theres text and no trailing comma, insure its parsed
                # as a tuple by adding a trailing comma
                code += ","
            expr = pyparser.parse(code, "exec", **exception_kwargs)
        else:
            expr = code

        f = pyparser.FindTuple(self, PythonCode, **exception_kwargs)
        f.visit(expr) 

def compile_rules(environment):
    """Compiles all the rules from the environment into a list of rules."""
    e = re.escape
    rules = [
        (len(environment.comment_start_string), 'comment',
         e(environment.comment_start_string)),
        (len(environment.block_start_string), 'block',
         e(environment.block_start_string)),
        (len(environment.variable_start_string), 'variable',
         e(environment.variable_start_string))
    ]

    if environment.line_statement_prefix is not None:
        rules.append((len(environment.line_statement_prefix), 'linestatement',
                      r'^[ \t\v]*' + e(environment.line_statement_prefix)))
    if environment.line_comment_prefix is not None:
        rules.append((len(environment.line_comment_prefix), 'linecomment',
                      r'(?:^|(?<=\S))[^\S\r\n]*' +
                      e(environment.line_comment_prefix)))

    return [x[1:] for x in sorted(rules, reverse=True)] 

def compile_rules(environment):
    """Compiles all the rules from the environment into a list of rules."""
    e = re.escape
    rules = [
        (len(environment.comment_start_string), 'comment',
         e(environment.comment_start_string)),
        (len(environment.block_start_string), 'block',
         e(environment.block_start_string)),
        (len(environment.variable_start_string), 'variable',
         e(environment.variable_start_string))
    ]

    if environment.line_statement_prefix is not None:
        rules.append((len(environment.line_statement_prefix), 'linestatement',
                      r'^[ \t\v]*' + e(environment.line_statement_prefix)))
    if environment.line_comment_prefix is not None:
        rules.append((len(environment.line_comment_prefix), 'linecomment',
                      r'(?:^|(?<=\S))[^\S\r\n]*' +
                      e(environment.line_comment_prefix)))

    return [x[1:] for x in sorted(rules, reverse=True)] 

def get_module_source_metadata(cls, module_source, full_line_map=False):
        source_map = re.search(
            r"__M_BEGIN_METADATA(.+?)__M_END_METADATA", module_source, re.S
        ).group(1)
        source_map = json.loads(source_map)
        source_map["line_map"] = dict(
            (int(k), int(v)) for k, v in source_map["line_map"].items()
        )
        if full_line_map:
            f_line_map = source_map["full_line_map"] = []
            line_map = source_map["line_map"]

            curr_templ_line = 1
            for mod_line in range(1, max(line_map)):
                if mod_line in line_map:
                    curr_templ_line = line_map[mod_line]
                f_line_map.append(curr_templ_line)
        return source_map 

def parse_page(self, response):
        pattern = re.compile('<tr><td>(.*?)</td><td>(.*?)</td><td>(.*?)</td><td>(.*?)</td><td>(.*?)</td></tr>',
                             re.S)
        items = re.findall(pattern, response.body.decode())
        for i, item in enumerate(items):
            if i >= 1:
                proxy = Proxy()
                proxy.set_value(
                        ip = item[0],
                        port = item[1],
                        country = item[2],
                        anonymity = item[3],
                        source = self.name
                )

                self.add_proxy(proxy = proxy) 

def __init__(self, code, **exception_kwargs):
        self.codeargs = []
        self.args = []
        self.declared_identifiers = set()
        self.undeclared_identifiers = set()
        if isinstance(code, compat.string_types):
            if re.match(r"\S", code) and not re.match(r",\s*$", code):
                # if theres text and no trailing comma, insure its parsed
                # as a tuple by adding a trailing comma
                code += ","
            expr = pyparser.parse(code, "exec", **exception_kwargs)
        else:
            expr = code

        f = pyparser.FindTuple(self, PythonCode, **exception_kwargs)
        f.visit(expr) 

def extract_domains(self, resp):
        tbl_regex = re.compile('<a name="hostanchor"><\/a>Host Records.*?<table.*?>(.*?)</table>', re.S)
        link_regex = re.compile('<td class="col-md-4">(.*?)<br>', re.S)
        links = []
        try:
            results_tbl = tbl_regex.findall(resp)[0]
        except IndexError:
            results_tbl = ''
        links_list = link_regex.findall(results_tbl)
        links = list(set(links_list))
        for link in links:
            subdomain = link.strip()
            if not subdomain.endswith(self.domain):
                continue
            if subdomain and subdomain not in self.subdomains and subdomain != self.domain:
                self.subdomains.append(subdomain.strip())
        return links 

def parse_page(self, response):
        pattern = re.compile(
		'<tr><td>(.*?)</td><td>(.*?)</td><td>(.*?)</td><td.+?>(.*?)</td><td>(.*?)</td><td.+?>(.*?)</td><td.+?>(.*?)</td><td.+?>(.*?)</td></tr>',
                re.S)
        items = re.findall(pattern, response.body.decode())

        if items is not None:
            for item in items:
                proxy = Proxy()
                proxy.set_value(
                        ip = item[0],
                        port = item[1],
                        country = item[3],
                        anonymity = item[4],
                        source = self.name,
                )

                self.add_proxy(proxy) 

def movie(name):
    param = dict(searchword=name.encode('gb2312'))
    response = requests.post(url + 'search.asp', data=param, headers={'User-Agent': random()})
    response.encoding = 'GBK'
    data = response.text
    find, = re.findall(pattern=r'<div class="list mb">(.+)</div>', string=data, flags=re.S)
    message = re.findall(pattern='a href="(.+)" title="(.+)" class', string=find)
    return message 

def versions_from_file(filename):
    """Try to determine the version from _version.py if present."""
    try:
        with open(filename) as f:
            contents = f.read()
    except EnvironmentError:
        raise NotThisMethod("unable to read _version.py")
    mo = re.search(r"version_json = '''\n(.*)'''  # END VERSION_JSON",
                   contents, re.M | re.S)
    if not mo:
        raise NotThisMethod("no version_json in _version.py")
    return json.loads(mo.group(1)) 

def splitpasswd(user):
    """splitpasswd('user:passwd') -> 'user', 'passwd'."""
    global _passwdprog
    if _passwdprog is None:
        import re
        _passwdprog = re.compile('^([^:]*):(.*)$',re.S)

    match = _passwdprog.match(user)
    if match: return match.group(1, 2)
    return user, None

# splittag('/path#tag') --> '/path', 'tag' 

def match_end(self):
        match = self.match(r"\Z", re.S)
        if match:
            string = match.group()
            if string:
                return string
            else:
                return True
        else:
            return False 

def match_comment(self):
        """matches the multiline version of a comment"""
        match = self.match(r"<%doc>(.*?)</%doc>", re.S)
        if match:
            self.append_node(parsetree.Comment, match.group(1))
            return True
        else:
            return False 

def _parse_attributes(self, expressions, nonexpressions):
        undeclared_identifiers = set()
        self.parsed_attributes = {}
        for key in self.attributes:
            if key in expressions:
                expr = []
                for x in re.compile(r"(\${.+?})", re.S).split(
                    self.attributes[key]
                ):
                    m = re.compile(r"^\${(.+?)}$", re.S).match(x)
                    if m:
                        code = ast.PythonCode(
                            m.group(1).rstrip(), **self.exception_kwargs
                        )
                        # we aren't discarding "declared_identifiers" here,
                        # which we do so that list comprehension-declared
                        # variables aren't counted.   As yet can't find a
                        # condition that requires it here.
                        undeclared_identifiers = undeclared_identifiers.union(
                            code.undeclared_identifiers
                        )
                        expr.append("(%s)" % m.group(1))
                    else:
                        if x:
                            expr.append(repr(x))
                self.parsed_attributes[key] = " + ".join(expr) or repr("")
            elif key in nonexpressions:
                if re.search(r"\${.+?}", self.attributes[key]):
                    raise exceptions.CompileException(
                        "Attibute '%s' in tag '%s' does not allow embedded "
                        "expressions" % (key, self.keyword),
                        **self.exception_kwargs
                    )
                self.parsed_attributes[key] = repr(self.attributes[key])
            else:
                raise exceptions.CompileException(
                    "Invalid attribute for tag '%s': '%s'"
                    % (self.keyword, key),
                    **self.exception_kwargs
                )
        self.expression_undeclared_identifiers = undeclared_identifiers 

def versions_from_file(filename):
    """Try to determine the version from _version.py if present."""
    try:
        with open(filename) as f:
            contents = f.read()
    except EnvironmentError:
        raise NotThisMethod("unable to read _version.py")
    mo = re.search(r"version_json = '''\n(.*)'''  # END VERSION_JSON",
                   contents, re.M | re.S)
    if not mo:
        mo = re.search(r"version_json = '''\r\n(.*)'''  # END VERSION_JSON",
                       contents, re.M | re.S)
    if not mo:
        raise NotThisMethod("no version_json in _version.py")
    return json.loads(mo.group(1)) 

def checkVul(self,res,server_addr,index):
        p=re.findall(VER_SIG[index], res, re.S)
        if len(p)>0:
            result = str(server_addr[0])+':'+str(server_addr[1])+'------is------'+VUL[index]
            #print result
            return result
        else:
            pass
            #print str(server_addr[0])+':'+str(server_addr[1])+'------is not vul------'+VUL[index]