Python validators.domain() Examples

def VTSearch(value, uas, proxies):
    server = "www.virustotal.com"
    results = ""

    try:
        url = "https://" + server + "/en/domain/" + value + "/information/"
        s = requests.Session()
        r = s.get(url, verify=False, headers={'User-Agent': PickRandomUA(uas)}, proxies=proxies)
        if r.status_code != 200:
            print("[-] Something is going wrong (status code: {})".format(r.status_code))
            return [], []
        results += r.text
    except Exception as e:
        print(e)

    return GetHostnames(results, value) 

def test_bitcoin_based_network_definitions(network):
    assert isinstance(network.API, bool)
    assert isinstance(network.name, str)
    assert isinstance(network.symbols, tuple)
    assert isinstance(network().default_symbol, str)
    assert getattr(network, 'seeds') or getattr(network, 'nodes'), f'[{network.__name__}] no seeds and nodes'
    if network.nodes:
        assert isinstance(network.nodes, tuple)
        assert not network.seeds, f'{network}: use nodes or seeds, not both.'
        for node in network.nodes:
            assert ipaddress.ip_address(node)
    else:
        assert isinstance(network.seeds, tuple)
        for seed in network.seeds:
            assert domain(seed)
    assert isinstance(network.port, int)
    assert isinstance(network.blacklist_nodes, dict)
    assert isinstance(network.message_start, bytes)
    assert isinstance(network.base58_prefixes, dict)
    assert isinstance(network.source_code_url, str)
    if network.bitcoin_based and network.API:
        assert isinstance(network.api_url, str)
    if network.blockexplorer_tx:
        assert isinstance(network.blockexplorer_tx, str) 

def __init__(self, project_name, server_name_or_ip, static_url,
                 gunicorn_file, nginx_file, project_file, virtualenv_file):
        """
        :param project_name: Django project folder
        :param server_name_or_ip: Server ip or domain
        :param static_url: Django settings.py STATIC_URL address
        :param gunicorn_file : gunicorn file name
        :param nginx_file : nginx file name
        :param virtualenv_file : virtualenv file directory
        """

        self.project_name = project_name
        self.server_name_or_ip = server_name_or_ip
        self.static_url = static_url
        self.gunicorn_file = gunicorn_file
        self.nginx_file = nginx_file
        self.project_file = project_file
        self.virtualenv_file = virtualenv_file

        # package.json read.
        with open("{}/client/file/package.json".format(BASE_DIR)) as data_file:
            self.data = json.load(data_file) 

def validate_hostname(value):
    values = value.split(':')

    if len(values) != 2:
        return False

    return validate_domain(values[0]) and values[1].isdigit() 

def query_crtsh(provided_ioc, session):
    """Search crt.sh for the given domain."""
    if sys.argv[1] == "subdomain":
        provided_ioc = "%25.{}".format(provided_ioc)
    elif sys.argv[1] == "wildcard":
        provided_ioc = "%25{}".format(provided_ioc)

    base_url  = "https://crt.sh/?q={}&output=json"
    url       = base_url.format(provided_ioc)
    resp      = session.get(url, timeout=180)
    crt_dicts = []

    if resp.status_code == 200 and resp.content != "":
        content      = resp.content.decode("UTF-8")
        cert_history = json.loads("{}".format(content.replace("}{", "},{")))
        # cert_history = json.loads("[{}]".format(content.replace("}{", "},{")))

        for cert in cert_history:
            cert = commons.lower_keys(cert)
            crt_dicts.append(cert)
    else:
        provided_ioc = provided_ioc.replace("%25.", "").replace("%25", "")
        crt_dicts.append({"no data": provided_ioc})
    return crt_dicts 

def to_dict(self):
        self.intel_dict['indicator_type'] = self.indicator_type
        self.intel_dict['ipaddress'] = self.ipaddress
        self.intel_dict['domain'] = self.domain
        self.intel_dict['domain_usage'] = self.domain_usage
        self.intel_dict['description'] = self.description
        self.intel_dict['url'] = self.url
        self.intel_dict['md5_hash'] = self.md5_hash
        self.intel_dict['sha1_hash'] = self.sha1_hash
        self.intel_dict['sha256_hash'] = self.sha256_hash
        self.intel_dict['src_port'] = self.src_port
        self.intel_dict['dst_port'] = self.dst_port
        self.intel_dict['proxy'] = self.proxy
        self.intel_dict['source'] = self.source
        self.intel_dict['submission_timestamp'] = self.submission_timestamp
        self.intel_dict['feed_date'] = self.feed_date
        self.intel_dict['country'] = self.country
        self.intel_dict['region'] = self.region
        self.intel_dict['city'] = self.city
        self.intel_dict['whois'] = self.whois
        return self.intel_dict 

def validate_domain(self):
        """
        Use domain validator to confirm domain name. 
        :return: BOOL
        """
        try:
            val = domain(str(self.subdomain))
            if val:
                # domain is valid
                return True
            else:
                # domain validation failed
                return False
        except Exception as e:
            # TODO: add in logger class for errors
            return False 

def process_iocs(results):
    """Return data formatted for Splunk from crt.sh."""
    if results != None:
        provided_iocs = [y for x in results for y in x.values()]
    elif sys.argv[1] != "subdomain" and sys.argv[1] != "wildcard":
        if len(sys.argv) > 1:
            provided_iocs = sys.argv[1:]
    elif sys.argv[1] == "subdomain" or sys.argv[1] == "wildcard":
        if len(sys.argv) > 2:
            provided_iocs = sys.argv[2:]

    session      = commons.create_session()
    splunk_table = []

    for provided_ioc in set(provided_iocs):
        provided_ioc = commons.deobfuscate_string(provided_ioc)

        if validators.domain(provided_ioc) or validators.ipv4(provided_ioc):
            crt_dicts = query_crtsh(provided_ioc, session)
        else:
            splunk_table.append({"invalid": provided_ioc})
            continue

        for crt_dict in crt_dicts:
            splunk_table.append(crt_dict)

    session.close()
    return splunk_table 

def SpyseSearch(value, apikey, limit=100, proxies=None):
    server = "api.spyse.com"
    data = {}

    try:
        url = f'https://{server}/v2/data/domain/subdomain?limit={limit}&domain={value}'
        headers = {
            'accept': 'application/json',
            'Authorization': f'Bearer {apikey}',
        }
        s = requests.Session()
        r = s.get(url, verify=False, headers=headers, proxies=proxies)
        if DEBUG: print(r.text)
        if r.status_code != 200:
            print(f'[-] Something is going wrong - status code: {r.status_code}')
        else:
            data = json.loads(r.text)
    except (ValueError, TypeError):
        print("Unrecognised data returned by spyse api")
    except NameError as e:
        print("Error: Insufficient data passed to SpyseSearch")
        print(e)
    except Exception as e:
        print(e)

    return data 

def __init__(self, ioc, type, config, queues):
        self.config = config
        self.module_name = __name__.split(".")[-1]
        self.types = ["domain", "IPv4", "URL"]
        self.search_method = "Online"
        self.description = "Search domain in Lehigh feeds"
        self.author = "Conix"
        self.creation_date = "15-09-2016"
        self.type = type
        self.ioc = ioc

        self.search() 

def __init__(self, ioc, type, config, queues):
        self.config = config
        self.module_name = __name__.split(".")[-1]
        self.types = ["domain", "URL", "IPv4"]
        self.search_method = "Online"
        self.description = "Search domain in Openphish feeds"
        self.author = "Conix"
        self.creation_date = "15-09-2016"
        self.type = type
        self.ioc = ioc

        self.search() 

def __init__(self, args, modules):
        redis_host, redis_port, redis_password = init_redis()
        try:
            conn = redis.StrictRedis(host=redis_host, port=redis_port,
                                     password=redis_password)
        except:
            mod.display("MAIN",
                        message_type="FATAL_ERROR",
                        string="Cannot establish connection with Redis")
            sys.exit()

        global observable_list
        queues = [working_queue, request_queue]

        if args.file == "False":
            observable_list = args.observables
        else:
            observable_list = []
            for file in args.observables:
                with open(file, "r") as f1:
                    try:
                        observable_list = list(set(observable_list +
                                                   f1.read().strip().splitlines()))
                    except:
                        mod.display("MAIN",
                                    message_type="FATAL_ERROR",
                                    string="Something went wrong with the argument file")
                    finally:
                        f1.close()

        for argument in observable_list:
            type = self.checkType(argument.lower())
            if "split_observable" in config and config["split_observable"]:
                if type == "URL" or type == "domain":
                    self.extend_IOC(argument.lower(), observable_list)
            matching_list = Utils.gen_matching_type_modules_list(modules, type)
            cluster.add_cluster(argument.lower(), matching_list, dictname, conn)
            self.run(argument.lower(), type, matching_list, queues)
        print("Every IOCs have been enqueued, BTG is processing ...\n") 

def checkType(self, argument):
        """
            Identify observable type
        """
        if not argument or len(argument.strip()) == 0:
            return None
        elif argument[0] is '#':
            return None
        elif validators.url(argument):
            return "URL"
        elif validators.md5(argument):
            return "MD5"
        elif validators.sha1(argument):
            return "SHA1"
        elif validators.sha256(argument):
            return "SHA256"
        elif validators.sha512(argument):
            return "SHA512"
        elif validators.ipv4(argument):
            return "IPv4"
        elif validators.ipv6(argument):
            return "IPv6"
        elif validators.domain(argument):
            return "domain"
        else:
            return None 

def parse_args():
        """
            Define the arguments
        """
        parser = argparse.ArgumentParser(description='Observable to qualify')
        parser.add_argument('observables',
                            metavar='observable',
                            type=str, nargs='+',
                            help='Type: [URL,MD5,SHA1,SHA256,SHA512,IPv4,IPv6,domain] or a file containing one observable per line')
        # TODO
        # parser.add_argument("-d",
        #                     "--debug",
        #                     action="store_true",
        #                     help="Display debug informations")
        parser.add_argument("-o",
                            "--offline",
                            action="store_true",
                            help=("Set BTG in offline mode, meaning all modules"
                                  "described as online (i.e. VirusTotal) are desactivated"))
        parser.add_argument("-s",
                            "--silent",
                            action="store_true",
                            help="Disable MOTD")
        parser.add_argument("-e",
                            "--extend",
                            action="store_true",
                            help=("Enable observable extension, "
                                  "meaning BTG will try to find related observable, "
                                  "for instance: domain -> subdomains"))
        parser.add_argument("-j",
                            "--json",
                            action="store_true",
                            help="Asking for a JSON output to the given path, "
                                 "at variable json_folder in btg.cfg. "
                                 "Otherwise, default folder is /tmp/BTG/json")
        return parser.parse_args() 

def process_iocs(results):
    """Return data formatted for Splunk from GreyNoise."""
    if results != None:
        provided_iocs = [y for x in results for y in x.values()]
    else:
        provided_iocs = sys.argv[1:]

    splunk_table = []
    lookup_path  = "{}/lookups".format(app_home)
    open_file    = open("{}/greynoise_feed.csv".format(lookup_path), "r")
    data_feed    = open_file.read().splitlines()
    header       = data_feed[0].split(",")
    open_file.close()

    open_file = open("{}/greynoise_scanners.csv".format(lookup_path), "r")
    scanners  = set(open_file.read().splitlines()[1:])
    scanners  = [x.lower() for x in scanners]
    open_file.close()

    for provided_ioc in set(provided_iocs):
        provided_ioc = commons.deobfuscate_string(provided_ioc)

        if not validators.ipv4(provided_ioc) and \
           not validators.domain(provided_ioc) and \
           provided_ioc.lower() not in scanners:
           splunk_table.append({"invalid": provided_ioc})
           continue

        line_found = False

        for line in data_feed:
            if provided_ioc.lower() in line.lower():
                line_found   = True
                scanner_data = line.split(",")
                scanner_dict = OrderedDict(zip(header, scanner_data))
                scanner_dict = commons.lower_keys(scanner_dict)
                splunk_table.append(scanner_dict)

        if line_found == False:
            splunk_table.append({"no data": provided_ioc})
    return splunk_table 

def process_iocs(results):
    """Return data formatted for Splunk from Malshare."""
    if results != None:
        provided_iocs = [y for x in results for y in x.values()]
    else:
        provided_iocs = sys.argv[1:]

    session = commons.create_session()
    api_key = commons.get_apikey("malshare")
    splunk_table = []

    for provided_ioc in set(provided_iocs):
        provided_ioc = commons.deobfuscate_string(provided_ioc)
        provided_ioc = provided_ioc.lower()

        if validators.ipv4(provided_ioc) or validators.domain(provided_ioc) or \
            re.match("^[a-f\d]{32}$", provided_ioc) or re.match("^[a-f\d]{64}$", provided_ioc):
            pass
        else:
            splunk_table.append({"invalid": provided_ioc})
            continue

        ioc_dicts = query_malshare(provided_ioc, api_key, session)

        for ioc_dict in ioc_dicts:
            splunk_table.append(ioc_dict)

    session.close()
    return splunk_table 

def process_iocs(results):
    """Return data formatted for Splunk from ThreatCrowd."""
    if results != None:
        provided_iocs = [y for x in results for y in x.values()]
    else:
        provided_iocs = sys.argv[1:]

    session = commons.create_session()
    splunk_table = []

    for provided_ioc in set(provided_iocs):
        provided_ioc = commons.deobfuscate_string(provided_ioc)
        provided_ioc = provided_ioc.lower()

        if validators.ipv4(provided_ioc):
            ioc_type = "ip"
        elif validators.domain(provided_ioc):
            ioc_type = "domain"
        elif validators.email(provided_ioc):
            ioc_type = "email"
        elif re.match("^[a-f\d]{32}$", provided_ioc) or re.match("^[a-f\d]{64}$", provided_ioc):
            ioc_type = "resource"
        else:
            splunk_table.append({"invalid": provided_ioc})
            continue

        ioc_dicts = query_threatcrowd(provided_ioc, ioc_type, session)

        for ioc_dict in ioc_dicts:
            splunk_table.append(ioc_dict)

        if len(provided_iocs) > 1:
            sleep(10)

    session.close()
    return splunk_table 

def query_threatcrowd(provided_ioc, ioc_type, session):
    """Pivot off an IP or domain and return data as an dictonary."""
    ioc_dicts = []

    if ioc_type == "resource":
        resp = session.get(api.format("file", ioc_type, provided_ioc), timeout=180)
    else:
        resp = session.get(api.format(ioc_type, ioc_type, provided_ioc), timeout=180)

    if resp.status_code == 200 and "permalink" in resp.json().keys() and \
       provided_ioc in resp.json()["permalink"]:
        for key in resp.json().keys():
            if key == "votes" or key == "permalink" or key == "response_code":
                continue
            elif key in ("md5", "sha1"):
                value = resp.json()[key]
                ioc_dicts.append({key: value})
            elif key == "resolutions":
                for res in resp.json()[key]:
                    res = commons.lower_keys(res)
                    ioc_dicts.append(res)
            else:
                for value in resp.json()[key]:
                    key = commons.lower_keys(key)
                    ioc_dicts.append({key: value})
    else:
        ioc_dicts.append({"no data": provided_ioc})
    return ioc_dicts 

def process_iocs(results):
    """Return data formatted for Splunk from urlscan.io."""
    if results != None:
        provided_iocs = [y for x in results for y in x.values()]
    elif sys.argv[1] in usfs.queries.keys():
        if len(sys.argv[1:]) < 3:
            return [{"error": "3 positional args needed. {} given.".format(str(len(sys.argv[1:])))}]
        provided_iocs = sys.argv[3:]
    else:
        provided_iocs = sys.argv[1:]

    session      = commons.create_session()
    splunk_table = []        

    for provided_ioc in set(provided_iocs):
        provided_ioc = commons.deobfuscate_string(provided_ioc)

        if provided_ioc == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855":
            splunk_table.append({"no data": provided_ioc})
            continue

        if provided_ioc.lower() in usfs.extensions.keys():
            ioc_dicts = query_urlscan_file(session, provided_ioc)
        elif validators.domain(provided_ioc) or validators.ipv4(provided_ioc) or \
           validators.sha256(provided_ioc) or "certstream-suspicious" in provided_ioc:
            ioc_dicts = query_urlscan(session, provided_ioc)
        else:
            splunk_table.append({"invalid": provided_ioc})
            continue

        for ioc_dict in ioc_dicts:
            if "ip" not in ioc_dict:
                splunk_table.append({"no data": provided_ioc})
                continue

            splunk_table.append(ioc_dict)

    session.close()
    return splunk_table 

def _create_payload(self, name: str, zone: str, address: str, address_type: str) -> dict:
        payload = {
            "name": name,
            "zone": zone
        }
        if address_type == "fqdn":
            payload["domain"] = address
            payload["dns_ttl"] = 0
        elif address_type == "cidr":
            payload["network"] = self._generate_subnet_netmask(address)
        else:
            payload["host"] = {
                "ip": address
            }

        object_type = address_type
        if address_type == "cidr":
            object_type = "ipv4"

        return {
            "address_objects": [{
                object_type: payload
            }]
        } 

def create(team):
    """
    Create a participant or host team.
    """
    """
    Invoked by running `evalai teams create`
    """
    is_host = False
    if team not in ("host", "participant"):
        echo(
            "Sorry, wrong argument. Please choose either "
            "{} or {}.".format(
                style("participant", bold=True, fg="yellow"),
                style("host", bold=True, fg="yellow"),
            )
        )
        sys.exit(1)

    team_name = click.prompt("Enter team name", type=str)
    if click.confirm(
        "Please confirm the team name - {}".format(team_name), abort=True
    ):
        team_url = ""
        if click.confirm("Do you want to enter the Team URL"):
            team_url = click.prompt("Team URL", type=str)
            while not (
                validators.url(team_url) or validators.domain(team_url)
            ):
                echo(
                    style(
                        "Sorry, please enter a valid link.",
                        bold=True,
                        fg="red",
                    )
                )
                team_url = click.prompt("Team URL", type=str)

        is_host = team == "host"
        create_team(team_name, team_url, is_host) 

def _determine_address_type(address: str) -> str:
        if validators.ipv4(address):
            return "ipv4"
        if validators.ipv6(address):
            return "ipv6"
        if validators.domain(address):
            return "fqdn"
        if re.search('/', address):
            return "cidr"
        raise PluginException(cause="Unknown address type provided.",
                              assistance=f"{address} is not one of the following: IPv4, IPv6, CIDR or domain name.") 

def get_data_type(indicator):
        if validators.ipv4(indicator) or validators.ipv6(indicator):
            return "IP"
        elif validators.url(indicator):
            return "URL"
        elif validators.domain(indicator):
            return "DOMAIN"
        elif validators.sha1(indicator):
            return "FILE_SHA1"

        raise PluginException(
            cause="Invalid indicator input provided.",
            assistance="Supported indicators are IP, URL, domain and SHA1 hash."
        ) 

def test_account_data(account):
    assert isinstance(account, Account)
    assert account.uid > 0
    assert account.username != ''
    assert validators.email(account.email)

    username, domain = account.email.split('@')
    assert account.username == username
    assert validators.domain(domain) 

def validate_url(url):
    """
    Validates the URL
    :param url:
    :return:
    """
    if validators.url(url):
        return url
    elif validators.domain(url):
        return "http://{}".format(url)
    return "" 

def test_json_is_valid(self):
        with open("../world_universities_and_domains.json", encoding='utf-8') as json_file:
            valid_json = json.load(json_file)
        for university in valid_json:
            self.assertIn("name", university)
            self.assertIn("domains", university)
            self.assertIsInstance(university["domains"], list)
            for domain in university["domains"]:
                self.assertTrue(validators.domain(domain))
            self.assertIn("web_pages", university)
            self.assertIn("alpha_two_code", university)
            self.assertIn("state-province", university)
            self.assertIn("country", university) 

def check_valid_url(url):

    """

    :type url: string

    """
    return validators.domain(url) 

def WhoisQuery(value):
    whoisData = collections.OrderedDict()
    whoisData["name"] = ["-", "Name:"]
    whoisData["org"] = ["-", "Organization:"]
    whoisData["address"] = ["-", "Address:"]
    whoisData["city"] = ["-", "City:"]
    whoisData["zipcode"] = ["-", "Zip code:"]
    whoisData["country"] = ["-", "Country:"]
    whoisData["emails"] = ["-", "Emails:"]
    whoisData["registrar"] = ["-", "Registrar:"]
    whoisData["whois_server"] = ["-", "Whois Server:"]
    whoisData["updated_date"] = ["-", "Updated Date:"]
    whoisData["expiration_date"] = ["-", "Expiration Date:"]
    whoisData["creation_date"] = ["-", "Creation Date:"]
    whoisData["name_servers"] = ["-", "Name Servers:"]

    domain = whois.whois(value)

    for rec in whoisData:
        if domain[rec]:
            if isinstance(domain[rec], list):
                if rec == 'name_servers':
                    whoisData[rec][0] = []
                    for val in domain[rec]:
                        whoisData[rec][0].append(val + ":" + VerifyHostname(val))
                else:
                    whoisData[rec][0] = []
                    for val in domain[rec]:
                        whoisData[rec][0].append(val)
            else:
                whoisData[rec][0] = str(domain[rec])

    return whoisData 

def checkDomain(value):
    domain_checked = validators.domain(value)
    if not domain_checked:
        raise argparse.ArgumentTypeError('Invalid {} domain.'.format(value))
    return value

################################################################### 

def validate_domain(self, domain=None):
        domain = domain or self.domain
        return domain_validator(domain, raise_errors=True)