Python mechanize.Browser() Examples

def mech_ops():
	br = mechanize.Browser()
	br.set_handle_robots(False)
	br.addheaders = [('user-agent', select_UA()), ('accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')]

	try:
		response = br.open("http://public-dns.info//nameservers.txt")
	except Exception as e:
		print "\n[" + t.red("!") + "]Critical, could not open public-dns.info"
		print "[" + t.green("+") + "]The following status code was recieved:"
		print "\n %s" % (e)
		sys.exit(1)
	
	result = response.read()
	proc = result.rstrip().split('\n')
		
	return proc


# If args, read list, else fetch 

def Key_Ops_HTTP():
	br = mechanize.Browser()
	br.set_handle_robots(False)
	br.addheaders = [('user-agent', '  Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.3) Gecko/20100423 Ubuntu/10.04 (lucid) Firefox/3.6.3'),
	('accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')]

	try:
		br.open("http://127.0.0.1:8000/admin/login/?next=/admin/")
	except Exception as e:
		# print "[!]Critical, could not open page."
		# print "\n %s" % (e)
		pass
		
	br.form = list(br.forms())[0]
	br["username"] = "RansomBot"
	br["password"] = "prettyflypassw0rd"

	br.submit()
	# If log in was succesful retrieve key and post ID
	###---@---###
######################################---NOT IMPLEMENTED---###################################### 

def wiglePrint(username, password, netid):
    browser = mechanize.Browser()
    browser.open('https://wigle.net')
    #构造请求数据
    reqData = urllib.urlencode({'credential_0':username, 'credential_1':password})
    #请求登录
    browser.open('https://wigle.net/gps/gps/main/login', reqData)
    params = {}
    params['netid'] = netid
    reqParams = urllib.urlencode(params)
    respURL = 'https://wigle.net/gps/gps/main/confirmquery/'
    resp = browser.open(respURL, reqParams).read()
    print str(resp)
    mapLat = 'N/A'
    mapLon = 'N/A'
    rLat = re.findall(search_lat, resp)
    if rLat:
        mapLat = rLat[0].split('&')[0].split('=')[1]
    rLon = re.findall(search_lon, resp)
    if rLon:
        mapLon = rLon[0].split
    print '[-] Lat: ' + mapLat + ', Lon: ' + mapLon 

def Findsubmit(link):
    global reqlist
    try:
        br = mechanize.Browser()  # initiating the browser
        br._factory.is_html = True
        br.addheaders = [('User-agent',
                          'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
        br.open(str(link), timeout=15)
        if br.forms():
            params = list(br.forms())
            for par in params:
                for p in par.controls:
                    ps = str(p)
                    # print p.name
                    if 'TextControl' in ps:
                        param = str(p.name)
                        reqstr = par.action + par.method + param
                        if reqstr not in reqlist:
                            reqlist.append(reqstr)
                            testxss(par.action, par.method, param)
    except Exception, e:
        print e
        pass 

def cli(scores, transfers):
	browser = mechanize.Browser()
	browser.set_handle_robots(False)	#Allows everything to be written
	cookies = mechanize.CookieJar()
	browser.set_cookiejar(cookies)
	browser.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.517.41 Safari/534.7')]
	browser.set_handle_refresh(False)	#Sometimes hangs without this
	if(scores):		#Called after score option is called
		soup = find_soup(browser,'http://www.goal.com/en-in/live-scores')	#Gets HTML of entire page
		score_box = soup.find_all('div',attrs={'class':'match-main-data'})	#Navigating to where the score is available in the page
		click.echo("\nThe scores of all matches being played currently is displayed below:")
		click.echo("--------------------------------------------------------------------")
		for i in score_box:		#To get the score of all live matches and recently done matches
			print i.text
			click.echo("--------------------------------------------------------------------")
		click.echo("\n\nNOTE: ALL THE MATCH TIMINGS ARE IN GMT\n\n")

	if(transfers):
		soup = find_soup(browser,'http://www.goal.com/en-us/transfer-rumours/1')	#Gets HTML of entire page
		rumours = soup.select(".transfer-card__desc p")
		click.echo("\nThe latest Transfer news & rumours are displayed below:")
		click.echo("--------------------------------------------------------------------")
		for i in rumours:
			print("->"+i.text)
			click.echo("--------------------------------------------------------------------") 

def cli(google,wiki):
	browser = mechanize.Browser()
	browser.set_handle_robots(False)	#Allows everything to be written
	cookies = mechanize.CookieJar()
	browser.set_cookiejar(cookies)
	browser.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.517.41 Safari/534.7')]
	browser.set_handle_refresh(False)	#Sometimes hangs without this
	if(google):
		query = raw_input("Enter the topic you want to search about: ")
		for link in search(query, tld="co.in", num=10, stop=1, pause=2):
			print link
	if(wiki):
		wiki_topic = raw_input('Enter the topic you want to read about: ')
		result = wikipedia.page(title=wiki_topic,auto_suggest=True,redirect=True, preload=False)
		wiki_content = result.content
		print wiki_content 

def wiglePrint(username, password, netid):
    browser = mechanize.Browser()
    browser.open('https://wigle.net')
    #构造请求数据
    reqData = urllib.urlencode({'credential_0':username, 'credential_1':password})
    #请求登录
    browser.open('https://wigle.net/gps/gps/main/login', reqData)
    params = {}
    params['netid'] = netid
    reqParams = urllib.urlencode(params)
    respURL = 'https://wigle.net/gps/gps/main/confirmquery/'
    resp = browser.open(respURL, reqParams).read()
    print str(resp)
    mapLat = 'N/A'
    mapLon = 'N/A'
    rLat = re.findall(search_lat, resp)
    if rLat:
        mapLat = rLat[0].split('&')[0].split('=')[1]
    rLon = re.findall(search_lon, resp)
    if rLon:
        mapLon = rLon[0].split
    print '[-] Lat: ' + mapLat + ', Lon: ' + mapLon

#打印网络信息 

def brute_server(server, user, password_list):
  br = mechanize.Browser()
  br.set_handle_equiv(False)
  br.set_handle_redirect(False)
  br.set_handle_referer(False)
  br.set_handle_robots(False)
  br.open(server)
  pass_list = open(password_list)
  for x in pass_list.readlines():
    try:  
      time.sleep(3)
      br.select_form( nr = 0 )
      br.form[form_username] = user
      br.form[form_password] = ''.join(x)
      print "Checking ",''.join(x)
      request = br.click(name=form_submit)
      response = br.open(request)
      if response.code == 200:
        print "No dice..."
    except HTTPError, e:
      if e.code == 302: # Redirect is our success case
	    print "Correct password is ",''.join(x) 

def sqli(target, sqli_list):
  
  sqli_list = open(sqli_list)
  for sqli in sqli_list.readlines():
    sqli = sqli.rstrip()
    br = mechanize.Browser()
    br.set_handle_equiv(False)
    br.set_handle_redirect(False)
    br.set_handle_referer(False)
    br.set_handle_robots(False)
    br.open(target)
    br.select_form(nr=0)
    time.sleep(2)
    for field in br.form.controls:
      if field.type == "text":
        br.form[field.name] = str(sqli)
    print br.form#"! injecting {0}, in the form {1}, on the page: {2}".format(str(sqli), str(br.form.name), str(target))
    request = br.click(type="submit")
    response = br.open(request)
    if response.code == 200:
      print "No dice... 200 OK response"
    if response.code == 500:
      print "500 Internal Error, potential SQL with {0}".format(str(field)) 

def run(self):
    while True:
      # Get target host
      target = self.host_queue.get()
      try:
        # Build URL
        url = "http://{0}/{1}".format(target, self.page)
        # Launch browser scraper
        br = mechanize.Browser()
        br.set_handle_equiv(False)
        br.set_handle_redirect(True)
        br.set_handle_referer(False)
        br.set_handle_robots(False)
        scraped = br.open(url)
        saved_name = str(target)+"."+str(self.page)
        with open(os.path.join(self.save, saved_name), 'wb') as temp_file:
          temp_file.write(str(scraped.read()))
        print "Successfully scraped {}".format(url)
      except:
        print "Error with {}".format(target)
      # Complete task in queue
      self.host_queue.task_done() 

def login(self,ids):
		try:
			global br
			br = mechanize.Browser()
			br.set_handle_equiv(True)
			br.set_handle_gzip(True)
			br.set_handle_redirect(True)
			br.set_handle_referer(True)
			br.set_handle_robots(False)
			br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
			br.addheaders = [('User-Agent','Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36')]
			br.open(self.u.format('/login'))
			br.select_form(nr=0)
			br.form['email']=ids.split('|')[0]
			br.form['pass']=ids.split('|')[1]
			sub=br.submit().read()
			if 'logout.php' in str(res) or 'mbasic_logout_button' in str(res):
				self.ganti(ids)
			else:
				print(f'[\033[91mFailed\033[0m] {ids}')
		except: pass 

def get_browser(self):
        '''Returns a mechanize.Browser object configured with the framework's global options.'''
        br = mechanize.Browser()
        # set the user-agent header
        br.addheaders = [('User-agent', self._global_options['user-agent'])]
        # set debug options
        if self._global_options['verbosity'] >= 2:
            br.set_debug_http(True)
            br.set_debug_redirects(True)
            br.set_debug_responses(True)
        # set proxy
        if self._global_options['proxy']:
            br.set_proxies({'http': self._global_options['proxy'], 'https': self._global_options['proxy']})
        # additional settings
        br.set_handle_robots(False)
        # set timeout
        socket.setdefaulttimeout(self._global_options['timeout'])
        return br 

def getBrowers():
    br = mechanize.Browser(history=NoHistory())
    #options
    br.set_handle_equiv(True)
    #br.set_handle_gzip(True)
    br.set_handle_redirect(True)
    br.set_handle_referer(True)
    br.set_handle_robots(False)
    cj = cookielib.LWPCookieJar()  
    br.set_cookiejar(cj)##关联cookies  
    br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
    br.set_debug_http(False)
    br.set_debug_redirects(False)
    br.set_debug_responses(False)
    br.addheaders = [("User-agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36")]
    return br 

def __init__(self, events_handler, rcs=1, firstevents=1, spid='', random_id=None, topics=[], lang='en', event_delay=3):
        self.events_handler = events_handler
        self.rcs = rcs
        self.firstevents = firstevents
        self.spid = spid
        self.topics = topics
        self.lang = lang
        self.event_delay = event_delay
        self.random_id = random_id or self._randID(9)

        self.connected = False

        self.server = random.choice(self.SERVER_LIST)
        self.client_id = None
        self.connected = False
        self.browser = mechanize.Browser()
        self.browser.addheaders = []

        # Call additional setup
        self.events_handler._setup(self) 

def aserver():
	gw = os.popen("ip -4 route show default").read().split()
	s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
	s.connect((gw[2], 0))
	ipaddr = s.getsockname()[0]
	print Y+"Starting Server\033[0m..."
	os.system("service apache2 start")
	br = raw_input(G+"Browser: \033[0m")
	a = '{0} {1}'.format(br,ipaddr)
	os.system(a)
	stop = raw_input("hit enter to stop server: ")
	print Y+"Stopping Server\033[0m..."
	os.system("service apache2 stop")
#-----------------#
#APACHE SERVER END#


#RUN A FILE START#
#----------------# 

def _init_browser(self):
        """
        Ovveride this method with the appropriate way to prepare a logged in
        browser.
        """
        self.browser = mechanize.Browser()
        self.browser.set_handle_robots(False)
        self.browser.open(self.server_url + "/youraccount/login")
        self.browser.select_form(nr=0)
        try:
            self.browser['nickname'] = self.user
            self.browser['password'] = self.password
        except:
            self.browser['p_un'] = self.user
            self.browser['p_pw'] = self.password
        # Set login_method to be writable
        self.browser.form.find_control('login_method').readonly = False
        self.browser['login_method'] = self.login_method
        self.browser.submit() 

def retDomains(ip):
    #   find parked domains by ip from Hurricane Electric, can use other source
    #   may be duplicate with get_parked(data)
    domains = []
    try:
        url = "http://bgp.he.net/ip/" + ip + "#_dns"
        userAgent = [('User-agent','Mozilla/5.0 (X11; U; '+\
                      'Linux 2.4.2-2 i586; en-US; m18) Gecko/20010131 Netscape6/6.01')]
        browser = mechanize.Browser()
        browser.addheaders = userAgent
        page = browser.open(url)
        html = page.read()
        link_finder = re.compile('href="(.*?)"')
        links = link_finder.findall(html)
        for i in range (0, len(links)):
          if links[i].find('/dns/') == 0:
              domains.append(links[i][5:])
        return domains
    except:
        return domains 

def __init__(self):
		self.br = mechanize.Browser()
		self.br.set_handle_equiv(True)
		self.br.set_handle_gzip(True)
		self.br.set_handle_redirect(True)
		self.br.set_handle_referer(True)
		self.br.set_handle_robots(False)
		self.br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
		self.br.addheaders =[('Connection','keep-alive'),
		('Pragma','no-cache'),
		('Cache-Control','no-cache'),
		('Origin','http://sms.payuterus.biz'),
		('Upgrade-Insecure-Requests','1'),
		('Content-Type','application/x-www-form-urlencoded'),
		('User-Agent','Mozilla/5.0 (Linux; Android 7.1.2; Redmi 4X Build/N2G47H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.91 Mobile Safari/537.36'),
		('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3'),
		('Referer','http://sms.payuterus.biz/alpha/'),
		('Accept-Encoding','gzip, deflate'),
		('Accept-Language','id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7'),
		('Cookie','_ga=GA1.2.131924726.1560439960; PHPSESSID=jjrqqaakmfcgfgbtjt8tve5595; _gid=GA1.2.1969561921.1561024035; _gat=1')
		]
		self.u='http://sms.payuterus.biz/alpha/'
		self.tkmain() 

def HTTPcode(self):
		try:
			if self.agent == True:
				br = Browser()

				UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0"
				header = {"User-Agent" : UserAgent}
				br.set_handle_robots(False)
				br.addheaders = [("User-agent", "Fifefox")]
				
				resp = br.open(self.target).code

			else:
				resp = u.urlopen(self.target).getcode()
	
			return(resp)
		except (u.HTTPError, u.URLError):
			return(404) 

def redirect(self):
		try:
			if self.agent == True:
				br = Browser()

				UserAgent = "Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0"
				header = {"User-Agent" : UserAgent}
				br.set_handle_robots(False)
				br.addheaders = [("User-agent", "Fifefox")]
				
				remote_url = br.open(self.target).geturl()

			else:
				remote_url = u.urlopen(self.target).geturl()

			return(remote_url)
		except Exception as e:
			print(e) 

def __init__(self):
		#install browser
		self.br = mechanize.Browser()
		self.br.set_handle_equiv(True)
		self.br.set_handle_gzip(True)
		self.br.set_handle_redirect(True)
		self.br.set_handle_referer(True)
		self.br.set_handle_robots(False)
		self.br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
		self.br.addheaders =[('Connection','keep-alive'),
		('Pragma','no-cache'),
		('Cache-Control','no-cache'),
		('Origin','http://sms.payuterus.biz'),
		('Upgrade-Insecure-Requests','1'),
		('Content-Type','application/x-www-form-urlencoded'),
		('User-Agent','Opera/9.80 (Android; Opera Mini/8.0.1807/36.1609; U; en) Presto/2.12.423 Version/12.16'),
		('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3'),
		('Referer','http://sms.payuterus.biz/alpha/'),
		('Accept-Encoding','gzip, deflate'),
		('Accept-Language','id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7'),
		('Cookie','_ga=GA1.2.131924726.1560439960; PHPSESSID=jjrqqaakmfcgfgbtjt8tve5595; _gid=GA1.2.1969561921.1561024035; _gat=1')
		]
		self.u='http://sms.payuterus.biz/alpha/'
		self.banner() 

def browser(url):
    import mechanize
    
    # Utilizamos Browser mechanize para saltar problemas con la busqueda en bing
    br = mechanize.Browser()
    # Browser options
    br.set_handle_equiv(False)
    br.set_handle_gzip(True)
    br.set_handle_redirect(True)
    br.set_handle_referer(False)
    br.set_handle_robots(False)
    # Follows refresh 0 but not hangs on refresh > 0
    br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
    # Want debugging messages?
    #br.set_debug_http(True)
    #br.set_debug_redirects(True)
    #br.set_debug_responses(True)
    
    # User-Agent (this is cheating, ok?)
    br.addheaders = [('User-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/600.7.12 (KHTML, like Gecko) Version/7.1.7 Safari/537.85.16')]
    #br.addheaders =[('Cookie','SRCHD=AF=QBRE; domain=.bing.com; expires=25 de febrero de 2018 13:00:28 GMT+1; MUIDB=3B942052D204686335322894D3086911; domain=www.bing.com;expires=24 de febrero de 2018 13:00:28 GMT+1')]
    # Open some site, let's pick a random one, the first that pops in mind
    r = br.open(url)
    response = r.read()
    print response
    if "img,divreturn" in response:
        r = br.open("http://ssl-proxy.my-addr.org/myaddrproxy.php/"+url)
        print "prooooxy"
        response = r.read()
    
    return response 

def browser(url):
    import mechanize
    
    # Utilizamos Browser mechanize para saltar problemas con la busqueda en bing
    br = mechanize.Browser()
    # Browser options
    br.set_handle_equiv(False)
    br.set_handle_gzip(True)
    br.set_handle_redirect(True)
    br.set_handle_referer(False)
    br.set_handle_robots(False)
    # Follows refresh 0 but not hangs on refresh > 0
    br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
    # Want debugging messages?
    #br.set_debug_http(True)
    #br.set_debug_redirects(True)
    #br.set_debug_responses(True)
    
    # User-Agent (this is cheating, ok?)
    #br.addheaders = [('User-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/600.7.12 (KHTML, like Gecko) Version/7.1.7 Safari/537.85.16')]
    #br.addheaders =[('Cookie','SRCHD=AF=QBRE; domain=.bing.com; expires=25 de febrero de 2018 13:00:28 GMT+1; MUIDB=3B942052D204686335322894D3086911; domain=www.bing.com;expires=24 de febrero de 2018 13:00:28 GMT+1')]
    # Open some site, let's pick a random one, the first that pops in mind
    r = br.open(url)
    response = r.read()
    print response
    if "img,divreturn" in response:
        r = br.open("http://ssl-proxy.my-addr.org/myaddrproxy.php/"+url)
        print "prooooxy"
        response = r.read()
    
    return response 

def new_browser(self, cookiejar = None, url = None):
        browser = mechanize.Browser()
        if cookiejar != None:
            browser.set_cookiejar(self.cookiejar)
        browser.set_handle_robots(False)
        if url != None:
            browser.open(url)
        return browser 

def setup_browser(self):

        # PEP 476: verify HTTPS certificates by default (implemented from Python 2.7.9)
        # https://bugs.python.org/issue22417
        if sys.hexversion >= 0x02070900:
            import ssl
            ssl._create_default_https_context = ssl._create_unverified_context

        # http://wwwsearch.sourceforge.net/mechanize/
        self.browser = mechanize.Browser()
        self.browser.set_cookiejar(cookielib.LWPCookieJar())
        self.browser.addheaders = [('User-Agent', regular_user_agent)]
        # ignore robots.txt
        self.browser.set_handle_robots(False) 

def query_url(url, br = None):
    if br == None:
        br = mechanize.Browser()
        br.set_handle_robots(False)

    br.open(url['url'].encode("ascii","ignore"))

    return 

def submit_form(form, inputs, br = None):
    if br == None:
        br = mechanize.Browser()
        cj = cookielib.LWPCookieJar()
        br.set_cookiejar(cj)
        br.set_handle_robots(False)

    br.open(form['url'].encode("ascii","ignore"))
    br.select_form(nr=get_form_index(br, form))

    for input in form['inputs']:
        if input['name'] in inputs:
            try:
                if br.find_control(name = input['name'], type = input['type']) == None:
                    continue
                if input['type'] == 'file':
                    filename = inputs[input['name']]['filename']
                    upload_filename = os.path.basename(filename)
                    mime_type = inputs[input['name']]['mime_type']
                    br.form.add_file(open(filename), mime_type, upload_filename, name = input['name'])
                    br.form.set_all_readonly(False)
                elif input['type'] == 'checkbox':
                    br.find_control(name = input['name'], type = input['type']).selected = inputs[input['name']]
                else:
                    if br.find_control(name = input['name'], type = input['type']).readonly:
                        continue
                    if input['type'] == 'radio':
                        continue
                    br[input['name']] = inputs[input['name']]
            except:
                # traceback.print_exc()
                pass

    response = br.submit().code

    return response, br 

def run():
    hits = []

    # Create browser
    br = mechanize.Browser()

    # Browser options
    br.set_handle_equiv(True)
    br.set_handle_redirect(True)
    br.set_handle_referer(True)
    br.set_handle_robots(False)
    br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
    br.addheaders = [('User-agent', USER_AGENT)]
    br.open(url)

    # Fill out form
    br.select_form(nr=0)
    br.form.set_all_readonly(False)  # allow changing the .value of all controls
    br.form["campingDate"] = date
    br.form["lengthOfStay"] = length_of_stay
    response = br.submit()

    # Scrape result
    soup = BeautifulSoup(response, "html.parser")
    table = soup.findAll("table", {"id": "shoppingitems"})

    if table:
        rows = table[0].findAll("tr", {"class": "br"})

        for row in rows:
            cells = row.findAll("td")
            l = len(cells)
            label = cells[0].findAll("div", {"class": "siteListLabel"})[0].text
            is_ada = bool(cells[3].findAll("img", {"title": "Accessible"}))
            is_group = bool('GROUP' in cells[2].text)
            status = cells[l - 1].text
            if not is_group and not is_ada and status.startswith('available'):
                hits.append(label)

    if hits:
        send_results(date, hits) 

def get_asn3(data):
    #   get as number from bgp.he.net(Hurricane)
    if chk_domain(data):
        ip, c_name = retIP(data)
    if chk_ip(data):
        ip = data

    as_number = 0
    subnet = ''

    url = "http://bgp.he.net/ip/" + ip + "#_ipinfo"
    userAgent = [('User-agent','Mozilla/5.0 (X11; U; '+\
                  'Linux 2.4.2-2 i586; en-US; m18) Gecko/20010131 Netscape6/6.01')]
    browser = mechanize.Browser()
    browser.addheaders = userAgent
    try:
        page = browser.open(url)
        
        if page.code == 200:
            html = page.read()
            link_finder = re.compile('href="(.*?)"')
            links = link_finder.findall(html)
            as_number = ''
            for i in range(0, len(links)):
                if links[i].find('/AS') == 0:
                    as_number = int(links[i][3:])
                if links[i].find('/net') == 0:
                    subnet = links[i][5:]
    except:
        msg ="[*] Exception: in accessing Hurrican (bgp.het.net)"
        logging.error(msg)
    return as_number, subnet 

def get_url(url):
    #   parse html by url supplied using mechanize
    userAgent = [('User-agent','Mozilla/5.0 (X11; U; '+\
                  'Linux 2.4.2-2 i586; en-US; m18) Gecko/20010131 Netscape6/6.01')]
    try:
        browser = mechanize.Browser()
        browser.addheaders = userAgent
        page = browser.open(url)
        html = page.read()
    except:
        html =''
    return html