Python urllib2() Examples

def retrieve_status_page(user, password, url):

    try:
        ssl._create_unverified_context
    except AttributeError:
        pass
    else:
        ssl._create_default_https_context = ssl._create_unverified_context

    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
    password_mgr.add_password(None, url, user, password)
    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    urllib2.install_opener(opener)

    req = urllib2.Request(url)
    try:
        response = urllib2.urlopen(req)
        return response.read()
    except Exception:
        raise CrawlError("can't access to http://%s", url) 

def run(self):
		username, password = getword()
		try:
			print "-"*12
			print "User:",username,"Password:",password
			req = urllib2.Request(sys.argv[1])
			passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
			passman.add_password(None, sys.argv[1], username, password)
			authhandler = urllib2.HTTPBasicAuthHandler(passman)
			opener = urllib2.build_opener(authhandler)
			fd = opener.open(req)
			print "\t\n\nUsername:",username,"Password:",password,"----- Login successful!!!\n\n"			
			print "Retrieved", fd.geturl()
			info = fd.info()
			for key, value in info.items():
    				print "%s = %s" % (key, value)
			sys.exit(2)
		except (urllib2.HTTPError, httplib.BadStatusLine,socket.error), msg: 
			print "An error occurred:", msg
			pass 

def _createUser(self, number):
        record = self._records[number]
        user = record.uid
        authBasic = HTTPBasicAuthHandler(password_mgr=HTTPPasswordMgrWithDefaultRealm())
        authBasic.add_password(
            realm=None,
            uri=self.servers[record.podID]["uri"],
            user=user.encode('utf-8'),
            passwd=record.password.encode('utf-8'))
        authDigest = HTTPDigestAuthHandler(passwd=HTTPPasswordMgrWithDefaultRealm())
        authDigest.add_password(
            realm=None,
            uri=self.servers[record.podID]["uri"],
            user=user.encode('utf-8'),
            passwd=record.password.encode('utf-8'))
        return record, user, {"basic": authBasic, "digest": authDigest, } 

def __init__(self, userCredentials=None, retries=3, sleepTime=5):
        DirectoryWalker.__init__(self, userCredentials, retries, sleepTime)
        if self.userCredentials:
            if self.userCredentials.httpProxy:
                os.environ['http_proxy'] = self.userCredentials.httpProxy
                # global kludge, default proxyHandler looks up proxy there
            passwordMgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
            for url, cred in self.userCredentials.credentials.iteritems():
                passwordMgr.add_password(None, url, cred.username, cred.password)
            authHandler = urllib2.HTTPBasicAuthHandler(passwordMgr)
            opener = urllib2.build_opener(authHandler)
        else:
#            opener = urllib2.build_opener()
            opener = None
#        opener.add_headers = [('User-agent', 'Mozilla/5.0')]
        self.opener = opener 

def get_api_opener_v1(url, username, password):
    """
    get the http api opener with base url and username,password

    :param url: http url
    :param username: username for api auth
    :param password: password for api auth
    """
    # create a password manager
    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()

    # Add the username and password.
    password_mgr.add_password(None, url, username, password)

    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    return opener 

def get_api_opener_v1(url, username, password):
    """
    get the http api opener with base url and username,password

    :param url: http url
    :param username: username for api auth
    :param password: password for api auth
    """
    # create a password manager
    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()

    # Add the username and password.
    password_mgr.add_password(None, url, username, password)

    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    return opener 

def get_api_opener_v1(url, username, password):
    """
    get the http api opener with base url and username,password

    :param url: http url
    :param username: username for api auth
    :param password: password for api auth
    """
    # create a password manager
    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()

    # Add the username and password.
    password_mgr.add_password(None, url, username, password)

    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    return opener 

def get_api_opener_v1(url, username, password):
    """
    get the http api opener with base url and username,password

    :param url: http url
    :param username: username for api auth
    :param password: password for api auth
    """
    # create a password manager
    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()

    # Add the username and password.
    password_mgr.add_password(None, url, username, password)

    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    return opener 

def get_api_opener_v1(url, username, password):
    """
    get the http api opener with base url and username,password

    :param url: http url
    :param username: username for api auth
    :param password: password for api auth
    """
    # create a password manager
    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()

    # Add the username and password.
    password_mgr.add_password(None, url, username, password)

    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    return opener 

def __init__(self, userCredentials=None, retries=3, sleepTime=5):
        DirectoryWalker.__init__(self, userCredentials, retries, sleepTime)
        if self.userCredentials:
            if self.userCredentials.httpProxy:
                os.environ['http_proxy'] = self.userCredentials.httpProxy
                # global kludge, default proxyHandler looks up proxy there
            passwordMgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
            for url, cred in self.userCredentials.credentials.iteritems():
                passwordMgr.add_password(None, url, cred.username, cred.password)
            authHandler = urllib2.HTTPBasicAuthHandler(passwordMgr)
            opener = urllib2.build_opener(authHandler)
        else:
#            opener = urllib2.build_opener()
            opener = None
#        opener.add_headers = [('User-agent', 'Mozilla/5.0')]
        self.opener = opener 

def run(self):
		password = getword()
		try:
			print "-"*12
			print "User:",username,"Password:",password
			req = urllib2.Request(sys.argv[1])
			passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
			passman.add_password(None, sys.argv[1], username, password)
			authhandler = urllib2.HTTPBasicAuthHandler(passman)
			opener = urllib2.build_opener(authhandler)
			fd = opener.open(req)
			print "\t\n\n[+] Login successful: Username:",username,"Password:",password,"\n"			
			print "[+] Retrieved", fd.geturl()
			info = fd.info()
			for key, value in info.items():
    				print "%s = %s" % (key, value)
			sys.exit(2)
		except (urllib2.HTTPError,socket.error):
			pass 

def prepare_us_proxy(cookie_handler):
	if (addon.getSetting('us_proxy_socks5') == 'true'):
		if ((addon.getSetting('us_proxy_pass') is not '') and (addon.getSetting('us_proxy_user') is not '')):
			print 'Using socks5 authenticated proxy: ' + addon.getSetting('us_proxy') + ':' + addon.getSetting('us_proxy_port')
			socks_handler = SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, addon.getSetting('us_proxy'), int(addon.getSetting('us_proxy_port')), True, addon.getSetting('us_proxy_user'), addon.getSetting('us_proxy_pass'))
			opener = urllib2.build_opener(socks_handler, cookie_handler)
		else:
			print 'Using socks5 proxy: ' + addon.getSetting('us_proxy') + ':' + addon.getSetting('us_proxy_port')
			socks_handler = SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, addon.getSetting('us_proxy'), int(addon.getSetting('us_proxy_port')), True)
			opener = urllib2.build_opener(socks_handler, cookie_handler)
	elif (addon.getSetting('us_proxy_socks5') == 'false'):
		us_proxy = 'http://' + addon.getSetting('us_proxy') + ':' + addon.getSetting('us_proxy_port')
		proxy_handler = urllib2.ProxyHandler({'http' : us_proxy})
		if ((addon.getSetting('us_proxy_pass') is not '') and (addon.getSetting('us_proxy_user') is not '')):
			print 'Using authenticated proxy: ' + us_proxy
			password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
			password_mgr.add_password(None, us_proxy, addon.getSetting('us_proxy_user'), addon.getSetting('us_proxy_pass'))
			proxy_auth_handler = urllib2.ProxyBasicAuthHandler(password_mgr)
			opener = urllib2.build_opener(proxy_handler, proxy_auth_handler, cookie_handler)
		else:
			print 'Using proxy: ' + us_proxy
			opener = urllib2.build_opener(proxy_handler, cookie_handler)
	return opener 

def run(self):
		username, password = getword()
		try:
			print "-"*12
			print "User:",username,"Password:",password
			req = urllib2.Request(sys.argv[1])
			passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
			passman.add_password(None, sys.argv[1], username, password)
			authhandler = urllib2.HTTPBasicAuthHandler(passman)
			opener = urllib2.build_opener(authhandler)
			fd = opener.open(req)
			print "\t\n\nUsername:",username,"Password:",password,"----- Login successful!!!\n\n"			
			print "Retrieved", fd.geturl()
			info = fd.info()
			for key, value in info.items():
    				print "%s = %s" % (key, value)
			sys.exit(2)
		except (urllib2.HTTPError, httplib.BadStatusLine,socket.error), msg: 
			print "An error occurred:", msg
			pass 

def run(self):
		password = getword()
		try:
			print "-"*12
			print "User:",username,"Password:",password
			req = urllib2.Request(sys.argv[1])
			passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
			passman.add_password(None, sys.argv[1], username, password)
			authhandler = urllib2.HTTPBasicAuthHandler(passman)
			opener = urllib2.build_opener(authhandler)
			fd = opener.open(req)
			print "\t\n\n[+] Login successful: Username:",username,"Password:",password,"\n"			
			print "[+] Retrieved", fd.geturl()
			info = fd.info()
			for key, value in info.items():
    				print "%s = %s" % (key, value)
			sys.exit(2)
		except (urllib2.HTTPError,socket.error):
			pass 

def __init__(self, base_url, exc_class=None, logger=None):
    """
    @param base_url: The base url to the API.
    @param exc_class: An exception class to handle non-200 results.

    Creates an HTTP(S) client to connect to the Cloudera Manager API.
    """
    self._base_url = base_url.rstrip('/')
    self._exc_class = exc_class or RestException
    self._logger = logger or LOG
    self._headers = { }

    # Make a basic auth handler that does nothing. Set credentials later.
    self._passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
    authhandler = urllib2.HTTPBasicAuthHandler(self._passmgr)

    # Make a cookie processor
    cookiejar = cookielib.CookieJar()

    self._opener = urllib2.build_opener(
        HTTPErrorProcessor(),
        urllib2.HTTPCookieProcessor(cookiejar),
        authhandler) 

def __init__(self, **kwargs):
        """
        @param kwargs: Keyword arguments.
            - B{proxy} - An HTTP proxy to be specified on requests.
                 The proxy is defined as {protocol:proxy,}
                    - type: I{dict}
                    - default: {}
            - B{timeout} - Set the URL open timeout (seconds).
                    - type: I{float}
                    - default: 90
            - B{username} - The username used for HTTP authentication.
                    - type: I{str}
                    - default: None
            - B{password} - The password used for HTTP authentication.
                    - type: I{str}
                    - default: None

        """
        HttpTransport.__init__(self, **kwargs)
        self.pm = urllib2.HTTPPasswordMgrWithDefaultRealm() 

def bing_search(query, key, offset, **kwargs):
    ''' Make the search '''
    username = ''
    baseURL = 'https://api.datamarket.azure.com/Bing/Search/'
    query = urllib.quote(query)
    user_agent = 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; FDM; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 1.1.4322)'
    credentials = (':%s' % key).encode('base64')[:-1]
    auth = 'Basic %s' % credentials
    url = baseURL+'Web?Query=%27'+query+'%27&$top=50&$format=json&$skip='+offset
    print '[*] Fetching '+url
    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
    password_mgr.add_password(None, url, username, key)
    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    urllib2.install_opener(opener)
    try:
        readURL = urllib2.urlopen(url, timeout=60).read()
    except Exception as e:
        sys.exit('[-] Failed to fetch bing results. Are you sure you have the right API key?\n      Error: '+str(e))
    return readURL 

def retrieve_status_page(hostname, port, user, password):
    statusPage = "http://%s:%s/manager/status?XML=true" % (hostname, port)

    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
    password_mgr.add_password(None, statusPage, user, password)
    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    urllib2.install_opener(opener)

    req = urllib2.Request(statusPage)
    try:
        response = urllib2.urlopen(req)
        return response.read()
    except Exception:
        raise CrawlError("can't access to http://%s:%s",
                         hostname, port) 

def get_opener():
    default_opener = urllib2.build_opener()
    if not exists(YUM_CONF):
        return default_opener
    config = RawConfigParser()
    config.read(YUM_CONF)
    if not config.has_section('main'):
        return default_opener
    if not config.has_option('main', 'proxy'):
        return default_opener
    proxy = {}
    url = config.get('main', 'proxy').strip()
    if not url:
        return default_opener
    http_proxy_handler = urllib2.ProxyHandler({'http': url, 'https': url})
    # urllib2 can open HTTPS ressources through a proxy since python 2.6.3
    # should be OK on Centos OS (Python 2.6.6)
    if config.has_option('main', 'proxy_username') and config.has_option(
            'main', 'proxy_password'):
        username = config.get('main', 'proxy_username').strip()
        password = config.get('main', 'proxy_password').strip()
        password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
        password_manager.add_password(None, url, username, password)
        proxy_auth_handler = urllib2.ProxyBasicAuthHandler(password_manager)
        return urllib2.build_opener(http_proxy_handler, proxy_auth_handler)
    return urllib2.build_opener(http_proxy_handler) 

def send(self, data):

        passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        passmgr.add_password(None, self._authurl, self._username, self._password)

        auth_handler = urllib2.HTTPDigestAuthHandler(passmgr)
        opener = urllib2.build_opener(auth_handler)
        urllib2.install_opener(opener)

        req = urllib2.Request(self._url, data, self._headers)

        try:
            self._fd = urllib2.urlopen(req)
        except:
            self._fd = None 

def get(self, uri, params={}, headers={}, with_status_code=False, timeout=10, user=None, password=None):
        data = None  # always none in GET
        
        if params:
            uri = "%s?%s" % (uri, urlencode(params))
        
        # SSL, user/password and basic
        # NOTE: currently don't manage ssl & user/password
        if uri.startswith('https://'):
            handler = HTTPSHandler(context=self.ssl_context)
        elif user and password:
            passwordMgr = HTTPPasswordMgrWithDefaultRealm()
            passwordMgr.add_password(None, uri, user, password)
            handler = HTTPBasicAuthHandler(passwordMgr)
        else:
            handler = HTTPHandler
        
        url_opener = build_opener(handler)
        
        req = Request(uri, data)
        req.get_method = lambda: 'GET'
        for (k, v) in headers.items():
            req.add_header(k, v)
        
        request = url_opener.open(req, timeout=timeout)
        
        response = request.read()
        status_code = request.code
        request.close()
        
        if not with_status_code:
            return response
        else:
            return (status_code, response) 

def auth (usr, pwd, dhusAlt=None):
  """Globally install Basic Auth, and set site specific string constants."""
  SITE["NAME"] = dhusAlt if dhusAlt is not None else "SciHub"
  site,collspec = "https://scihub.copernicus.eu/", "(producttype:S2MSI%s)"
  if SITE["NAME"]=="CODE-DE": site,collspec = "https://code-de.org/", "platformname:Sentinel-2"
  SITE["BASE"] = site+"dhus/"
#  pm=urllib2.HTTPPasswordMgrWithDefaultRealm()
#  pm.add_password(None, SITE["BASE"], usr, pwd)
#  urllib2.install_opener(urllib2.build_opener(urllib2.HTTPBasicAuthHandler(pm)))
#...does not work transparently in combination with proxy support => workaround: urlOpen().
  import base64; SITE["AUTH"] = "Basic " + base64.b64encode("%s:%s" % (usr, pwd))
  SITE["SEARCH"] = SITE["BASE"] + "search?format=xml&sortedby=beginposition&order=desc&rows=%d&q=%s" % (ROWSSTEP, collspec)
  product = SITE["BASE"] + "odata/v1/Products('%s')/"
  SITE["CHECKSUM"], SITE["SAFEZIP"], SITE["SAFEROOT"] = product+"Checksum/Value/$value", product+"$value", product+"Nodes('%s.SAFE')/" 

def __init__(self, password_mgr=None, debuglevel=0):
        """Initialize an instance of a AbstractNtlmAuthHandler.

Verify operation with all default arguments.
>>> abstrct = AbstractNtlmAuthHandler()

Verify "normal" operation.
>>> abstrct = AbstractNtlmAuthHandler(urllib2.HTTPPasswordMgrWithDefaultRealm())
"""
        if password_mgr is None:
            password_mgr = urllib2.HTTPPasswordMgr()
        self.passwd = password_mgr
        self.add_password = self.passwd.add_password
        self._debuglevel = debuglevel 

def threader(site):
	username, password = getword()
	global logins
	try:
		print "-"*12
		print "User:",username,"Password:",password
		req = urllib2.Request(site)
		passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
		passman.add_password(None, site, username, password)
		authhandler = urllib2.HTTPBasicAuthHandler(passman)
		opener = urllib2.build_opener(authhandler)
		fd = opener.open(req)
		site = urllib2.urlopen(fd.geturl()).read()
		print "\n[+] Checking the authenticity of the login...\n"
		if not re.search(('denied'), site.lower()):
			print "\t\n\n[+] Username:",username,"Password:",password,"----- Login successful!!!\n\n"
			print "[+] Writing Successful Login:",sys.argv[5],"\n"
			logins +=1
			file = open(sys.argv[5], "a")
			file.writelines("Site: "+site+" Username: "+username+ " Password: "+password+"\n")
			file.close()			
			print "Retrieved", fd.geturl()
			info = fd.info()
			for key, value in info.items():
    				print "%s = %s" % (key, value)
		else: 
			print "- Redirection"
	except (urllib2.HTTPError, httplib.BadStatusLine,socket.error), msg: 
		print "An error occurred:", msg
		pass 

def threader(site):
	username, password = getword()
	global logins
	try:
		print "-"*12
		print "User:",username,"Password:",password
		req = urllib2.Request(site)
		passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
		passman.add_password(None, site, username, password)
		authhandler = urllib2.HTTPBasicAuthHandler(passman)
		opener = urllib2.build_opener(authhandler)
		fd = opener.open(req)
		site = urllib2.urlopen(fd.geturl()).read()
		print "\n[+] Checking the authenticity of the login...\n"
		if not re.search(('denied'), site.lower()):
			print "\t\n\n[+] Username:",username,"Password:",password,"----- Login successful!!!\n\n"
			print "[+] Writing Successful Login:",sys.argv[5],"\n"
			logins +=1
			file = open(sys.argv[5], "a")
			file.writelines("Site: "+site+" Username: "+username+ " Password: "+password+"\n")
			file.close()			
			print "Retrieved", fd.geturl()
			info = fd.info()
			for key, value in info.items():
    				print "%s = %s" % (key, value)
		else: 
			print "- Redirection\n"
	except (urllib2.HTTPError,httplib.BadStatusLine,socket.error), msg: 
		print "An error occurred:", msg
		pass 

def _login_BA(self):
        try:
            # Creates a PasswordMgr instance
            passmanager = urllib2.HTTPPasswordMgrWithDefaultRealm()
            passmanager.add_password(None, self.url, self.username, self.password)
            # Creates an auth handling object and builds it with opener
            auth = urllib2.HTTPBasicAuthHandler(passmanager)
            opener = urllib2.build_opener(auth)
            response = opener.open(self.url, timeout=8)
            data = response.read()
            response.close()
            return data
        except Exception, e:
            if 'Error 401' in str(e):
                raise Exception('Login credentials incorrect.') 

def __build_url_opener(self, uri):
        """
        Build the url opener for managing HTTP Basic Athentication
        """
        # Create an OpenerDirector with support
        # for Basic HTTP Authentication...
        password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        password_mgr.add_password(realm=None,
                                  uri=uri,
                                  user=self.client_id,
                                  passwd=self.client_secret)
        handler = urllib2.HTTPBasicAuthHandler(password_mgr)
        opener = urllib2.build_opener(handler)
        return opener 

def __init__(self, uri, login=None, password=None):
        if login and password:
            passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
            passman.add_password(None, uri, login, password)
            auth_NTLM = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(passman)
            self.opener = urllib2.build_opener(auth_NTLM)
        else:
            self.opener = urllib2.build_opener() 

def fetch_file(uri, file=None, username=None, password=None):
    """
    Fetch a file based on the URI provided. If you do not pass in a file pointer
    a tempfile.NamedTemporaryFile, or None if the file could not be 
    retrieved is returned.
    The URI can be either an HTTP url, or "s3://bucket_name/key_name"
    """
    boto.log.info('Fetching %s' % uri)
    if file == None:
        file = tempfile.NamedTemporaryFile()
    try:
        if uri.startswith('s3://'):
            bucket_name, key_name = uri[len('s3://'):].split('/', 1)
            c = boto.connect_s3(aws_access_key_id=username, aws_secret_access_key=password)
            bucket = c.get_bucket(bucket_name)
            key = bucket.get_key(key_name)
            key.get_contents_to_file(file)
        else:
            if username and password:
                passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
                passman.add_password(None, uri, username, password)
                authhandler = urllib2.HTTPBasicAuthHandler(passman)
                opener = urllib2.build_opener(authhandler)
                urllib2.install_opener(opener)
            s = urllib2.urlopen(uri)
            file.write(s.read())
        file.seek(0)
    except:
        raise
        boto.log.exception('Problem Retrieving file: %s' % uri)
        file = None
    return file 

def threader(site):
	username, password = getword()
	global logins
	try:
		print "-"*12
		print "User:",username,"Password:",password
		req = urllib2.Request(site)
		passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
		passman.add_password(None, site, username, password)
		authhandler = urllib2.HTTPBasicAuthHandler(passman)
		opener = urllib2.build_opener(authhandler)
		fd = opener.open(req)
		site = urllib2.urlopen(fd.geturl()).read()
		print "\n[+] Checking the authenticity of the login...\n"
		if not re.search(('denied'), site.lower()):
			print "\t\n\n[+] Username:",username,"Password:",password,"----- Login successful!!!\n\n"
			print "[+] Writing Successful Login:",sys.argv[5],"\n"
			logins +=1
			file = open(sys.argv[5], "a")
			file.writelines("Site: "+site+" Username: "+username+ " Password: "+password+"\n")
			file.close()			
			print "Retrieved", fd.geturl()
			info = fd.info()
			for key, value in info.items():
    				print "%s = %s" % (key, value)
		else: 
			print "- Redirection"
	except (urllib2.HTTPError, httplib.BadStatusLine,socket.error), msg: 
		print "An error occurred:", msg
		pass