Python urllib2.HTTPBasicAuthHandler() Examples

def retrieve_status_page(user, password, url):

    try:
        ssl._create_unverified_context
    except AttributeError:
        pass
    else:
        ssl._create_default_https_context = ssl._create_unverified_context

    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
    password_mgr.add_password(None, url, user, password)
    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    urllib2.install_opener(opener)

    req = urllib2.Request(url)
    try:
        response = urllib2.urlopen(req)
        return response.read()
    except Exception:
        raise CrawlError("can't access to http://%s", url) 

def get_api_opener_v1(url, username, password):
    """
    get the http api opener with base url and username,password

    :param url: http url
    :param username: username for api auth
    :param password: password for api auth
    """
    # create a password manager
    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()

    # Add the username and password.
    password_mgr.add_password(None, url, username, password)

    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    return opener 

def run(self):
		password = getword()
		try:
			print "-"*12
			print "User:",username,"Password:",password
			req = urllib2.Request(sys.argv[1])
			passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
			passman.add_password(None, sys.argv[1], username, password)
			authhandler = urllib2.HTTPBasicAuthHandler(passman)
			opener = urllib2.build_opener(authhandler)
			fd = opener.open(req)
			print "\t\n\n[+] Login successful: Username:",username,"Password:",password,"\n"			
			print "[+] Retrieved", fd.geturl()
			info = fd.info()
			for key, value in info.items():
    				print "%s = %s" % (key, value)
			sys.exit(2)
		except (urllib2.HTTPError,socket.error):
			pass 

def send(self, data):
        """
        Send data via sendall.

        @type	data: string
        @param	data: Data to send
        """

        passmgr = urllib2.HTTPPasswordMgr()
        passmgr.add_password(self._realm, self._url, self._username, self._password)

        auth_handler = urllib2.HTTPBasicAuthHandler(passmgr)
        opener = urllib2.build_opener(auth_handler)
        urllib2.install_opener(opener)

        req = urllib2.Request(self._url, data, self._headers)

        try:
            self._fd = urllib2.urlopen(req)
        except:
            self._fd = None 

def _createUser(self, number):
        record = self._records[number]
        user = record.uid
        authBasic = HTTPBasicAuthHandler(password_mgr=HTTPPasswordMgrWithDefaultRealm())
        authBasic.add_password(
            realm=None,
            uri=self.servers[record.podID]["uri"],
            user=user.encode('utf-8'),
            passwd=record.password.encode('utf-8'))
        authDigest = HTTPDigestAuthHandler(passwd=HTTPPasswordMgrWithDefaultRealm())
        authDigest.add_password(
            realm=None,
            uri=self.servers[record.podID]["uri"],
            user=user.encode('utf-8'),
            passwd=record.password.encode('utf-8'))
        return record, user, {"basic": authBasic, "digest": authDigest, } 

def test_basic_auth_with_unquoted_realm(self):
        opener = OpenerDirector()
        password_manager = MockPasswordManager()
        auth_handler = urllib2.HTTPBasicAuthHandler(password_manager)
        realm = "ACME Widget Store"
        http_handler = MockHTTPHandler(
            401, 'WWW-Authenticate: Basic realm=%s\r\n\r\n' % realm)
        opener.add_handler(auth_handler)
        opener.add_handler(http_handler)
        msg = "Basic Auth Realm was unquoted"
        with test_support.check_warnings((msg, UserWarning)):
            self._test_basic_auth(opener, auth_handler, "Authorization",
                                  realm, http_handler, password_manager,
                                  "http://acme.example.com/protected",
                                  "http://acme.example.com/protected"
                                 ) 

def __init__(self, configuration):
        self.setup(configuration)
        self.echo = None
        if "ECHO" in configuration:
            self.echo = configuration['ECHO']
        if self.proxy_scheme is not None and self.proxy_host is not None and \
                        self.proxy_port is not None:
            credentials = ""
            if self.proxy_username is not None and self.proxy_password is not None:
                credentials = self.proxy_username + ":" + self.proxy_password + "@"
            proxyDict = {
                self.proxy_scheme: self.proxy_scheme + "://" + credentials +
                                                    self.proxy_host + ":" + self.proxy_port
            }

            proxy = urllib2.ProxyHandler(proxyDict)

            if credentials != '':
                auth = urllib2.HTTPBasicAuthHandler()
                opener = urllib2.build_opener(proxy, auth, urllib2.HTTPHandler)
            else:
                opener = urllib2.build_opener(proxy)
            urllib2.install_opener(opener) 

def __init__(self, userCredentials=None, retries=3, sleepTime=5):
        DirectoryWalker.__init__(self, userCredentials, retries, sleepTime)
        if self.userCredentials:
            if self.userCredentials.httpProxy:
                os.environ['http_proxy'] = self.userCredentials.httpProxy
                # global kludge, default proxyHandler looks up proxy there
            passwordMgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
            for url, cred in self.userCredentials.credentials.iteritems():
                passwordMgr.add_password(None, url, cred.username, cred.password)
            authHandler = urllib2.HTTPBasicAuthHandler(passwordMgr)
            opener = urllib2.build_opener(authHandler)
        else:
#            opener = urllib2.build_opener()
            opener = None
#        opener.add_headers = [('User-agent', 'Mozilla/5.0')]
        self.opener = opener 

def __init__(self, userCredentials=None, retries=3, sleepTime=5):
        DirectoryWalker.__init__(self, userCredentials, retries, sleepTime)
        if self.userCredentials:
            if self.userCredentials.httpProxy:
                os.environ['http_proxy'] = self.userCredentials.httpProxy
                # global kludge, default proxyHandler looks up proxy there
            passwordMgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
            for url, cred in self.userCredentials.credentials.iteritems():
                passwordMgr.add_password(None, url, cred.username, cred.password)
            authHandler = urllib2.HTTPBasicAuthHandler(passwordMgr)
            opener = urllib2.build_opener(authHandler)
        else:
#            opener = urllib2.build_opener()
            opener = None
#        opener.add_headers = [('User-agent', 'Mozilla/5.0')]
        self.opener = opener 

def run(self):
		username, password = getword()
		try:
			print "-"*12
			print "User:",username,"Password:",password
			req = urllib2.Request(sys.argv[1])
			passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
			passman.add_password(None, sys.argv[1], username, password)
			authhandler = urllib2.HTTPBasicAuthHandler(passman)
			opener = urllib2.build_opener(authhandler)
			fd = opener.open(req)
			print "\t\n\nUsername:",username,"Password:",password,"----- Login successful!!!\n\n"			
			print "Retrieved", fd.geturl()
			info = fd.info()
			for key, value in info.items():
    				print "%s = %s" % (key, value)
			sys.exit(2)
		except (urllib2.HTTPError, httplib.BadStatusLine,socket.error), msg: 
			print "An error occurred:", msg
			pass 

def test_basic_auth_with_unquoted_realm(self):
        opener = OpenerDirector()
        password_manager = MockPasswordManager()
        auth_handler = urllib2.HTTPBasicAuthHandler(password_manager)
        realm = "ACME Widget Store"
        http_handler = MockHTTPHandler(
            401, 'WWW-Authenticate: Basic realm=%s\r\n\r\n' % realm)
        opener.add_handler(auth_handler)
        opener.add_handler(http_handler)
        msg = "Basic Auth Realm was unquoted"
        with test_support.check_warnings((msg, UserWarning)):
            self._test_basic_auth(opener, auth_handler, "Authorization",
                                  realm, http_handler, password_manager,
                                  "http://acme.example.com/protected",
                                  "http://acme.example.com/protected"
                                 ) 

def bing_search(query, key, offset, **kwargs):
    ''' Make the search '''
    username = ''
    baseURL = 'https://api.datamarket.azure.com/Bing/Search/'
    query = urllib.quote(query)
    user_agent = 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; FDM; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 1.1.4322)'
    credentials = (':%s' % key).encode('base64')[:-1]
    auth = 'Basic %s' % credentials
    url = baseURL+'Web?Query=%27'+query+'%27&$top=50&$format=json&$skip='+offset
    print '[*] Fetching '+url
    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
    password_mgr.add_password(None, url, username, key)
    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    urllib2.install_opener(opener)
    try:
        readURL = urllib2.urlopen(url, timeout=60).read()
    except Exception as e:
        sys.exit('[-] Failed to fetch bing results. Are you sure you have the right API key?\n      Error: '+str(e))
    return readURL 

def retrieve_status_page(hostname, port, user, password):
    statusPage = "http://%s:%s/manager/status?XML=true" % (hostname, port)

    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
    password_mgr.add_password(None, statusPage, user, password)
    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    urllib2.install_opener(opener)

    req = urllib2.Request(statusPage)
    try:
        response = urllib2.urlopen(req)
        return response.read()
    except Exception:
        raise CrawlError("can't access to http://%s:%s",
                         hostname, port) 

def run(self):
		password = getword()
		try:
			print "-"*12
			print "User:",username,"Password:",password
			req = urllib2.Request(sys.argv[1])
			passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
			passman.add_password(None, sys.argv[1], username, password)
			authhandler = urllib2.HTTPBasicAuthHandler(passman)
			opener = urllib2.build_opener(authhandler)
			fd = opener.open(req)
			print "\t\n\n[+] Login successful: Username:",username,"Password:",password,"\n"			
			print "[+] Retrieved", fd.geturl()
			info = fd.info()
			for key, value in info.items():
    				print "%s = %s" % (key, value)
			sys.exit(2)
		except (urllib2.HTTPError,socket.error):
			pass 

def get_api_opener_v1(url, username, password):
    """
    get the http api opener with base url and username,password

    :param url: http url
    :param username: username for api auth
    :param password: password for api auth
    """
    # create a password manager
    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()

    # Add the username and password.
    password_mgr.add_password(None, url, username, password)

    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    return opener 

def get_api_opener_v1(url, username, password):
    """
    get the http api opener with base url and username,password

    :param url: http url
    :param username: username for api auth
    :param password: password for api auth
    """
    # create a password manager
    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()

    # Add the username and password.
    password_mgr.add_password(None, url, username, password)

    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    return opener 

def run(self):
		username, password = getword()
		try:
			print "-"*12
			print "User:",username,"Password:",password
			req = urllib2.Request(sys.argv[1])
			passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
			passman.add_password(None, sys.argv[1], username, password)
			authhandler = urllib2.HTTPBasicAuthHandler(passman)
			opener = urllib2.build_opener(authhandler)
			fd = opener.open(req)
			print "\t\n\nUsername:",username,"Password:",password,"----- Login successful!!!\n\n"			
			print "Retrieved", fd.geturl()
			info = fd.info()
			for key, value in info.items():
    				print "%s = %s" % (key, value)
			sys.exit(2)
		except (urllib2.HTTPError, httplib.BadStatusLine,socket.error), msg: 
			print "An error occurred:", msg
			pass 

def get_api_opener_v1(url, username, password):
    """
    get the http api opener with base url and username,password

    :param url: http url
    :param username: username for api auth
    :param password: password for api auth
    """
    # create a password manager
    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()

    # Add the username and password.
    password_mgr.add_password(None, url, username, password)

    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    return opener 

def get_api_opener_v1(url, username, password):
    """
    get the http api opener with base url and username,password

    :param url: http url
    :param username: username for api auth
    :param password: password for api auth
    """
    # create a password manager
    password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()

    # Add the username and password.
    password_mgr.add_password(None, url, username, password)

    handler = urllib2.HTTPBasicAuthHandler(password_mgr)
    opener = urllib2.build_opener(handler)
    return opener 

def run(self):
		username, password = getword()
		try:
			print "-"*12
			print "User:",username,"Password:",password
			auth_handler = urllib2.HTTPBasicAuthHandler()
			auth_handler.add_password("cPanel", server, base64encodestring(username)[:-1], base64encodestring(password)[:-1])
			opener = urllib2.build_opener(auth_handler)
			urllib2.install_opener(opener)
			urllib2.urlopen(server)
			print "\t\n\nUsername:",username,"Password:",password,"----- Login successful!!!\n\n"			
		except (urllib2.HTTPError, httplib.BadStatusLine), msg: 
			#print "An error occurred:", msg
			pass 

def threader(site):
	username, password = getword()
	global logins
	try:
		print "-"*12
		print "User:",username,"Password:",password
		req = urllib2.Request(site)
		passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
		passman.add_password(None, site, username, password)
		authhandler = urllib2.HTTPBasicAuthHandler(passman)
		opener = urllib2.build_opener(authhandler)
		fd = opener.open(req)
		site = urllib2.urlopen(fd.geturl()).read()
		print "\n[+] Checking the authenticity of the login...\n"
		if not re.search(('denied'), site.lower()):
			print "\t\n\n[+] Username:",username,"Password:",password,"----- Login successful!!!\n\n"
			print "[+] Writing Successful Login:",sys.argv[5],"\n"
			logins +=1
			file = open(sys.argv[5], "a")
			file.writelines("Site: "+site+" Username: "+username+ " Password: "+password+"\n")
			file.close()			
			print "Retrieved", fd.geturl()
			info = fd.info()
			for key, value in info.items():
    				print "%s = %s" % (key, value)
		else: 
			print "- Redirection"
	except (urllib2.HTTPError, httplib.BadStatusLine,socket.error), msg: 
		print "An error occurred:", msg
		pass 

def threader(site):
	username, password = getword()
	global logins
	try:
		print "-"*12
		print "User:",username,"Password:",password
		req = urllib2.Request(site)
		passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
		passman.add_password(None, site, username, password)
		authhandler = urllib2.HTTPBasicAuthHandler(passman)
		opener = urllib2.build_opener(authhandler)
		fd = opener.open(req)
		site = urllib2.urlopen(fd.geturl()).read()
		print "\n[+] Checking the authenticity of the login...\n"
		if not re.search(('denied'), site.lower()):
			print "\t\n\n[+] Username:",username,"Password:",password,"----- Login successful!!!\n\n"
			print "[+] Writing Successful Login:",sys.argv[5],"\n"
			logins +=1
			file = open(sys.argv[5], "a")
			file.writelines("Site: "+site+" Username: "+username+ " Password: "+password+"\n")
			file.close()			
			print "Retrieved", fd.geturl()
			info = fd.info()
			for key, value in info.items():
    				print "%s = %s" % (key, value)
		else: 
			print "- Redirection"
	except (urllib2.HTTPError, httplib.BadStatusLine,socket.error), msg: 
		print "An error occurred:", msg
		pass 

def threader(site):
	username, password = getword()
	global logins
	try:
		print "-"*12
		print "User:",username,"Password:",password
		req = urllib2.Request(site)
		passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
		passman.add_password(None, site, username, password)
		authhandler = urllib2.HTTPBasicAuthHandler(passman)
		opener = urllib2.build_opener(authhandler)
		fd = opener.open(req)
		site = urllib2.urlopen(fd.geturl()).read()
		print "\n[+] Checking the authenticity of the login...\n"
		if not re.search(('denied'), site.lower()):
			print "\t\n\n[+] Username:",username,"Password:",password,"----- Login successful!!!\n\n"
			print "[+] Writing Successful Login:",sys.argv[5],"\n"
			logins +=1
			file = open(sys.argv[5], "a")
			file.writelines("Site: "+site+" Username: "+username+ " Password: "+password+"\n")
			file.close()			
			print "Retrieved", fd.geturl()
			info = fd.info()
			for key, value in info.items():
    				print "%s = %s" % (key, value)
		else: 
			print "- Redirection\n"
	except (urllib2.HTTPError,httplib.BadStatusLine,socket.error), msg: 
		print "An error occurred:", msg
		pass 

def test_basic_auth(self, quote_char='"'):
        opener = OpenerDirector()
        password_manager = MockPasswordManager()
        auth_handler = urllib2.HTTPBasicAuthHandler(password_manager)
        realm = "ACME Widget Store"
        http_handler = MockHTTPHandler(
            401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' %
            (quote_char, realm, quote_char) )
        opener.add_handler(auth_handler)
        opener.add_handler(http_handler)
        self._test_basic_auth(opener, auth_handler, "Authorization",
                              realm, http_handler, password_manager,
                              "http://acme.example.com/protected",
                              "http://acme.example.com/protected"
                             ) 

def __open(self, url, headers={}, data=None, baseurl=""):
        """Raw urlopen command"""
        if not baseurl:
            baseurl = self.baseurl
        req = Request("%s%s" % (baseurl, url), headers=headers)
        try:
            req.data = urlencode(data).encode('utf-8') # Python 3
        except:
            try:
                req.add_data(urlencode(data)) # Python 2
            except:
                pass

        # Proxy support
        if self.proxy_url:
            if self.proxy_user:
                proxy = ProxyHandler({'https': 'https://%s:%s@%s' % (self.proxy_user,
                                                                     self.proxy_password,
                                                                     self.proxy_url)})
                auth = HTTPBasicAuthHandler()
                opener = build_opener(proxy, auth, HTTPHandler)
            else:
                handler = ProxyHandler({'https': self.proxy_url})
                opener = build_opener(handler)
        else:
            opener = build_opener()
        resp = opener.open(req)
        charset = resp.info().get('charset', 'utf-8')
        return json.loads(resp.read().decode(charset)) 

def urlretrieve(url, filename, data=None, auth=None):
    if auth is not None:
        # https://docs.python.org/2.7/howto/urllib2.html#id6
        password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()

        # Add the username and password.
        # If we knew the realm, we could use it instead of None.
        username, password = auth
        top_level_url = urlparse(url).netloc
        password_mgr.add_password(None, top_level_url, username, password)

        handler = urllib2.HTTPBasicAuthHandler(password_mgr)

        # create "opener" (OpenerDirector instance)
        opener = urllib2.build_opener(handler)
    else:
        opener = urllib2.build_opener()

    res = opener.open(url, data=data)

    headers = res.info()

    with open(filename, "wb") as fp:
        fp.write(res.read())

    return filename, headers 

def test_urlopen():
    # urllib
    url = urllib.quote('file:///bin/ls')
    urllib.urlopen(url, 'blah', 32)
    urllib.urlretrieve('file:///bin/ls', '/bin/ls2')
    opener = urllib.URLopener()
    opener.open('file:///bin/ls')
    opener.retrieve('file:///bin/ls')
    opener = urllib.FancyURLopener()
    opener.open('file:///bin/ls')
    opener.retrieve('file:///bin/ls')

    # urllib2
    handler = urllib2.HTTPBasicAuthHandler()
    handler.add_password(realm='test',
                         uri='http://mysite.com',
                         user='bob')
    opener = urllib2.build_opener(handler)
    urllib2.install_opener(opener)
    urllib2.urlopen('file:///bin/ls')
    urllib2.Request('file:///bin/ls')

    # Python 3
    urllib.request.urlopen('file:///bin/ls')
    urllib.request.urlretrieve('file:///bin/ls', '/bin/ls2')
    opener = urllib.request.URLopener()
    opener.open('file:///bin/ls')
    opener.retrieve('file:///bin/ls')
    opener = urllib.request.FancyURLopener()
    opener.open('file:///bin/ls')
    opener.retrieve('file:///bin/ls')

    # Six
    six.moves.urllib.request.urlopen('file:///bin/ls')
    six.moves.urllib.request.urlretrieve('file:///bin/ls', '/bin/ls2')
    opener = six.moves.urllib.request.URLopener()
    opener.open('file:///bin/ls')
    opener.retrieve('file:///bin/ls')
    opener = six.moves.urllib.request.FancyURLopener()
    opener.open('file:///bin/ls')
    opener.retrieve('file:///bin/ls') 

def test_urlopen():
    # urllib
    url = urllib.quote('file:///bin/ls')
    urllib.urlopen(url, 'blah', 32)
    urllib.urlretrieve('file:///bin/ls', '/bin/ls2')
    opener = urllib.URLopener()
    opener.open('file:///bin/ls')
    opener.retrieve('file:///bin/ls')
    opener = urllib.FancyURLopener()
    opener.open('file:///bin/ls')
    opener.retrieve('file:///bin/ls')

    # urllib2
    handler = urllib2.HTTPBasicAuthHandler()
    handler.add_password(realm='test',
                         uri='http://mysite.com',
                         user='bob')
    opener = urllib2.build_opener(handler)
    urllib2.install_opener(opener)
    urllib2.urlopen('file:///bin/ls')
    urllib2.Request('file:///bin/ls')

    # Python 3
    urllib.request.urlopen('file:///bin/ls')
    urllib.request.urlretrieve('file:///bin/ls', '/bin/ls2')
    opener = urllib.request.URLopener()
    opener.open('file:///bin/ls')
    opener.retrieve('file:///bin/ls')
    opener = urllib.request.FancyURLopener()
    opener.open('file:///bin/ls')
    opener.retrieve('file:///bin/ls')

    # Six
    six.moves.urllib.request.urlopen('file:///bin/ls')
    six.moves.urllib.request.urlretrieve('file:///bin/ls', '/bin/ls2')
    opener = six.moves.urllib.request.URLopener()
    opener.open('file:///bin/ls')
    opener.retrieve('file:///bin/ls')
    opener = six.moves.urllib.request.FancyURLopener()
    opener.open('file:///bin/ls')
    opener.retrieve('file:///bin/ls') 

def http_error_auth_reqed(self, auth_header, host, req, headers):
        # Reset the retry counter once for each request.
        if hash(req) not in self.retried_req:
            self.retried_req.add(hash(req))
            self.retried_count = 0
        else:
            if self.retried_count > 5:
                raise urllib2.HTTPError(req.get_full_url(), 401, "basic auth failed",
                                headers, None)
            else:
                self.retried_count += 1

        return urllib2.HTTPBasicAuthHandler.http_error_auth_reqed(
                        self, auth_header, host, req, headers) 

def __init__(self, *args, **kwargs):
        urllib2.HTTPBasicAuthHandler.__init__(self, *args, **kwargs)
        self.retried_req = set()
        self.retried_count = 0