Python urllib2.HTTPCookieProcessor() Examples

def read_openload(url):
    default_headers = dict()
    default_headers[
        "User-Agent"] = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3163.100 Safari/537.36"
    default_headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
    default_headers["Accept-Language"] = "es-ES,es;q=0.8,en-US;q=0.5,en;q=0.3"
    default_headers["Accept-Charset"] = "UTF-8"
    default_headers["Accept-Encoding"] = "gzip"
    cj = cookielib.MozillaCookieJar()
    request_headers = default_headers.copy()
    url = urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]")
    handlers = [urllib2.HTTPHandler(debuglevel=False)]
    handlers.append(NoRedirectHandler())
    handlers.append(urllib2.HTTPCookieProcessor(cj))
    opener = urllib2.build_opener(*handlers)
    req = urllib2.Request(url, None, request_headers)
    handle = opener.open(req, timeout=None)

    return handle.headers.dict.get('location') 

def urlopen(self, url, redirect=True, **args):
        if 'data' in args and type(args['data']) == dict:
            args['data'] = json.dumps(args['data'])
            # arg['data'] = urllib.urlencode(args['data'])
            self.headers['Content-Type'] = 'application/json'
        if not redirect:
            self.opener = urllib2.build_opener(
                self.SmartRedirectHandler(),
                urllib2.HTTPCookieProcessor(self.cookiejar))
        rs = self.opener.open(
            urllib2.Request(url, headers=self.headers, **args), timeout=30)
        if 'Location' in rs.headers:
            return rs.headers.get('Location', '')
        if rs.headers.get('content-encoding', '') == 'gzip':
            content = gzip.GzipFile(fileobj=StringIO(rs.read())).read()
        else:
            content = rs.read()
        return content 

def __init__(self, user, pwd, softId="110614",
                 softKey="469c0d8a805a40f39d3c1ec3c9281e9c",
                 codeType="1004"):
        self.softId = softId
        self.softKey = softKey
        self.user = user
        self.pwd = pwd
        self.codeType = codeType
        self.uid = "100"
        self.initUrl = "http://common.taskok.com:9000/Service/ServerConfig.aspx"
        self.version = '1.1.1.2'
        self.cookieJar = cookielib.CookieJar()
        self.opener = urllib2.build_opener(
            urllib2.HTTPCookieProcessor(self.cookieJar))
        self.loginUrl = None
        self.uploadUrl = None
        self.codeUrl = None
        self.params = []
        self.uKey = None 

def read_openload(url):
    default_headers = dict()
    default_headers[
        "User-Agent"] = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3163.100 Safari/537.36"
    default_headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
    default_headers["Accept-Language"] = "es-ES,es;q=0.8,en-US;q=0.5,en;q=0.3"
    default_headers["Accept-Charset"] = "UTF-8"
    default_headers["Accept-Encoding"] = "gzip"
    cj = cookielib.MozillaCookieJar()
    request_headers = default_headers.copy()
    url = urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]")
    handlers = [urllib2.HTTPHandler(debuglevel=False)]
    handlers.append(NoRedirectHandler())
    handlers.append(urllib2.HTTPCookieProcessor(cj))
    opener = urllib2.build_opener(*handlers)
    req = urllib2.Request(url, None, request_headers)
    handle = opener.open(req, timeout=None)

    return handle.headers.dict.get('location') 

def login():
	cookie_filename = "cookies.txt"
	cookiejar = cookielib.MozillaCookieJar(cookie_filename)
	opener = urllib2.build_opener(urllib2.HTTPRedirectHandler(),urllib2.HTTPHandler(debuglevel=0),urllib2.HTTPSHandler(debuglevel=0),urllib2.HTTPCookieProcessor(cookiejar))
	page = loadPage(opener, "https://www.linkedin.com/")
	parse = BeautifulSoup(page, "html.parser")

	csrf = parse.find(id="loginCsrfParam-login")['value']
	
	login_data = urllib.urlencode({'session_key': username, 'session_password': password, 'loginCsrfParam': csrf})
	page = loadPage(opener,"https://www.linkedin.com/uas/login-submit", login_data)
	
	parse = BeautifulSoup(page, "html.parser")
	cookie = ""
	
	try:
		cookie = cookiejar._cookies['.www.linkedin.com']['/']['li_at'].value
	except:
		sys.exit(0)
	
	cookiejar.save()
	os.remove(cookie_filename)
	return cookie 

def read_openload(url):
    default_headers = dict()
    default_headers[
        "User-Agent"] = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3163.100 Safari/537.36"
    default_headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
    default_headers["Accept-Language"] = "es-ES,es;q=0.8,en-US;q=0.5,en;q=0.3"
    default_headers["Accept-Charset"] = "UTF-8"
    default_headers["Accept-Encoding"] = "gzip"
    cj = cookielib.MozillaCookieJar()
    request_headers = default_headers.copy()
    url = urllib.quote(url, safe="%/:=&?~#+!$,;'@()*[]")
    handlers = [urllib2.HTTPHandler(debuglevel=False)]
    handlers.append(NoRedirectHandler())
    handlers.append(urllib2.HTTPCookieProcessor(cj))
    opener = urllib2.build_opener(*handlers)
    req = urllib2.Request(url, None, request_headers)
    handle = opener.open(req, timeout=None)

    return handle.headers.dict.get('location') 

def send_post(url, form_data_dict):
        "pass value by POST method, return response string"
        #set headers
        user_agent = \
            'Mozilla/5.0 (Linux; Android 4.2.2; GT-I9505 Build/JDQ39) ' +\
            'AppleWebKit/537.36 (KHTML, like Gecko) ' +\
            'Chrome/31.0.1650.59 Mobile Safari/537.36'
        headers = {'User-Agent': user_agent}
        #convert form dict data
        data = urllib.urlencode(form_data_dict)
        #get request object
        req = urllib2.Request(url, data, headers)

        #set cookie and create a general opener instead urlopen()
        cookie = cookielib.CookieJar()  # CookieJar object to store cookie
        handler = urllib2.HTTPCookieProcessor(cookie)  # create cookie processor
        opener = urllib2.build_opener(handler)  # a general opener
        #return response page content
        response = opener.open(req, timeout=100)
        page = response.read()

        return page 

def get_cookie(self):
        """
        获取cookie
        :return:
        """
        cookie = cookielib.CookieJar()
        handler = urllib2.HTTPCookieProcessor(cookie)
        opener = urllib2.build_opener(handler)
        req = urllib2.Request(self.base_url, headers=self.headers)
        response = opener.open(req).read()
        self.cookie = cookie 

def getHttpData(url):
    if url[:2] == '//':
        url = 'http:' + url
    print "getHttpData: " + url
    # setup proxy support
    proxy = __addon__.getSetting('http_proxy')
    type = 'http'
    if proxy <> '':
        ptype = re.split(':', proxy)
        if len(ptype)<3:
            # full path requires by Python 2.4
            proxy = type + '://' + proxy 
        else: type = ptype[0]
        httpProxy = {type: proxy}
    else:
        httpProxy = {}
    proxy_support = urllib2.ProxyHandler(httpProxy)
    
    # setup cookie support
    cj = cookielib.MozillaCookieJar(cookieFile)
    if os.path.isfile(cookieFile):
        cj.load(ignore_discard=True, ignore_expires=True)
    else:
        if not os.path.isdir(os.path.dirname(cookieFile)):
            os.makedirs(os.path.dirname(cookieFile))
   
    # create opener for both proxy and cookie
    opener = urllib2.build_opener(proxy_support, urllib2.HTTPCookieProcessor(cj))
    charset=''
    req = urllib2.Request(url)
    req.add_header('User-Agent', UserAgent)
    try:
        response = opener.open(req)
    except urllib2.HTTPError, e:
        httpdata = e.read() 

def get_html(self, url):
        """
        获取网页html
        :return:
        """
        req = urllib2.Request(url, headers=self.headers)
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookie))
        response = opener.open(req)
        return response.read() 

def __init__(self):
        self.cookiejar = CookieJar()
        self._cookie_processor = HTTPCookieProcessor(self.cookiejar)
        self.form = None

        self.url = "http://0.0.0.0:8080/"
        self.path = "/"
        
        self.status = None
        self.data = None
        self._response = None
        self._forms = None 

def get_cookie(self):
        """
        获取cookie
        :return:
        """
        cookie = cookielib.CookieJar()
        handler = urllib2.HTTPCookieProcessor(cookie)
        opener = urllib2.build_opener(handler)
        req = urllib2.Request(self.base_url, headers=self.headers)
        response = opener.open(req).read()
        self.cookie = cookie 

def get_html(self, url):
        """
        获取网页html
        :return:
        """
        req = urllib2.Request(url, headers=self.headers)
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookie))
        response = opener.open(req)
        return response.read() 

def get_cookie(self):
        """
        获取cookie
        :return:
        """
        cookie = cookielib.CookieJar()
        handler = urllib2.HTTPCookieProcessor(cookie)
        opener = urllib2.build_opener(handler)
        req = urllib2.Request(self.base_url, headers=self.headers)
        response = opener.open(req).read()
        self.cookie = cookie 

def __init__(self, cookie_jar=None, vocde_handler=util.default_vcode_handler):
        # 初始化urlopener
        self._cookie_jar = cookielib.CookieJar() if cookie_jar is None else cookie_jar
        self._url_opener = urllib2.build_opener(
            urllib2.HTTPCookieProcessor(self._cookie_jar)
        )
        # 验证码处理函数
        self.vcode_handler = vocde_handler
        # 获取登陆信息
        self.is_login = False
        self.user_name = self.bdstoken = self.bduss = None
        self.timpstamp = self.download_sign = None
        self._get_login_info() 

def login_validate(self, stucode, passwd):
        """
        登录校验
        :param stucode: 学号
        :param passwd: 密码
        :return:
        """
        header = {
            'User-Agent': 'WeJudge.ESConnector/1.0',
            'Accept': 'text/html',
            "Content-Type": "application/x-www-form-urlencoded"
        }
        data = '__EVENTTARGET=&__EVENTARGUMENT=&__VIEWSTATE=%%2FwEPDwUKMTI1OTcwODc0Ng9kFgICAQ9kFgQCCA8PZBYCHgdvbmNsaWNrBQ93aW5kb3cuY2xvc2UoKTtkAg0PFgQeCWlubmVyaHRtbAUP5a%%2BG56CB5LiN5q2j56GuHgdWaXNpYmxlZ2Rk7YWMllHqV6LbwNWGxr43Cf425jw%%3D&__VIEWSTATEGENERATOR=09394A33&__PREVIOUSPAGE=P41Qx-bOUYMcrSUDsalSZQ66PXL-H_8xeQ4t7bJ3gWnYCDI-j8Z8SOoK8eM1&__EVENTVALIDATION=%%2FwEWCwL7joKJCgLs0bLrBgLs0fbZDAK%%2FwuqQDgKAqenNDQLN7c0VAveMotMNAu6ImbYPArursYYIApXa%%2FeQDAoixx8kBnW6rorZ5%%2FA8zs6L4Q7VnbZC%%2BVVQ%%3D&TextBox1=%s&TextBox2=%s&RadioButtonList1=%%E5%%AD%%A6%%E7%%94%%9F&Button4_test='
        req = urllib2.Request("http://es.bnuz.edu.cn/default2.aspx", headers=header, data=data % (stucode, passwd))
        debug_handler = urllib2.HTTPHandler()
        opener = urllib2.build_opener(debug_handler, RedirctHandler, urllib2.HTTPCookieProcessor(self.cookie))
        try:
            web = opener.open(req)
            ctx = web.read()
            if u"密码不正确" in ctx:
                self.login_msg = "密码不正确"
            elif u"5分钟" in ctx:
                self.login_msg = "因验证接口密码错误限制，请5分钟以后再试"
            else:
                self.login_msg = "登录失败"
            self.is_login = False
            return False
        except urllib2.URLError as e:
            if hasattr(e, 'code') and e.code == 302:
                if hasattr(e, 'info') and 'error' in e.info().get('Location'):
                    self.login_msg = "登录失败"
                    self.is_login = False
                    return False
                self.login_msg = ""
                self.is_login = True
                return True
            else:
                print e.read()
                self.login_msg = "未知错误"
                self.is_login = False
                return False 

def login(self):
        print "*** Login"
        login = self.addon.getSetting('login')
        if login:
            password = self.addon.getSetting('password')
            headers = {
                "Host" : "seasonvar.ru",
                "Referer" : self.url + '/',
                "Origin" : self.url,
                "User-Agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"
                }                    
            values = {
                "login": login,
                "password": password
                }
        
            cj = cookielib.CookieJar()
            opener = build_opener(HTTPCookieProcessor(cj), HTTPHandler())
            req = Request(self.url + "/?mod=login", urllib.urlencode(values), headers)
            f = opener.open(req)
            
            for cookie in cj:
                cookie = str(cookie).split('svid1=')[-1].split(' ')[0].strip()
                if cookie and (cookie > ""):
                    self.addon.setSetting('cookie', cookie)
                    #self.authcookie = "svid1=" + cookie
                    #self.vip = True 

def _GetOpener(self):
    """Returns an OpenerDirector that supports cookies and ignores redirects.

    Returns:
      A urllib2.OpenerDirector object.
    """
    opener = urllib2.OpenerDirector()
    opener.add_handler(urllib2.ProxyHandler())
    opener.add_handler(urllib2.UnknownHandler())
    opener.add_handler(urllib2.HTTPHandler())
    opener.add_handler(urllib2.HTTPDefaultErrorHandler())
    opener.add_handler(urllib2.HTTPSHandler())
    opener.add_handler(urllib2.HTTPErrorProcessor())
    if self.save_cookies:
      self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies")
      self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
      if os.path.exists(self.cookie_file):
        try:
          self.cookie_jar.load()
          self.authenticated = True
          StatusUpdate("Loaded authentication cookies from %s" %
                       self.cookie_file)
        except (cookielib.LoadError, IOError):
          # Failed to load cookies - just ignore them.
          pass
      else:
        # Create an empty cookie file with mode 600
        fd = os.open(self.cookie_file, os.O_CREAT, 0600)
        os.close(fd)
      # Always chmod the cookie file
      os.chmod(self.cookie_file, 0600)
    else:
      # Don't save cookies across runs of update.py.
      self.cookie_jar = cookielib.CookieJar()
    opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
    return opener 

def get_html(self, url):
        """
        获取网页html
        :return:
        """
        req = urllib2.Request(url, headers=self.headers)
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookie))
        response = opener.open(req)
        return response.read() 

def _GetOpener(self):
    """Returns an OpenerDirector that supports cookies and ignores redirects.

    Returns:
      A urllib2.OpenerDirector object.
    """
    opener = urllib2.OpenerDirector()
    opener.add_handler(urllib2.ProxyHandler())
    opener.add_handler(urllib2.UnknownHandler())
    opener.add_handler(urllib2.HTTPHandler())
    opener.add_handler(urllib2.HTTPDefaultErrorHandler())
    opener.add_handler(urllib2.HTTPSHandler())
    opener.add_handler(urllib2.HTTPErrorProcessor())
    if self.save_cookies:
      self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies")
      self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
      if os.path.exists(self.cookie_file):
        try:
          self.cookie_jar.load()
          self.authenticated = True
          StatusUpdate("Loaded authentication cookies from %s" %
                       self.cookie_file)
        except (cookielib.LoadError, IOError):
          # Failed to load cookies - just ignore them.
          pass
      else:
        # Create an empty cookie file with mode 600
        fd = os.open(self.cookie_file, os.O_CREAT, 0600)
        os.close(fd)
      # Always chmod the cookie file
      os.chmod(self.cookie_file, 0600)
    else:
      # Don't save cookies across runs of update.py.
      self.cookie_jar = cookielib.CookieJar()
    opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
    return opener 

def _urllib2Opener():
    """
    This function creates the urllib2 OpenerDirector.
    """

    debugMsg = "creating HTTP requests opener object"
    logger.debug(debugMsg)

    handlers = [proxyHandler, authHandler, redirectHandler, rangeHandler, httpsHandler]

    if not conf.dropSetCookie:
        if not conf.loadCookies:
            conf.cj = cookielib.CookieJar()
        else:
            conf.cj = cookielib.MozillaCookieJar()
            resetCookieJar(conf.cj)

        handlers.append(urllib2.HTTPCookieProcessor(conf.cj))

    # Reference: http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html
    if conf.keepAlive:
        warnMsg = "persistent HTTP(s) connections, Keep-Alive, has "
        warnMsg += "been disabled because of its incompatibility "

        if conf.proxy:
            warnMsg += "with HTTP(s) proxy"
            logger.warn(warnMsg)
        elif conf.authType:
            warnMsg += "with authentication methods"
            logger.warn(warnMsg)
        else:
            handlers.append(keepAliveHandler)

    opener = urllib2.build_opener(*handlers)
    urllib2.install_opener(opener) 

def get_all_user_by_async(self):
        user_link = urlparse.urljoin(self.url if self.url[len(self.url)-1] == '/' else self.url+'/', self.user_link)
        cookiejar = cookielib.CookieJar()
        #httpHandler = urllib2.HTTPHandler(debuglevel=1)
        #opener = urllib2.build_opener(httpHandler, urllib2.HTTPCookieProcessor(cookiejar))
        opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar))

        opener.addheaders = [('User-Agent', HTTP_HEADERS['User-Agent'])]
        urllib2.install_opener(opener)

        try:
            html = urllib2.urlopen(user_link, timeout = self.timeout).read()
            result = re.findall(u'makeStaplerProxy\(\'(.*);</script>', html)
            if len(result) != 0:
                re_list = result[0].split(',')
                proxy_num = re_list[0][re_list[0].rfind('/')+1:-1]
                crumb = re_list[1].strip('\'')

                if len(re_list) == 4 and re_list[2].find('start') == -1:
                    self.user_list.extend(self.__get_peopel_waiting_done(urllib2, user_link ,crumb, proxy_num))
                else:
                    start_url = '%s/$stapler/bound/%s/start' % (self.url, proxy_num)
                    req = urllib2.Request(start_url, data = '[]')
                    req.add_header("Content-type", 'application/x-stapler-method-invocation;charset=UTF-8')
                    req.add_header("X-Prototype-Version", "1.7")
                    req.add_header("Origin", self.url)
                    req.add_header("Crumb", crumb)
                    req.add_header("Accept", 'text/javascript, text/html, application/xml, text/xml, */*')
                    req.add_header("X-Requested-With", "XMLHttpRequest")
                    req.add_header("Referer", user_link)
                    resp = urllib2.urlopen(req, timeout = self.timeout)

                    if resp.getcode() == 200:
                        self.user_list.extend(self.__get_peopel_waiting_done(urllib2, user_link, crumb, proxy_num))

        except urllib2.HTTPError,e:
            color_output('[-]....get_all_user_by_async failed, retcode:%d' % e.code, False)
            return False 

def __init__(self, host, port, timeout=120, ssl=False, opener=None):
        self._root_url = "{http}{host}:{port}".format(
            http=(ssl and "https://" or "http://"), host=host, port=port)
        self._timeout = timeout
        self._builder = URLBuilder(self)
        self._opener = opener
        if not opener:
            cookie_jar = CookieJar()
            self._opener = build_opener(HTTPCookieProcessor(cookie_jar)) 

def nse_opener(self):
        """
        builds opener for urllib2
        :return: opener object
        """
        cj = CookieJar()
        return build_opener(HTTPCookieProcessor(cj)) 

def test_cookie_redirect(self):
        # cookies shouldn't leak into redirected requests
        from cookielib import CookieJar

        from test.test_cookielib import interact_netscape

        cj = CookieJar()
        interact_netscape(cj, "http://www.example.com/", "spam=eggs")
        hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n")
        hdeh = urllib2.HTTPDefaultErrorHandler()
        hrh = urllib2.HTTPRedirectHandler()
        cp = urllib2.HTTPCookieProcessor(cj)
        o = build_test_opener(hh, hdeh, hrh, cp)
        o.open("http://www.example.com/")
        self.assertTrue(not hh.req.has_header("Cookie")) 

def test_cookies(self):
        cj = MockCookieJar()
        h = urllib2.HTTPCookieProcessor(cj)
        o = h.parent = MockOpener()

        req = Request("http://example.com/")
        r = MockResponse(200, "OK", {}, "")
        newreq = h.http_request(req)
        self.assertTrue(cj.ach_req is req is newreq)
        self.assertEqual(req.get_origin_req_host(), "example.com")
        self.assertTrue(not req.is_unverifiable())
        newr = h.http_response(req, r)
        self.assertTrue(cj.ec_req is req)
        self.assertTrue(cj.ec_r is r is newr) 

def _cookies_init(self):
        if self.CJ is None:
            return

        urllib2.install_opener(
            urllib2.build_opener(
                urllib2.HTTPCookieProcessor(self.CJ)
            )
        )

        self.cookie_path = os.path.join(self.path, 'cookies')
        if not os.path.exists(self.cookie_path):
            os.makedirs(self.cookie_path)
            # print '[%s]: os.makedirs(cookie_path=%s)' % (addon_id, cookie_path) 

def test_cookie_redirect(self):
        # cookies shouldn't leak into redirected requests
        from cookielib import CookieJar

        from test.test_cookielib import interact_netscape

        cj = CookieJar()
        interact_netscape(cj, "http://www.example.com/", "spam=eggs")
        hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n")
        hdeh = urllib2.HTTPDefaultErrorHandler()
        hrh = urllib2.HTTPRedirectHandler()
        cp = urllib2.HTTPCookieProcessor(cj)
        o = build_test_opener(hh, hdeh, hrh, cp)
        o.open("http://www.example.com/")
        self.assertTrue(not hh.req.has_header("Cookie")) 

def test_cookies(self):
        cj = MockCookieJar()
        h = urllib2.HTTPCookieProcessor(cj)
        o = h.parent = MockOpener()

        req = Request("http://example.com/")
        r = MockResponse(200, "OK", {}, "")
        newreq = h.http_request(req)
        self.assertTrue(cj.ach_req is req is newreq)
        self.assertEqual(req.get_origin_req_host(), "example.com")
        self.assertTrue(not req.is_unverifiable())
        newr = h.http_response(req, r)
        self.assertTrue(cj.ec_req is req)
        self.assertTrue(cj.ec_r is r is newr) 

def do_login(self):
    #set cookie
        cj = cookielib.CookieJar()
        url_login = self.url_login
        form_data = self.form_data_dict
        try:
            opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
            urllib2.install_opener(opener)
            req = urllib2.Request(url_login, urllib.urlencode(form_data))
            u = urllib2.urlopen(req)
            return u.read().decode('utf-8').encode('gbk')
        except:
            print "Ooops! Failed to log in !>_< there may be a problem."
            return