Python http.cookiejar.CookieJar() Examples

def __init__(self):
        super(WebService, self).__init__()
        self.cache = defaultdict(defaultdict)
        self._cookie = CookieJar()
        self._opener = urllib2.build_opener(
            urllib2.HTTPCookieProcessor(self._cookie))
        self.query_interval = 1 

def __request__(self):

        Cookiejar = CookieJar()
        opener = build_opener(HTTPCookieProcessor(Cookiejar))
        _header = dict(self.headers.items())
        if self.cookie:
            _header.update({'Cookie': self.cookie})
        req = Request(self.url, headers=_header, origin_req_host=self.host)

        res = None

        try:
            res = opener.open(req)
            # break
        except HTTPError as e:
            # if e.code >= 400 and e.code < 500:
            return e, None

        except (socket.timeout, URLError) as e:
            return e, None
        except Exception as e:
            traceback.print_exc()
            return e, None

        return res, Cookiejar._cookies 

def cookie_friendly_download(referer_url, file_url, store_dir='.', timeout=1000):
    from http.cookiejar import CookieJar
    from urllib import request
    cj = CookieJar()
    cp = request.HTTPCookieProcessor(cj)
    opener = request.build_opener(cp)
    with opener.open(referer_url) as fin:
        fin.headers.items()
    import os
    from os import path
    with opener.open(file_url, timeout=timeout) as fin:
        file_bin = fin.read()
        filename = fin.headers['Content-Disposition']
        filename = filename.split(';')[-1].split('=')[1]
        os.makedirs(store_dir, exist_ok=True)
        with open(path.join(store_dir, filename), mode='wb') as fout:
            fout.write(file_bin)
            return path.join(store_dir, filename) 

def set_cookies(firefox_cookies_path: Optional[str]):
    """
    显式设置Cookies
    :param firefox_cookies_path:
    :return:
    """
    global __COOKIES_PATH
    __COOKIES_PATH = firefox_cookies_path
    if firefox_cookies_path is None:
        warn("Haven't set a valid cookies path, use default.")
        get_request_session().cookies = CookieJar()
    else:
        try:
            get_request_session().cookies = _init_cookies(CookieJar(), firefox_cookies_path)
        except:
            warn("Error while loading firefox cookies from: {}, please check it.".format(__COOKIES_PATH))


# If we have default cookies path, set here 

def __init__(self, host, auth_header, debug=0):
        self._host = host
        self._headers['Authorization'] = auth_header

        self._opener = build_opener(HTTPHandler(debuglevel=debug),
                                    HTTPSHandler(debuglevel=debug),
                                    HTTPCookieProcessor(CookieJar()),
                                    LoggingHandler(),
                                    HTTPDefaultErrorHandler()) 

def __init__(self, app: "Quart", use_cookies: bool = True) -> None:
        self.cookie_jar: Optional[CookieJar]
        if use_cookies:
            self.cookie_jar = CookieJar()
        else:
            self.cookie_jar = None
        self.app = app
        self.push_promises: List[Tuple[str, Headers]] = []
        self.preserve_context = False 

def setOpener(self):
        """ 初始化opener
        """
        cj = cookiejar.CookieJar()
        pro = request.HTTPCookieProcessor(cj)
        self.opener = request.build_opener(pro)
        header = []
        for key, value in self._headers.items():
            elem = (key, value)
            header.append(elem)
        self.opener.addheaders = header 

def load_html(url, with_cookies=False, headers={}):
    """Attempts to load an HTML page, returning a BeautifulSoup instance. Raises
    any networking or parsing exceptions"""
    if with_cookies:
        cj = CookieJar()
        opener = urlopen.build_opener(urlopen.HTTPCookieProcessor(cj))
    else:
        opener = urlopen.build_opener()

    request = urlopen.Request(url, headers=headers)

    response = opener.open(request)
    html = response.read().decode('utf-8', errors='replace')

    soup = BeautifulSoup(html, 'html.parser')
    return soup 

def create_cookie_opener(self):
        '''
        设置启用Cookie
        :return: 返回一个自定义的opener
        '''
        cookie = cookiejar.CookieJar()
        cookie_process = request.HTTPCookieProcessor(cookie)
        opener = request.build_opener(cookie_process)
        return opener 

def download(self, url, retry_count=3, headers=None, proxy=None, data=None):
        if url is None:
            return None
        try:
            req = request.Request(url, headers=headers, data=data)
            cookie = cookiejar.CookieJar()
            cookie_process = request.HTTPCookieProcessor(cookie)
            opener = request.build_opener()
            if proxy:
                proxies = {urlparse(url).scheme: proxy}
                opener.add_handler(request.ProxyHandler(proxies))
            content = opener.open(req).read()
        except error.URLError as e:
            print('HtmlDownLoader download error:', e.reason)
            content = None
            if retry_count > 0:
                if hasattr(e, 'code') and 500 <= e.code < 600:
                    #说明是 HTTPError 错误且 HTTP CODE 为 5XX 范围说明是服务器错误，可以尝试再次下载
                    return self.download(url, retry_count-1, headers, proxy, data)
        return content 

def get_html(web_url):
    cookie_jar = cookielib.CookieJar()
    if mode == 'FAVS':
        cookie_jar = auth(cookie_jar)
    if antizapret_enabled:
        opener = urllib_request.build_opener(urllib_request.HTTPCookieProcessor(cookie_jar),
                                             az.AntizapretProxyHandler())
    else:
        opener = urllib_request.build_opener(urllib_request.HTTPCookieProcessor(cookie_jar))
    opener.addheaders = [("User-Agent", USER_AGENT)]
    connection = opener.open(web_url)
    html = connection.read()
    connection.close()
    return html.decode('utf-8') 

def get_html_with_referer(page_url, referer):
    cookie_jar = cookielib.CookieJar()
    if mode == 'FAVS':
        cookie_jar = auth(cookie_jar)
    if antizapret_enabled:
        opener = urllib_request.build_opener(urllib_request.HTTPCookieProcessor(cookie_jar),
                                             az.AntizapretProxyHandler())
    else:
        opener = urllib_request.build_opener(urllib_request.HTTPCookieProcessor(cookie_jar))
    if referer is not None:
        opener.addheaders = [("Referer", referer)]
    connection = opener.open(page_url)
    html = connection.read()
    connection.close()
    return html.decode('utf-8') 

def add_bookmark(bookmark_id):
    cookie_jar = auth(cookielib.CookieJar())
    fav_url = site_url + '/engine/ajax/favorites.php?fav_id=' + bookmark_id + '&action=plus&skin=Baskino'
    if antizapret_enabled:
        opener = urllib_request.build_opener(urllib_request.HTTPCookieProcessor(cookie_jar),
                                             az.AntizapretProxyHandler())
    else:
        opener = urllib_request.build_opener(urllib_request.HTTPCookieProcessor(cookie_jar))
    connection = opener.open(fav_url)
    connection.close()
    notificator('Добавление закладки', 'Закладка добавлена', 3000) 

def remove_bookmark(bookmark_id):
    cookie_jar = auth(cookielib.CookieJar())
    fav_url = site_url + '/engine/ajax/favorites.php?fav_id=' + bookmark_id + '&action=minus&skin=Baskino'
    if antizapret_enabled:
        opener = urllib_request.build_opener(urllib_request.HTTPCookieProcessor(cookie_jar),
                                             az.AntizapretProxyHandler())
    else:
        opener = urllib_request.build_opener(urllib_request.HTTPCookieProcessor(cookie_jar))
    connection = opener.open(fav_url)
    connection.close()
    notificator('Удаление закладки', 'Закладка удалена', 3000)
    xbmc.executebuiltin('Container.Refresh()') 

def login(username, password, client_id, perms):
    opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(CookieJar()))
    url = ('https://oauth.vk.com/authorize?redirect_uri=https://oauth.vk.com/blank.html&'
           'client_id={}&scope={}&response_type=token&display=mobile'.format(client_id, perms))
    try:
        auth_page = opener.open(url)
    except urllib.error.HTTPError as e:
        logger.exception(e)
        print(url)
        return None
    auth_page = auth_page.read().decode()
    fields = dict(FORM_ELEMENT.findall(auth_page))
    fields['email'], fields['pass'] = username, password
    redir_page = opener.open('https://login.vk.com/?act=login&soft=1&utf8=1',
                            data=urllib.parse.urlencode(fields).encode())
    token_url = redir_page.geturl()
    match = ACCESS_TOKEN.search(token_url)
    if not match:
        content = redir_page.read().decode()
        match = ALLOW_FORM.search(content)
        if not match:
            print(url)
            return None
        token_url = opener.open(match.group(1)).geturl()
        match = ACCESS_TOKEN.search(token_url)
        if not match:
            print(url)
            return None
    return match.group(1) 

def _getCookies(headers):
    cj = CookieJar()
    cookies = headers.split(',')
    for cookie in cookies:
        attrs = cookie.split(';')
        cookieNameValue = name = attrs[0].strip().split('=')
        pathNameValue = name = attrs[1].strip().split('=')
        ck = Cookie(version=1, name=cookieNameValue[0],
                              value=cookieNameValue[1],
                              port=443,
                              port_specified=False,
                              domain=swaDomain,
                              domain_specified=True,
                              domain_initial_dot=False,
                              path=pathNameValue[1],
                              path_specified=True,
                              secure=True,
                              expires=None,
                              discard=True,
                              comment=None,
                              comment_url=None,
                              rest={'HttpOnly': None},
                              rfc2109=False)
        cj.set_cookie(ck)
    return cj 

def _getNewSession(cj):
    if( cj == None):
        cj = CookieJar()
        cj = _isSessionExists()
    return cj 

def _isSessionExists():
    cookieJar = CookieJar()
    data = {'serviceID':'isSessionExists', 'isLoggedIn':'false'}
    reqData = merge_two_dicts(defaultData, data)    
    req = requests.post(urlBase,
                       headers=headers, data=reqData, cookies=cookieJar)
    #print(r1.status_code)
    cookieJar = _getCookies(req.headers['set-cookie'])
    return cookieJar 

def __init__(self, **kwargs):
        """
        @param kwargs: Keyword arguments.
            - B{proxy} - An http proxy to be specified on requests.
                 The proxy is defined as {protocol:proxy,}
                    - type: I{dict}
                    - default: {}
            - B{timeout} - Set the url open timeout (seconds).
                    - type: I{float}
                    - default: 90
        """
        Transport.__init__(self)
        Unskin(self.options).update(kwargs)
        self.cookiejar = CookieJar()
        self.proxy = {}
        self.urlopener = None 

def test_cookie_redirect(self):
        # cookies shouldn't leak into redirected requests
        from http.cookiejar import CookieJar
        from test.test_http_cookiejar import interact_netscape

        cj = CookieJar()
        interact_netscape(cj, "http://www.example.com/", "spam=eggs")
        hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n")
        hdeh = urllib.request.HTTPDefaultErrorHandler()
        hrh = urllib.request.HTTPRedirectHandler()
        cp = urllib.request.HTTPCookieProcessor(cj)
        o = build_test_opener(hh, hdeh, hrh, cp)
        o.open("http://www.example.com/")
        self.assertFalse(hh.req.has_header("Cookie")) 

def _init_cookies(cookie_jar: CookieJar, firefox_cookies_path: str):
    """
    Initialize cookies from firefox
    :param cookie_jar:
    :param firefox_cookies_path:
    :return:
    """
    if firefox_cookies_path is None:
        firefox_cookies_path = __COOKIES_PATH
    con = sqlite3.connect(firefox_cookies_path)
    cur = con.cursor()
    # noinspection SqlResolve
    cur.execute("SELECT host, path, isSecure, expiry, name, value FROM moz_cookies")
    for item in cur.fetchall():
        c = Cookie(
            0,
            item[4],
            item[5],
            None,
            False,
            item[0],
            item[0].startswith('.'),
            item[0].startswith('.'),
            item[1],
            False,
            item[2],
            item[3],
            item[3] == "",
            None, None, {}
        )
        cookie_jar.set_cookie(c)
    return cookie_jar 

def __init__(self):
        super(WebService, self).__init__()
        self._cookie = CookieJar()
        self._opener = urllib2.build_opener(
            urllib2.HTTPCookieProcessor(self._cookie))
        self.query_interval = 1.0 

def initOpener(self):
        self.cookiejar = CookieJar()
        self.opener = build_opener(HTTPCookieProcessor(self.cookiejar))
        self.opener.addheaders = list(self.headers.items()) 

def getrawsource(url, ua):
    if url == "": # Empty freakin shit
        r = ['0','Empty URL Provided','', '']
        return r
    try:
        ckreq = urllib.request.Request(
        url,
        data=None,
        headers={
            'User-Agent': ua,
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
            #'Accept-Encoding': 'gzip, deflate, sdch',
            'Accept-Language': 'en-US,en;q=0.8',
            'Connection': 'keep-alive'
        }
        )
        cj = CookieJar()
        opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
        with opener.open(ckreq, timeout=8) as response:
            scode = response.read().decode("utf-8", 'ignore')
            headers = str(response.info())
            rurl = response.geturl()
            r = ['1', scode, headers, rurl] ## 'success code', 'source code', 'http headers', 'redirect url'
            return r
    except Exception as e:
        ef = str(e)
        try:
            ecode = str(e.code)
            ehed = str(e.info())
            r = ['2', ef, ecode, ehed] ## will come in handy evading good guys
            return r
        except Exception as f:
            r = ['2', ef, '', ''] ## 'error code', 'error message', 'empty'
            return r 

def __init__(self, email, password):
        self.email = email
        self.password = password
        self.cj = CookieJar() 

def _create_token_dictionary(cookie_data, cookie_name='X-VRT-Token'):
        """Create a dictionary with token name and token expirationDate from a Python cookielib.CookieJar or urllib2 Set-Cookie http header"""
        if cookie_data is None:
            return None
        if isinstance(cookie_data, cookielib.CookieJar):
            # Get token dict from cookiejar
            token_cookie = next((cookie for cookie in cookie_data if cookie.name == cookie_name), None)
            if token_cookie is None:
                return None
            from datetime import datetime
            token_dictionary = {
                token_cookie.name: token_cookie.value,
                'expirationDate': datetime.utcfromtimestamp(token_cookie.expires).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
            }
        elif cookie_name in cookie_data:
            # Get token dict from http header
            import dateutil.parser
            import re
            cookie_data = cookie_data.split('X-VRT-Token=')[1].split('; ')
            expires_regex = re.compile(r'(?P<expires>[A-Za-z]{3}, \d{2} [A-Za-z]{3} \d{4} \d{2}:\d{2}:\d{2} [A-Za-z]{3})')
            expires = re.search(expires_regex, cookie_data[2]).group('expires')
            token_dictionary = {
                cookie_name: cookie_data[0],
                'expirationDate': dateutil.parser.parse(expires).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
            }
        else:
            return None
        return token_dictionary 

def _get_usertoken(self, name=None, login_json=None):
        """Get a user X-VRT-Token, vrtlogin-at, vrtlogin-expiry, vrtlogin-rt, SESSION, OIDCXSRF or state token"""
        if not login_json:
            login_json = self._get_login_json()
        cookiejar = cookielib.CookieJar()
        opener = build_opener(HTTPCookieProcessor(cookiejar), ProxyHandler(self._proxies))
        log(2, 'URL get: {url}', url=unquote(self._USER_TOKEN_GATEWAY_URL))
        opener.open(self._USER_TOKEN_GATEWAY_URL)
        xsrf = next((cookie for cookie in cookiejar if cookie.name == 'OIDCXSRF'), None)
        if xsrf is None:
            return None
        payload = dict(
            UID=login_json.get('UID'),
            UIDSignature=login_json.get('UIDSignature'),
            signatureTimestamp=login_json.get('signatureTimestamp'),
            client_id='vrtnu-site',
            _csrf=xsrf.value
        )
        data = urlencode(payload).encode()
        log(2, 'URL post: {url}', url=unquote(self._VRT_LOGIN_URL))
        opener.open(self._VRT_LOGIN_URL, data=data)

        # Cache additional tokens for later use
        refreshtoken = TokenResolver._create_token_dictionary(cookiejar, cookie_name='vrtlogin-rt')
        accesstoken = TokenResolver._create_token_dictionary(cookiejar, cookie_name='vrtlogin-at')
        if refreshtoken is not None:
            from json import dumps
            cache_file = self._get_token_filename('vrtlogin-rt')
            update_cache(cache_file, dumps(refreshtoken), self._TOKEN_CACHE_DIR)
        if accesstoken is not None:
            from json import dumps
            cache_file = self._get_token_filename('vrtlogin-at')
            update_cache(cache_file, dumps(accesstoken), self._TOKEN_CACHE_DIR)

        return TokenResolver._create_token_dictionary(cookiejar, name) 

def _get_fresh_token(self, refresh_token, name):
        """Refresh an expired X-VRT-Token, vrtlogin-at or vrtlogin-rt token"""
        refresh_url = self._TOKEN_GATEWAY_URL + '/refreshtoken?legacy=true'
        cookie_value = 'vrtlogin-rt=' + refresh_token
        headers = {'Cookie': cookie_value}
        cookiejar = cookielib.CookieJar()
        opener = build_opener(HTTPCookieProcessor(cookiejar), ProxyHandler(self._proxies))
        req = Request(refresh_url, headers=headers)
        log(2, 'URL get: {url}', url=refresh_url)
        opener.open(req)
        return TokenResolver._create_token_dictionary(cookiejar, name) 

def __init__(self):
        self.cookiejar = CookieJar()
        self._cookie_processor = HTTPCookieProcessor(self.cookiejar)
        self.form = None

        self.url = "http://0.0.0.0:8080/"
        self.path = "/"
        
        self.status = None
        self.data = None
        self._response = None
        self._forms = None 

def get_response(self, query):
        response = None
        try:
            # we can't use helpers.retrieve_url because of redirects
            # we need the cookie processor to handle redirects
            opener = urllib_request.build_opener(urllib_request.HTTPCookieProcessor(CookieJar()))
            response = opener.open(query).read().decode('utf-8')
        except urllib_request.HTTPError as e:
            # if the page returns a magnet redirect, used in download_torrent
            if e.code == 302:
                response = e.url
        except Exception:
            pass
        return response