Python requests.session() Examples

def fetch_access_rules(session, page_num=1, zone_id=None, per_page=50):
    """
    Fetch current access rules from the CloudFlare API
    """

    # If zone_id, only apply rule to current zone/domain
    params = {'page': page_num, 'per_page': per_page}
    if zone_id:
        r = session.get('https://api.cloudflare.com/client/v4/zones/{}'
                        '/firewall/access_rules/rules'.format(
                            zone_id), params=params)
    else:
        r = session.get('https://api.cloudflare.com/client/v4/user'
                        '/firewall/access_rules/rules', params=params)
    r.raise_for_status()
    res = r.json()
    if not res['success']:
        raise CloudFlareAPIError(res['errors'])
    else:
        return res 

def head_object(self, container_name, key):
        """
        Head object from Swift with a key. Throws StorageNoSuchKeyError if the given key does not exist.
        :param key: key of the object
        :return: Data of the object
        :rtype: str/bytes
        """
        url = '/'.join([self.endpoint, container_name, key])
        try:
            res = self.session.head(url)
            if res.status_code == 200:
                return res.headers
            elif res.status_code == 404:
                raise StorageNoSuchKeyError(container_name, key)
            else:
                raise Exception('{} - {}'.format(res.status_code, key))
        except Exception as e:
            raise StorageNoSuchKeyError(container_name, key) 

def viewRemainingImgurRequests(self):
        if self._rddtDataExtractor.currentlyDownloading:
            QMessageBox.warning(QMessageBox(), "Data Extractor for reddit",
                                "Cannot view imgur requests while currently downloading. Please wait.")
            return
        msgBox = QMessageBox()
        msgBox.setWindowTitle("Data Extractor for reddit")
        if self._rddtDataExtractor.imgurAPIClientID is not None:
            headers = {'Authorization': 'Client-ID ' + self._rddtDataExtractor.imgurAPIClientID}
            apiURL = "https://api.imgur.com/3/credits"
            requestsSession = requests.session()
            requestsSession.headers[
                'User-Agent'] = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36'
            json = exceptionSafeJsonRequest(requestsSession, apiURL, headers=headers, stream=True,
                                            verify='RedditDataExtractor/cacert.pem')
            if json is not None and json.get('data') is not None and json.get('data').get('ClientRemaining'):
                msgBox.setText("You have " + str(json.get('data').get('ClientRemaining')) + " requests remaining.")
            else:
                msgBox.setText(
                    "A problem occurred using the Imgur API. Check that you are connected to the internet and make sure your client-id is correct.")
        else:
            msgBox.setText(
                "You do not currently have an Imgur client-id set. To set one, go to settings and check 'Change / Reset Client-id'")
        msgBox.exec() 

def __init__(self):
        """
        A simple dialog for letting the user know that a client-id needs to be set to use album / gallery / page
        downloads from Imgur
        """
        QDialog.__init__(self)

        # Set up the user interface from Designer.
        self.setupUi(self)

        self.enterClientIdBtn.clicked.connect(self._checkClientIdLineEdit)
        self.enterLaterBtn.clicked.connect(self._enterLater)

        self._requestsSession = requests.session()
        self._requestsSession.headers[
            'User-Agent'] = 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36'

        self.imgurAPIClientID = None 

def yt(query):
    with requests.session() as s:
         isi = []
         if query == "":
             query = "S1B tanysyz"   
         s.headers['user-agent'] = 'Mozilla/5.0'
         url    = 'http://www.youtube.com/results'
         params = {'search_query': query}
         r    = s.get(url, params=params)
         soup = BeautifulSoup(r.content, 'html5lib')
         for a in soup.select('.yt-lockup-title > a[title]'):
            if '&list=' not in a['href']:
                if 'watch?v' in a['href']:
                    b = a['href'].replace('watch?v=', '')
                    isi += ['youtu.be' + b]
         return isi 

def __init__(self, product_link, domain):
        '''
        (str, str) -> eBae
        Given a link to an eBay product <product_link> and a catch-all domain
        address <domain>, a random email address is generated and an eBae
        object is returned.
        
        REQ: domain is a catch-all domain
        REQ: product_link is a link to a product listed on eBay
        '''
        self.s = requests.session()
        self.product_link = product_link
        self.s.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36"
        }
        self.proxy_list = read_from_txt("proxies.txt")
        self.email = gen_email(domain) 

def remove_access_rule(session, rule_id, zone_id=None):
    """
    Remove an existing access rule via the CloudFlare API
    """
    if zone_id:
        r = session.delete('https://api.cloudflare.com/client/v4/zones/{}'
                           '/firewall/access_rules/rules/{}'.format(
                               zone_id, rule_id))
    else:
        # Apply rule across all zones
        r = session.delete('https://api.cloudflare.com/client/v4/user'
                           '/firewall/access_rules/rules/{}'.format(
                               rule_id))
    r.raise_for_status()
    res = r.json()
    if not res['success']:
        raise CloudFlareAPIError(res['errors']) 

def __init__(self, swift_config, **kwargs):
        self.auth_url = swift_config['swift_auth_url']
        self.user_id = swift_config['swift_user_id']
        self.project_id = swift_config['swift_project_id']
        self.password = swift_config['swift_password']
        self.region = swift_config['swift_region']
        self.endpoint = None

        if 'token' in swift_config:
            self.token = swift_config['token']
            self.endpoint = swift_config['endpoint']
        else:
            self.token = self.generate_swift_token()
            swift_config['token'] = self.token
            swift_config['endpoint'] = self.endpoint

        self.session = requests.session()
        self.session.headers.update({'X-Auth-Token': self.token})
        adapter = requests.adapters.HTTPAdapter(pool_maxsize=64, max_retries=3)
        self.session.mount('http://', adapter)
        self.session.mount('https://', adapter) 

def put_object(self, container_name, key, data):
        """
        Put an object in Swift. Override the object if the key already exists.
        :param key: key of the object.
        :param data: data of the object
        :type data: str/bytes
        :return: None
        """
        url = '/'.join([self.endpoint, container_name, key])
        try:
            res = self.session.put(url, data=data)
            status = 'OK' if res.status_code == 201 else 'Error'
            try:
                logger.debug('PUT Object {} - Size: {} - {}'.format(key, sizeof_fmt(len(data)), status))
            except Exception:
                logger.debug('PUT Object {} - {}'.format(key, status))
        except Exception as e:
            print(e) 

def bucket_exists(self, container_name):
        """
        Head container from Swift with a name. Throws StorageNoSuchKeyError if the given container does not exist.
        :param container_name: name of the container
        :return: Data of the bucket
        :rtype: str/bytes
        """
        url = '/'.join([self.endpoint, container_name])
        try:
            res = self.session.head(url)
            if res.status_code == 204:
                return res.headers
            elif res.status_code == 404:
                raise StorageNoSuchKeyError(container_name, '')
            else:
                raise Exception('{} - {}'.format(res.status_code))
        except Exception as e:
            raise StorageNoSuchKeyError(container_name, '') 

def list_objects(self, container_name, prefix=''):
        """
        Lists the objects in a bucket. Throws StorageNoSuchKeyError if the given bucket does not exist.
        :param key: key of the object
        :return: Data of the object
        :rtype: str/bytes
        """
        if prefix:
            url = '/'.join([self.endpoint, container_name, '?format=json&prefix='+prefix])
        else:
            url = '/'.join([self.endpoint, container_name, '?format=json'])
        try:
            res = self.session.get(url)
            objects = res.json()

            # TODO: Adapt to Key and Size
            return objects
        except Exception as e:
            raise e 

def upload_doc(session, section_id, data_path, doc):
    doc_path = get_doc_path(data_path, doc)

    print('Processing %s%s (%s)' % (doc.location, doc.title, doc.guid))

    with ZipFile(doc_path) as zip_file:
        html_content, src_file_names = clean_html(zip_file.read('index.html'), doc)

        if len(src_file_names) > 5:
            print('Upload may failed if images more than 5')

        data_send = {
            'Presentation': (None, html_content, mimetypes.guess_type('index.html')[0])
        }

        for name in src_file_names:
            data_send[name] = (None, zip_file.read('index_files/' + name), mimetypes.guess_type(name)[0])

    resp = session.post(API_BASE + '/sections/%s/pages' % section_id, files=data_send)
    resp.raise_for_status() 

def dojo_get_address(addr, at):
    # Request details about a collection of HD accounts and/or loose addresses and/or public keys.
    # Takes arguments:
    #   addr:   address or list of addresses
    #   at:     authentication token

    onion_address = dojo_get_settings()["onion"]

    url = "http://" + onion_address + "/v2/address/" + addr + "/info?at=" + at

    session = requests.session()
    session.proxies = {
        "http": "socks5h://127.0.0.1:9150",
        "https": "socks5h://127.0.0.1:9150",
    }
    try:
        auth_response = session.get(url)
    except requests.exceptions.ConnectionError:
        auth_response = {"status": "error", "error": "Connection Error"}
    return auth_response 

def dojo_get_hd(xpub, at):
    # Request details about an HD account. If account does not exist, it must be created.
    # https://github.com/Samourai-Wallet/samourai-dojo/blob/master/doc/GET_xpub.md
    onion_address = dojo_get_settings()["onion"]
    url = "http://" + onion_address + "/v2/xpub/"
    session = requests.session()
    session.proxies = {
        "http": "socks5h://127.0.0.1:9150",
        "https": "socks5h://127.0.0.1:9150",
    }
    try:
        url = url + xpub + "?at=" + at
        auth_response = session.get(url)
    except requests.exceptions.ConnectionError:
        auth_response = {"status": "error", "error": "Connection Error"}
    return auth_response 

def get_search_suggestions(query: str) -> List[str]:
        """Returns a list of suggestions for the given query from Youtube."""
        with youtube_session() as session:
            params = {
                "client": "youtube",
                "q": query[:100],  # queries longer than 100 characters are not accepted
                "xhr": "t",  # this makes the response be a json file
            }
            response = session.get(
                "https://clients1.google.com/complete/search", params=params
            )
        suggestions = json.loads(response.text)
        # first entry is the query, the second one contains the suggestions
        suggestions = suggestions[1]
        # suggestions are given as tuples
        # extract the string and skip the query if it occurs identically
        suggestions = [entry[0] for entry in suggestions if entry[0] != query]
        return suggestions 

def youtube_session() -> Iterator[requests.Session]:
    """This context opens a requests session and loads the youtube cookies file."""
    session = requests.session()
    try:
        with open(
            os.path.join(settings.BASE_DIR, "config/youtube_cookies.pickle"), "rb"
        ) as f:
            session.cookies.update(pickle.load(f))
    except FileNotFoundError:
        pass

    headers = {
        "User-Agent": youtube_dl.utils.random_user_agent(),
    }
    session.headers.update(headers)
    yield session

    with open(
        os.path.join(settings.BASE_DIR, "config/youtube_cookies.pickle"), "wb"
    ) as f:
        pickle.dump(session.cookies, f) 

def _load_cookies(self):
        cookies_path = os.path.join(consts.HOST_PATH, self.cookies_file)
        if os.path.exists(cookies_path):
            with open(cookies_path, 'r') as cookies_file:
                load_cookies = pickle.load(cookies_file)
                cookies = requests.utils.cookiejar_from_dict(load_cookies)
                self.session.cookies = cookies
                if 'BDUSS' in load_cookies:
                    self.user['BDUSS'] = load_cookies['BDUSS']
                if 'BAIDUID' in load_cookies:
                    self.user['BAIDUID'] = load_cookies['BAIDUID']
                return True
        else:
            return False 

def main():
    data_dir, docs = get_documents()

    with requests.session() as session:
        token = get_token(session)
        session.auth = BearerAuth(token)

        notebook_id = create_notebook(session)

        for location, docs in docs.items():
            section_name = location.strip('/').replace('/', '-')
            section_id = create_section(session, notebook_id, section_name)

            for doc in docs:
                upload_doc(session, section_id, data_dir, doc) 

def _request(self, path, **kwargs):
        resp = self.session.get(urljoin(self.base_url, path), **kwargs)
        
        if resp.status_code != requests.codes.ok:
            resp.raise_for_status()
        
        self.session.close()  # VLC doesn't support keep-alive
        
        return resp 

def create_section(session, notebook_id, name):
    print('Creating section: "%s"' % name)
    resp = session.post(API_BASE + '/notebooks/%s/sections' % notebook_id, json={'name': name})
    resp.raise_for_status()

    return resp.json()['id'] 

def create_notebook(session):
    name = input('Pleas input new notebook name(such as WizNote, can\'t be empty).\nName: ')
    print('Creating notebook: "%s"' % name)
    resp = session.post(API_BASE + '/notebooks', json={'name': name})
    resp.raise_for_status()

    return resp.json()['id'] 

def get_token(session):
    print('Sign in via: ')
    print(AUTH_URL)
    print('Copy the url of blank page after signed in,\n'
          'url should starts with "https://login.live.com/oauth20_desktop.srf"')

    while True:
        url = input('URL: ')

        match = re.match(r'https://login.live.com/oauth20_desktop\.srf\?code=([\w-]{37})', url)
        if not match:
            print('Invalid URL!')
            continue

        code = match.group(1)
        break

    resp = session.post(OAUTH_URL, data={
        'grant_type': 'authorization_code',
        'client_id': CLIENT_ID,
        'client_secret': '',
        'code': code,
        'redirect_uri': REDIRECT_URI,
    }).json()
    
    print('Token result:', resp)

    return resp['access_token'] 

def host_test(users, passes, target):
    with open(users) as f:
        usernames = f.readlines()
    with open(passes) as g:
        passwords = g.readlines()
    login = {'Login' : 'Login'}
    for user in usernames:
        for passwd in passwords:
            print("[*] Testing username %s and password %s against %s") % (user.rstrip('\n'), passwd.rstrip('\n'), target.rstrip('\n'))
            payload = {'username':user.rstrip('\n'), 'password':passwd.rstrip('\n')}
            session = requests.session()
            postrequest = session.post(target, payload)
            print("[*] The response size is: %s") % (len(postrequest.text))
            print("[*] The cookie for this attempt is: %s") % (str(requests.utils.dict_from_cookiejar(session.cookies))) 

def __init__(self, cookies={}):
        self.client = requests.session()
        for name, value in cookies.items():
            self.client.cookies[name] = value 

def create_client(stage_info, use_accelerate_endpoint: bool = False):
        """Creates a client object with a stage credential.

        Args:
            stage_info: Information about the stage.
            use_accelerate_endpoint: Not used for Azure client.

        Returns:
            The client to communicate with GCS.
        """
        stage_credentials = stage_info['creds']
        sas_token = stage_credentials['AZURE_SAS_TOKEN']
        if sas_token and sas_token.startswith('?'):
            sas_token = sas_token[1:]
        end_point = stage_info['endPoint']
        if end_point.startswith('blob.'):
            end_point = end_point[len('blob.'):]
        if use_new_azure_api:
            client = BlobServiceClient(
                account_url="https://{}.blob.{}".format(
                    stage_info['storageAccount'],
                    end_point
                ),
                credential=sas_token)
            client._config.retry_policy = ExponentialRetry(
                initial_backoff=1,
                increment_base=2,
                max_attempts=60,
                random_jitter_range=2
            )
        else:
            client = BlockBlobService(account_name=stage_info['storageAccount'],
                                      sas_token=sas_token,
                                      endpoint_suffix=end_point)
            client._httpclient = RawBodyReadingClient(session=requests.session(), protocol="https", timeout=2000)
            client.retry = ExponentialRetry(
                initial_backoff=1, increment_base=2, max_attempts=60, random_jitter_range=2).retry

        return client 

def perform_request(self, request):
            """Sends an HTTPRequest to Azure Storage and returns an HTTPResponse.

            If the response code indicates an error, raise an HTTPError.
            """
            # Verify whether the body is in bytes or either a file-like/stream object
            if request.body:
                request.body = _get_data_bytes_or_stream_only('request.body', request.body)

            # Construct the URI
            uri = self.protocol.lower() + '://' + request.host + request.path

            # Send the request
            response = self.session.request(request.method,
                                            uri,
                                            params=request.query,
                                            headers=request.headers,
                                            data=request.body or None,
                                            timeout=self.timeout,
                                            proxies=self.proxies,
                                            stream=True)

            # Parse the response
            status = int(response.status_code)
            response_headers = {}
            for key, name in response.headers.items():
                # Preserve the case of metadata
                if key.lower().startswith('x-ms-meta-'):
                    response_headers[key] = name
                else:
                    response_headers[key.lower()] = name

            wrap = HTTPResponse(status, response.reason, response_headers, response.raw.read())
            response.close()

            return wrap 

def search_id(self) -> Optional[str]:
        with youtube_session() as session:
            params = {
                "search_query": self.query,
                # this is the value that youtube uses to filter for playlists only
                "sp": "EgQQA1AD",
            }
            response = session.get("https://www.youtube.com/results", params=params)

        initial_data = Youtube._get_initial_data(response.text)

        path = [
            "contents",
            "twoColumnSearchResultsRenderer",
            "primaryContents",
            "sectionListRenderer",
            "contents",
        ]
        section_renderers = initial_data
        for step in path:
            section_renderers = section_renderers[step]

        list_id = None
        for section_renderer in cast(List[Dict[str, Any]], section_renderers):
            search_results = section_renderer["itemSectionRenderer"]["contents"]

            try:
                list_id = next(
                    res["playlistRenderer"]["playlistId"]
                    for res in search_results
                    if "playlistRenderer" in res
                )
                break
            except StopIteration:
                # the search result did not contain the list id
                pass

        return list_id 

def get_suggestion(self) -> str:
        with youtube_session() as session:
            response = session.get(self.get_external_url())

        initial_data = Youtube._get_initial_data(response.text)

        path = [
            "contents",
            "twoColumnWatchNextResults",
            "secondaryResults",
            "secondaryResults",
            "results",
            0,
            "compactAutoplayRenderer",
            "contents",
            0,
            "compactVideoRenderer",
            "navigationEndpoint",
            "commandMetadata",
            "webCommandMetadata",
            "url",
        ]
        url = initial_data
        for step in path:
            url = url[cast(str, step)]
        return "https://www.youtube.com" + cast(str, url) 

def _init_session(session):
    if session is None:
        session = requests.session()
    return session 

def post_credentials(session, username, password):
    # Credentials POSTing and tokens retrieval
    # POST https://secure.zwift.com/auth/realms/zwift/tokens/access/codes

    try:
        response = session.post(
            url="https://secure.zwift.com/auth/realms/zwift/tokens/access/codes",
            headers={
                "Accept": "*/*",
                "Accept-Encoding": "gzip, deflate",
                "Connection": "keep-alive",
                "Content-Type": "application/x-www-form-urlencoded",
                "Host": "secure.zwift.com",
                "User-Agent": "Zwift/1.5 (iPhone; iOS 9.0.2; Scale/2.00)",
                "Accept-Language": "en-US;q=1",
            },
            data={
                "client_id": "Zwift_Mobile_Link",
                "username": username,
                "password": password,
                "grant_type": "password",
            },
            allow_redirects = False,
            verify = args.verifyCert,
        )

        if args.verbose:
            print('Response HTTP Status Code: {status_code}'.format(
                status_code=response.status_code))
            print('Response HTTP Response Body: {content}'.format(
                content=response.content))

        json_dict = json.loads(response.content)

        return (json_dict["access_token"], json_dict["refresh_token"], json_dict["expires_in"])

    except requests.exceptions.RequestException as e:
        print('HTTP Request failed: %s' % e)