Python urllib.request.add_header() Examples

def put(self, location, params=None):
        """Dispatch a PUT request to a SeaMicro chassis.

        The seamicro box has order-dependent HTTP parameters, so we build
        our own get URL, and use a list vs. a dict for data, as the order is
        implicit.
        """
        opener = urllib.request.build_opener(urllib.request.HTTPHandler)
        url = self.build_url(location, params)
        request = urllib.request.Request(url)
        request.get_method = lambda: "PUT"
        request.add_header("content-type", "text/json")
        response = opener.open(request)
        json_data = self.parse_response(url, response)

        return json_data["result"] 

def makePut(self,url):
        """Puts data to Canvas (passes token as header)"""
        try:
            # Tack on http://.../ to the beginning of the url if needed
            if self.CANVAS_API not in url:
                urlString = self.CANVAS_API+url
            else:
                urlString = url
        
            print("Putting: " +urlString)
            request = urllib.request.Request(urlString, method='PUT')
            request.add_header("Authorization", "Bearer " + self.CANVAS_TOKEN);
            response = urllib.request.urlopen(request)
            json_string = response.read().decode('utf-8');
            retVal = dict(json.loads(json_string))
            #print (retVal)
            if(response.status == 200):
                return True
            else:
                return False
        except Exception as ex:
            print(ex)
            e = sys.exc_info()[0]
            print(e)
            raise 

def downloadUrls(self, urls):
        url_data = {}
        for u in urls:
            url = self.base_url + u

            request = urllib.request.Request(url)


            # the .htaccess file checks for the header, and if it exists returns unprocessed data.
            request.add_header('User-agent', 'our-web-crawler')
            try:
                response = urllib.request.urlopen(request)
                data = response.read()
            except urllib.request.HTTPError:
                log (url)
                raise
            except urllib.request.URLError:
                log (url)
                raise
            yield (u,data) 

def get_blob(self, thread_id, blob_id):
        """Returns a file-like object with the contents of the given blob from
        the given thread.

        The object is described in detail here:
        https://docs.python.org/2/library/urllib2.html#urllib2.urlopen
        """
        request = Request(
            url=self._url("blob/%s/%s" % (thread_id, blob_id)))
        if self.access_token:
            request.add_header("Authorization", "Bearer " + self.access_token)
        try:
            return urlopen(request, timeout=self.request_timeout)
        except HTTPError as error:
            try:
                # Extract the developer-friendly error message from the response
                message = json.loads(error.read().decode())["error_description"]
            except Exception:
                raise error
            raise QuipError(error.code, message, error) 

def _fetch_json(self, path, post_data=None, **args):
        request = Request(url=self._url(path, **args))
        if post_data:
            post_data = dict((k, v) for k, v in post_data.items()
                             if v or isinstance(v, int))
            request_data = urlencode(self._clean(**post_data))
            if PY3:
                request.data = request_data.encode()
            else:
                request.data = request_data

        if self.access_token:
            request.add_header("Authorization", "Bearer " + self.access_token)
        try:
            return json.loads(
                urlopen(
                    request, timeout=self.request_timeout).read().decode())
        except HTTPError as error:
            try:
                # Extract the developer-friendly error message from the response
                message = json.loads(error.read().decode())["error_description"]
            except Exception:
                raise error
            raise QuipError(error.code, message, error) 

def get_blob(self, thread_id, blob_id):
        """Returns a file-like object with the contents of the given blob from
        the given thread.

        The object is described in detail here:
        https://docs.python.org/2/library/urllib2.html#urllib2.urlopen
        """
        request = Request(
            url=self._url("blob/%s/%s" % (thread_id, blob_id)))
        if self.access_token:
            request.add_header("Authorization", "Bearer " + self.access_token)
        try:
            return urlopen(request, timeout=self.request_timeout)
        except HTTPError as error:
            try:
                # Extract the developer-friendly error message from the response
                message = json.loads(error.read().decode())["error_description"]
            except Exception:
                raise error
            raise QuipError(error.code, message, error) 

def _fetch_json(self, path, post_data=None, **args):
        request = Request(url=self._url(path, **args))
        if post_data:
            post_data = dict((k, v) for k, v in post_data.items()
                             if v or isinstance(v, int))
            request_data = urlencode(self._clean(**post_data))
            if PY3:
                request.data = request_data.encode()
            else:
                request.data = request_data

        if self.access_token:
            request.add_header("Authorization", "Bearer " + self.access_token)
        try:
            return json.loads(
                urlopen(
                    request, timeout=self.request_timeout).read().decode())
        except HTTPError as error:
            try:
                # Extract the developer-friendly error message from the response
                message = json.loads(error.read().decode())["error_description"]
            except Exception:
                raise error
            raise QuipError(error.code, message, error) 

def get_blob(self, thread_id, blob_id):
        """Returns a file-like object with the contents of the given blob from
        the given thread.

        The object is described in detail here:
        https://docs.python.org/2/library/urllib2.html#urllib2.urlopen
        """
        request = Request(
            url=self._url("blob/%s/%s" % (thread_id, blob_id)))
        if self.access_token:
            request.add_header("Authorization", "Bearer " + self.access_token)
        try:
            return urlopen(request, timeout=self.request_timeout)
        except HTTPError as error:
            try:
                # Extract the developer-friendly error message from the response
                message = json.loads(error.read().decode())["error_description"]
            except Exception:
                raise error
            raise QuipError(error.code, message, error) 

def get_blob(self, thread_id, blob_id):
        """Returns a file-like object with the contents of the given blob from
        the given thread.

        The object is described in detail here:
        https://docs.python.org/2/library/urllib2.html#urllib2.urlopen
        """
        request = Request(
            url=self._url("blob/%s/%s" % (thread_id, blob_id)))
        if self.access_token:
            request.add_header("Authorization", "Bearer " + self.access_token)
        try:
            return urlopen(request, timeout=self.request_timeout)
        except HTTPError as error:
            try:
                # Extract the developer-friendly error message from the response
                message = json.loads(error.read().decode())["error_description"]
            except Exception:
                raise error
            raise QuipError(error.code, message, error) 

def _fetch_json(self, path, post_data=None, **args):
        request = Request(url=self._url(path, **args))
        if post_data:
            post_data = dict((k, v) for k, v in post_data.items()
                             if v or isinstance(v, int))
            request_data = urlencode(self._clean(**post_data))
            if PY3:
                request.data = request_data.encode()
            else:
                request.data = request_data

        if self.access_token:
            request.add_header("Authorization", "Bearer " + self.access_token)
        try:
            return json.loads(
                urlopen(
                    request, timeout=self.request_timeout).read().decode())
        except HTTPError as error:
            try:
                # Extract the developer-friendly error message from the response
                message = json.loads(error.read().decode())["error_description"]
            except Exception:
                raise error
            raise QuipError(error.code, message, error) 

def get_blob(self, thread_id, blob_id):
        """Returns a file-like object with the contents of the given blob from
        the given thread.

        The object is described in detail here:
        https://docs.python.org/2/library/urllib2.html#urllib2.urlopen
        """
        request = Request(
            url=self._url("blob/%s/%s" % (thread_id, blob_id)))
        if self.access_token:
            request.add_header("Authorization", "Bearer " + self.access_token)
        try:
            return urlopen(request, timeout=self.request_timeout)
        except HTTPError as error:
            try:
                # Extract the developer-friendly error message from the response
                message = json.loads(error.read().decode())["error_description"]
            except Exception:
                raise error
            raise QuipError(error.code, message, error) 

def _fetch_json(self, path, post_data=None, **args):
        request = Request(url=self._url(path, **args))
        if post_data:
            post_data = dict((k, v) for k, v in post_data.items()
                             if v or isinstance(v, int))
            request_data = urlencode(self._clean(**post_data))
            if PY3:
                request.data = request_data.encode()
            else:
                request.data = request_data

        if self.access_token:
            request.add_header("Authorization", "Bearer " + self.access_token)
        try:
            return json.loads(
                urlopen(
                    request, timeout=self.request_timeout).read().decode())
        except HTTPError as error:
            try:
                # Extract the developer-friendly error message from the response
                message = json.loads(error.read().decode())["error_description"]
            except Exception:
                raise error
            raise QuipError(error.code, message, error) 

def get_blob(self, thread_id, blob_id):
        """Returns a file-like object with the contents of the given blob from
        the given thread.

        The object is described in detail here:
        https://docs.python.org/2/library/urllib2.html#urllib2.urlopen
        """
        request = Request(
            url=self._url("blob/%s/%s" % (thread_id, blob_id)))
        if self.access_token:
            request.add_header("Authorization", "Bearer " + self.access_token)
        try:
            return urlopen(request, timeout=self.request_timeout)
        except HTTPError as error:
            try:
                # Extract the developer-friendly error message from the response
                message = json.loads(error.read().decode())["error_description"]
            except Exception:
                raise error
            raise QuipError(error.code, message, error) 

def get_blob(self, thread_id, blob_id):
        """Returns a file-like object with the contents of the given blob from
        the given thread.

        The object is described in detail here:
        https://docs.python.org/2/library/urllib2.html#urllib2.urlopen
        """
        request = Request(
            url=self._url("blob/%s/%s" % (thread_id, blob_id)))
        if self.access_token:
            request.add_header("Authorization", "Bearer " + self.access_token)
        try:
            return urlopen(request, timeout=self.request_timeout)
        except HTTPError as error:
            try:
                # Extract the developer-friendly error message from the response
                message = json.loads(error.read().decode())["error_description"]
            except Exception:
                raise error
            raise QuipError(error.code, message, error) 

def _fetch_json(self, path, post_data=None, **args):
        request = Request(url=self._url(path, **args))
        if post_data:
            post_data = dict((k, v) for k, v in post_data.items()
                             if v or isinstance(v, int))
            request_data = urlencode(self._clean(**post_data))
            if PY3:
                request.data = request_data.encode()
            else:
                request.data = request_data

        if self.access_token:
            request.add_header("Authorization", "Bearer " + self.access_token)
        try:
            return json.loads(
                urlopen(
                    request, timeout=self.request_timeout).read().decode())
        except HTTPError as error:
            try:
                # Extract the developer-friendly error message from the response
                message = json.loads(error.read().decode())["error_description"]
            except Exception:
                raise error
            raise QuipError(error.code, message, error) 

def download(self, source: str, destination: str) -> int:
        destination_path: str = os.path.join(DATA_BASE_PATH, destination)
        url: str = os.path.join(STORAGE_BASE_URL, source)
        request = urllib.request.Request(url)
        request.add_header('User-Agent', self.USERAGENT)
        response = urllib.request.urlopen(request)
        with open(destination_path, 'wb') as output:
            filesize: int = 0
            while source:
                chunk = response.read(self.CHUNK_SIZE)
                if not chunk:
                    break
                filesize += len(chunk)
                output.write(chunk)
        # assume that we always distribute data as .tar.xz archives
        with lzma.open(destination_path) as f:
            with tarfile.open(fileobj=f) as tar:
                tar.extractall(os.path.dirname(destination_path))
        return filesize 

def ip_adress_proxies(url='https://www.ip-adress.com/proxy_list/'):
    # Downloading without proxy
    opener = urllib.request.build_opener(urllib.request.ProxyHandler())
    urllib.request.install_opener(opener)
    request = urllib.request.Request(url)
    request.add_header('user-agent', USER_AGENT)
    parsed_uri = urlparse(url)
    host = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
    request.add_header('referer', host)
    s = False
    try:
        context = ssl._create_unverified_context()
        with urlopen(request, context=context, timeout=3000) as response:
            s = response.read().decode('utf-8')
    except Exception as er:
        print(er)
    pattern = r'\d*\.\d*\.\d*\.\d*\</a>:\d*'
    found = [i.replace('</a>', '') + '\n' for i in re.findall(pattern, s)]
    return found 

def perform_metadata_exchange(self, endpoint, xaddr):
        if not (xaddr.startswith('http://') or xaddr.startswith('https://')):
            logger.debug('invalid XAddr: {}'.format(xaddr))
            return

        host = None
        url = xaddr
        if self.mch.family == socket.AF_INET6:
            host = '[{}]'.format(url.partition('[')[2].partition(']')[0])
            url = url.replace(']', '%{}]'.format(self.mch.interface.name))

        body = self.build_getmetadata_message(endpoint)
        request = urllib.request.Request(url, data=body, method='POST')
        request.add_header('Content-Type', 'application/soap+xml')
        request.add_header('User-Agent', 'wsdd')
        if host is not None:
            request.add_header('Host', host)

        try:
            with urllib.request.urlopen(request, None, 2.0) as stream:
                self.handle_metadata(stream.read(), endpoint, xaddr)
        except urllib.error.URLError as e:
            logger.warn('could not fetch metadata from: {}'.format(url, e)) 

def getHtml(url, headers=None, encode=None, maxError=3, timeout=10):
        error = 0
        while error < maxError:
            try:
                if not headers:
                    headers = c_spider.defaultHeaders
                    headers.__setitem__('Referer', url)

                request = urllib.request.Request(url)
                for key in headers:
                    request.add_header(key, headers[key])

                response = urllib.request.urlopen(request, timeout=timeout)
                html = response.read()
                if encode:
                    return html.decode(encode)
                else:
                    return html
            except:
                error += 1

    # 获取网页源代码 

def get_web_page(url, headers, cookies):
    try:
        logging.info(f'Fetching {url}')
        request = urllib.request.Request(url, None, headers)
        request.add_header('Authorization', cookies)
        response = urllib.request.urlopen(request)
        if response.info().get('Content-Encoding') == 'gzip':
            buf = BytesIO(response.read())
            f = gzip.GzipFile(fileobj=buf)
            r = f.read()
        else:
            r = response.read()
        return r
    except urllib.error.HTTPError as e:
        logging.info(f"Error processing webpage: {e}")
        if e.code == ALREADY_CLICKED_CODE:
            return ALREADY_CLICKED_CODE
        if e.code == UNAUTHORIZED:
            return UNAUTHORIZED
        return None 

def request(self, host, handler, request_body, verbose):
        self.verbose = verbose
        url = 'http://{0}{1}'.format(host, handler)
        request = urllib.Request(url)
        request.add_data(request_body)
        request.add_header("User-Agent", self.user_agent)
        request.add_header("Content-Type", "text/html")
        f = urllib.urlopen(request)
        return self.parse_response(f) 

def post_api(self, api_path):
        options = json.dumps(self.options)
        options = options if isinstance(options, bytes) else options.encode('utf8')
        request = urllib.request.Request(self.arachni_url + api_path, options)
        request.add_header('Content-Type', 'application/json')
        return urllib.request.urlopen(request).read().decode('utf8') 

def downloadIfNotModifiedSince(url, timestamp):
    logger.info("(" + url + "," + time.ctime(timestamp) + ")")

    # Convierte la fecha a GMT
    fecha_formateada = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(timestamp))
    logger.info("fechaFormateada=%s" % fecha_formateada)

    # Comprueba si ha cambiado
    inicio = time.clock()
    req = urllib.request.Request(url)
    req.add_header('If-Modified-Since', fecha_formateada)
    req.add_header('User-Agent',
                   'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; es-ES; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12')

    updated = False

    try:
        response = urllib.request.urlopen(req)
        data = response.read()

        # Si llega hasta aquí, es que ha cambiado
        updated = True
        response.close()

    except urllib.error.URLError as e:
        # Si devuelve 304 es que no ha cambiado
        if hasattr(e, 'code'):
            logger.info("Codigo de respuesta HTTP : %d" % e.code)
            if e.code == 304:
                logger.info("No ha cambiado")
                updated = False
        # Agarra los errores con codigo de respuesta del servidor externo solicitado     
        else:
            for line in sys.exc_info():
                logger.error("%s" % line)
        data = ""

    fin = time.clock()
    logger.info("Descargado en %d segundos " % (fin - inicio + 1))

    return updated, data 

def geturl_req(url):
    """get request that returns 302"""
    request = urllib.request.Request(url)
    request.add_header('Authorization', 'token %s' % API_TOKEN)
    try:
        response_url = urllib.request.urlopen(request).geturl()
        return response_url
    except urllib.error.HTTPError:
        exception()
        sys.exit(0) 

def get_req(url):
    """simple get request"""
    request = urllib.request.Request(url)
    request.add_header('Authorization', 'token %s' % API_TOKEN)
    try:
        response = urllib.request.urlopen(request).read().decode('utf-8')
        return response
    except urllib.error.HTTPError:
        exception()
        sys.exit(0)

# RETURNS 302 

def _get_url(url, as_string=True):
    """Get the content from a URL."""
    user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:49.0) '\
                 'Gecko/20100101 Firefox/49.0'
    request = urllib.request.Request(url)
    request.add_header('User-Agent', user_agent)
    body = ''
    try:
        body = urllib.request.urlopen(request).read()
        if as_string:
            body = body.decode('utf-8')
    except Exception:
        logger.warning('Fail URL %s' % url)
    return body 

def upload_image(self, image_uri, sync, username, userid, channel_name):
        token = self.apikey
        logger.info('downloading %s', image_uri)
        filename = os.path.basename(image_uri)
        request = urllib.request.Request(image_uri)
        request.add_header("Authorization", "Bearer %s" % token)
        image_response = urllib.request.urlopen(request)
        content_type = image_response.info().get_content_type()

        filename_extension = mimetypes.guess_extension(content_type).lower() # returns with "."
        physical_extension = "." + filename.rsplit(".", 1).pop().lower()

        if physical_extension == filename_extension:
            pass
        elif filename_extension == ".jpe" and physical_extension in [ ".jpg", ".jpeg", ".jpe", ".jif", ".jfif" ]:
            # account for mimetypes idiosyncrancy to return jpe for valid jpeg
            pass
        else:
            logger.warning("unable to determine extension: {} {}".format(filename_extension, physical_extension))
            filename += filename_extension

        logger.info('uploading as %s', filename)
        image_id = yield from self.bot._client.upload_image(image_response, filename=filename)

        logger.info('sending HO message, image_id: %s', image_id)
        yield from sync._bridgeinstance._send_to_internal_chat(
            sync.hangoutid,
            "shared media from slack",
            {   "sync": sync,
                "source_user": username,
                "source_uid": userid,
                "source_title": channel_name },
            image_id=image_id ) 

def get_page(url, torclient_host='127.0.0.1', torclient_port=9050):

    request = urllib.request.Request(url)
    # UA of the Tor browser bundle
    request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; rv:24.0) Gecko/20100101 Firefox/24.0')
    return urllib.request.urlopen(request, timeout=5).read(max_size * 100000)

#FIXME don't work at all 

def cors0x00(url, ssltest, firstrun=False):

    try:

        request = urllib.request.Request(url)
        print(C+' [!] Setting origin : '+url)
        request.add_header('Origin', url)
        request.add_header('Cookie', "")
        print(C+' [!] Setting user agent...')
        request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64)')
        if 'https://' not in url:
            response = urllib.request.urlopen(request, timeout=10)
        else:
            response = urllib.request.urlopen(request, timeout=10, context=ssl._create_unverified_context())
        acao = response.info().getheader('Access-Control-Allow-Origin')
        acac = str(response.info().getheader('Access-Control-Allow-Credentials')).lower() == "true"
        vary = "Origin" in str(response.info().getheader('Vary'))

        if not acac:
            print(O+' [*] Checking whether Access-Control-Allow-Credentials header value present...')
            acao = "no_acac"
        if acac and acao != '*':
            print(G+" [+] Access-Control-Allow-Credentials present...")
        if vary:
            print(O+ " [!] Access-Control-Allow-Origin dynamically generated...")

        return acao

    except Exception as e:
        print(R+' [-] Something happened...')
        print(R+' [-] Error : '+str(e)) 

def http_put(settings, url, data):
    json_data = json.dumps(data).encode('utf8');
    request = urllib.request.Request(url, json_data, method="PUT")
    request.add_header("Authorization", settings['auth'])
    request.add_header("Accept", "application/json")
    request.add_header("Content-type", "application/json")    

    return http_call(request);