Python urllib.request.request() Examples

def _make_request(self, opener, request, timeout=None):
        """Make the API call and return the response. This is separated into
           it's own function, so we can mock it easily for testing.

        :param opener:
        :type opener:
        :param request: url payload to request
        :type request: urllib.Request object
        :param timeout: timeout value or None
        :type timeout: float
        :return: urllib response
        """
        timeout = timeout or self.timeout
        try:
            return opener.open(request, timeout=timeout)
        except HTTPError as err:
            exc = handle_error(err)
            return exc 

def get_output_path(request, path_format, path_format_var):
    # 取得输出文件的路径
    OutputPathFormat = (request.GET.get(path_format, USettings.UEditorSettings["defaultPathFormat"]) % path_format_var).replace("\\", "/")
    # 分解OutputPathFormat
    OutputPath, OutputFile = os.path.split(OutputPathFormat)
    OutputPath = os.path.join(USettings.gSettings.MEDIA_ROOT, OutputPath)
    # 如果OutputFile为空说明传入的OutputPathFormat没有包含文件名，因此需要用默认的文件名
    if not OutputFile:
        OutputFile = USettings.UEditorSettings["defaultPathFormat"] % path_format_var
        OutputPathFormat = os.path.join(OutputPathFormat, OutputFile)
    if not os.path.exists(OutputPath):
        os.makedirs(OutputPath)
    return (OutputPathFormat, OutputPath, OutputFile)


# 涂鸦功能上传处理 

def make_request(self,
                     method,
                     request_body=None,
                     query_params=None,
                     request_headers=None):
        method = method.upper()
        if request_headers:
            self._set_headers(request_headers)
            request_body = json.dumps(request_body) if request_body else None
            query_params = query_params if query_params else None
            opener = urllib.build_opener()
            request = urllib.Request(self._build_url(query_params),
                                     data=request_body)
            for key, value in self.request_headers.iteritems():
                request.add_header(key, value)
            request.get_method = lambda: method
            self._response = opener.open(request)
            self._set_response(self._response)
            self._reset() 

def RetrieveHeader(Target):

    ReplyHeaders = ""
    print(O+' [*] Making request to retrieve HHTP headers...')
    if "https" in Target[:5]:
        sslcontext = ssl.create_default_context()
        n = input(O+' [§] Ignore SSL certificate errors? (Y/n) :> ')
        if n == 'y' or n == 'Y':
            print(GR+" [*] Ignoring certificate errors...")
            sslcontext = ssl._create_unverified_context()
        try:
            ReplyHeaders = urllib.urlopen(Target,context=sslcontext).headers.headers
        except ssl.CertificateError:
            print(R+" [-] SSL Certificate authentication error...")
        return ReplyHeaders
    else:
        ReplyHeaders = urllib.urlopen(Target).headers.headers
        return ReplyHeaders 

def get_output_path(request, path_format, path_format_var):
    # 取得输出文件的路径
    OutputPathFormat = (request.GET.get(path_format, USettings.UEditorSettings["defaultPathFormat"]) % path_format_var).replace("\\", "/")
    # 分解OutputPathFormat
    OutputPath, OutputFile = os.path.split(OutputPathFormat)
    OutputPath = os.path.join(USettings.gSettings.MEDIA_ROOT, OutputPath)
    # 如果OutputFile为空说明传入的OutputPathFormat没有包含文件名，因此需要用默认的文件名
    if not OutputFile:
        OutputFile = USettings.UEditorSettings["defaultPathFormat"] % path_format_var
        OutputPathFormat = os.path.join(OutputPathFormat, OutputFile)
    if not os.path.exists(OutputPath):
        os.makedirs(OutputPath)
    return (OutputPathFormat, OutputPath, OutputFile)


# 涂鸦功能上传处理 

def http(method, url, body=None, headers=None):
    url_info = urlparse.urlparse(url)
    if url_info.scheme == "https":
        con = httplib.HTTPSConnection(url_info.hostname, url_info.port or 443)
    else:
        con = httplib.HTTPConnection(url_info.hostname, url_info.port or 80)

    con.request(method, url_info.path, body, headers)
    response = con.getresponse()

    try:
        if 400 <= response.status < 500:
            raise HttpClientError(response.status, response.reason,
                                  response.read())
        elif 500 <= response.status < 600:
            raise HttpServerError(response.status, response.reason,
                                  response.read())
        else:
            yield response
    finally:
        con.close() 

def check_ros_version(architecture: str, version: str) -> bool:
    """

    :param architecture:
    :param version:
    :return:
    """

    if not all(isinstance(var, str) for var in (architecture, version)):
        raise TypeError("Expected str type for architecture and version, got {0} and {1}".format(
            type(architecture), type(version)))

    url = "{}/{}/routeros-{}-{}.npk".format(MK_DOWNLOAD_CDN, version, architecture, version)

    request = urllib.Request(url)
    request.get_method = lambda: "HEAD"

    try:
        urllib.urlopen(request)
    except IOError:
        return False

    return True 

def _update_headers(self, request_headers):
        """Update the headers for the request

        :param request_headers: headers to set for the API call
        :type request_headers: dictionary
        :return: dictionary
        """
        self.request_headers.update(request_headers) 

def get_ueditor_controller(request):
    """获取ueditor的后端URL地址    """

    action = request.GET.get("action", "")
    reponseAction = {
        "config": get_ueditor_settings,
        "uploadimage": UploadFile,
        "uploadscrawl": UploadFile,
        "uploadvideo": UploadFile,
        "uploadfile": UploadFile,
        "catchimage": catcher_remote_image,
        "listimage": list_files,
        "listfile": list_files
    }
    return reponseAction[action](request) 

def get_ueditor_settings(request):
    return HttpResponse(json.dumps(USettings.UEditorUploadSettings, ensure_ascii=False), content_type="application/javascript") 

def _update_headers(self, request_headers):
        """Update the headers for the request

        :param request_headers: headers to set for the API call
        :type request_headers: dictionary
        :return: dictionary
        """
        self.request_headers.update(request_headers) 

def test__urllib_method(self, maker):
        self.client.delete()
        request = maker.call_args[0][1]
        self.assertEqual(request.get_method(), 'DELETE') 

def test__urllib_headers(self, maker):
        self.client._update_headers({'X-test': 'Test'})
        self.client.get()
        request = maker.call_args[0][1]
        self.assertIn('X-test', request.headers) 

def _make_request(self, opener, request, timeout=None):
        if 200 <= self.response_code < 299:  # if successful code
            return MockResponse(self.response_code)
        else:
            raise handle_error(MockException(self.response_code)) 

def download(url, server_fname, local_fname=None, progress_update_percentage=5):
    """
    An internet download utility modified from
    http://stackoverflow.com/questions/22676/
    how-do-i-download-a-file-over-http-using-python/22776#22776
    """
    try:
        import urllib
        urllib.urlretrieve('http://google.com')
    except AttributeError:
        import urllib.request as urllib
    u = urllib.urlopen(url)
    if local_fname is None:
        local_fname = server_fname
    full_path = local_fname
    meta = u.info()
    with open(full_path, 'wb') as f:
        try:
            file_size = int(meta.get("Content-Length"))
        except TypeError:
            print("WARNING: Cannot get file size, displaying bytes instead!")
            file_size = 100
        print("Downloading: %s Bytes: %s" % (server_fname, file_size))
        file_size_dl = 0
        block_sz = int(1E7)
        p = 0
        while True:
            buffer = u.read(block_sz)
            if not buffer:
                break
            file_size_dl += len(buffer)
            f.write(buffer)
            if (file_size_dl * 100. / file_size) > p:
                status = r"%10d  [%3.2f%%]" % (file_size_dl, file_size_dl *
                                               100. / file_size)
                print(status)
                p += progress_update_percentage 

def stage2discharge(gage, usgs_id):
    '''
    Function to convert stage data to discharge using usgs reference table

    Input
    -------
    gage : masked array,
                array of stage data to be converted
    usgs_id : int,
                8 digit identifier for river on usgs

    Output
    ---------
    flux : masked array,
                discharge data in cubic feet
    '''
    import urllib.request as urllib

    # Load lookup table
    url = discharge_url + usgs_id
    stage = []
    discharge = []
    header = 0
    for line in urllib.urlopen(url):
        if not line.startswith(b'#'):
            if header < 2:
                header += 1
            else:
                a = line.split(b'\t')
                stage.append(float(line.split(b'\t')[0]))
                discharge.append(float(line.split(b'\t')[2]))
    stage = np.asarray(stage).astype(float)
    discharge = np.asarray(discharge).astype(float)

    # Convert data
    flux = np.ma.masked_array(np.zeros(len(gage)), mask=gage.mask)
    for i, f in enumerate(gage):
        if f:
            flux[i] = discharge[(np.abs(stage - f)).argmin()]
    return flux 

def player_request(self, line, player_id=None,
                       raw=False, wait=True) -> Union[str, None]:
        """Makes a single request to a particular player (or the current)"""
        try:
            player_id = (player_id or
                         self.cur_player_id or
                         list(self.players.values())[0]["playerid"])
            return self._request(["%s %s" % (player_id, line)],
                                 raw=raw, wait=wait)[0]
        except IndexError:
            return None 

def get_ueditor_controller(request):
    """获取ueditor的后端URL地址    """

    action = request.GET.get("action", "")
    reponseAction = {
        "config": get_ueditor_settings,
        "uploadimage": UploadFile,
        "uploadscrawl": UploadFile,
        "uploadvideo": UploadFile,
        "uploadfile": UploadFile,
        "catchimage": catcher_remote_image,
        "listimage": list_files,
        "listfile": list_files
    }
    return reponseAction[action](request) 

def get_ueditor_settings(request):
    return HttpResponse(json.dumps(USettings.UEditorUploadSettings, ensure_ascii=False), content_type="application/javascript") 

def extractArticlePublishedDate(articleLink, html = None):

    print("Extracting date from " + articleLink)

    articleDate = None

    try:
        articleDate = _extractFromURL(articleLink)

        if html is None:
            request = urllib.Request(articleLink)
            # Using a browser user agent, decreases the change of sites blocking this request - just a suggestion
            # request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36')
            html = urllib.build_opener().open(request).read()

        parsedHTML = BeautifulSoup(html,"lxml")

        possibleDate = _extractFromLDJson(parsedHTML)
        if possibleDate is None:
            possibleDate = _extractFromMeta(parsedHTML)
        if possibleDate is None:
            possibleDate = _extractFromHTMLTag(parsedHTML)


        articleDate = possibleDate

    except Exception as e:
        print("Exception in extractArticlePublishedDate for " + articleLink)
        print(e.args)   

    return articleDate 

def create_metadata(self, key, secret, ds_name, data_frequency, desc,
                        has_history=True, has_live=True):
        """

        Returns
        -------

        """
        headers = get_signed_headers(ds_name, key, secret)
        r = requests.post(
            '{}/marketplace/register'.format(AUTH_SERVER),
            json=dict(
                ds_name=ds_name,
                desc=desc,
                data_frequency=data_frequency,
                has_history=has_history,
                has_live=has_live,
            ),
            headers=headers,
        )

        if r.status_code != 200:
            raise MarketplaceHTTPRequest(
                request='register', error=r.status_code
            )

        if 'error' in r.json():
            raise MarketplaceHTTPRequest(
                request='upload file', error=r.json()['error']
            ) 

def POST(path):
    url = getURL()
    opener = getOpener()
    request = urllib.Request(url, GLOBALS['POSTDATA'])
    response = opener.open(request)
    setValues(response) 

def GET(path):
    url = getURL()
    opener = getOpener()
    request = urllib.Request(url)
    response = opener.open(request)
    setValues(response) 

def download(url, server_fname, local_fname=None, progress_update_percentage=5):
    """
    An internet download utility modified from
    http://stackoverflow.com/questions/22676/
    how-do-i-download-a-file-over-http-using-python/22776#22776
    """
    try:
        import urllib
        urllib.urlretrieve('http://google.com')
    except AttributeError:
        import urllib.request as urllib
    u = urllib.urlopen(url)
    if local_fname is None:
        local_fname = server_fname
    full_path = local_fname
    meta = u.info()
    with open(full_path, 'wb') as f:
        try:
            file_size = int(meta.get("Content-Length"))
        except TypeError:
            print("WARNING: Cannot get file size, displaying bytes instead!")
            file_size = 100
        print("Downloading: %s Bytes: %s" % (server_fname, file_size))
        file_size_dl = 0
        block_sz = int(1E7)
        p = 0
        while True:
            buffer = u.read(block_sz)
            if not buffer:
                break
            file_size_dl += len(buffer)
            f.write(buffer)
            if (file_size_dl * 100. / file_size) > p:
                status = r"%10d  [%3.2f%%]" % (file_size_dl, file_size_dl *
                                               100. / file_size)
                print(status)
                p += progress_update_percentage 

def download(url, server_fname, local_fname=None, progress_update_percentage=5):
    """
    An internet download utility modified from
    http://stackoverflow.com/questions/22676/
    how-do-i-download-a-file-over-http-using-python/22776#22776
    """
    try:
        import urllib
        urllib.urlretrieve('http://google.com')
    except AttributeError:
        import urllib.request as urllib
    u = urllib.urlopen(url)
    if local_fname is None:
        local_fname = server_fname
    full_path = local_fname
    meta = u.info()
    with open(full_path, 'wb') as f:
        try:
            file_size = int(meta.get("Content-Length"))
        except TypeError:
            print("WARNING: Cannot get file size, displaying bytes instead!")
            file_size = 100
        print("Downloading: %s Bytes: %s" % (server_fname, file_size))
        file_size_dl = 0
        block_sz = int(1E7)
        p = 0
        while True:
            buffer = u.read(block_sz)
            if not buffer:
                break
            file_size_dl += len(buffer)
            f.write(buffer)
            if (file_size_dl * 100. / file_size) > p:
                status = r"%10d  [%3.2f%%]" % (file_size_dl, file_size_dl *
                                               100. / file_size)
                print(status)
                p += progress_update_percentage 

def download(url, server_fname, local_fname=None, progress_update_percentage=5):
    """
    An internet download utility modified from
    http://stackoverflow.com/questions/22676/
    how-do-i-download-a-file-over-http-using-python/22776#22776
    """
    try:
        import urllib
        urllib.urlretrieve('http://google.com')
    except AttributeError:
        import urllib.request as urllib
    u = urllib.urlopen(url)
    if local_fname is None:
        local_fname = server_fname
    full_path = local_fname
    meta = u.info()
    with open(full_path, 'wb') as f:
        try:
            file_size = int(meta.get("Content-Length"))
        except TypeError:
            print("WARNING: Cannot get file size, displaying bytes instead!")
            file_size = 100
        print("Downloading: %s Bytes: %s" % (server_fname, file_size))
        file_size_dl = 0
        block_sz = int(1E7)
        p = 0
        while True:
            buffer = u.read(block_sz)
            if not buffer:
                break
            file_size_dl += len(buffer)
            f.write(buffer)
            if (file_size_dl * 100. / file_size) > p:
                status = r"%10d  [%3.2f%%]" % (file_size_dl, file_size_dl *
                                               100. / file_size)
                print(status)
                p += progress_update_percentage 

def catcher_remote_image(request):
    """远程抓图，当catchRemoteImageEnable:true时，
        如果前端插入图片地址与当前web不在同一个域，则由本函数从远程下载图片到本地
    """
    if not request.method == "POST":
        return HttpResponse(json.dumps(u"{'state:'ERROR'}"), content_type="application/javascript")

    state = "SUCCESS"

    allow_type = list(request.GET.get("catcherAllowFiles", USettings.UEditorUploadSettings.get("catcherAllowFiles", "")))
    max_size = int(request.GET.get("catcherMaxSize", USettings.UEditorUploadSettings.get("catcherMaxSize", 0)))

    remote_urls = request.POST.getlist("source[]", [])
    catcher_infos = []
    path_format_var = get_path_format_vars()

    for remote_url in remote_urls:
        # 取得上传的文件的原始名称
        remote_file_name = os.path.basename(remote_url)
        remote_original_name, remote_original_ext = os.path.splitext(remote_file_name)
        # 文件类型检验
        if remote_original_ext in allow_type:
            path_format_var.update({
                "basename": remote_original_name,
                "extname": remote_original_ext[1:],
                "filename": remote_original_name
            })
            # 计算保存的文件名
            o_path_format, o_path, o_file = get_output_path(request, "catcherPathFormat", path_format_var)
            o_filename = os.path.join(o_path, o_file).replace("\\", "/")
            # 读取远程图片文件
            try:
                remote_image = urllib.urlopen(remote_url)
                # 将抓取到的文件写入文件
                try:
                    f = open(o_filename, 'wb')
                    f.write(remote_image.read())
                    f.close()
                    state = "SUCCESS"
                except Exception as e:
                    state = u"写入抓取图片文件错误:%s" % e
            except Exception as e:
                state = u"抓取图片错误：%s" % e

            catcher_infos.append({
                "state": state,
                "url": urljoin(USettings.gSettings.MEDIA_URL, o_path_format),
                "size": os.path.getsize(o_filename),
                "title": os.path.basename(o_file),
                "original": remote_file_name,
                "source": remote_url
            })

    return_info = {
        "state": "SUCCESS" if len(catcher_infos) > 0 else "ERROR",
        "list": catcher_infos
    }

    return HttpResponse(json.dumps(return_info, ensure_ascii=False), content_type="application/javascript") 

def list_files(request):
    """列出文件"""
    if request.method != "GET":
        return HttpResponse(json.dumps(u"{'state:'ERROR'}"), content_type="application/javascript")
    # 取得动作
    action = request.GET.get("action", "listimage")

    allowFiles = {
        "listfile": USettings.UEditorUploadSettings.get("fileManagerAllowFiles", []),
        "listimage": USettings.UEditorUploadSettings.get("imageManagerAllowFiles", [])
    }
    listSize = {
        "listfile": USettings.UEditorUploadSettings.get("fileManagerListSize", ""),
        "listimage": USettings.UEditorUploadSettings.get("imageManagerListSize", "")
    }
    listpath = {
        "listfile": USettings.UEditorUploadSettings.get("fileManagerListPath", ""),
        "listimage": USettings.UEditorUploadSettings.get("imageManagerListPath", "")
    }
    # 取得参数
    list_size = int(request.GET.get("size", listSize[action]))
    list_start = int(request.GET.get("start", 0))

    files = []
    root_path = os.path.join(USettings.gSettings.MEDIA_ROOT, listpath[action]).replace("\\", "/")
    files = get_files(root_path, root_path, allowFiles[action])

    if (len(files) == 0):
        return_info = {
            "state": u"未找到匹配文件！",
            "list": [],
            "start": list_start,
            "total": 0
        }
    else:
        return_info = {
            "state": "SUCCESS",
            "list": files[list_start:list_start + list_size],
            "start": list_start,
            "total": len(files)
        }

    return HttpResponse(json.dumps(return_info), content_type="application/javascript") 

def publish(self, dataset, datadir, watch):
        dataset = dataset.lower()
        provider_info = self.mkt_contract.functions.getDataProviderInfo(
            Web3.toHex(dataset.encode())
        ).call()

        if not provider_info[4]:
            raise MarketplaceDatasetNotFound(dataset=dataset)

        match = next(
            (l for l in self.addresses if l['pubAddr'] == provider_info[0]),
            None
        )
        if not match:
            raise MarketplaceNoAddressMatch(
                dataset=dataset,
                address=provider_info[0])

        print('Using address: {} to publish this dataset.'.format(
            provider_info[0]))

        if 'key' in match:
            key = match['key']
            secret = match['secret']
        else:
            key, secret = get_key_secret(provider_info[0], match['wallet'])

        filenames = glob.glob(os.path.join(datadir, '*.csv'))

        if not filenames:
            raise MarketplaceNoCSVFiles(datadir=datadir)

        def read_file(pathname):
            with open(pathname, 'rb') as f:
                return f.read()

        files = []
        for idx, file in enumerate(filenames):
            log.info('Uploading file {} of {}: {}'.format(
                idx + 1, len(filenames), file))
            files.append(('file', (os.path.basename(file), read_file(file))))

        headers = get_signed_headers(dataset, key, secret)
        r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
                          files=files,
                          headers=headers)

        if r.status_code != 200:
            raise MarketplaceHTTPRequest(request='upload file',
                                         error=r.status_code)

        if 'error' in r.json():
            raise MarketplaceHTTPRequest(request='upload file',
                                         error=r.json()['error'])

        log.info('File processed successfully.')

        print('\nDataset {} uploaded and processed successfully.'.format(
            dataset)) 

def list_files(request):
    """列出文件"""
    if request.method != "GET":
        return HttpResponse(json.dumps(u"{'state:'ERROR'}"), content_type="application/javascript")
    # 取得动作
    action = request.GET.get("action", "listimage")

    allowFiles = {
        "listfile": USettings.UEditorUploadSettings.get("fileManagerAllowFiles", []),
        "listimage": USettings.UEditorUploadSettings.get("imageManagerAllowFiles", [])
    }
    listSize = {
        "listfile": USettings.UEditorUploadSettings.get("fileManagerListSize", ""),
        "listimage": USettings.UEditorUploadSettings.get("imageManagerListSize", "")
    }
    listpath = {
        "listfile": USettings.UEditorUploadSettings.get("fileManagerListPath", ""),
        "listimage": USettings.UEditorUploadSettings.get("imageManagerListPath", "")
    }
    # 取得参数
    list_size = int(request.GET.get("size", listSize[action]))
    list_start = int(request.GET.get("start", 0))

    files = []
    root_path = os.path.join(USettings.gSettings.MEDIA_ROOT, listpath[action]).replace("\\", "/")
    files = get_files(root_path, root_path, allowFiles[action])

    if (len(files) == 0):
        return_info = {
            "state": u"未找到匹配文件！",
            "list": [],
            "start": list_start,
            "total": 0
        }
    else:
        return_info = {
            "state": "SUCCESS",
            "list": files[list_start:list_start + list_size],
            "start": list_start,
            "total": len(files)
        }

    return HttpResponse(json.dumps(return_info), content_type="application/javascript")