Python pycurl.EFFECTIVE_URL Examples
The following are 19
code examples of pycurl.EFFECTIVE_URL().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
pycurl
, or try the search function
.
Example #1
Source File: pycurldownload.py From QMusic with GNU Lesser General Public License v2.1 | 6 votes |
def url_check(self, url): '''下载地址检查''' url_info = {} proto = urlparse.urlparse(url)[0] if proto not in VALIDPROTOCOL: print 'Valid protocol should be http or ftp, but % s found < %s >!' % (proto, url) else: ss = StringIO() curl = pycurl.Curl() curl.setopt(pycurl.FOLLOWLOCATION, 1) curl.setopt(pycurl.MAXREDIRS, 5) curl.setopt(pycurl.CONNECTTIMEOUT, 30) curl.setopt(pycurl.TIMEOUT, 300) curl.setopt(pycurl.NOSIGNAL, 1) curl.setopt(pycurl.NOPROGRESS, 1) curl.setopt(pycurl.NOBODY, 1) curl.setopt(pycurl.HEADERFUNCTION, ss.write) curl.setopt(pycurl.URL, url) try: curl.perform() except: pass if curl.errstr() == '' and curl.getinfo(pycurl.RESPONSE_CODE) in STATUS_OK: url_info['url'] = curl.getinfo(pycurl.EFFECTIVE_URL) url_info['file'] = os.path.split(url_info['url'])[1] url_info['size'] = int( curl.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD)) url_info['partible'] = (ss.getvalue().find('Accept - Ranges') != -1) return url_info
Example #2
Source File: httpclient.py From honeything with GNU General Public License v3.0 | 6 votes |
def fetch(self, request, **kwargs): """Executes an HTTPRequest, returning an HTTPResponse. If an error occurs during the fetch, we raise an HTTPError. """ if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) buffer = cStringIO.StringIO() headers = httputil.HTTPHeaders() try: _curl_setup_request(self._curl, request, buffer, headers) self._curl.perform() code = self._curl.getinfo(pycurl.HTTP_CODE) effective_url = self._curl.getinfo(pycurl.EFFECTIVE_URL) buffer.seek(0) response = HTTPResponse( request=request, code=code, headers=headers, buffer=buffer, effective_url=effective_url) if code < 200 or code >= 300: raise HTTPError(code, response=response) return response except pycurl.error, e: buffer.close() raise CurlError(*e)
Example #3
Source File: httpclient.py From honeything with GNU General Public License v3.0 | 5 votes |
def _finish(self, curl, curl_error=None, curl_message=None): info = curl.info curl.info = None self._multi.remove_handle(curl) self._free_list.append(curl) buffer = info["buffer"] if curl_error: error = CurlError(curl_error, curl_message) code = error.code body = None effective_url = None buffer.close() buffer = None else: error = None code = curl.getinfo(pycurl.HTTP_CODE) effective_url = curl.getinfo(pycurl.EFFECTIVE_URL) buffer.seek(0) try: info["callback"](HTTPResponse( request=info["request"], code=code, headers=info["headers"], buffer=buffer, effective_url=effective_url, error=error, request_time=time.time() - info["start_time"])) except (KeyboardInterrupt, SystemExit): raise except: logging.error("Exception in callback %r", info["callback"], exc_info=True)
Example #4
Source File: CurlHelper.py From AdvancedDownloader with GNU General Public License v3.0 | 5 votes |
def get_final_location(self): try: self._init_base_headers_curl() self._follow_location_and_auto_referer() self._perform_curl() return self._curl.getinfo(pycurl.EFFECTIVE_URL) except Exception as e: self._make_message_and_send(str(e), True) return None
Example #5
Source File: CurlHelper.py From AdvancedDownloader with GNU General Public License v3.0 | 5 votes |
def _update_redirected_link(self): try: self._init_base_headers_curl() self._follow_location_and_auto_referer() self._perform_curl() self._link = self._curl.getinfo(pycurl.EFFECTIVE_URL) self._release_self_curl() return True except Exception as e: self._make_message_and_send(str(e), True) return False
Example #6
Source File: __init__.py From EventGhost with GNU General Public License v2.0 | 5 votes |
def info(self): "Return a dictionary with all info on the last response." m = {} m['effective-url'] = self.handle.getinfo(pycurl.EFFECTIVE_URL) m['http-code'] = self.handle.getinfo(pycurl.HTTP_CODE) m['total-time'] = self.handle.getinfo(pycurl.TOTAL_TIME) m['namelookup-time'] = self.handle.getinfo(pycurl.NAMELOOKUP_TIME) m['connect-time'] = self.handle.getinfo(pycurl.CONNECT_TIME) m['pretransfer-time'] = self.handle.getinfo(pycurl.PRETRANSFER_TIME) m['redirect-time'] = self.handle.getinfo(pycurl.REDIRECT_TIME) m['redirect-count'] = self.handle.getinfo(pycurl.REDIRECT_COUNT) m['size-upload'] = self.handle.getinfo(pycurl.SIZE_UPLOAD) m['size-download'] = self.handle.getinfo(pycurl.SIZE_DOWNLOAD) m['speed-upload'] = self.handle.getinfo(pycurl.SPEED_UPLOAD) m['header-size'] = self.handle.getinfo(pycurl.HEADER_SIZE) m['request-size'] = self.handle.getinfo(pycurl.REQUEST_SIZE) m['content-length-download'] = self.handle.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD) m['content-length-upload'] = self.handle.getinfo(pycurl.CONTENT_LENGTH_UPLOAD) m['content-type'] = self.handle.getinfo(pycurl.CONTENT_TYPE) m['response-code'] = self.handle.getinfo(pycurl.RESPONSE_CODE) m['speed-download'] = self.handle.getinfo(pycurl.SPEED_DOWNLOAD) m['ssl-verifyresult'] = self.handle.getinfo(pycurl.SSL_VERIFYRESULT) m['filetime'] = self.handle.getinfo(pycurl.INFO_FILETIME) m['starttransfer-time'] = self.handle.getinfo(pycurl.STARTTRANSFER_TIME) m['redirect-time'] = self.handle.getinfo(pycurl.REDIRECT_TIME) m['redirect-count'] = self.handle.getinfo(pycurl.REDIRECT_COUNT) m['http-connectcode'] = self.handle.getinfo(pycurl.HTTP_CONNECTCODE) m['httpauth-avail'] = self.handle.getinfo(pycurl.HTTPAUTH_AVAIL) m['proxyauth-avail'] = self.handle.getinfo(pycurl.PROXYAUTH_AVAIL) m['os-errno'] = self.handle.getinfo(pycurl.OS_ERRNO) m['num-connects'] = self.handle.getinfo(pycurl.NUM_CONNECTS) m['ssl-engines'] = self.handle.getinfo(pycurl.SSL_ENGINES) m['cookielist'] = self.handle.getinfo(pycurl.INFO_COOKIELIST) m['lastsocket'] = self.handle.getinfo(pycurl.LASTSOCKET) m['ftp-entry-path'] = self.handle.getinfo(pycurl.FTP_ENTRY_PATH) return m
Example #7
Source File: curl_httpclient.py From EventGhost with GNU General Public License v2.0 | 5 votes |
def _finish(self, curl, curl_error=None, curl_message=None): info = curl.info curl.info = None self._multi.remove_handle(curl) self._free_list.append(curl) buffer = info["buffer"] if curl_error: error = CurlError(curl_error, curl_message) code = error.code effective_url = None buffer.close() buffer = None else: error = None code = curl.getinfo(pycurl.HTTP_CODE) effective_url = curl.getinfo(pycurl.EFFECTIVE_URL) buffer.seek(0) # the various curl timings are documented at # http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html time_info = dict( queue=info["curl_start_time"] - info["request"].start_time, namelookup=curl.getinfo(pycurl.NAMELOOKUP_TIME), connect=curl.getinfo(pycurl.CONNECT_TIME), pretransfer=curl.getinfo(pycurl.PRETRANSFER_TIME), starttransfer=curl.getinfo(pycurl.STARTTRANSFER_TIME), total=curl.getinfo(pycurl.TOTAL_TIME), redirect=curl.getinfo(pycurl.REDIRECT_TIME), ) try: info["callback"](HTTPResponse( request=info["request"], code=code, headers=info["headers"], buffer=buffer, effective_url=effective_url, error=error, reason=info['headers'].get("X-Http-Reason", None), request_time=time.time() - info["curl_start_time"], time_info=time_info)) except Exception: self.handle_callback_exception(info["callback"])
Example #8
Source File: Request.py From wfuzz with GNU General Public License v2.0 | 5 votes |
def response_from_conn_object(self, conn, header, body): # followlocation if conn.getinfo(pycurl.EFFECTIVE_URL) != self.completeUrl: self.setFinalUrl(conn.getinfo(pycurl.EFFECTIVE_URL)) self.totaltime = conn.getinfo(pycurl.TOTAL_TIME) self.response = Response() self.response.parseResponse(header, rawbody=body) return self.response
Example #9
Source File: httpclient.py From honeything with GNU General Public License v3.0 | 5 votes |
def _finish(self, curl, curl_error=None, curl_message=None): info = curl.info curl.info = None self._multi.remove_handle(curl) self._free_list.append(curl) buffer = info["buffer"] if curl_error: error = CurlError(curl_error, curl_message) code = error.code effective_url = None buffer.close() buffer = None else: error = None code = curl.getinfo(pycurl.HTTP_CODE) effective_url = curl.getinfo(pycurl.EFFECTIVE_URL) buffer.seek(0) try: info["callback"](HTTPResponse( request=info["request"], code=code, headers=info["headers"], buffer=buffer, effective_url=effective_url, error=error, request_time=time.time() - info["start_time"])) except (KeyboardInterrupt, SystemExit): raise except: logging.error("Exception in callback %r", info["callback"], exc_info=True)
Example #10
Source File: curl_httpclient.py From tornado-zh with MIT License | 5 votes |
def _finish(self, curl, curl_error=None, curl_message=None): info = curl.info curl.info = None self._multi.remove_handle(curl) self._free_list.append(curl) buffer = info["buffer"] if curl_error: error = CurlError(curl_error, curl_message) code = error.code effective_url = None buffer.close() buffer = None else: error = None code = curl.getinfo(pycurl.HTTP_CODE) effective_url = curl.getinfo(pycurl.EFFECTIVE_URL) buffer.seek(0) # the various curl timings are documented at # http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html time_info = dict( queue=info["curl_start_time"] - info["request"].start_time, namelookup=curl.getinfo(pycurl.NAMELOOKUP_TIME), connect=curl.getinfo(pycurl.CONNECT_TIME), pretransfer=curl.getinfo(pycurl.PRETRANSFER_TIME), starttransfer=curl.getinfo(pycurl.STARTTRANSFER_TIME), total=curl.getinfo(pycurl.TOTAL_TIME), redirect=curl.getinfo(pycurl.REDIRECT_TIME), ) try: info["callback"](HTTPResponse( request=info["request"], code=code, headers=info["headers"], buffer=buffer, effective_url=effective_url, error=error, reason=info['headers'].get("X-Http-Reason", None), request_time=time.time() - info["curl_start_time"], time_info=time_info)) except Exception: self.handle_callback_exception(info["callback"])
Example #11
Source File: curl_httpclient.py From honeything with GNU General Public License v3.0 | 5 votes |
def _finish(self, curl, curl_error=None, curl_message=None): info = curl.info curl.info = None self._multi.remove_handle(curl) self._free_list.append(curl) buffer = info["buffer"] if curl_error: error = CurlError(curl_error, curl_message) code = error.code effective_url = None buffer.close() buffer = None else: error = None code = curl.getinfo(pycurl.HTTP_CODE) effective_url = curl.getinfo(pycurl.EFFECTIVE_URL) buffer.seek(0) # the various curl timings are documented at # http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html time_info = dict( queue=info["curl_start_time"] - info["request"].start_time, namelookup=curl.getinfo(pycurl.NAMELOOKUP_TIME), connect=curl.getinfo(pycurl.CONNECT_TIME), pretransfer=curl.getinfo(pycurl.PRETRANSFER_TIME), starttransfer=curl.getinfo(pycurl.STARTTRANSFER_TIME), total=curl.getinfo(pycurl.TOTAL_TIME), redirect=curl.getinfo(pycurl.REDIRECT_TIME), ) try: info["callback"](HTTPResponse( request=info["request"], code=code, headers=info["headers"], buffer=buffer, effective_url=effective_url, error=error, request_time=monotime() - info["curl_start_time"], time_info=time_info)) except Exception: self.handle_callback_exception(info["callback"])
Example #12
Source File: client.py From pycopia with Apache License 2.0 | 5 votes |
def finalize(self, c): """finalize a Curl object and extract information from it.""" self._url = c.getinfo(pycurl.EFFECTIVE_URL) # timing info nt = c.getinfo(pycurl.NAMELOOKUP_TIME) ct = c.getinfo(pycurl.CONNECT_TIME) pt = c.getinfo(pycurl.PRETRANSFER_TIME) st = c.getinfo(pycurl.STARTTRANSFER_TIME) tt = c.getinfo(pycurl.TOTAL_TIME) rt = c.getinfo(pycurl.REDIRECT_TIME) self._timing = TimingInfo(nt, ct, pt, st, tt, rt) self._redirectcount = c.getinfo(pycurl.REDIRECT_COUNT) self._cookielist = c.getinfo(pycurl.INFO_COOKIELIST) c.close() self._downloadfile = None
Example #13
Source File: curl_httpclient.py From viewfinder with Apache License 2.0 | 5 votes |
def _finish(self, curl, curl_error=None, curl_message=None): info = curl.info curl.info = None self._multi.remove_handle(curl) self._free_list.append(curl) buffer = info["buffer"] if curl_error: error = CurlError(curl_error, curl_message) code = error.code effective_url = None buffer.close() buffer = None else: error = None code = curl.getinfo(pycurl.HTTP_CODE) effective_url = curl.getinfo(pycurl.EFFECTIVE_URL) buffer.seek(0) # the various curl timings are documented at # http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html time_info = dict( queue=info["curl_start_time"] - info["request"].start_time, namelookup=curl.getinfo(pycurl.NAMELOOKUP_TIME), connect=curl.getinfo(pycurl.CONNECT_TIME), pretransfer=curl.getinfo(pycurl.PRETRANSFER_TIME), starttransfer=curl.getinfo(pycurl.STARTTRANSFER_TIME), total=curl.getinfo(pycurl.TOTAL_TIME), redirect=curl.getinfo(pycurl.REDIRECT_TIME), ) try: info["callback"](HTTPResponse( request=info["request"], code=code, headers=info["headers"], buffer=buffer, effective_url=effective_url, error=error, request_time=time.time() - info["curl_start_time"], time_info=time_info)) except Exception: self.handle_callback_exception(info["callback"])
Example #14
Source File: curl_httpclient.py From viewfinder with Apache License 2.0 | 5 votes |
def _finish(self, curl, curl_error=None, curl_message=None): info = curl.info curl.info = None self._multi.remove_handle(curl) self._free_list.append(curl) buffer = info["buffer"] if curl_error: error = CurlError(curl_error, curl_message) code = error.code effective_url = None buffer.close() buffer = None else: error = None code = curl.getinfo(pycurl.HTTP_CODE) effective_url = curl.getinfo(pycurl.EFFECTIVE_URL) buffer.seek(0) # the various curl timings are documented at # http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html time_info = dict( queue=info["curl_start_time"] - info["request"].start_time, namelookup=curl.getinfo(pycurl.NAMELOOKUP_TIME), connect=curl.getinfo(pycurl.CONNECT_TIME), pretransfer=curl.getinfo(pycurl.PRETRANSFER_TIME), starttransfer=curl.getinfo(pycurl.STARTTRANSFER_TIME), total=curl.getinfo(pycurl.TOTAL_TIME), redirect=curl.getinfo(pycurl.REDIRECT_TIME), ) try: info["callback"](HTTPResponse( request=info["request"], code=code, headers=info["headers"], buffer=buffer, effective_url=effective_url, error=error, request_time=time.time() - info["curl_start_time"], time_info=time_info)) except Exception: self.handle_callback_exception(info["callback"])
Example #15
Source File: homura.py From homura with BSD 2-Clause "Simplified" License | 5 votes |
def _move_path(self): """ Move the downloaded file to the authentic path (identified by effective URL) """ if is_temp_path(self._path) and self._pycurl is not None: eurl = self._pycurl.getinfo(pycurl.EFFECTIVE_URL) er = get_resource_name(eurl) r = get_resource_name(self.url) if er != r and os.path.exists(self.path): new_path = self._get_path(self._path, eurl) shutil.move(self.path, new_path) self.path = new_path
Example #16
Source File: curl_httpclient.py From tornado-zh with MIT License | 5 votes |
def _finish(self, curl, curl_error=None, curl_message=None): info = curl.info curl.info = None self._multi.remove_handle(curl) self._free_list.append(curl) buffer = info["buffer"] if curl_error: error = CurlError(curl_error, curl_message) code = error.code effective_url = None buffer.close() buffer = None else: error = None code = curl.getinfo(pycurl.HTTP_CODE) effective_url = curl.getinfo(pycurl.EFFECTIVE_URL) buffer.seek(0) # the various curl timings are documented at # http://curl.haxx.se/libcurl/c/curl_easy_getinfo.html time_info = dict( queue=info["curl_start_time"] - info["request"].start_time, namelookup=curl.getinfo(pycurl.NAMELOOKUP_TIME), connect=curl.getinfo(pycurl.CONNECT_TIME), pretransfer=curl.getinfo(pycurl.PRETRANSFER_TIME), starttransfer=curl.getinfo(pycurl.STARTTRANSFER_TIME), total=curl.getinfo(pycurl.TOTAL_TIME), redirect=curl.getinfo(pycurl.REDIRECT_TIME), ) try: info["callback"](HTTPResponse( request=info["request"], code=code, headers=info["headers"], buffer=buffer, effective_url=effective_url, error=error, reason=info['headers'].get("X-Http-Reason", None), request_time=time.time() - info["curl_start_time"], time_info=time_info)) except Exception: self.handle_callback_exception(info["callback"])
Example #17
Source File: curl_loop.py From falsy with MIT License | 4 votes |
def perform(cls): if cls._futures: while True: status, num_active = cls._multi.perform() if status != pycurl.E_CALL_MULTI_PERFORM: break while True: num_ready, success, fail = cls._multi.info_read() for c in success: cc = cls._futures.pop(c) result = curl_result(c) result['url'] = c._raw_url result['id'] = c._raw_id result['state'] = 'normal' result['spider'] = 'pycurl' result['payload'] = payload = c._raw_payload # post_func = payload.get('post_func') # if type(post_func) == str: # post_func = load(post_func) # if post_func: # result = post_func(payload, result) cc.set_result(result) for c, err_num, err_msg in fail: print('error:', err_num, err_msg, c.getinfo(pycurl.EFFECTIVE_URL)) result = curl_result(c) result['url'] = c._raw_url result['id'] = c._raw_id result['state'] = 'error' result['spider'] = 'pycurl' result['error_code'] = err_num result['error_desc'] = err_msg result['payload'] = payload = c._raw_payload # post_func = payload.get('post_func') # if type(post_func) == str: # post_func = load(post_func) # if post_func: # result2 = post_func(payload, result) # if type(result2) is dict and len(result2) >= len(result): # result = result2 cls._futures.pop(c).set_exception(CurlLoop.CurlException(code=err_num, desc=err_msg, data=result)) if num_ready == 0: break
Example #18
Source File: download.py From launcher with GNU General Public License v3.0 | 4 votes |
def run(self): c = pycurl.Curl() c.setopt(pycurl.URL, self.url) c.setopt(pycurl.FOLLOWLOCATION, True) c.setopt(pycurl.MAXREDIRS, 4) #c.setopt(pycurl.NOBODY, 1) c.setopt(c.VERBOSE, True) #c.setopt(pycurl.CONNECTTIMEOUT, 20) if self.useragent: c.setopt(pycurl.USERAGENT, self.useragent) # add cookies, if available if self.cookies: c.setopt(pycurl.COOKIE, self.cookies) #realurl = c.getinfo(pycurl.EFFECTIVE_URL) realurl = self.url print("realurl",realurl) self.filename = realurl.split("/")[-1].strip() c.setopt(pycurl.URL, realurl) c.setopt(pycurl.FOLLOWLOCATION, True) c.setopt(pycurl.NOPROGRESS, False) c.setopt(pycurl.XFERINFOFUNCTION, self.getProgress) c.setopt(pycurl.SSL_VERIFYPEER, False) c.setopt(pycurl.SSL_VERIFYHOST, False) # configure pycurl output file if self.path == False: self.path = os.getcwd() filepath = os.path.join(self.path, self.filename) if os.path.exists(filepath):## remove old file,restart download os.system("rm -rf " + filepath) buffer = StringIO() c.setopt(pycurl.WRITEDATA, buffer) self._dst_path = filepath # add cookies, if available if self.cookies: c.setopt(pycurl.COOKIE, self.cookies) self._stop = False self.progress["stopped"] = False # download file try: c.perform() except pycurl.error, error: errno,errstr = error print("curl error: %s" % errstr) self._errors.append(errstr) self._stop = True self.progress["stopped"] = True self.stop()
Example #19
Source File: curl_result.py From falsy with MIT License | 3 votes |
def curl_result(c): effective_url = c.getinfo(pycurl.EFFECTIVE_URL) primary_ip = c.getinfo(pycurl.PRIMARY_IP) primary_port = c.getinfo(pycurl.PRIMARY_PORT) local_ip = c.getinfo(pycurl.LOCAL_IP) local_port = c.getinfo(pycurl.LOCAL_PORT) speed_download = c.getinfo(pycurl.SPEED_DOWNLOAD) size_download = c.getinfo(pycurl.SIZE_DOWNLOAD) redirect_time = c.getinfo(pycurl.REDIRECT_TIME) redirect_count = c.getinfo(pycurl.REDIRECT_COUNT) redirect_url = c.getinfo(pycurl.REDIRECT_URL) http_code = c.getinfo(pycurl.HTTP_CODE) response_code = c.getinfo(pycurl.RESPONSE_CODE) total_time = c.getinfo(pycurl.TOTAL_TIME) content_type = c.getinfo(pycurl.CONTENT_TYPE) namelookup_time = c.getinfo(pycurl.NAMELOOKUP_TIME) info_filetime = c.getinfo(pycurl.INFO_FILETIME) http_connectcode = c.getinfo(pycurl.HTTP_CONNECTCODE) starttransfer_time = c.getinfo(pycurl.STARTTRANSFER_TIME) pretransfer_time = c.getinfo(pycurl.PRETRANSFER_TIME) header_size = c.getinfo(pycurl.HEADER_SIZE) request_size = c.getinfo(pycurl.REQUEST_SIZE) ssl_verifyresult = c.getinfo(pycurl.SSL_VERIFYRESULT) num_connects = c.getinfo(pycurl.NUM_CONNECTS) return { 'effective_url': effective_url, 'primary_ip': primary_ip, 'primary_port': primary_port, 'local_ip': local_ip, 'local_port': local_port, 'speed_download': speed_download, 'size_download': size_download, 'redirect_time': redirect_time, 'redirect_count': redirect_count, 'redirect_url': redirect_url, 'http_code': http_code, 'response_code': response_code, 'total_time': total_time, 'content_type': content_type, 'namelookup_time': namelookup_time, 'info_filetime': info_filetime, 'http_connectcode': http_connectcode, 'starttransfer_time': starttransfer_time, 'pretransfer_time': pretransfer_time, 'header_size': header_size, 'request_size': request_size, 'ssl_verifyresult': ssl_verifyresult, 'num_connects': num_connects, # 'proxy_ssl_verifyresult': proxy_ssl_verifyresult, # 'app_connecttime': app_connecttime, }