Python pycurl.URL Examples
The following are 30
code examples of pycurl.URL().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
pycurl
, or try the search function
.
.
Example #1
| Source File: downloader.py From Paradrop with Apache License 2.0 | 7 votes |
|
def _create_curl_conn(self, url):
"""
Create a cURL connection object with useful default settings.
"""
headers = []
if self.user is not None and self.secret is not None:
b64cred = base64.b64encode("{}:{}".format(self.user, self.secret))
headers.append("Authorization: Basic {}".format(b64cred))
conn = pycurl.Curl()
if len(headers) > 0:
conn.setopt(pycurl.HTTPHEADER, headers)
conn.setopt(pycurl.URL, url)
# github often redirects
conn.setopt(pycurl.FOLLOWLOCATION, 1)
return conn
Example #2
| Source File: fetch.py From landscape-client with GNU General Public License v2.0 | 6 votes |
|
def fetch_many_async(urls, callback=None, errback=None, **kwargs):
"""
Retrieve a list of URLs asynchronously.
@param callback: Optionally, a function that will be fired one time for
each successful URL, and will be passed its content and the URL itself.
@param errback: Optionally, a function that will be fired one time for each
failing URL, and will be passed the failure and the URL itself.
@return: A C{DeferredList} whose callback chain will be fired as soon as
all downloads have terminated. If an error occurs, the errback chain
of the C{DeferredList} will be fired immediatly.
"""
results = []
for url in urls:
result = fetch_async(url, **kwargs)
if callback:
result.addCallback(callback, url)
if errback:
result.addErrback(errback, url)
results.append(result)
return DeferredList(results, fireOnOneErrback=True, consumeErrors=True)
Example #3
| Source File: http.py From Paradrop with Apache License 2.0 | 6 votes |
|
def urlEncodeParams(data):
"""
Return data URL-encoded.
This function specifically handles None and boolean values
to convert them to JSON-friendly strings (e.g. None -> 'null').
"""
copy = dict()
for key, value in six.iteritems(data):
if value is None:
copy[key] = 'null'
elif isinstance(value, bool):
copy[key] = json.dumps(value)
else:
copy[key] = value
return urllib.urlencode(copy, doseq=True)
Example #4
| Source File: downloader.py From Paradrop with Apache License 2.0 | 6 votes |
|
def _create_curl_conn(self, url):
"""
Create a cURL connection object with useful default settings.
"""
headers = []
if self.user is not None and self.secret is not None:
b64cred = base64.b64encode("{}:{}".format(self.user, self.secret))
headers.append("Authorization: Basic {}".format(b64cred))
conn = pycurl.Curl()
if len(headers) > 0:
conn.setopt(pycurl.HTTPHEADER, headers)
conn.setopt(pycurl.URL, url)
# github often redirects
conn.setopt(pycurl.FOLLOWLOCATION, 1)
return conn
Example #5
| Source File: custom_path_selection.py From stem with GNU Lesser General Public License v3.0 | 6 votes |
|
def query(url):
"""
Uses pycurl to fetch a site using the proxy on the SOCKS_PORT.
"""
output = StringIO.StringIO()
query = pycurl.Curl()
query.setopt(pycurl.URL, url)
query.setopt(pycurl.PROXY, 'localhost')
query.setopt(pycurl.PROXYPORT, SOCKS_PORT)
query.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
query.setopt(pycurl.CONNECTTIMEOUT, CONNECTION_TIMEOUT)
query.setopt(pycurl.WRITEFUNCTION, output.write)
try:
query.perform()
return output.getvalue()
except pycurl.error as exc:
raise ValueError("Unable to reach %s (%s)" % (url, exc))
Example #6
| Source File: client_usage_using_pycurl.py From stem with GNU Lesser General Public License v3.0 | 6 votes |
|
def query(url):
"""
Uses pycurl to fetch a site using the proxy on the SOCKS_PORT.
"""
output = io.BytesIO()
query = pycurl.Curl()
query.setopt(pycurl.URL, url)
query.setopt(pycurl.PROXY, 'localhost')
query.setopt(pycurl.PROXYPORT, SOCKS_PORT)
query.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
query.setopt(pycurl.WRITEFUNCTION, output.write)
try:
query.perform()
return output.getvalue()
except pycurl.error as exc:
return "Unable to reach %s (%s)" % (url, exc)
# Start an instance of Tor configured to only exit through Russia. This prints
# Tor's bootstrap information as it starts. Note that this likely will not
# work if you have another Tor instance running.
Example #7
| Source File: psurl.py From pscheduler with Apache License 2.0 | 6 votes |
|
def url_post( url, # GET URL
params={}, # GET parameters
data=None, # Data to post
content_type=None, # Content type
bind=None, # Bind request to specified address
json=True, # Interpret result as JSON
throw=True, # Throw if status isn't 200
timeout=None, # Seconds before giving up
allow_redirects=True, #Allows URL to be redirected
headers={}, # Hash of HTTP headers
verify_keys=verify_keys_default # Verify SSL keys
):
"""
Post to a URL, returning whatever came back.
"""
content_type, data = __content_type_data(content_type, headers, data)
headers["Content-Type"] = content_type
curl = PycURLRunner(url, params, bind, timeout, allow_redirects, headers, verify_keys)
curl.curl.setopt(pycurl.POSTFIELDS, data)
return curl(json, throw)
Example #8
| Source File: torutil.py From onion-expose with BSD 3-Clause "New" or "Revised" License | 6 votes |
|
def _check_status(self):
"""
For internal use only. Only members of :class:`Tunnel`
"""
try:
output = io.BytesIO()
query = pycurl.Curl()
query.setopt(pycurl.URL, "%s:%s" % (self.addr, self.status_server.port))
query.setopt(pycurl.PROXY, "127.0.0.1")
query.setopt(pycurl.PROXYPORT, config.tor_proxy_port)
query.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
query.setopt(pycurl.WRITEFUNCTION, output.write)
query.perform()
return "ONIONGROK_CLIENT_STATUS_SUCCESS" in output.getvalue().decode("utf8")
except Exception as e:
return False
Example #9
| Source File: reqresp.py From darkc0de-old-stuff with GNU General Public License v3.0 | 6 votes |
|
def getXML(self,obj):
r=obj.createElement("request")
r.setAttribute("method",self.method)
url=obj.createElement("URL")
url.appendChild(obj.createTextNode(self.completeUrl))
r.appendChild(url)
if self.method=="POST":
pd=obj.createElement("PostData")
pd.appendChild(obj.createTextNode(self.postdata))
r.appendChild(pd)
if "Cookie" in self.__headers:
ck=obj.createElement("Cookie")
ck.appendChild(obj.createTextNode(self.__headers["Cookie"]))
r.appendChild(ck)
return r
Example #10
| Source File: psurl.py From pscheduler with Apache License 2.0 | 6 votes |
|
def url_put( url, # GET URL
params={}, # GET parameters
data=None, # Data for body
content_type=None, # Content type
bind=None, # Bind request to specified address
json=True, # Interpret result as JSON
throw=True, # Throw if status isn't 200
timeout=None, # Seconds before giving up
allow_redirects=True, #Allows URL to be redirected
headers={}, # Hash of HTTP headers
verify_keys=verify_keys_default # Verify SSL keys
):
"""
PUT to a URL, returning whatever came back.
"""
content_type, data = __content_type_data(content_type, headers, data)
headers["Content-Type"] = content_type
curl = PycURLRunner(url, params, bind, timeout, allow_redirects, headers, verify_keys)
curl.curl.setopt(pycurl.CUSTOMREQUEST, "PUT")
curl.curl.setopt(pycurl.POSTFIELDS, data)
return curl(json, throw)
Example #11
| Source File: server_pre_install.py From contrail-server-manager with Apache License 2.0 | 6 votes |
|
def send_REST_request(ip, port, object, payload):
try:
response = StringIO()
headers = ["Content-Type:application/json"]
url = "http://%s:%s/%s" %(
ip, port, object)
conn = pycurl.Curl()
conn.setopt(pycurl.URL, url)
conn.setopt(pycurl.HTTPHEADER, headers)
conn.setopt(pycurl.POST, 1)
conn.setopt(pycurl.POSTFIELDS, '%s'%json.dumps(payload))
conn.setopt(pycurl.CUSTOMREQUEST, "PUT")
conn.setopt(pycurl.WRITEFUNCTION, response.write)
conn.perform()
return response.getvalue()
except:
return None
Example #12
| Source File: smgr_inventory.py From contrail-server-manager with Apache License 2.0 | 6 votes |
|
def send_REST_request(self, server_ip, port):
try:
response = StringIO()
url = "http://%s:%s/%s" % (server_ip, port, 'InventoryInfo')
headers = ["Content-Type:application/json"]
conn = pycurl.Curl()
conn.setopt(pycurl.URL, str(url))
conn.setopt(pycurl.HTTPHEADER, headers)
conn.setopt(conn.WRITEFUNCTION, response.write)
conn.setopt(pycurl.HTTPGET, 1)
conn.perform()
data_dict = response.getvalue()
data_dict = dict(json.loads(data_dict))
data_list = list(data_dict["__ServerInventoryInfoUve_list"]["ServerInventoryInfoUve"])
return data_list
except Exception as e:
print "Inventory Error is: " + str(e)
return None
# end def send_REST_request
# Filters the data returned from REST API call for requested information
Example #13
| Source File: restClient.py From recipebook with MIT License | 6 votes |
|
def put(url, data, encoding, headers=None):
"""Make a PUT request to the url, using data in the message body,
with the additional headers, if any"""
if headers is None:
headers = {}
reply = -1 # default, non-http response
curl = pycurl.Curl()
curl.setopt(pycurl.URL, url)
if len(headers) > 0:
curl.setopt(pycurl.HTTPHEADER, [k + ': ' + v for k, v in list(headers.items())])
curl.setopt(pycurl.PUT, 1)
curl.setopt(pycurl.INFILESIZE, len(data))
databuffer = BytesIO(data.encode(encoding))
curl.setopt(pycurl.READDATA, databuffer)
try:
curl.perform()
reply = curl.getinfo(pycurl.HTTP_CODE)
except Exception:
pass
curl.close()
return reply
Example #14
| Source File: smgr_delete.py From contrail-server-manager with Apache License 2.0 | 6 votes |
|
def send_REST_request(ip, port, object, key, value, force=False):
try:
response = StringIO()
headers = ["Content-Type:application/json"]
url = "http://%s:%s/%s?%s=%s" %(
ip, port, object,
urllib.quote_plus(key),
urllib.quote_plus(value))
if force:
url += "&force"
conn = pycurl.Curl()
conn.setopt(pycurl.URL, url)
conn.setopt(pycurl.HTTPHEADER, headers)
conn.setopt(pycurl.CUSTOMREQUEST, "delete")
conn.setopt(pycurl.WRITEFUNCTION, response.write)
conn.perform()
return response.getvalue()
except:
return None
# end def send_REST_request
Example #15
| Source File: smgr_provision_server.py From contrail-server-manager with Apache License 2.0 | 6 votes |
|
def send_REST_request(ip, port, payload):
try:
response = StringIO()
headers = ["Content-Type:application/json"]
url = "http://%s:%s/server/provision" %(
ip, port)
conn = pycurl.Curl()
conn.setopt(pycurl.URL, url)
conn.setopt(pycurl.HTTPHEADER, headers)
conn.setopt(pycurl.POST, 1)
conn.setopt(pycurl.POSTFIELDS, '%s'%json.dumps(payload))
conn.setopt(pycurl.WRITEFUNCTION, response.write)
conn.perform()
return response.getvalue()
except:
return None
Example #16
| Source File: smgr_monitoring.py From contrail-server-manager with Apache License 2.0 | 6 votes |
|
def send_REST_request(self, server_ip, port):
try:
response = StringIO()
headers = ["Content-Type:application/json"]
url = "http://%s:%s/%s" % (server_ip, port, 'MonitorInfo')
args_str = ''
conn = pycurl.Curl()
conn.setopt(pycurl.URL, url)
conn.setopt(pycurl.HTTPHEADER, headers)
conn.setopt(pycurl.HTTPGET, 1)
conn.setopt(pycurl.WRITEFUNCTION, response.write)
conn.perform()
data_dict = response.getvalue()
data_dict = dict(json.loads(data_dict))
data_list = list(data_dict["__ServerMonitoringInfoTrace_list"]["ServerMonitoringInfoTrace"])
return data_list
except Exception as e:
print "Error is: " + str(e)
return None
# end def send_REST_request
Example #17
| Source File: psurl.py From pscheduler with Apache License 2.0 | 6 votes |
|
def url_delete_list(
urls,
params=None,
bind=None,
timeout=None, # Seconds before giving up
allow_redirects=True, #Allows URL to be redirected
headers=None, # Hash of HTTP headers
verify_keys=verify_keys_default # Verify SSL keys
):
"""
Delete a list of URLs and return tuples of the status and error for
each. Note that the timeout is per delete, not for the aggregated
operation.
"""
return [ url_delete(url, throw=False, timeout=timeout, params=params,
bind=bind, headers=headers, verify_keys=verify_keys,
allow_redirects=allow_redirects)
for url in urls ]
Example #18
| Source File: smgr_contrail_status.py From contrail-server-manager with Apache License 2.0 | 6 votes |
|
def send_REST_request(ip, port, object, match_key,
match_value, detail):
try:
response = StringIO()
headers = ["Content-Type:application/json"]
url = "http://%s:%s/%s" % (ip, port, object)
args_str = ''
if match_key:
args_str += match_key + "=" + match_value
if detail:
args_str += "&detail"
if args_str != '':
url += "?" + args_str
conn = pycurl.Curl()
conn.setopt(pycurl.TIMEOUT, 1)
conn.setopt(pycurl.URL, url)
conn.setopt(pycurl.HTTPHEADER, headers)
conn.setopt(pycurl.HTTPGET, 1)
conn.setopt(pycurl.WRITEFUNCTION, response.write)
conn.perform()
return response.getvalue()
except:
return None
# end def send_REST_request
Example #19
| Source File: fetch.py From landscape-client with GNU General Public License v2.0 | 6 votes |
|
def fetch_to_files(urls, directory, logger=None, **kwargs):
"""
Retrieve a list of URLs and save their content as files in a directory.
@param urls: The list URLs to fetch.
@param directory: The directory to save the files to, the name of the file
will equal the last fragment of the URL.
@param logger: Optional function to be used to log errors for failed URLs.
"""
def write(data, url):
filename = url_to_filename(url, directory=directory)
fd = open(filename, "wb")
fd.write(data)
fd.close()
def log_error(failure, url):
if logger:
logger("Couldn't fetch file from %s (%s)" % (
url, str(failure.value)))
return failure
return fetch_many_async(urls, callback=write, errback=log_error, **kwargs)
Example #20
| Source File: smgr_reimage_server.py From contrail-server-manager with Apache License 2.0 | 6 votes |
|
def send_REST_request(ip, port, payload):
try:
response = StringIO()
headers = ["Content-Type:application/json"]
url = "http://%s:%s/server/reimage" %(
ip, port)
conn = pycurl.Curl()
conn.setopt(pycurl.URL, url)
conn.setopt(pycurl.HTTPHEADER, headers)
conn.setopt(pycurl.POST, 1)
conn.setopt(pycurl.POSTFIELDS, '%s'%json.dumps(payload))
conn.setopt(pycurl.WRITEFUNCTION, response.write)
conn.perform()
return response.getvalue()
except:
return None
Example #21
| Source File: test_fetch.py From landscape-client with GNU General Public License v2.0 | 6 votes |
|
def test_post(self):
curl = CurlStub(b"result")
result = fetch("http://example.com", post=True, curl=curl)
self.assertEqual(result, b"result")
self.assertEqual(curl.options,
{pycurl.URL: b"http://example.com",
pycurl.FOLLOWLOCATION: 1,
pycurl.MAXREDIRS: 5,
pycurl.CONNECTTIMEOUT: 30,
pycurl.LOW_SPEED_LIMIT: 1,
pycurl.LOW_SPEED_TIME: 600,
pycurl.NOSIGNAL: 1,
pycurl.WRITEFUNCTION: Any(),
pycurl.POST: True,
pycurl.DNS_CACHE_TIMEOUT: 0,
pycurl.ENCODING: b"gzip,deflate"})
Example #22
| Source File: test_fetch.py From landscape-client with GNU General Public License v2.0 | 6 votes |
|
def test_post_data(self):
curl = CurlStub(b"result")
result = fetch("http://example.com", post=True, data="data", curl=curl)
self.assertEqual(result, b"result")
self.assertEqual(curl.options[pycurl.READFUNCTION](), b"data")
self.assertEqual(curl.options,
{pycurl.URL: b"http://example.com",
pycurl.FOLLOWLOCATION: 1,
pycurl.MAXREDIRS: 5,
pycurl.CONNECTTIMEOUT: 30,
pycurl.LOW_SPEED_LIMIT: 1,
pycurl.LOW_SPEED_TIME: 600,
pycurl.NOSIGNAL: 1,
pycurl.WRITEFUNCTION: Any(),
pycurl.POST: True,
pycurl.POSTFIELDSIZE: 4,
pycurl.READFUNCTION: Any(),
pycurl.DNS_CACHE_TIMEOUT: 0,
pycurl.ENCODING: b"gzip,deflate"})
Example #23
| Source File: test_fetch.py From landscape-client with GNU General Public License v2.0 | 6 votes |
|
def test_cainfo(self):
curl = CurlStub(b"result")
result = fetch("https://example.com", cainfo="cainfo", curl=curl)
self.assertEqual(result, b"result")
self.assertEqual(curl.options,
{pycurl.URL: b"https://example.com",
pycurl.FOLLOWLOCATION: 1,
pycurl.MAXREDIRS: 5,
pycurl.CONNECTTIMEOUT: 30,
pycurl.LOW_SPEED_LIMIT: 1,
pycurl.LOW_SPEED_TIME: 600,
pycurl.NOSIGNAL: 1,
pycurl.WRITEFUNCTION: Any(),
pycurl.CAINFO: b"cainfo",
pycurl.DNS_CACHE_TIMEOUT: 0,
pycurl.ENCODING: b"gzip,deflate"})
Example #24
| Source File: test_fetch.py From landscape-client with GNU General Public License v2.0 | 6 votes |
|
def test_headers(self):
curl = CurlStub(b"result")
result = fetch("http://example.com",
headers={"a": "1", "b": "2"}, curl=curl)
self.assertEqual(result, b"result")
self.assertEqual(curl.options,
{pycurl.URL: b"http://example.com",
pycurl.FOLLOWLOCATION: 1,
pycurl.MAXREDIRS: 5,
pycurl.CONNECTTIMEOUT: 30,
pycurl.LOW_SPEED_LIMIT: 1,
pycurl.LOW_SPEED_TIME: 600,
pycurl.NOSIGNAL: 1,
pycurl.WRITEFUNCTION: Any(),
pycurl.HTTPHEADER: ["a: 1", "b: 2"],
pycurl.DNS_CACHE_TIMEOUT: 0,
pycurl.ENCODING: b"gzip,deflate"})
Example #25
| Source File: test_fetch.py From landscape-client with GNU General Public License v2.0 | 6 votes |
|
def test_timeouts(self):
curl = CurlStub(b"result")
result = fetch("http://example.com", connect_timeout=5,
total_timeout=30, curl=curl)
self.assertEqual(result, b"result")
self.assertEqual(curl.options,
{pycurl.URL: b"http://example.com",
pycurl.FOLLOWLOCATION: 1,
pycurl.MAXREDIRS: 5,
pycurl.CONNECTTIMEOUT: 5,
pycurl.LOW_SPEED_LIMIT: 1,
pycurl.LOW_SPEED_TIME: 30,
pycurl.NOSIGNAL: 1,
pycurl.WRITEFUNCTION: Any(),
pycurl.DNS_CACHE_TIMEOUT: 0,
pycurl.ENCODING: b"gzip,deflate"})
Example #26
| Source File: test_fetch.py From landscape-client with GNU General Public License v2.0 | 6 votes |
|
def test_pycurl_insecure(self):
curl = CurlStub(b"result")
result = fetch("http://example.com/get-ca-cert", curl=curl,
insecure=True)
self.assertEqual(result, b"result")
self.assertEqual(curl.options,
{pycurl.URL: b"http://example.com/get-ca-cert",
pycurl.FOLLOWLOCATION: 1,
pycurl.MAXREDIRS: 5,
pycurl.CONNECTTIMEOUT: 30,
pycurl.LOW_SPEED_LIMIT: 1,
pycurl.LOW_SPEED_TIME: 600,
pycurl.NOSIGNAL: 1,
pycurl.WRITEFUNCTION: Any(),
pycurl.SSL_VERIFYPEER: False,
pycurl.DNS_CACHE_TIMEOUT: 0,
pycurl.ENCODING: b"gzip,deflate"})
Example #27
| Source File: smgr_status.py From contrail-server-manager with Apache License 2.0 | 6 votes |
|
def send_REST_request(ip, port, object, match_key,
match_value, detail):
try:
response = StringIO()
headers = ["Content-Type:application/json"]
url = "http://%s:%s/%s" % (ip, port, object)
args_str = ''
if match_key:
args_str += match_key + "=" + match_value
if detail:
args_str += "&detail"
if args_str != '':
url += "?" + args_str
conn = pycurl.Curl()
conn.setopt(pycurl.URL, url)
conn.setopt(pycurl.HTTPHEADER, headers)
conn.setopt(pycurl.HTTPGET, 1)
conn.setopt(pycurl.WRITEFUNCTION, response.write)
conn.perform()
return response.getvalue()
except:
return None
# end def send_REST_request
Example #28
| Source File: smgr_restart_server.py From contrail-server-manager with Apache License 2.0 | 6 votes |
|
def send_REST_request(ip, port, payload):
try:
response = StringIO()
headers = ["Content-Type:application/json"]
url = "http://%s:%s/server/restart" %(
ip, port)
conn = pycurl.Curl()
conn.setopt(pycurl.URL, url)
conn.setopt(pycurl.HTTPHEADER, headers)
conn.setopt(pycurl.POST, 1)
conn.setopt(pycurl.POSTFIELDS, '%s'%json.dumps(payload))
conn.setopt(pycurl.WRITEFUNCTION, response.write)
conn.perform()
return response.getvalue()
except:
return None
# end def send_REST_request
Example #29
| Source File: sm_ansible_utils.py From contrail-server-manager with Apache License 2.0 | 6 votes |
|
def send_REST_request(self, ip, port, endpoint, payload,
method='POST', urlencode=False):
try:
response = StringIO()
headers = ["Content-Type:application/json"]
url = "http://%s:%s/%s" %(ip, port, endpoint)
conn = pycurl.Curl()
if method == "PUT":
conn.setopt(pycurl.CUSTOMREQUEST, method)
if urlencode == False:
first = True
for k,v in payload.iteritems():
if first:
url = url + '?'
first = False
else:
url = url + '&'
url = url + ("%s=%s" % (k,v))
else:
url = url + '?' + payload
if self.logger:
self.logger.log(self.logger.INFO,
"Sending post request to %s" % url)
conn.setopt(pycurl.URL, url)
conn.setopt(pycurl.HTTPHEADER, headers)
conn.setopt(pycurl.POST, 1)
if urlencode == False:
conn.setopt(pycurl.POSTFIELDS, '%s'%json.dumps(payload))
conn.setopt(pycurl.WRITEFUNCTION, response.write)
conn.perform()
return response.getvalue()
except:
return None
Example #30
| Source File: Request.py From wfuzz with GNU General Public License v2.0 | 6 votes |
|
def getXML(self, obj):
r = obj.createElement("request")
r.setAttribute("method", self.method)
url = obj.createElement("URL")
url.appendChild(obj.createTextNode(self.completeUrl))
r.appendChild(url)
if self.postdata:
pd = obj.createElement("PostData")
pd.appendChild(obj.createTextNode(self.postdata))
r.appendChild(pd)
if "Cookie" in self._headers:
ck = obj.createElement("Cookie")
ck.appendChild(obj.createTextNode(self._headers["Cookie"]))
r.appendChild(ck)
return r
