Python pycurl.Curl() Examples

def _create_curl_conn(self, url):
        """
        Create a cURL connection object with useful default settings.
        """
        headers = []
        if self.user is not None and self.secret is not None:
            b64cred = base64.b64encode("{}:{}".format(self.user, self.secret))
            headers.append("Authorization: Basic {}".format(b64cred))

        conn = pycurl.Curl()

        if len(headers) > 0:
            conn.setopt(pycurl.HTTPHEADER, headers)

        conn.setopt(pycurl.URL, url)

        # github often redirects
        conn.setopt(pycurl.FOLLOWLOCATION, 1)

        return conn 

def curl_init(self, url = False):

        self.curl = pycurl.Curl()
        self.set_url(url = url)
        self.curl.setopt(self.curl.SSL_VERIFYPEER, False)

        if DEBUG:

            self._log(
                'Following HTTP redirects: %s' % (
                    str(self.follow_http_redirect)
                )
            )

        self.curl.setopt(self.curl.FOLLOWLOCATION, self.follow_http_redirect)
        self.curl.setopt(self.curl.CONNECTTIMEOUT, self.connect_timeout)
        self.curl.setopt(self.curl.TIMEOUT, self.timeout)
        self.curl.setopt(self.curl.TCP_KEEPALIVE, 1)
        self.curl.setopt(self.curl.TCP_KEEPIDLE, 2)

        if self.ignore_content_length:

            self.curl.setopt(self.curl.IGNORE_CONTENT_LENGTH, 136) 

def query(url):
  """
  Uses pycurl to fetch a site using the proxy on the SOCKS_PORT.
  """

  output = io.BytesIO()

  query = pycurl.Curl()
  query.setopt(pycurl.URL, url)
  query.setopt(pycurl.PROXY, 'localhost')
  query.setopt(pycurl.PROXYPORT, SOCKS_PORT)
  query.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
  query.setopt(pycurl.WRITEFUNCTION, output.write)

  try:
    query.perform()
    return output.getvalue()
  except pycurl.error as exc:
    return "Unable to reach %s (%s)" % (url, exc)


# Start an instance of Tor configured to only exit through Russia. This prints
# Tor's bootstrap information as it starts. Note that this likely will not
# work if you have another Tor instance running. 

def query(url):
  """
  Uses pycurl to fetch a site using the proxy on the SOCKS_PORT.
  """

  output = StringIO.StringIO()

  query = pycurl.Curl()
  query.setopt(pycurl.URL, url)
  query.setopt(pycurl.PROXY, 'localhost')
  query.setopt(pycurl.PROXYPORT, SOCKS_PORT)
  query.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
  query.setopt(pycurl.CONNECTTIMEOUT, CONNECTION_TIMEOUT)
  query.setopt(pycurl.WRITEFUNCTION, output.write)

  try:
    query.perform()
    return output.getvalue()
  except pycurl.error as exc:
    raise ValueError("Unable to reach %s (%s)" % (url, exc)) 

def _check_status(self):
        """
        For internal use only. Only members of :class:`Tunnel`
        """
        try:
            output = io.BytesIO()

            query = pycurl.Curl()
            query.setopt(pycurl.URL, "%s:%s" % (self.addr, self.status_server.port))
            query.setopt(pycurl.PROXY, "127.0.0.1")
            query.setopt(pycurl.PROXYPORT, config.tor_proxy_port)
            query.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME)
            query.setopt(pycurl.WRITEFUNCTION, output.write)
            query.perform()

            return "ONIONGROK_CLIENT_STATUS_SUCCESS" in output.getvalue().decode("utf8")
        except Exception as e:
            return False 

def url_check(self, url):
        '''下载地址检查'''

        url_info = {}
        proto = urlparse.urlparse(url)[0]
        if proto not in VALIDPROTOCOL:
            print 'Valid protocol should be http or ftp, but % s found < %s >!' % (proto, url)
        else:
            ss = StringIO()
            curl = pycurl.Curl()
            curl.setopt(pycurl.FOLLOWLOCATION, 1)
            curl.setopt(pycurl.MAXREDIRS, 5)
            curl.setopt(pycurl.CONNECTTIMEOUT, 30)
            curl.setopt(pycurl.TIMEOUT, 300)
            curl.setopt(pycurl.NOSIGNAL, 1)
            curl.setopt(pycurl.NOPROGRESS, 1)
            curl.setopt(pycurl.NOBODY, 1)
            curl.setopt(pycurl.HEADERFUNCTION, ss.write)
            curl.setopt(pycurl.URL, url)

        try:
            curl.perform()
        except:
            pass

        if curl.errstr() == '' and curl.getinfo(pycurl.RESPONSE_CODE) in STATUS_OK:
            url_info['url'] = curl.getinfo(pycurl.EFFECTIVE_URL)
            url_info['file'] = os.path.split(url_info['url'])[1]
            url_info['size'] = int(
                curl.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD))
            url_info['partible'] = (ss.getvalue().find('Accept - Ranges') != -1)

        return url_info 

def get_form_poster(self):
        """Initialze a Curl object for a single POST request.

        This sends a multipart/form-data, which allows you to upload files.

        Returns a tuple of initialized Curl and HTTPResponse objects.
        """
        data = self._data.items()
        c = pycurl.Curl()
        resp = HTTPResponse(self._encoding)
        c.setopt(c.URL, str(self._url))
        c.setopt(pycurl.HTTPPOST, data)
        c.setopt(c.WRITEFUNCTION, resp._body_callback)
        c.setopt(c.HEADERFUNCTION, resp._header_callback)
        self._set_common(c)
        return c, resp 

def get_raw_poster(self):
        """Initialze a Curl object for a single POST request.

        This sends whatever data you give it, without specifying the content
        type.

        Returns a tuple of initialized Curl and HTTPResponse objects.
        """
        ld = len(self._data)
        c = pycurl.Curl()
        resp = HTTPResponse(self._encoding)
        c.setopt(c.URL, str(self._url))
        c.setopt(pycurl.POST, 1)
        c.setopt(c.READFUNCTION, DataWrapper(self._data).read)
        c.setopt(c.POSTFIELDSIZE, ld)
        c.setopt(c.WRITEFUNCTION, resp._body_callback)
        c.setopt(c.HEADERFUNCTION, resp._header_callback)
        headers = self._headers[:]
        headers.append(httputils.ContentType("")) # removes libcurl internal header
        headers.append(httputils.ContentLength(str(ld)))
        c.setopt(c.HTTPHEADER, map(str, headers))
        self._set_common(c)
        return c, resp 

def get_deleter(self):
        """Initialze a Curl object for a single DELETE request.

        Returns a tuple of initialized Curl and HTTPResponse objects.
        """
        c = pycurl.Curl()
        resp = HTTPResponse(self._encoding)
        c.setopt(pycurl.CUSTOMREQUEST, "DELETE")
        c.setopt(c.URL, str(self._url))
        c.setopt(c.WRITEFUNCTION, resp._body_callback)
        c.setopt(c.HEADERFUNCTION, resp._header_callback)
        c.setopt(c.HTTPHEADER, map(str, self._headers))
        self._set_common(c)
        return c, resp

    # sets options common to all operations 

def put(url, data, encoding, headers=None):
    """Make a PUT request to the url, using data in the message body,
    with the additional headers, if any"""

    if headers is None:
        headers = {}
    reply = -1  # default, non-http response

    curl = pycurl.Curl()
    curl.setopt(pycurl.URL, url)
    if len(headers) > 0:
        curl.setopt(pycurl.HTTPHEADER, [k + ': ' + v for k, v in list(headers.items())])
    curl.setopt(pycurl.PUT, 1)
    curl.setopt(pycurl.INFILESIZE, len(data))
    databuffer = BytesIO(data.encode(encoding))
    curl.setopt(pycurl.READDATA, databuffer)
    try:
        curl.perform()
        reply = curl.getinfo(pycurl.HTTP_CODE)
    except Exception:
        pass
    curl.close()

    return reply 

def _curl(self, payload, computer_id, exchange_token, message_api):
        # There are a few "if _PY3" checks below, because for Python 3 we
        # want to convert a number of values from bytes to string, before
        # assigning them to the headers.
        if _PY3 and isinstance(message_api, bytes):
            message_api = message_api.decode("ascii")
        headers = {"X-Message-API": message_api,
                   "User-Agent": "landscape-client/%s" % VERSION,
                   "Content-Type": "application/octet-stream"}
        if computer_id:
            if _PY3 and isinstance(computer_id, bytes):
                computer_id = computer_id.decode("ascii")
            headers["X-Computer-ID"] = computer_id
        if exchange_token:
            if _PY3 and isinstance(exchange_token, bytes):
                exchange_token = exchange_token.decode("ascii")
            headers["X-Exchange-Token"] = str(exchange_token)
        curl = pycurl.Curl()
        return (curl, fetch(self._url, post=True, data=payload,
                            headers=headers, cainfo=self._pubkey, curl=curl)) 

def _create_curl_conn(self, url):
        """
        Create a cURL connection object with useful default settings.
        """
        headers = []
        if self.user is not None and self.secret is not None:
            b64cred = base64.b64encode("{}:{}".format(self.user, self.secret))
            headers.append("Authorization: Basic {}".format(b64cred))

        conn = pycurl.Curl()

        if len(headers) > 0:
            conn.setopt(pycurl.HTTPHEADER, headers)

        conn.setopt(pycurl.URL, url)

        # github often redirects
        conn.setopt(pycurl.FOLLOWLOCATION, 1)

        return conn 

def init_request(self):

        if self.init_url is not None:

            if self.progress is not None:

                self.progress.set_status('requesting cookie')

            self.init_curl = Curl(
                self.init_url,
                silent = True,
                debug = self.debug,
                cache = self.init_use_cache,
                req_headers = self.req_headers,
                follow = False,
            )
            headers = getattr(self.init_curl, self.init_fun)()
            self.req_headers.extend(headers)


    # caching: 

def _initialize(self):
        # pycurl Connection pool
        self.m = pycurl.CurlMulti()
        self.handles = []

        for i in range(self.options.get("concurrent")):
            curl_h = pycurl.Curl()
            self.handles.append(curl_h)
            self.curlh_freelist.append(curl_h)

        # create threads
        self.ths = []

        for fn in ("_read_multi_stack",):
            th = Thread(target=getattr(self, fn))
            th.setName(fn)
            self.ths.append(th)
            th.start() 

def send_REST_request(ip, port, object, payload):
    try:
        response = StringIO()
        headers = ["Content-Type:application/json"]
        url = "http://%s:%s/%s" %(
            ip, port, object)
        conn = pycurl.Curl()
        conn.setopt(pycurl.URL, url)
        conn.setopt(pycurl.HTTPHEADER, headers)
        conn.setopt(pycurl.POST, 1)
        conn.setopt(pycurl.POSTFIELDS, '%s'%json.dumps(payload))
        conn.setopt(pycurl.CUSTOMREQUEST, "PUT")
        conn.setopt(pycurl.WRITEFUNCTION, response.write)
        conn.perform()
        return response.getvalue()
    except:
        return None 

def send_REST_request(self, server_ip, port):
        try:
            response = StringIO()
            url = "http://%s:%s/%s" % (server_ip, port, 'InventoryInfo')
            headers = ["Content-Type:application/json"]
            conn = pycurl.Curl()
            conn.setopt(pycurl.URL, str(url))
            conn.setopt(pycurl.HTTPHEADER, headers)
            conn.setopt(conn.WRITEFUNCTION, response.write)
            conn.setopt(pycurl.HTTPGET, 1)
            conn.perform()
            data_dict = response.getvalue()
            data_dict = dict(json.loads(data_dict))
            data_list = list(data_dict["__ServerInventoryInfoUve_list"]["ServerInventoryInfoUve"])
            return data_list
        except Exception as e:
            print "Inventory Error is: " + str(e)
            return None

    # end def send_REST_request

    # Filters the data returned from REST API call for requested information 

def send_REST_request(ip, port, object, key, value, force=False):
    try:
        response = StringIO()
        headers = ["Content-Type:application/json"]
        url = "http://%s:%s/%s?%s=%s" %(
            ip, port, object, 
            urllib.quote_plus(key), 
            urllib.quote_plus(value))
        if force:
            url += "&force"
        conn = pycurl.Curl()
        conn.setopt(pycurl.URL, url)
        conn.setopt(pycurl.HTTPHEADER, headers)
        conn.setopt(pycurl.CUSTOMREQUEST, "delete")
        conn.setopt(pycurl.WRITEFUNCTION, response.write)
        conn.perform()
        return response.getvalue()
    except:
        return None
# end def send_REST_request 

def send_REST_request(ip, port, payload):
    try:
        response = StringIO()
        headers = ["Content-Type:application/json"]
        url = "http://%s:%s/server/provision" %(
            ip, port)
        conn = pycurl.Curl()
        conn.setopt(pycurl.URL, url)
        conn.setopt(pycurl.HTTPHEADER, headers)
        conn.setopt(pycurl.POST, 1)
        conn.setopt(pycurl.POSTFIELDS, '%s'%json.dumps(payload))
        conn.setopt(pycurl.WRITEFUNCTION, response.write)
        conn.perform()
        return response.getvalue()
    except:
        return None 

def send_REST_request(self, server_ip, port):
        try:
            response = StringIO()
            headers = ["Content-Type:application/json"]
            url = "http://%s:%s/%s" % (server_ip, port, 'MonitorInfo')
            args_str = ''
            conn = pycurl.Curl()
            conn.setopt(pycurl.URL, url)
            conn.setopt(pycurl.HTTPHEADER, headers)
            conn.setopt(pycurl.HTTPGET, 1)
            conn.setopt(pycurl.WRITEFUNCTION, response.write)
            conn.perform()
            data_dict = response.getvalue()
            data_dict = dict(json.loads(data_dict))
            data_list = list(data_dict["__ServerMonitoringInfoTrace_list"]["ServerMonitoringInfoTrace"])
            return data_list
        except Exception as e:
            print "Error is: " + str(e)
            return None
    # end def send_REST_request 

def send_REST_request(ip, port, object, match_key,
                      match_value, detail):
    try:
        response = StringIO()
        headers = ["Content-Type:application/json"]
        url = "http://%s:%s/%s" % (ip, port, object)
        args_str = ''
        if match_key:
            args_str += match_key + "=" + match_value
        if detail:
            args_str += "&detail"
        if args_str != '':
            url += "?" + args_str
        conn = pycurl.Curl()
        conn.setopt(pycurl.TIMEOUT, 1)
        conn.setopt(pycurl.URL, url)
        conn.setopt(pycurl.HTTPHEADER, headers)
        conn.setopt(pycurl.HTTPGET, 1)
        conn.setopt(pycurl.WRITEFUNCTION, response.write)
        conn.perform()
        return response.getvalue()
    except:
        return None
# end def send_REST_request 

def send_REST_request(ip, port, payload):
    try:
        response = StringIO()
        headers = ["Content-Type:application/json"]
        url = "http://%s:%s/server/reimage" %(
            ip, port)
        conn = pycurl.Curl()
        conn.setopt(pycurl.URL, url)
        conn.setopt(pycurl.HTTPHEADER, headers)
        conn.setopt(pycurl.POST, 1)
        conn.setopt(pycurl.POSTFIELDS, '%s'%json.dumps(payload))
        conn.setopt(pycurl.WRITEFUNCTION, response.write)
        conn.perform()
        return response.getvalue()
    except:
        return None 

def send_REST_request(ip, port, object, match_key,
                      match_value, detail):
    try:
        response = StringIO()
        headers = ["Content-Type:application/json"]
        url = "http://%s:%s/%s" % (ip, port, object)
        args_str = ''
        if match_key:
            args_str += match_key + "=" + match_value
        if detail:
            args_str += "&detail"
        if args_str != '':
            url += "?" + args_str
        conn = pycurl.Curl()
        conn.setopt(pycurl.URL, url)
        conn.setopt(pycurl.HTTPHEADER, headers)
        conn.setopt(pycurl.HTTPGET, 1)
        conn.setopt(pycurl.WRITEFUNCTION, response.write)
        conn.perform()
        return response.getvalue()
    except:
        return None
# end def send_REST_request 

def send_REST_request(ip, port, payload):
    try:
        response = StringIO()
        headers = ["Content-Type:application/json"]
        url = "http://%s:%s/server/restart" %(
            ip, port)
        conn = pycurl.Curl()
        conn.setopt(pycurl.URL, url)
        conn.setopt(pycurl.HTTPHEADER, headers)
        conn.setopt(pycurl.POST, 1)
        conn.setopt(pycurl.POSTFIELDS, '%s'%json.dumps(payload))
        conn.setopt(pycurl.WRITEFUNCTION, response.write)
        conn.perform()
        return response.getvalue()
    except:
        return None
# end def send_REST_request 

def send_REST_request(self, ip, port, endpoint, payload,
                          method='POST', urlencode=False):
        try:
            response = StringIO()
            headers = ["Content-Type:application/json"]
            url = "http://%s:%s/%s" %(ip, port, endpoint)
            conn = pycurl.Curl()
            if method == "PUT":
                conn.setopt(pycurl.CUSTOMREQUEST, method)
                if urlencode == False:
                    first = True
                    for k,v in payload.iteritems():
                        if first:
                            url = url + '?'
                            first = False
                        else:
                            url = url + '&'
                        url = url + ("%s=%s" % (k,v))
                else:
                    url = url + '?' + payload
            if self.logger:
                self.logger.log(self.logger.INFO,
                                "Sending post request to %s" % url)
            conn.setopt(pycurl.URL, url)
            conn.setopt(pycurl.HTTPHEADER, headers)
            conn.setopt(pycurl.POST, 1)
            if urlencode == False:
                conn.setopt(pycurl.POSTFIELDS, '%s'%json.dumps(payload))
            conn.setopt(pycurl.WRITEFUNCTION, response.write)
            conn.perform()
            return response.getvalue()
        except:
            return None 

def perform(self):
        self.__performHead = ""
        self.__performBody = ""
        self.__headersSent = ""

        try:
            conn = Request.to_pycurl_object(pycurl.Curl(), self)
            conn.perform()
            self.response_from_conn_object(conn, self.__performHead, self.__performBody)
        except pycurl.error as error:
            errno, errstr = error
            raise ReqRespException(ReqRespException.FATAL, errstr)
        finally:
            conn.close()

    # ######## ESTE conjunto de funciones no es necesario para el uso habitual de la clase 

def test_create_curl(self):
        curls = []

        def pycurl_Curl():
            curl = CurlStub(b"result")
            curls.append(curl)
            return curl
        Curl = pycurl.Curl
        try:
            pycurl.Curl = pycurl_Curl
            result = fetch("http://example.com")
            curl = curls[0]
            self.assertEqual(result, b"result")
            self.assertEqual(curl.options,
                             {pycurl.URL: b"http://example.com",
                              pycurl.FOLLOWLOCATION: 1,
                              pycurl.MAXREDIRS: 5,
                              pycurl.CONNECTTIMEOUT: 30,
                              pycurl.LOW_SPEED_LIMIT: 1,
                              pycurl.LOW_SPEED_TIME: 600,
                              pycurl.NOSIGNAL: 1,
                              pycurl.WRITEFUNCTION: Any(),
                              pycurl.DNS_CACHE_TIMEOUT: 0,
                              pycurl.ENCODING: b"gzip,deflate"})
        finally:
            pycurl.Curl = Curl 

def request(self, endpoint, post=None):
        buffer = BytesIO()

        ch = pycurl.Curl()
        ch.setopt(pycurl.URL, Constants.API_URL + endpoint)
        ch.setopt(pycurl.USERAGENT, self.userAgent)
        ch.setopt(pycurl.WRITEFUNCTION, buffer.write)
        ch.setopt(pycurl.FOLLOWLOCATION, True)
        ch.setopt(pycurl.HEADER, True)
        ch.setopt(pycurl.VERBOSE, False)
        ch.setopt(pycurl.COOKIEFILE, os.path.join(self.IGDataPath, self.username, self.username + "-cookies.dat"))
        ch.setopt(pycurl.COOKIEJAR, os.path.join(self.IGDataPath, self.username, self.username + "-cookies.dat"))

        if post is not None:
            ch.setopt(pycurl.POST, True)
            ch.setopt(pycurl.POSTFIELDS, post)

        if self.proxy:
            ch.setopt(pycurl.PROXY, self.proxyHost)
            if self.proxyAuth:
                ch.setopt(pycurl.PROXYUSERPWD, self.proxyAuth)

        ch.perform()
        resp = buffer.getvalue()
        header_len = ch.getinfo(pycurl.HEADER_SIZE)
        header = resp[0: header_len]
        body = resp[header_len:]

        ch.close()

        if self.debug:
            print("REQUEST: " + endpoint)
            if post is not None:
                if not isinstance(post, list):
                    print("DATA: " + str(post))
            print("RESPONSE: " + body)

        return [header, json_decode(body)] 

def head(self):
        conn = pycurl.Curl()
        conn.setopt(pycurl.SSL_VERIFYPEER, False)
        conn.setopt(pycurl.SSL_VERIFYHOST, 0)
        conn.setopt(pycurl.URL, self.completeUrl)

        conn.setopt(pycurl.NOBODY, True)  # para hacer un pedido HEAD

        conn.setopt(pycurl.WRITEFUNCTION, self.header_callback)
        conn.perform()

        rp = Response()
        rp.parseResponse(self.__performHead)
        self.response = rp 

def open_file(self):

        if not self.silent:

            self.print_status('Opening file `%s`' % self.outfile)

        super(Curl, self).__init__(self.outfile, extract = False) 

def _curl_create(max_simultaneous_connections=None):
    curl = pycurl.Curl()
    if logging.getLogger().isEnabledFor(logging.DEBUG):
        curl.setopt(pycurl.VERBOSE, 1)
        curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug)
    curl.setopt(pycurl.MAXCONNECTS, max_simultaneous_connections or 5)
    return curl