Python pycurl.SSL_VERIFYPEER Examples

The following are 25 code examples of pycurl.SSL_VERIFYPEER(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module pycurl , or try the search function .
Example #1
Source File: reqresp.py    From wfuzz with GNU General Public License v2.0 6 votes vote down vote up
def head(self):
		conn=pycurl.Curl()
		conn.setopt(pycurl.SSL_VERIFYPEER,False)
		conn.setopt(pycurl.SSL_VERIFYHOST,1)
		conn.setopt(pycurl.URL,self.completeUrl)

		conn.setopt(pycurl.HEADER, True) # estas dos lineas son las que importan
		conn.setopt(pycurl.NOBODY, True) # para hacer un pedido HEAD

		conn.setopt(pycurl.WRITEFUNCTION, self.header_callback)
		conn.perform()

		rp=Response()
		rp.parseResponse(self.__performHead)
		self.response=rp 
Example #2
Source File: test_fetch.py    From landscape-client with GNU General Public License v2.0 6 votes vote down vote up
def test_pycurl_insecure(self):
        curl = CurlStub(b"result")
        result = fetch("http://example.com/get-ca-cert", curl=curl,
                       insecure=True)
        self.assertEqual(result, b"result")
        self.assertEqual(curl.options,
                         {pycurl.URL: b"http://example.com/get-ca-cert",
                          pycurl.FOLLOWLOCATION: 1,
                          pycurl.MAXREDIRS: 5,
                          pycurl.CONNECTTIMEOUT: 30,
                          pycurl.LOW_SPEED_LIMIT: 1,
                          pycurl.LOW_SPEED_TIME: 600,
                          pycurl.NOSIGNAL: 1,
                          pycurl.WRITEFUNCTION: Any(),
                          pycurl.SSL_VERIFYPEER: False,
                          pycurl.DNS_CACHE_TIMEOUT: 0,
                          pycurl.ENCODING: b"gzip,deflate"}) 
Example #3
Source File: Request.py    From wfuzz with GNU General Public License v2.0 5 votes vote down vote up
def head(self):
        conn = pycurl.Curl()
        conn.setopt(pycurl.SSL_VERIFYPEER, False)
        conn.setopt(pycurl.SSL_VERIFYHOST, 0)
        conn.setopt(pycurl.URL, self.completeUrl)

        conn.setopt(pycurl.NOBODY, True)  # para hacer un pedido HEAD

        conn.setopt(pycurl.WRITEFUNCTION, self.header_callback)
        conn.perform()

        rp = Response()
        rp.parseResponse(self.__performHead)
        self.response = rp 
Example #4
Source File: transport.py    From termite-visualizations with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def request(self, url, method, body, headers):
            c = pycurl.Curl()
            c.setopt(pycurl.URL, str(url))
            if 'proxy_host' in self.proxy:
                c.setopt(pycurl.PROXY, self.proxy['proxy_host'])
            if 'proxy_port' in self.proxy:
                c.setopt(pycurl.PROXYPORT, self.proxy['proxy_port'])
            if 'proxy_user' in self.proxy:
                c.setopt(pycurl.PROXYUSERPWD, "%(proxy_user)s:%(proxy_pass)s" % self.proxy)
            self.buf = StringIO()
            c.setopt(pycurl.WRITEFUNCTION, self.buf.write)
            #c.setopt(pycurl.READFUNCTION, self.read)
            #self.body = StringIO(body)
            #c.setopt(pycurl.HEADERFUNCTION, self.header)
            if self.cacert:
                c.setopt(c.CAINFO, str(self.cacert)) 
            c.setopt(pycurl.SSL_VERIFYPEER, self.cacert and 1 or 0)
            c.setopt(pycurl.SSL_VERIFYHOST, self.cacert and 2 or 0)
            c.setopt(pycurl.CONNECTTIMEOUT, self.timeout/6) 
            c.setopt(pycurl.TIMEOUT, self.timeout)
            if method=='POST':
                c.setopt(pycurl.POST, 1)
                c.setopt(pycurl.POSTFIELDS, body)            
            if headers:
                hdrs = ['%s: %s' % (str(k), str(v)) for k, v in headers.items()]
                ##print hdrs
                c.setopt(pycurl.HTTPHEADER, hdrs)
            c.perform()
            ##print "pycurl perform..."
            c.close()
            return {}, self.buf.getvalue() 
Example #5
Source File: pycurllib.py    From BitTorrent with GNU General Public License v3.0 5 votes vote down vote up
def __init__(self, url):
        self.c = pycurl.Curl()
        self.headers = {}
        self.c.setopt(self.c.URL, url)

        if use_cert_authority:
            self.c.setopt(pycurl.CAINFO, CA_PATH)
            self.c.setopt(pycurl.CAPATH, CA_PATH)
            #self.c.setopt(pycurl.CA_BUNDLE, CA_PATH)
        else:
            self.c.setopt(pycurl.SSL_VERIFYHOST, 0)
            self.c.setopt(pycurl.SSL_VERIFYPEER, 0)

        if http_bindaddr:
            self.c.setopt(self.c.INTERFACE, http_bindaddr)

        if user_agent:
            self.c.setopt(pycurl.USERAGENT, user_agent)

        if use_compression:
            if _pycurl_compression:
                # If a zero-length string is set, then an Accept-Encoding header
                # containing all supported encodings is sent.
                self.c.setopt(pycurl.ENCODING, "")
            # someday, gzip manually with GzipFile
            #else:
            #    self.add_header("Accept-Encoding", "gzip")

        if timeout:
            self.c.setopt(self.c.TIMEOUT, timeout)
        if connect_timeout:
            self.c.setopt(self.c.CONNECTTIMEOUT, timeout)
        if max_connects:
            self.c.setopt(self.c.MAXCONNECTS, max_connects) 
Example #6
Source File: pycurllib.py    From BitTorrent with GNU General Public License v3.0 5 votes vote down vote up
def __init__(self, url):
        self.c = pycurl.Curl()
        self.headers = {}
        self.c.setopt(self.c.URL, url)

        if use_cert_authority:
            self.c.setopt(pycurl.CAINFO, CA_PATH)
            self.c.setopt(pycurl.CAPATH, CA_PATH)
            #self.c.setopt(pycurl.CA_BUNDLE, CA_PATH)
        else:
            self.c.setopt(pycurl.SSL_VERIFYHOST, 0)
            self.c.setopt(pycurl.SSL_VERIFYPEER, 0)

        if http_bindaddr:
            self.c.setopt(self.c.INTERFACE, http_bindaddr)

        if user_agent:
            self.c.setopt(pycurl.USERAGENT, user_agent)

        if use_compression:
            if _pycurl_compression:
                # If a zero-length string is set, then an Accept-Encoding header
                # containing all supported encodings is sent.
                self.c.setopt(pycurl.ENCODING, "")
            # someday, gzip manually with GzipFile
            #else:
            #    self.add_header("Accept-Encoding", "gzip")

        if timeout:
            self.c.setopt(self.c.TIMEOUT, timeout)
        if connect_timeout:
            self.c.setopt(self.c.CONNECTTIMEOUT, timeout)
        if max_connects:
            self.c.setopt(self.c.MAXCONNECTS, max_connects) 
Example #7
Source File: toolClass.py    From Video-Downloader with GNU General Public License v2.0 5 votes vote down vote up
def getPage (self, url, requestHeader = []) :
		resultFormate = StringIO.StringIO()

		fakeIp = self.fakeIp()
		requestHeader.append('CLIENT-IP:' + fakeIp)
		requestHeader.append('X-FORWARDED-FOR:' + fakeIp)

		try:
			curl = pycurl.Curl()
			curl.setopt(pycurl.URL, url.strip())
			curl.setopt(pycurl.ENCODING, 'gzip,deflate')
			curl.setopt(pycurl.HEADER, 1)
			curl.setopt(pycurl.TIMEOUT, 120)
			curl.setopt(pycurl.SSL_VERIFYPEER, 0)   
			curl.setopt(pycurl.SSL_VERIFYHOST, 0)
			curl.setopt(pycurl.HTTPHEADER, requestHeader)
			curl.setopt(pycurl.WRITEFUNCTION, resultFormate.write)
			curl.perform()
			headerSize = curl.getinfo(pycurl.HEADER_SIZE)
			curl.close()

			header = resultFormate.getvalue()[0 : headerSize].split('\r\n')
			body = resultFormate.getvalue()[headerSize : ]
		except Exception, e:
			header = ''
			body = '' 
Example #8
Source File: patator_ext.py    From project-black with GNU General Public License v2.0 5 votes vote down vote up
def connect(self, host, port, scheme):
    fp = pycurl.Curl()
    fp.setopt(pycurl.SSL_VERIFYPEER, 0)
    fp.setopt(pycurl.SSL_VERIFYHOST, 0)
    fp.setopt(pycurl.HEADER, 1)
    fp.setopt(pycurl.USERAGENT, 'Mozilla/5.0')
    fp.setopt(pycurl.NOSIGNAL, 1)

    return TCP_Connection(fp) 
Example #9
Source File: test_helpers.py    From the-new-hotness with GNU Lesser General Public License v2.1 5 votes vote down vote up
def test_secure_download_no_cainfo(self):
        import pycurl
        import io

        # Required mocks
        mock_curl_instance = Mock()
        mock_stringio_instance = Mock()

        mock_stringio_instance.getvalue.return_value = "mock StringIO value"

        mock_stringio_instance.getvalue.return_value = "mock StringIO value"

        expected_setopt_calls = [
            call(pycurl.URL, b"https://example.com/file.extension"),
            call(pycurl.SSL_VERIFYPEER, 1),
            call(pycurl.SSL_VERIFYHOST, 2),
            call(pycurl.WRITEFUNCTION, mock_stringio_instance.write),
            call(pycurl.FOLLOWLOCATION, 1),
            call(pycurl.MAXREDIRS, 10),
        ]

        with patch.object(pycurl, "Curl") as mock_curl_constructor:
            with patch.object(io, "StringIO") as mock_stringio_constructor:
                mock_curl_constructor.return_value = mock_curl_instance
                mock_stringio_constructor.return_value = mock_stringio_instance

                secure_download_result = helpers.secure_download(
                    "https://example.com/file.extension"
                )

        # Check curl calls
        mock_curl_instance.setopt.assert_has_calls(expected_setopt_calls)
        mock_curl_instance.perform.assert_called_once_with()
        mock_curl_instance.close.assert_called_once_with()

        # Check StringIO call
        mock_stringio_instance.getvalue.assert_called_once_with()

        self.assertEqual(secure_download_result, "mock StringIO value") 
Example #10
Source File: test_helpers.py    From the-new-hotness with GNU Lesser General Public License v2.1 5 votes vote down vote up
def test_secure_download(self):
        import pycurl
        import io

        # Required mocks
        mock_curl_instance = Mock()
        mock_stringio_instance = Mock()

        mock_stringio_instance.getvalue.return_value = "mock StringIO value"

        # Expected pycurl setopt calls
        expected_setopt_calls = [
            call(pycurl.URL, b"https://example.com/file.extension"),
            call(pycurl.SSL_VERIFYPEER, 1),
            call(pycurl.SSL_VERIFYHOST, 2),
            call(pycurl.CAINFO, "/etc/certs/cabundle.pem"),
            call(pycurl.WRITEFUNCTION, mock_stringio_instance.write),
            call(pycurl.FOLLOWLOCATION, 1),
            call(pycurl.MAXREDIRS, 10),
        ]

        with patch.object(pycurl, "Curl") as mock_curl_constructor:
            with patch.object(io, "StringIO") as mock_stringio_constructor:
                mock_curl_constructor.return_value = mock_curl_instance
                mock_stringio_constructor.return_value = mock_stringio_instance

                secure_download_result = helpers.secure_download(
                    "https://example.com/file.extension", "/etc/certs/cabundle.pem"
                )

        # Check curl calls
        mock_curl_instance.setopt.assert_has_calls(expected_setopt_calls)
        mock_curl_instance.perform.assert_called_once_with()
        mock_curl_instance.close.assert_called_once_with()

        # Check StringIO call
        mock_stringio_instance.getvalue.assert_called_once_with()

        self.assertEqual(secure_download_result, "mock StringIO value") 
Example #11
Source File: helpers.py    From the-new-hotness with GNU Lesser General Public License v2.1 5 votes vote down vote up
def secure_download(url, cainfo=""):
    import pycurl
    import io

    c = pycurl.Curl()
    c.setopt(pycurl.URL, url.encode("ascii"))

    # -k / --insecure
    # c.setopt(pycurl.SSL_VERIFYPEER, 0)

    # Verify certificate
    c.setopt(pycurl.SSL_VERIFYPEER, 1)

    # Verify CommonName or Subject Alternate Name
    c.setopt(pycurl.SSL_VERIFYHOST, 2)

    # --cacert
    if cainfo:
        c.setopt(pycurl.CAINFO, cainfo)

    res = io.StringIO()

    c.setopt(pycurl.WRITEFUNCTION, res.write)

    # follow up to 10 http location: headers
    c.setopt(pycurl.FOLLOWLOCATION, 1)
    c.setopt(pycurl.MAXREDIRS, 10)

    c.perform()
    c.close()
    data = res.getvalue()
    res.close()

    return data 
Example #12
Source File: patator.py    From patator with GNU General Public License v2.0 5 votes vote down vote up
def connect(self, host, port, scheme):
    fp = pycurl.Curl()
    fp.setopt(pycurl.SSL_VERIFYPEER, 0)
    fp.setopt(pycurl.SSL_VERIFYHOST, 0)
    fp.setopt(pycurl.HEADER, 1)
    fp.setopt(pycurl.USERAGENT, 'Mozilla/5.0')
    fp.setopt(pycurl.NOSIGNAL, 1)

    return TCP_Connection(fp) 
Example #13
Source File: client.py    From pycopia with Apache License 2.0 5 votes vote down vote up
def _set_common(self, c):
        c.setopt(pycurl.FOLLOWLOCATION, 1)
        c.setopt(pycurl.AUTOREFERER, 1)
        c.setopt(pycurl.ENCODING, self._accept_encoding)
        c.setopt(pycurl.MAXREDIRS, 255)
        c.setopt(pycurl.CONNECTTIMEOUT, 30)
        c.setopt(pycurl.TIMEOUT, 300)
        c.setopt(pycurl.NOSIGNAL, 1)
        if self._proxy:
            c.setopt(pycurl.PROXY, self._proxy)
        if self._url.scheme == 'https':
            c.setopt(pycurl.SSLVERSION, 3)
            c.setopt(pycurl.SSL_VERIFYPEER, 0) 
Example #14
Source File: psurl.py    From pscheduler with Apache License 2.0 5 votes vote down vote up
def __init__(self, url, params, bind, timeout, allow_redirects, headers, verify_keys):
        """Constructor"""

        self.curl = pycurl.Curl()

        full_url = url if params is None else "%s?%s" % (url, urllib.urlencode(params))
        self.curl.setopt(pycurl.URL, str(full_url))

        if bind is not None:
            self.curl.setopt(pycurl.INTERFACE, str(bind))

        self.curl.setopt(pycurl.FOLLOWLOCATION, allow_redirects)

        if headers is not None:            
            self.curl.setopt(pycurl.HTTPHEADER, [
                "%s: %s" % (str(key), str(value))
                for (key, value) in headers.items()
            ])

        if timeout is not None:
            self.curl.setopt(pycurl.TIMEOUT_MS, int(timeout * 1000.0))

        self.curl.setopt(pycurl.SSL_VERIFYHOST, verify_keys)
        self.curl.setopt(pycurl.SSL_VERIFYPEER, verify_keys)

        self.buf = StringIO.StringIO()
        self.curl.setopt(pycurl.WRITEFUNCTION, self.buf.write) 
Example #15
Source File: Checkpoint.py    From Instagram-API with MIT License 5 votes vote down vote up
def request(self, endpoint, headers=None, post=None, first=True):
        buffer = BytesIO()

        ch = pycurl.Curl()

        ch.setopt(pycurl.URL, endpoint)
        ch.setopt(pycurl.USERAGENT, self.userAgent)
        ch.setopt(pycurl.WRITEFUNCTION, buffer.write)
        ch.setopt(pycurl.FOLLOWLOCATION, True)
        ch.setopt(pycurl.HEADER, True)
        if headers:
            ch.setopt(pycurl.HTTPHEADER, headers)

        ch.setopt(pycurl.VERBOSE, self.debug)
        ch.setopt(pycurl.SSL_VERIFYPEER, False)
        ch.setopt(pycurl.SSL_VERIFYHOST, False)
        ch.setopt(pycurl.COOKIEFILE, self.settingsPath + self.username + '-cookies.dat')
        ch.setopt(pycurl.COOKIEJAR, self.settingsPath + self.username + '-cookies.dat')

        if post:
            import urllib
            ch.setopt(pycurl.POST, len(post))
            ch.setopt(pycurl.POSTFIELDS, urllib.urlencode(post))

        ch.perform()
        resp = buffer.getvalue()
        header_len = ch.getinfo(pycurl.HEADER_SIZE)
        header = resp[0: header_len]
        body = resp[header_len:]
        ch.close()

        if self.debug:
            import urllib
            print("REQUEST: " + endpoint)
            if post is not None:
                if not isinstance(post, list):
                    print('DATA: ' + urllib.unquote_plus(json.dumps(post)))
            print("RESPONSE: " + body + "\n")

        return [header, json_decode(body)] 
Example #16
Source File: analyze_packages.py    From pyCAF with GNU General Public License v3.0 4 votes vote down vote up
def get_rh_sso(self):
        """
        Function wich used login and password to connect at www.redhat.com and return
        the "rh_sso" proxy used for request authentication.
        """
        self._logger.debug("RHEL rh_sso getting in progress...")
        username = self.config_server.redhat["login_username"]
        password = self.config_server.redhat["login_pass"]
        url = 'https://www.redhat.com/wapps/sso/login.html'
        values={'username':username,'password':password,'_flowId':'legacy-login-flow','redirect':'https://www.redhat.com/wapps/ugc/protected/account.html','failureRedirect':'http://www.redhat.com/wapps/sso/login.html'}
        data_post = urllib.urlencode(values)
      
        headers = Storage()
        
        c = pycurl.Curl()
        c.setopt(pycurl.URL, url)
#==============================================================================
#     DEBUG comments in order to use the Burp Proxy software 
#==============================================================================
    #c.setopt(pycurl.PROXY, '127.0.0.1')
    #c.setopt(pycurl.PROXYPORT, 8080)
    #c.setopt(pycurl.SSL_VERIFYPEER, 1)
    #c.setopt(pycurl.SSL_VERIFYHOST, 2)
    #c.setopt(pycurl.CAINFO, "/home/thierry/Divers/Burp/cacert.crt")
#==============================================================================
        c.setopt(pycurl.USERAGENT, "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0")
        c.setopt(pycurl.HTTPHEADER, ['Accept-Language: en-US,en;q=0.5',
                                     'Connection: keep-alive',
                                     'www.whitehouse.gov: 200'])
        c.setopt(pycurl.POST, 1)
        c.setopt(pycurl.POSTFIELDS, data_post)
        
        c.setopt(c.HEADERFUNCTION, headers.store)
        
        c.perform()
        c.close()
        headers = str(headers)
        
        expression = r"(?P<var1>.*)(?P<rh_sso>Set-Cookie: rh_sso=)(?P<sso_key>(?!\").*?)(?P<end>;)"
        header_lines = headers.split('\n')
        for head in header_lines:
            result_re = re.match(expression, head)
            
            if result_re is not None:
                sso_key = "rh_sso=" + str(result_re.group('sso_key'))
        self._logger.debug("rh_sso value : "+ str(sso_key))
        return sso_key 
Example #17
Source File: utils.py    From APIFuzzer with GNU General Public License v3.0 4 votes vote down vote up
def init_pycurl(debug=False):
    """
    Provides an instances of pycurl with basic configuration
    :return: pycurl instance
    """
    _curl = pycurl.Curl()
    _curl.setopt(pycurl.SSL_OPTIONS, pycurl.SSLVERSION_TLSv1_2)
    _curl.setopt(pycurl.SSL_VERIFYPEER, False)
    _curl.setopt(pycurl.SSL_VERIFYHOST, False)
    _curl.setopt(pycurl.VERBOSE, debug)
    _curl.setopt(pycurl.TIMEOUT, 10)
    _curl.setopt(pycurl.COOKIEFILE, "")
    _curl.setopt(pycurl.USERAGENT, 'APIFuzzer')
    return _curl 
Example #18
Source File: download.py    From launcher with GNU General Public License v3.0 4 votes vote down vote up
def run(self):
        c = pycurl.Curl()
        c.setopt(pycurl.URL, self.url)
        c.setopt(pycurl.FOLLOWLOCATION, True)
        c.setopt(pycurl.MAXREDIRS, 4)
        #c.setopt(pycurl.NOBODY, 1)

        c.setopt(c.VERBOSE, True)

        #c.setopt(pycurl.CONNECTTIMEOUT, 20)
        
        if self.useragent:
            c.setopt(pycurl.USERAGENT, self.useragent)
        
        # add cookies, if available
        if self.cookies:
            c.setopt(pycurl.COOKIE, self.cookies)
        
        #realurl = c.getinfo(pycurl.EFFECTIVE_URL)
        realurl = self.url
        print("realurl",realurl)
        self.filename = realurl.split("/")[-1].strip()
        c.setopt(pycurl.URL, realurl)
        c.setopt(pycurl.FOLLOWLOCATION, True)
        c.setopt(pycurl.NOPROGRESS, False)
        c.setopt(pycurl.XFERINFOFUNCTION, self.getProgress)
        
        c.setopt(pycurl.SSL_VERIFYPEER, False)
        c.setopt(pycurl.SSL_VERIFYHOST, False)
        
        # configure pycurl output file
        if self.path == False:
            self.path = os.getcwd()
        filepath = os.path.join(self.path, self.filename)
        
         
        if os.path.exists(filepath):## remove old file,restart download 
            os.system("rm -rf " + filepath)
        
        buffer = StringIO() 
        c.setopt(pycurl.WRITEDATA, buffer)
        
        self._dst_path = filepath

        # add cookies, if available
        if self.cookies:
            c.setopt(pycurl.COOKIE, self.cookies)
        
        self._stop = False
        self.progress["stopped"] = False 
        # download file
        try:
            c.perform()
        except pycurl.error, error:
            errno,errstr = error
            print("curl error: %s" % errstr)
            self._errors.append(errstr)
            self._stop = True
            self.progress["stopped"] = True
            self.stop() 
Example #19
Source File: utils.py    From marathon-lb with Apache License 2.0 4 votes vote down vote up
def __init__(self, url, auth, verify):
        self.url = url
        self.received_buffer = BytesIO()

        headers = ['Cache-Control: no-cache', 'Accept: text/event-stream']

        self.curl = pycurl.Curl()
        self.curl.setopt(pycurl.URL, url)
        self.curl.setopt(pycurl.ENCODING, 'gzip')
        self.curl.setopt(pycurl.CONNECTTIMEOUT, 10)
        self.curl.setopt(pycurl.WRITEDATA, self.received_buffer)

        # Marathon >= 1.7.x returns 30x responses for /v2/events responses
        # when they're coming from a non-leader. So we follow redirects.
        self.curl.setopt(pycurl.FOLLOWLOCATION, True)
        self.curl.setopt(pycurl.MAXREDIRS, 1)
        self.curl.setopt(pycurl.UNRESTRICTED_AUTH, True)

        # The below settings are to prevent the connection from hanging if the
        # connection breaks silently. Since marathon-lb only listens, silent
        # connection failure results in marathon-lb waiting infinitely.
        #
        # Minimum bytes/second below which it is considered "low speed". So
        # "low speed" here refers to 0 bytes/second.
        self.curl.setopt(pycurl.LOW_SPEED_LIMIT, 1)
        # How long (in seconds) it's allowed to go below the speed limit
        # before it times out
        self.curl.setopt(pycurl.LOW_SPEED_TIME, 300)

        if auth and type(auth) is DCOSAuth:
            auth.refresh_auth_header()
            headers.append('Authorization: %s' % auth.auth_header)
        elif auth:
            self.curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
            self.curl.setopt(pycurl.USERPWD, '%s:%s' % auth)
        if verify:
            self.curl.setopt(pycurl.CAINFO, verify)
        else:
            self.curl.setopt(pycurl.SSL_VERIFYHOST, 0)
            self.curl.setopt(pycurl.SSL_VERIFYPEER, 0)

        self.curl.setopt(pycurl.HTTPHEADER, headers)

        self.curlmulti = pycurl.CurlMulti()
        self.curlmulti.add_handle(self.curl)

        self.status_code = 0 
Example #20
Source File: reqresp.py    From darkc0de-old-stuff with GNU General Public License v3.0 4 votes vote down vote up
def perform(self):
		self.__performHead=""
		self.__performBody=""

		conn=pycurl.Curl()
		conn.setopt(pycurl.SSL_VERIFYPEER,False)
		conn.setopt(pycurl.SSL_VERIFYHOST,1)
		conn.setopt(pycurl.URL,self.completeUrl)

		if self.__method or self.__userpass:
			if self.__method=="basic":
				conn.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
			elif self.__method=="ntlm":
				conn.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_NTLM)
			elif self.__method=="digest":
				conn.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST)
			conn.setopt(pycurl.USERPWD, self.__userpass)

		if self.__timeout:
			conn.setopt(pycurl.CONNECTTIMEOUT, self.__timeout)
			conn.setopt(pycurl.NOSIGNAL, 1)

		if self.__totaltimeout:
			conn.setopt(pycurl.TIMEOUT, self.__totaltimeout)
			conn.setopt(pycurl.NOSIGNAL, 1)

		conn.setopt(pycurl.WRITEFUNCTION, self.body_callback)
		conn.setopt(pycurl.HEADERFUNCTION, self.header_callback)

		if self.__proxy!=None:
			conn.setopt(pycurl.PROXY,self.__proxy)
			if self.__headers.has_key("Proxy-Connection"):
				del self.__headers["Proxy-Connection"]

		conn.setopt(pycurl.HTTPHEADER,self.__getHeaders())
		if self.method=="POST":
			conn.setopt(pycurl.POSTFIELDS,self.__postdata)
		conn.perform()

		rp=Response()
		rp.parseResponse(self.__performHead)
		rp.addContent(self.__performBody)

		self.response=rp

	######### ESTE conjunto de funciones no es necesario para el uso habitual de la clase 
Example #21
Source File: mycurl.py    From QMusic with GNU Lesser General Public License v2.1 4 votes vote down vote up
def upload(self,
               url,
               data,
               header=None,
               proxy_host=None,
               proxy_port=None,
               cookie_file=None):
        '''
        open url with upload
        @param url: the url to visit
        @param data: the data to upload
        @param header: the http header
        @param proxy_host: the proxy host name
        @param proxy_port: the proxy port
        '''
        crl = pycurl.Curl()
        #crl.setopt(pycurl.VERBOSE,1)
        crl.setopt(pycurl.NOSIGNAL, 1)

        # set proxy
        rel_proxy_host = proxy_host or self.proxy_host
        if rel_proxy_host:
            crl.setopt(pycurl.PROXY, rel_proxy_host)
        rel_proxy_port = proxy_port or self.proxy_port
        if rel_proxy_port:
            crl.setopt(pycurl.PROXYPORT, rel_proxy_port)

            # set cookie
        rel_cookie_file = cookie_file or self.cookie_file
        if rel_cookie_file:
            crl.setopt(pycurl.COOKIEFILE, rel_cookie_file)
            crl.setopt(pycurl.COOKIEJAR, rel_cookie_file)

            # set ssl
        crl.setopt(pycurl.SSL_VERIFYPEER, 0)
        crl.setopt(pycurl.SSL_VERIFYHOST, 0)
        crl.setopt(pycurl.SSLVERSION, 3)

        crl.setopt(pycurl.CONNECTTIMEOUT, 10)
        crl.setopt(pycurl.TIMEOUT, 300)
        crl.setopt(pycurl.HTTPPROXYTUNNEL, 1)

        rel_header = header or self.header
        if rel_header:
            crl.setopt(pycurl.HTTPHEADER, rel_header)

        crl.fp = StringIO.StringIO()

        if isinstance(url, unicode):
            url = str(url)
        crl.setopt(pycurl.URL, url)
        crl.setopt(pycurl.HTTPPOST, data)  # upload file
        crl.setopt(crl.WRITEFUNCTION, crl.fp.write)
        try:
            crl.perform()
        except Exception, e:
            raise CurlException(e) 
Example #22
Source File: mycurl.py    From QMusic with GNU Lesser General Public License v2.1 4 votes vote down vote up
def post(self,
             url,
             data,
             header=None,
             proxy_host=None,
             proxy_port=None,
             cookie_file=None):
        '''
        open url width post method
        @param url: the url to visit
        @param data: the data to post
        @param header: the http header
        @param proxy_host: the proxy host name
        @param proxy_port: the proxy port
        '''
        crl = pycurl.Curl()
        #crl.setopt(pycurl.VERBOSE,1)
        crl.setopt(pycurl.NOSIGNAL, 1)

        # set proxy
        rel_proxy_host = proxy_host or self.proxy_host
        if rel_proxy_host:
            crl.setopt(pycurl.PROXY, rel_proxy_host)
        rel_proxy_port = proxy_port or self.proxy_port
        if rel_proxy_port:
            crl.setopt(pycurl.PROXYPORT, rel_proxy_port)

            # set cookie
        rel_cookie_file = cookie_file or self.cookie_file
        if rel_cookie_file:
            crl.setopt(pycurl.COOKIEFILE, rel_cookie_file)
            crl.setopt(pycurl.COOKIEJAR, rel_cookie_file)

            # set ssl
        crl.setopt(pycurl.SSL_VERIFYPEER, 0)
        crl.setopt(pycurl.SSL_VERIFYHOST, 0)
        crl.setopt(pycurl.SSLVERSION, 3)

        crl.setopt(pycurl.CONNECTTIMEOUT, 10)
        crl.setopt(pycurl.TIMEOUT, 300)
        crl.setopt(pycurl.HTTPPROXYTUNNEL, 1)

        rel_header = header or self.header
        if rel_header:
            crl.setopt(pycurl.HTTPHEADER, rel_header)

        crl.fp = StringIO.StringIO()

        crl.setopt(crl.POSTFIELDS, data)  # post data

        if isinstance(url, unicode):
            url = str(url)
        crl.setopt(pycurl.URL, url)
        crl.setopt(crl.WRITEFUNCTION, crl.fp.write)
        try:
            crl.perform()
        except Exception, e:
            raise CurlException(e) 
Example #23
Source File: Download_data_CFSR.py    From wa with Apache License 2.0 4 votes vote down vote up
def Download_data(Date, Version, output_folder, Var):        
    """
    This function downloads CFSR data from the FTP server
				For - CFSR:    https://nomads.ncdc.noaa.gov/data/cfsr/
				    - CFSRv2:  http://nomads.ncdc.noaa.gov/modeldata/cfsv2_analysis_timeseries/

    Keyword arguments:
    Date -- pandas timestamp day
    Version -- 1 or 2 (1 = CFSR, 2 = CFSRv2)			
    output_folder -- The directory for storing the downloaded files
    Var -- The variable that must be downloaded from the server ('dlwsfc','uswsfc','dswsfc','ulwsfc')
    """
    # Define the filename that must be downloaded     
    if Version == 1:
        filename = Var + '.gdas.' + str(Date.strftime('%Y')) + str(Date.strftime('%m')) + '.grb2'
    if Version == 2:
        filename = Var + '.gdas.' + str(Date.strftime('%Y')) + str(Date.strftime('%m')) + '.grib2'

    try:
         # download the file when it not exist					
        local_filename = os.path.join(output_folder, filename)
        if not os.path.exists(local_filename):
            Downloaded = 0
            Times = 0
            while Downloaded == 0:				
                # Create the command and run the command in cmd
                if Version == 1:
                    FTP_name = 'https://nomads.ncdc.noaa.gov/data/cfsr/' + Date.strftime('%Y') + Date.strftime('%m')+ '/' + filename
    							
                if Version == 2:
                    FTP_name = 'https://nomads.ncdc.noaa.gov/modeldata/cfsv2_analysis_timeseries/' + Date.strftime('%Y') + '/' + Date.strftime('%Y') + Date.strftime('%m')+ '/' + filename
    
                curl = pycurl.Curl()
                curl.setopt(pycurl.URL, FTP_name)
                fp = open(local_filename, "wb")
                curl.setopt(pycurl.SSL_VERIFYPEER, 0)
                curl.setopt(pycurl.SSL_VERIFYHOST, 0)
                curl.setopt(pycurl.WRITEDATA, fp)
                curl.perform()
                curl.close()
                fp.close()	
                statinfo = os.stat(local_filename)		
                if int(statinfo.st_size) > 10000:
                    Downloaded = 1	
                else:
                    Times += 1
                    if Times == 10:
                        Downloaded = 1
    except:
        print 'Was not able to download the CFSR file from the FTP server'
    
    return(local_filename) 
Example #24
Source File: http_helper.py    From centinel with MIT License 4 votes vote down vote up
def request(self, path="/", header=None, ssl=False, timeout=None):

        if timeout is None:
            timeout = self.timeout

        buf = StringIO()
        c = pycurl.Curl()

        if header:
            slist = []
            for key, value in header.iteritems():
                slist.append(key+": "+value)
            c.setopt(pycurl.HTTPHEADER, slist)

        c.setopt(pycurl.HEADERFUNCTION, self.header_function)
        c.setopt(pycurl.FOLLOWLOCATION, True)
        c.setopt(pycurl.WRITEDATA, buf)
        c.setopt(pycurl.TIMEOUT, timeout)
        c.setopt(pycurl.ENCODING, 'identity')
        c.setopt(pycurl.NOSIGNAL, 1)

        if ssl:
            if self.port is None:
                self.port = 443
            c.setopt(pycurl.URL, "https://"+self.host+":"+str(self.port)+path)
            c.setopt(pycurl.SSL_VERIFYPEER, 1)
            c.setopt(pycurl.SSL_VERIFYHOST, 2)
        else:
            if self.port is None:
                self.port = 80
            c.setopt(pycurl.URL,"http://"+self.host + ":" + str(self.port) + path)

        c.perform()

        self.status = c.getinfo(pycurl.RESPONSE_CODE)

        c.close()

        encoding = None
        if 'content-type' in self.headers:
            content_type = self.headers['content-type'].lower()
            match = re.search('charset=(\S+)', content_type)
            if match:
                encoding = match.group(1)
        if encoding is None:
            # Default encoding for HTML is iso-8859-1.
            # Other content types may have different default encoding,
            # or in case of binary data, may have no encoding at all.
            encoding = 'iso-8859-1'

        self.body = buf.getvalue().decode(encoding) 
Example #25
Source File: mycurl.py    From QMusic with GNU Lesser General Public License v2.1 4 votes vote down vote up
def get(self,
            url,
            header=None,
            proxy_host=None,
            proxy_port=None,
            cookie_file=None):
        '''
        open url width get method
        @param url: the url to visit
        @param header: the http header
        @param proxy_host: the proxy host name
        @param proxy_port: the proxy port
        '''
        crl = pycurl.Curl()
        #crl.setopt(pycurl.VERBOSE,1)
        crl.setopt(pycurl.NOSIGNAL, 1)

        # set proxy
        # crl.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5)

        rel_proxy_host = proxy_host or self.proxy_host
        if rel_proxy_host:
            crl.setopt(pycurl.PROXY, rel_proxy_host)

        rel_proxy_port = proxy_port or self.proxy_port
        if rel_proxy_port:
            crl.setopt(pycurl.PROXYPORT, rel_proxy_port)

            # set cookie
        rel_cookie_file = cookie_file or self.cookie_file
        if rel_cookie_file:
            crl.setopt(pycurl.COOKIEFILE, rel_cookie_file)
            crl.setopt(pycurl.COOKIEJAR, rel_cookie_file)

            # set ssl
        crl.setopt(pycurl.SSL_VERIFYPEER, 0)
        crl.setopt(pycurl.SSL_VERIFYHOST, 0)
        crl.setopt(pycurl.SSLVERSION, 3)

        crl.setopt(pycurl.CONNECTTIMEOUT, 10)
        crl.setopt(pycurl.TIMEOUT, 300)
        crl.setopt(pycurl.HTTPPROXYTUNNEL, 1)

        rel_header = header or self.header
        if rel_header:
            crl.setopt(pycurl.HTTPHEADER, rel_header)

        crl.fp = StringIO.StringIO()

        if isinstance(url, unicode):
            url = str(url)
        crl.setopt(pycurl.URL, url)
        crl.setopt(crl.WRITEFUNCTION, crl.fp.write)
        try:
            crl.perform()
        except Exception, e:
            raise CurlException(e)