Python pycurl.WRITEDATA Examples

def Download_ETens_from_WA_FTP(output_folder, Lat_tiles, Lon_tiles):           
    """
    This function retrieves ETensV1.0 data for a given date from the
    ftp.wateraccounting.unesco-ihe.org server.
				
    Restrictions:
    The data and this python file may not be distributed to others without
    permission of the WA+ team.

    Keyword arguments:
    output_folder -- Directory of the outputs
    Lat_tiles -- [Lat_min, Lat_max] Tile number of the max and min latitude tile number
    Lon_tiles -- [Lon_min, Lon_max] Tile number of the max and min longitude tile number				
    """      
    for v_tile in range(Lat_tiles[0], Lat_tiles[1]+1):
        for h_tile in range(Lon_tiles[0], Lon_tiles[1]+1):													

            Tilename = "h%sv%s.zip" %(h_tile, v_tile)
            if not os.path.exists(os.path.join(output_folder,Tilename)): 
                try:  
                    # Collect account and FTP information			
                    username, password = WebAccounts.Accounts(Type = 'FTP_WA')
                    FTP_name = "ftp://ftp.wateraccounting.unesco-ihe.org//WaterAccounting_Guest/ETensV1.0/%s" % Tilename
                    local_filename = os.path.join(output_folder, Tilename)   
			
                    # Download data from FTP 	
                    curl = pycurl.Curl()
                    curl.setopt(pycurl.URL, FTP_name)	
                    curl.setopt(pycurl.USERPWD, '%s:%s' %(username, password))								
                    fp = open(local_filename, "wb")								 
                    curl.setopt(pycurl.WRITEDATA, fp)
                    curl.perform()
                    curl.close()
                    fp.close()	
																
                except:
                    print "tile %s is not found and will be replaced by NaN values"	% Tilename
																
    return() 

def get(url, encoding, user_agent=UA, referrer=None):
    """Make a GET request of the url using pycurl and return the data
    (which is None if unsuccessful)"""

    data = None
    databuffer = BytesIO()

    curl = pycurl.Curl()
    curl.setopt(pycurl.URL, url)
    curl.setopt(pycurl.FOLLOWLOCATION, 1)
    curl.setopt(pycurl.CONNECTTIMEOUT, 5)
    curl.setopt(pycurl.TIMEOUT, 8)
    curl.setopt(pycurl.WRITEDATA, databuffer)
    curl.setopt(pycurl.COOKIEFILE, '')
    if user_agent:
        curl.setopt(pycurl.USERAGENT, user_agent)
    if referrer is not None:
        curl.setopt(pycurl.REFERER, referrer)
    try:
        curl.perform()
        data = databuffer.getvalue().decode(encoding)
    except Exception:
        pass
    curl.close()

    return data 

def _set_only_receive_headers(self):
        self._curl.setopt(pycurl.VERBOSE, True)
        self._curl.setopt(pycurl.RANGE, "0-0")
        self._curl.setopt(pycurl.HEADERFUNCTION, self._response_header_handler.response_header_handler)
        self._curl.setopt(pycurl.WRITEDATA, None) 

def test_download_file_unexpected(self):

        curl = pycurl.Curl()
        curl.setopt = MagicMock()
        curl.perform = MagicMock()
        curl.getinfo = MagicMock(return_value=500)
        curl.close = MagicMock()
        pycurl.Curl = MagicMock(return_value=curl)

        url = 'http://localhost:4502/.cqactions.html'
        params = {'foo1': 'bar1', 'foo2': 'bar2'}
        handlers = {}

        try:
            pyaem.bagofrequests.download_file(url, params, handlers, file='/tmp/somefile')
            self.fail('An exception should have been raised')
        except pyaem.PyAemException as exception:
            self.assertEqual(exception.code, 500)
            self.assertEqual(exception.message,
                'Unexpected response\nhttp code: 500\nbody:\n' +
                'Download http://localhost:4502/.cqactions.html?foo1=bar1&foo2=bar2 to /tmp/somefile')

        curl.setopt.assert_any_call(pycurl.URL, 'http://localhost:4502/.cqactions.html?foo1=bar1&foo2=bar2')
        curl.setopt.assert_any_call(pycurl.FOLLOWLOCATION, 1)
        curl.setopt.assert_any_call(pycurl.FRESH_CONNECT, 1)

        # 5 calls including the one with pycurl.WRITEDATA and pycurl.WRITEFUNCTION
        self.assertEqual(curl.setopt.call_count, 4)

        curl.perform.assert_called_once_with()
        curl.getinfo.assert_called_once_with(pycurl.HTTP_CODE)
        curl.close.assert_called_once_with() 

def __init__(self, url, auth, verify):
        self.url = url
        self.received_buffer = BytesIO()

        headers = ['Cache-Control: no-cache', 'Accept: text/event-stream']

        self.curl = pycurl.Curl()
        self.curl.setopt(pycurl.URL, url)
        self.curl.setopt(pycurl.ENCODING, 'gzip')
        self.curl.setopt(pycurl.CONNECTTIMEOUT, 10)
        self.curl.setopt(pycurl.WRITEDATA, self.received_buffer)

        # Marathon >= 1.7.x returns 30x responses for /v2/events responses
        # when they're coming from a non-leader. So we follow redirects.
        self.curl.setopt(pycurl.FOLLOWLOCATION, True)
        self.curl.setopt(pycurl.MAXREDIRS, 1)
        self.curl.setopt(pycurl.UNRESTRICTED_AUTH, True)

        # The below settings are to prevent the connection from hanging if the
        # connection breaks silently. Since marathon-lb only listens, silent
        # connection failure results in marathon-lb waiting infinitely.
        #
        # Minimum bytes/second below which it is considered "low speed". So
        # "low speed" here refers to 0 bytes/second.
        self.curl.setopt(pycurl.LOW_SPEED_LIMIT, 1)
        # How long (in seconds) it's allowed to go below the speed limit
        # before it times out
        self.curl.setopt(pycurl.LOW_SPEED_TIME, 300)

        if auth and type(auth) is DCOSAuth:
            auth.refresh_auth_header()
            headers.append('Authorization: %s' % auth.auth_header)
        elif auth:
            self.curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
            self.curl.setopt(pycurl.USERPWD, '%s:%s' % auth)
        if verify:
            self.curl.setopt(pycurl.CAINFO, verify)
        else:
            self.curl.setopt(pycurl.SSL_VERIFYHOST, 0)
            self.curl.setopt(pycurl.SSL_VERIFYPEER, 0)

        self.curl.setopt(pycurl.HTTPHEADER, headers)

        self.curlmulti = pycurl.CurlMulti()
        self.curlmulti.add_handle(self.curl)

        self.status_code = 0 

def run(self):
        c = pycurl.Curl()
        c.setopt(pycurl.URL, self.url)
        c.setopt(pycurl.FOLLOWLOCATION, True)
        c.setopt(pycurl.MAXREDIRS, 4)
        #c.setopt(pycurl.NOBODY, 1)

        c.setopt(c.VERBOSE, True)

        #c.setopt(pycurl.CONNECTTIMEOUT, 20)
        
        if self.useragent:
            c.setopt(pycurl.USERAGENT, self.useragent)
        
        # add cookies, if available
        if self.cookies:
            c.setopt(pycurl.COOKIE, self.cookies)
        
        #realurl = c.getinfo(pycurl.EFFECTIVE_URL)
        realurl = self.url
        print("realurl",realurl)
        self.filename = realurl.split("/")[-1].strip()
        c.setopt(pycurl.URL, realurl)
        c.setopt(pycurl.FOLLOWLOCATION, True)
        c.setopt(pycurl.NOPROGRESS, False)
        c.setopt(pycurl.XFERINFOFUNCTION, self.getProgress)
        
        c.setopt(pycurl.SSL_VERIFYPEER, False)
        c.setopt(pycurl.SSL_VERIFYHOST, False)
        
        # configure pycurl output file
        if self.path == False:
            self.path = os.getcwd()
        filepath = os.path.join(self.path, self.filename)
        
         
        if os.path.exists(filepath):## remove old file,restart download 
            os.system("rm -rf " + filepath)
        
        buffer = StringIO() 
        c.setopt(pycurl.WRITEDATA, buffer)
        
        self._dst_path = filepath

        # add cookies, if available
        if self.cookies:
            c.setopt(pycurl.COOKIE, self.cookies)
        
        self._stop = False
        self.progress["stopped"] = False 
        # download file
        try:
            c.perform()
        except pycurl.error, error:
            errno,errstr = error
            print("curl error: %s" % errstr)
            self._errors.append(errstr)
            self._stop = True
            self.progress["stopped"] = True
            self.stop() 

def Download_data(Date, Version, output_folder, Var):        
    """
    This function downloads CFSR data from the FTP server
				For - CFSR:    https://nomads.ncdc.noaa.gov/data/cfsr/
				    - CFSRv2:  http://nomads.ncdc.noaa.gov/modeldata/cfsv2_analysis_timeseries/

    Keyword arguments:
    Date -- pandas timestamp day
    Version -- 1 or 2 (1 = CFSR, 2 = CFSRv2)			
    output_folder -- The directory for storing the downloaded files
    Var -- The variable that must be downloaded from the server ('dlwsfc','uswsfc','dswsfc','ulwsfc')
    """
    # Define the filename that must be downloaded     
    if Version == 1:
        filename = Var + '.gdas.' + str(Date.strftime('%Y')) + str(Date.strftime('%m')) + '.grb2'
    if Version == 2:
        filename = Var + '.gdas.' + str(Date.strftime('%Y')) + str(Date.strftime('%m')) + '.grib2'

    try:
         # download the file when it not exist					
        local_filename = os.path.join(output_folder, filename)
        if not os.path.exists(local_filename):
            Downloaded = 0
            Times = 0
            while Downloaded == 0:				
                # Create the command and run the command in cmd
                if Version == 1:
                    FTP_name = 'https://nomads.ncdc.noaa.gov/data/cfsr/' + Date.strftime('%Y') + Date.strftime('%m')+ '/' + filename
    							
                if Version == 2:
                    FTP_name = 'https://nomads.ncdc.noaa.gov/modeldata/cfsv2_analysis_timeseries/' + Date.strftime('%Y') + '/' + Date.strftime('%Y') + Date.strftime('%m')+ '/' + filename
    
                curl = pycurl.Curl()
                curl.setopt(pycurl.URL, FTP_name)
                fp = open(local_filename, "wb")
                curl.setopt(pycurl.SSL_VERIFYPEER, 0)
                curl.setopt(pycurl.SSL_VERIFYHOST, 0)
                curl.setopt(pycurl.WRITEDATA, fp)
                curl.perform()
                curl.close()
                fp.close()	
                statinfo = os.stat(local_filename)		
                if int(statinfo.st_size) > 10000:
                    Downloaded = 1	
                else:
                    Times += 1
                    if Times == 10:
                        Downloaded = 1
    except:
        print 'Was not able to download the CFSR file from the FTP server'
    
    return(local_filename) 

def request(self, path="/", header=None, ssl=False, timeout=None):

        if timeout is None:
            timeout = self.timeout

        buf = StringIO()
        c = pycurl.Curl()

        if header:
            slist = []
            for key, value in header.iteritems():
                slist.append(key+": "+value)
            c.setopt(pycurl.HTTPHEADER, slist)

        c.setopt(pycurl.HEADERFUNCTION, self.header_function)
        c.setopt(pycurl.FOLLOWLOCATION, True)
        c.setopt(pycurl.WRITEDATA, buf)
        c.setopt(pycurl.TIMEOUT, timeout)
        c.setopt(pycurl.ENCODING, 'identity')
        c.setopt(pycurl.NOSIGNAL, 1)

        if ssl:
            if self.port is None:
                self.port = 443
            c.setopt(pycurl.URL, "https://"+self.host+":"+str(self.port)+path)
            c.setopt(pycurl.SSL_VERIFYPEER, 1)
            c.setopt(pycurl.SSL_VERIFYHOST, 2)
        else:
            if self.port is None:
                self.port = 80
            c.setopt(pycurl.URL,"http://"+self.host + ":" + str(self.port) + path)

        c.perform()

        self.status = c.getinfo(pycurl.RESPONSE_CODE)

        c.close()

        encoding = None
        if 'content-type' in self.headers:
            content_type = self.headers['content-type'].lower()
            match = re.search('charset=(\S+)', content_type)
            if match:
                encoding = match.group(1)
        if encoding is None:
            # Default encoding for HTML is iso-8859-1.
            # Other content types may have different default encoding,
            # or in case of binary data, may have no encoding at all.
            encoding = 'iso-8859-1'

        self.body = buf.getvalue().decode(encoding)