Python wget.download() Examples

def download_recoding(recording, dest_folder=None):
    # construct url
    url = "https:%s%s/%s.%s" % (recording["mediaPrefix"],
                                recording["sitekey"],
                                recording["info"]["movie_exported_name"],
                                recording["info"]["movie_type"])

    if dest_folder is None:
        dest_folder = os.path.realpath(recording["sitename"])

    title = recording["title"].replace("/", "-")
    filename = "%s.%s" % (title, recording["info"]["movie_type"])
    destination = os.path.join(dest_folder, filename)

    # download!
    wget.download(url, out=destination) 

def download_file(local_path, link, checksum_reference=None):
    """Checks if a local file is present and downloads it from the specified path otherwise.
    If checksum_reference is specified, the file's md5 checksum is compared against the
    expected value.

    Keyword arguments:
    local_path -- path of the file whose checksum shall be generated
    link -- link where the file shall be downloaded from if it is not found locally
    checksum_reference -- expected MD5 checksum of the file
    """
    if not os.path.exists(local_path):
        print('Downloading from %s, this may take a while...' % link)
        wget.download(link, local_path)
        print()
    if checksum_reference is not None:
        checksum = generate_md5_checksum(local_path)
        if checksum != checksum_reference:
            raise ValueError(
                'The MD5 checksum of local file %s differs from %s, please manually remove \
                 the file and try again.' %
                (local_path, checksum_reference))
    return local_path 

def download_bnpp_data(dir="."):
    """
    Download BNP Paribas' dataset

    Parameters
    ----------
    dir: str
        Directory where the dataset will be stored

    """

    dir = os.path.expanduser(dir)
    if not os.path.exists(dir):
        os.makedirs(dir)

    url = "https://github.com/cdqa-suite/cdQA/releases/download/bnpp_newsroom_v1.1/bnpp_newsroom-v1.1.csv"

    print("\nDownloading BNP data...")

    file = url.split("/")[-1]
    if os.path.exists(os.path.join(dir, file)):
        print(file, "already downloaded")
    else:
        wget.download(url=url, out=dir) 

def load_mnist():
    if not os.path.exists(os.path.join(os.curdir, "data")):
        os.mkdir(os.path.join(os.curdir, "data"))
        wget.download("http://deeplearning.net/data/mnist/mnist.pkl.gz", out="data")

    data_file = gzip.open(os.path.join(os.curdir, "data", "mnist.pkl.gz"), "rb")
    train_data, val_data, test_data = pickle.load(data_file, encoding="latin1")
    data_file.close()

    train_inputs = [np.reshape(x, (784, 1)) for x in train_data[0]]
    train_results = [vectorized_result(y) for y in train_data[1]]
    train_data = list(zip(train_inputs, train_results))

    val_inputs = [np.reshape(x, (784, 1)) for x in val_data[0]]
    val_results = val_data[1]
    val_data = list(zip(val_inputs, val_results))

    test_inputs = [np.reshape(x, (784, 1)) for x in test_data[0]]
    test_data = list(zip(test_inputs, test_data[1]))
    return train_data, val_data, test_data 

def download_test_assets(tmpdir_factory):
    assets_urls = [
        # PDF
        "https://invest.bnpparibas.com/documents/1q19-pr-12648",
        "https://invest.bnpparibas.com/documents/4q18-pr-18000",
        "https://invest.bnpparibas.com/documents/4q17-pr",
        # MD
        "https://raw.githubusercontent.com/cdqa-suite/cdQA/master/README.md",
        "https://raw.githubusercontent.com/huggingface/pytorch-transformers/master/docs/source/quickstart.md",
        "https://raw.githubusercontent.com/huggingface/pytorch-transformers/master/docs/source/migration.md",
    ]

    print("\nDownloading assets...")
    fn = tmpdir_factory.mktemp("assets_data")
    for url in assets_urls:
        wget.download(url=url, out=str(fn))
    return fn 

def download_file(local_path, link, checksum_reference=None):
    """Checks if a local file is present and downloads it from the specified path otherwise.
    If checksum_reference is specified, the file's md5 checksum is compared against the
    expected value.

    Keyword arguments:
    local_path -- path of the file whose checksum shall be generated
    link -- link where the file shall be downloaded from if it is not found locally
    checksum_reference -- expected MD5 checksum of the file
    """
    if not os.path.exists(local_path):
        print('Downloading from %s, this may take a while...' % link)
        wget.download(link, local_path)
        print()
    if checksum_reference is not None:
        checksum = generate_md5_checksum(local_path)
        if checksum != checksum_reference:
            raise ValueError(
                'The MD5 checksum of local file %s differs from %s, please manually remove \
                 the file and try again.' %
                (local_path, checksum_reference))
    return local_path 

def download_models(output_dir):
    output_dir = FileManager.handle_output_directory(output_dir)
    sys.stderr.write(TextColor.YELLOW + "DOWNLOADING MODEL DESCRIPTION FILE" + TextColor.END + "\n")
    description_file = "https://storage.googleapis.com/kishwar-helen/models_helen/mp_helen_model_description.csv"
    wget.download(description_file, output_dir)
    sys.stderr.write("\n")
    sys.stderr.flush()

    with open(output_dir+'/mp_helen_model_description.csv') as f:
        models = [line.rstrip() for line in f]

    os.remove(output_dir+'/mp_helen_model_description.csv')

    for model in models:
        model_name, model_url = model.split(',')
        sys.stderr.write("INFO: DOWNLOADING FILE: " + str(model_name) + ".pkl\n")
        sys.stderr.write("INFO: DOWNLOADING LINK: " + str(model_url) + "\n")
        wget.download(model_url, output_dir)
        sys.stderr.write("\n")
        sys.stderr.flush() 

def download(source, target, force_clear=False):
    if force_clear and os.path.exists(target):
        print('Removing {}...'.format(target))
        shutil.rmtree(target)

    check_dir(target)
    
    targt_file = str(Path(target).joinpath('data.zip'))
    if os.path.exists(targt_file) and not force_clear:
        print('data already exists, skipping download')
        return

    if source.startswith('http'):
        print("Downloading from {} to {}".format(source, target))
        wget.download(source, targt_file)  
        print("Done!")
    else:
        print("Copying from {} to {}".format(source, target))
        shutil.copyfile(source, targt_file)

    print('Unzipping {}'.format(targt_file))
    zipr = zipfile.ZipFile(targt_file)
    zipr.extractall(target)
    zipr.close() 

def initSpraykatz():
    logging.warning("%sHey, did you read the code?\n" % (debugBlue))

    # Ensure procdump binaries are available to be used by Spraykatz.
    procdumpPath = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'misc', 'procdump')
    procdumpZip = os.path.join(procdumpPath, 'procdump.zip')
    procdump32 = os.path.join(procdumpPath, 'procdump32.exe')
    procdump64 = os.path.join(procdumpPath, 'procdump64.exe')
    if not os.path.isfile(procdump32) or not os.path.isfile(procdump64):
        choices = ['y','yes','Y','Yes','YES']
        choice = input("%sProcDump binaries have not been found. Do you want Spraykatz to download them? [y/N]" % (infoYellow)).lower()

        if choice in choices:
            url = 'https://download.sysinternals.com/files/Procdump.zip'
            wget.download(url, procdumpZip)
            with zipfile.ZipFile(procdumpZip, 'r') as zip_ref:
                zip_ref.extractall(procdumpPath)
            os.rename(os.path.join(procdumpPath, 'procdump.exe'), procdump32)
            os.remove(procdumpZip)
            logging.warning("\n")
        else:
            logging.warning("\n%sYou can manually download and put 'procdump32.exe' and 'procdump64.exe' into misc/procdump folder." % (warningRed))
            sys.exit(2) 

def Downloadfiles(urls_metadata):
	print "\nDo you like downloading these files to analyze metadata(Y/N)?"
	#try:
	resp = raw_input()
	if (resp == 'N'):
		print "Exiting"
		exit(1)
	if ((resp != 'Y') and (resp != 'N')):
		print "The option is not valided. Please, try again it"
	if (resp =='Y'):
		print "Indicate the location where keep the files downloaded"
		path = raw_input()
		try:
			for url in urls_metadata:
				try:
					filename = wget.download(url,path)
					Analyze_Metadata(filename)
				except:
					pass
			Display_Export_Metadata(metadata_files)
		except:
			pass
#********************************************************#
#Definition and treatment of the parameters 

def get_data(experiment_name, experiment_numbers=None,overwrite=False):
    assert experiment_name in experiments
    if type(experiment_numbers)==int:
        experiment_numbers=[experiment_numbers]
    elif type(experiment_numbers)==list:
        pass
    elif experiment_numbers is None:
        experiment_numbers = range(0, number_of_runs[experiments.index(experiment_name)])
    else:
        raise TypeError("Unsupported type "+type(experiment_numbers))

    base_url = os.path.join(MVSEC_URL, experiment_name, experiment_name)
    full_urls = [base_url+str(n)+"_data.bag" for n in experiment_numbers]

    base_path = os.path.join(TMP_FOLDER, experiment_name, experiment_name)
    full_paths = [base_path+str(n)+"_data.bag" for n in experiment_numbers]

    download(full_urls, full_paths, overwrite)

    return full_paths 

def Downloadfiles(urls_metadata):
	print "\nDo you like downloading these files to analyze metadata(Y/N)?"
	#try:
	resp = raw_input().lower()
	if (resp == 'n'):
		print "Exiting"
		exit(1)
	if ((resp != 'y') and (resp != 'n')):
		print "The option is not valided. Please, try again it"
	if (resp =='y'):
		print "Indicate the location where you want to keep the files downloaded"
		path = raw_input()
		try:
			for url in urls_metadata:
				try:
					filename= wget.download(url,path)
					Analyze_Metadata(filename)
				except Exception, e:
					print e
		except:
			pass
#********************************************************#
#Definition and treatment of the parameters 

def Downloadfiles(urls_metadata):
	print "\nDo you like downloading these files to analyze metadata(Y/N)?"
	#try:
	resp = raw_input()
	if (resp == 'N'):
		print "Exiting"
		exit(1)
	if ((resp != 'Y') and (resp != 'N')):
		print "The option is not valided. Please, try again it"
	if (resp =='Y'):
		try:
			for url in urls_metadata:
				try:
					filename= wget.download(url,"/opt/")
					Analyze_Metadata(filename)
				except Exception, e:
					print e
		except:
			pass
#********************************************************#
#Definition and treatment of the parameters 

def Downloadfiles(urls_metadata,output,target):
	path = None
	try:
		filename = None
		print "\nDo you like downloading these files to analyze metadata(Y/N)?"
		#Convert to lower the input
		resp = raw_input().lower()
		if (resp == 'n'):
			print "Exiting"
			exit(1)
		if ((resp != 'y') and (resp != 'n')):
			print "The option is not valided. Please, try again it"
		if (resp =='y'):
			path = str(target) + '/temp'
			for url in urls_metadata:
				filename = wget.download(url,str(path))
				Analyze_Metadata(filename)
			time.sleep(3)
			#Delete temp folder
			os.system('rm -r ' + str(path))
			Display_Export_Metadata(metadata_files,output,target)
	except Exception as e:
		print str(e) 

def pageview_download():
    import wget

    qtime = datetime.datetime.now()
    one_hour = datetime.timedelta(hours=1)
    qtime -= one_hour
    for _ in range(30*24):
        qtime -= one_hour
        file_name = get_file_name(qtime.month, qtime.day, qtime.hour)
        rlink = link_path_root + file_name
        local_path = save_path_root + file_name

        dir_path = os.path.dirname(local_path)
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)

        print(f"Downloading {file_name}")
        wget.download(rlink, out=local_path) 

def Downloadfiles(urls_metadata):
	print "\nDo you like downloading these files to analyze metadata(Y/N)?"
	#try:
	resp = raw_input().lower()
	if (resp == 'n'):
		print "Exiting"
		exit(1)
	if ((resp != 'y') and (resp != 'n')):
		print "The option is not valided. Please, try again it"
	if (resp =='y'):
		print "Indicate the location where you want to keep the files downloaded"
		path = raw_input()
		try:
			for url in urls_metadata:
				try:
					filename= wget.download(url,path)
					Analyze_Metadata(filename)
				except Exception, e:
					print e
		except:
			pass
#********************************************************#
#Definition and treatment of the parameters 

def get_ground_truth(experiment_name, experiment_numbers=None,overwrite=False):
    assert experiment_name in experiments
    if type(experiment_numbers)==int:
        experiment_numbers=[experiment_numbers]
    elif type(experiment_numbers)==list:
        pass
    elif experiment_numbers is None:
        experiment_numbers = range(0, number_of_runs[experiments.index(experiment_name)])
    else:
        raise TypeError("Unsupported type "+type(experiment_numbers))

    base_url = os.path.join(MVSEC_URL, experiment_name, experiment_name)
    full_urls = [base_url+str(n)+"_gt.bag" for n in experiment_numbers]

    base_path = os.path.join(TMP_FOLDER, experiment_name, experiment_name)
    full_paths = [base_path+str(n)+"_gt.bag" for n in experiment_numbers]

    download(full_urls, full_paths, overwrite)

    return full_paths 

def Downloadfiles(urls_metadata):
	print "\nDo you like downloading these files to analyze metadata(Y/N)?"
	#try:
	resp = raw_input()
	if (resp == 'N'):
		print "Exiting"
		exit(1)
	if ((resp != 'Y') and (resp != 'N')):
		print "The option is not valided. Please, try again it"
	if (resp =='Y'):
		try:
			for url in urls_metadata:
				try:
					filename= wget.download(url,"/opt/")
					Analyze_Metadata(filename)
				except Exception, e:
					print e
		except:
			pass
#********************************************************#
#Definition and treatment of the parameters 

def download_figshare(file_name, file_ext, dir_path='./', change_name = None):
    prepare_data_dir(dir_path)
    url = 'https://ndownloader.figshare.com/files/' + file_name
    wget.download(url, out=dir_path)
    file_path = os.path.join(dir_path, file_name)

    if file_ext == '.zip':
        zip_ref = zipfile.ZipFile(file_path,'r')
        if change_name is not None:
            dir_path = os.path.join(dir_path, change_name)
        zip_ref.extractall(dir_path)
        zip_ref.close()
        os.remove(file_path)
    elif file_ext == '.tar.bz2':
        tar_ref = tarfile.open(file_path,'r:bz2')
        if change_name is not None:
            dir_path = os.path.join(dir_path, change_name)
        tar_ref.extractall(dir_path)
        tar_ref.close()
        os.remove(file_path)
    elif change_name is not None:
        os.rename(file_path, os.path.join(dir_path, change_name))

# Download QM9 dataset 

def downloadTheFiles(jsonData, hashes, elementsPerDir):
    seen = set()
    i = 0
    paginate = False
    outputDir = ""
    elementNum = 1
    if ((elementsPerDir > 0) and (len(jsonData) > elementsPerDir)):
        paginate = True
        pageNum = 1
        outputDir = makeOutputDir(pageNum)
    for element in jsonData:
        url = element["url"]
        ext = element["ext"]
        hash = element["md5"].upper()
        if (hash in hashes and not hash in seen):
            seen.add(hash)
            i += 1
            fileName = hash + "." + ext
            if (paginate):
                if (elementNum > elementsPerDir):
                    elementNum = 1
                    pageNum += 1
                    outputDir = makeOutputDir(pageNum)
                fileName = os.path.join(outputDir, fileName)
            print("[%d] %s -> %s" % (i, url, fileName), file=sys.stderr)
            try:
                outputFile = wget.download(url, out=fileName)
            except Exception as e:
                error(e)
            print("")
            elementNum += 1 

def download_file(url, file_ext, dir_path='./'):
    file_name = wget.download(url, out=dir_path)
    file_path = os.path.join(dir_path, file_name)
    if file_ext == '.zip':
        zip_ref = zipfile.ZipFile(file_path,'r')
        zip_ref.extractall(dir_path)
        zip_ref.close()
        os.remove(file_path)

# Download data from figshare 

def Downloadfiles(urls_metadata,output,target):
	path = None
	try:
		filename = None
		print "\nDo you like downloading these files to analyze metadata(Y/N)?"
		#Convert to lower the input
		resp = raw_input().lower()
		if (resp == 'n'):
			print "Exiting"
			exit(1)
		if ((resp != 'y') and (resp != 'n')):
			print "The option is not valided. Please, try again it"
		if (resp =='y'):
			print "Indicate the location where you want to keep the files downloaded.by default in the target folder",
			path = raw_input()
			#path = createdir.CreateDir(target)
			for url in urls_metadata:
				filename = wget.download(url,str(path))
				Analyze_Metadata(filename)
			Display_Export_Metadata(metadata_files,output,target)
			if count_pdf > 1:
				os.system('mv *pdf '+ str(target))
			if count_word > 1:
				os.system('mv *doc '+ str(target))
				os.system('mv *docx '+ str(target))
			if count_others > 1:
				os.system('mv *xlsx '+ str(target))
				os.system('mv *ppt '+ str(target))
	except Exception as e:
		print str(e) 

def download_vpl_interactively(vpl_app_name, vpl_app_url, extract=False):
    """
    Download the specified Visual Programming langage web app and returns its path.
    If it couldn't be downloaded, return None
    """
    pypot_datadir = get_pypot_datadir()
    vpl_dir = pypot_datadir / vpl_app_name
    actual_vpl_dir = vpl_dir / vpl_app_name if extract else vpl_dir
    
    if vpl_dir.is_dir():
        return actual_vpl_dir
    else:
        while True:
            response = input("This is the first time you are launching {}, it needs to be downloaded first. Proceed? [Y/n] ".format(vpl_app_name))
            if response.lower() in ["y", ""]:
                try:
                    vpl_dir.mkdir(parents=True)
                except FileExistsError:
                    pass
                print("Downloading...")
                try:
                    downloaded_app = download(vpl_app_url, tempfile.gettempdir())
                except URLError as e:
                    print("Cannot download the {] app from {}: {}".format(vpl_app_name, vpl_app_url, str(e)), file=sys.stderr)
                else:
                    try:
                        with ZipFile(downloaded_app, 'r') as archive:
                            archive.extractall(vpl_dir)
                    except FileNotFoundError:
                        print("Couldn't extract {} from zipfile".format(vpl_app_name))
                    else:
                        return actual_vpl_dir
            else:
                print("Download aborted by user", file=sys.stderr)
                return None 

def ShowResults(newlist,num_files,target):
	print "Files in the target "+target+" are:\n"
	print "Files indexed:", len (urls_final)
	for i in urls_final:
		if i not in newlist:
			newlist.append(i)
			print i		
	#verify if the user wants to export results
	if output == 'Y':
		#Only it can enter if -j is put in the execution
		ExportResults(newlist)
	#Call to function to download the files		
	Downloadfiles(newlist)
#INICIO MAIN 

def Downloadfiles(urls_metadata,output):
	try:
		print "\nDo you like downloading these files to analyze metadata(Y/N)?"
		#Convert to lower the input
		resp = raw_input().lower()
		if (resp == 'n'):
			print "Exiting"
			exit(1)
		if ((resp != 'y') and (resp != 'n')):
			print "The option is not valided. Please, try again it"
		if (resp =='y'):
			print "Indicate the location where you want to keep the files downloaded",
			path = raw_input()
			try:
				for url in urls_metadata:
					try:
						filename = wget.download(url,path)
						Analyze_Metadata(filename)
					except:
						pass
				Display_Export_Metadata(metadata_files,output)
			except:
				pass
	except Exception as e:
		print str(e)
#********************************************************#
#Definition and treatment of the parameters 

def ShowResults(newlist,num_files,target,output,option,captcha):
	if option == 1:
		print "Files into the target "+target+" are:\n"
		if captcha == False:
			print "Files indexed:", len (url_google)
			for i in url_google:
				#if i not in url_google:
				newlist.append(i)
				print i		
		else: ##Catpcha == True, try results of Bing
			print "Files indexed:", len (urls_final)
			for i in urls_final:
				#if i not in url_google:
				newlist.append(i)
				print i	
	else:
		#option ==2
		print "ShowResults outside target"
		print "Files outside target "+target+" are:\n"
		print "Files indexed:", len (url_google)
		for i in url_google:
		#if i not in url_google:
			newlist.append(i)
			print i	
	#verify if the user wants to export results
	if (output == 'Y') or (output =='y'):
		#Only it can enter if -j is put in the execution
		ExportResults(newlist)
	#Call to function to download the files		
	Downloadfiles(newlist,output)

#MAIN 

def Downloadfiles(urls_metadata,output):
	try:
		print "\nDo you like downloading these files to analyze metadata(Y/N)?"
		#Convert to lower the input
		resp = raw_input().lower()
		if (resp == 'n'):
			print "Exiting"
			exit(1)
		if ((resp != 'y') and (resp != 'n')):
			print "The option is not valided. Please, try again it"
		if (resp =='y'):
			print "Indicate the location where you want to keep the files downloaded",
			path = raw_input()
			try:
				for url in urls_metadata:
					try:
						filename = wget.download(url,path)
						Analyze_Metadata(filename)
					except:
						pass
				Display_Export_Metadata(metadata_files,output)
			except:
				pass
	except Exception as e:
		print str(e)
#********************************************************#
#Definition and treatment of the parameters 

def ShowResults(newlist,num_files,target,output):
	print "Files in the target "+target+" are:\n"
	print "Files indexed:", len (urls_final)
	for i in urls_final:
		if i not in newlist:
			newlist.append(i)
			print i		
	#verify if the user wants to export results
	if output == 'Y':
		#Only it can enter if -j is put in the execution
		ExportResults(newlist)
	#Call to function to download the files		
	Downloadfiles(newlist,output)
#MAIN 

def ShowResults(newlist,num_files,target,output):
	print "Files in the target "+target+" are:\n"
	print "Files indexed:", len (urls_final)
	for i in urls_final:
		if i not in newlist:
			newlist.append(i)
			print i		
	#verify if the user wants to export results
	if output == 'Y':
		#Only it can enter if -j is put in the execution
		ExportResults(newlist)
	#Call to function to download the files		
	Downloadfiles(newlist,output)
#MAIN 

def ShowResults(newlist,num_files,target):
	print "Files in the target "+target+" are:\n"
	print "Files indexed:", len (urls_final)
	for i in urls_final:
		if i not in newlist:
			newlist.append(i)
			print i		
	#verify if the user wants to export results
	if output == 'Y':
		#Only it can enter if -j is put in the execution
		ExportResults(newlist)
	#Call to function to download the files		
	Downloadfiles(newlist)
#MAIN