Python progressbar.FileTransferSpeed() Examples

def download_model(model):
    if model_in_cache(model): return
    model_url = get_model_url(model)
    tmp_zip = tempfile.NamedTemporaryFile(suffix=".zip")
    prompt = "Downloading model {}".format(model)
    def cb(count, block_size, total_size):
        global progress_bar
        if not progress_bar:
            widgets = [prompt, Percentage(), ' ', Bar(), ' ', FileTransferSpeed(), ' ', ETA()]
            progress_bar = ProgressBar(widgets=widgets, maxval=int(total_size)).start()
        progress_bar.update(min(total_size, count * block_size))
    urllib.urlretrieve(model_url, tmp_zip.name, cb)
    z = zipfile.ZipFile(tmp_zip)
    out_path = get_model_local_path(model)
    try:
        os.mkdir(out_path)
    except:
        pass
    for name in z.namelist():
        if name.startswith("_"): continue
        z.extract(name, out_path) 

def main():
    uri, outfile, dataset = get_arguments()
    fd = tempfile.NamedTemporaryFile()
    progress = ProgressBar(widgets=[Percentage(), ' ', Bar(), ' ', ETA(), ' ', FileTransferSpeed()])

    def update(count, blockSize, totalSize):
        if progress.maxval is None:
            progress.maxval = totalSize
            progress.start()
        progress.update(min(count * blockSize, totalSize))

    urllib.urlretrieve(uri, fd.name, reporthook = update)
    if dataset == 'zinc12':
        df = pandas.read_csv(fd.name, delimiter = '\t')
        df = df.rename(columns={'SMILES':'structure'})
        df.to_hdf(outfile, 'table', format = 'table', data_columns = True)
    elif dataset == 'chembl22':
        df = pandas.read_table(fd.name,compression='gzip')
        df = df.rename(columns={'canonical_smiles':'structure'})
        df.to_hdf(outfile, 'table', format = 'table', data_columns = True)
        pass
    else:
        df = pandas.read_csv(fd.name, delimiter = '\t')
        df.to_hdf(outfile, 'table', format = 'table', data_columns = True) 

def download(url, dst_file_path):
    # Download a file, showing progress
    bar_wrap = [None]

    def reporthook(count, block_size, total_size):
        bar = bar_wrap[0]
        if bar is None:
            bar = progressbar.ProgressBar(
                maxval=total_size,
                widgets=[
                    progressbar.Percentage(),
                    ' ',
                    progressbar.Bar(),
                    ' ',
                    progressbar.FileTransferSpeed(),
                    ' | ',
                    progressbar.ETA(),
                ])
            bar.start()
            bar_wrap[0] = bar
        bar.update(min(count * block_size, total_size))

    request.urlretrieve(url, dst_file_path, reporthook=reporthook) 

def __call__(self, *args, **kwargs):
        if self.first_call:
            self.widgets = [progressbar.Percentage(), ' ', progressbar.Bar(marker=progressbar.RotatingMarker('>')),
                            ' ', progressbar.FileTransferSpeed()]
            self.pbar = progressbar.ProgressBar(widgets=self.widgets, maxval=kwargs['size']).start()
            self.first_call = False

        if kwargs['size'] <= kwargs['progress']:
            self.pbar.finish()
        else:
            self.pbar.update(kwargs['progress']) 

def __init__(self, remote_addr, mem_size, progressbar=False,
                 recv_size=1048576, sock_timeout=1):
        """
        :type remote_addr: str
        :param remote_addr: hostname or ip address of target server
        :type mem_size: int
        :param mem_size: target server memory size in bytes
        :type progressbar: bool
        :param progressbar: ncurses progress bar toggle
        :type recv_size: int
        :param recv_size: transfer socket max receive size
        :type sock_timeout: int
        :param sock_timeout: transfer socket receive timeout
        """
        self.mem_size = mem_size
        self.progressbar = progressbar
        self.recv_size = recv_size
        self.sock_timeout = sock_timeout
        self.padding_percentage = 0.03
        self.max_size = self.max_size(mem_size, self.padding_percentage)
        self.update_interval = 5
        self.update_threshold = recv_size * self.update_interval
        self.remote_addr = remote_addr
        self.transfered = 0
        self.progress = 0
        self.widgets = [' {0} '.format(remote_addr), Percentage(), ' ', Bar(),
                        ' ', ETA(), ' ', FileTransferSpeed()]
        self.sock = None
        self.outfile = None
        self.bar = None 

def _addresses_to_check_with_caching(self, show_progress=True):
        num_addrs = len(list(self._addresses_to_check()))
        widgets = ['ROP: ', progressbar.Percentage(), ' ',
                   progressbar.Bar(marker=progressbar.RotatingMarker()),
                   ' ', progressbar.ETA(), ' ', progressbar.FileTransferSpeed()]
        progress = progressbar.ProgressBar(widgets=widgets, maxval=num_addrs)
        if show_progress:
            progress.start()
        self._cache = dict()
        seen = dict()
        for i, a in enumerate(self._addresses_to_check()):
            if show_progress:
                progress.update(i)
            try:
                bl = self.project.factory.block(a)
                if bl.size > self._max_block_size:
                    continue
                block_data = bl.bytes
            except (SimEngineError, SimMemoryError):
                continue
            if block_data in seen:
                self._cache[seen[block_data]].add(a)
                continue
            else:
                if self._is_jumpkind_valid(bl.vex.jumpkind) and \
                        len(bl.vex.constant_jump_targets) == 0 and \
                        not self._block_has_ip_relative(a, bl):
                    seen[block_data] = a
                    self._cache[a] = set()
                yield a
        if show_progress:
            progress.finish() 

def dnl_vid(url, filename, size):
    try:
        file = open(filename, 'wb')
    except IOError:
        sys.exit('cannot access file '+filename)
    size = int(size)
    dsize = 0

    widgets = ['progress: ', pb.Percentage(), ' ', pb.Bar(marker=pb.RotatingMarker()), ' ', pb.ETA(), ' ', pb.FileTransferSpeed()]
    pbar = pb.ProgressBar(widgets=widgets, maxval=size).start()

    try:
        h_url = urllib2.urlopen(url)
    except urllib2.URLError:
        sys.exit('error : cannot open url')
    try:
        while True:
            info = h_url.read(8192)
            if len(info) < 1 :
                break
            dsize += len(info)
            file.write(info)
            pbar += len(info)

        pbar.finish()
    except IOError:
        sys.exit('error : unable to download the video')

    print 'done'
    pass 

def write_to_csv(self):
        if self.num_results > 0:
            self.num_results = sum(1 for line in codecs.open(self.tmp_file, mode='r', encoding='utf-8'))
            if self.num_results > 0:
                output_file = codecs.open(self.opts.output_file, mode='a', encoding='utf-8')
                csv_writer = csv.DictWriter(output_file, fieldnames=self.csv_headers)
                csv_writer.writeheader()
                timer = 0
                widgets = ['Write to csv ',
                           progressbar.Bar(left='[', marker='#', right=']'),
                           progressbar.FormatLabel(' [%(value)i/%(max)i] ['),
                           progressbar.Percentage(),
                           progressbar.FormatLabel('] [%(elapsed)s] ['),
                           progressbar.ETA(), '] [',
                           progressbar.FileTransferSpeed(unit='lines'), ']'
                           ]
                bar = progressbar.ProgressBar(widgets=widgets, maxval=self.num_results).start()

                for line in codecs.open(self.tmp_file, mode='r', encoding='utf-8'):
                    timer += 1
                    bar.update(timer)
                    csv_writer.writerow(json.loads(line))
                output_file.close()
                bar.finish()
            else:
                print('There is no docs with selected field(s): {}.'.format(','.join(self.opts.fields)))
            os.remove(self.tmp_file) 

def __call__(self, *args, **kwargs):
        if self.first_call:
            self.widgets = [progressbar.Percentage(), ' ', progressbar.Bar(marker=progressbar.RotatingMarker('>')),
                            ' ', progressbar.FileTransferSpeed()]
            self.pbar = progressbar.ProgressBar(widgets=self.widgets, maxval=kwargs['size']).start()
            self.first_call = False

        if kwargs['size'] <= kwargs['progress']:
            self.pbar.finish()
        else:
            self.pbar.update(kwargs['progress']) 

def __init__(self, msg, maxval):
        try:
            # see docs for other options
            self.__widgets = [msg, progressbar.Percentage(),
                              ' ', progressbar.Bar(marker='#', left='[', right=']'),
                              ' ', progressbar.ETA(), ' ', progressbar.FileTransferSpeed()]

            self.__bar = progressbar.ProgressBar(maxval=maxval, widgets=self.__widgets)
        except Exception as e:
            raise Exception("Failed to init progressbar: " + str(e)) 

def progress_bar(block_num, block_size, total_size):
    global pbar
    if pbar is None:

        # pbar = progressbar.ProgressBar(maxval = total_size)
        # Customized progress bar
        widgets = [progressbar.Percentage(), ' ', progressbar.Bar(marker = '>', left = '[', right = ']'), ' ', progressbar.ETA(), ' ', progressbar.FileTransferSpeed()] 
        pbar = progressbar.ProgressBar(widgets = widgets, maxval = total_size)

    downloaded = block_num * block_size
    if downloaded < total_size:
        pbar.update(downloaded)
    else:
        pbar.finish()
        pbar = None 

def progress_bar(block_num, block_size, total_size):
    global pbar
    if pbar is None:

        # pbar = progressbar.ProgressBar(maxval = total_size)
        # Customized progress bar
        widgets = [progressbar.Percentage(), ' ', progressbar.Bar(marker = '>', left = '[', right = ']'), ' ', progressbar.ETA(), ' ', progressbar.FileTransferSpeed()] 
        pbar = progressbar.ProgressBar(widgets = widgets, maxval = total_size)

    downloaded = block_num * block_size
    if downloaded < total_size:
        pbar.update(downloaded)
    else:
        pbar.finish()
        pbar = None 

def __init__(self, msg, maxval):
        try:
            # see docs for other options
            self.__widgets = [msg, progressbar.Percentage(), ' ', progressbar.Bar(marker='#', left='[', right=']'),
                              ' ', progressbar.ETA(), ' ', progressbar.FileTransferSpeed()]

            self.__bar = progressbar.ProgressBar(maxval=maxval, widgets=self.__widgets)
        except Exception as e:
            raise Exception("Failed to init progressbar: " + str(e)) 

def download(number=-1, name="", save_dir='./'):
    """Download pre-trained word vector
    :param number: integer, default ``None``
    :param save_dir: str, default './'
    :return: file path for downloaded file
    """
    df = load_datasets()

    if number > -1:
        row = df.iloc[[number]]
    elif name:
        row = df.loc[df["Name"] == name]

    url = ''.join(row.URL)
    if not url:
        print('The word vector you specified was not found. Please specify correct name.')

    widgets = ['Test: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ', ETA(), ' ', FileTransferSpeed()]
    pbar = ProgressBar(widgets=widgets)

    def dlProgress(count, blockSize, totalSize):
        if pbar.max_value is None:
            pbar.max_value = totalSize
            pbar.start()

        pbar.update(min(count * blockSize, totalSize))

    file_name = url.split('/')[-1]
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
    save_path = os.path.join(save_dir, file_name)
    path, _ = urlretrieve(url, save_path, reporthook=dlProgress)
    pbar.finish()
    return path 

def __init__(self, msg, maxval):
		try:
			#see docs for other options
			self.__widgets = [msg, progressbar.Percentage(), \
							' ', progressbar.Bar(marker='#', \
							left='[',right=']'),
							' ', progressbar.ETA(), ' ', \
							progressbar.FileTransferSpeed()]

			self.__bar = progressbar.ProgressBar(maxval=maxval, widgets=self.__widgets)
		except Exception as e:
			raise Exception("Failed to init progressbar: " + str(e)) 

def __init__(self, msg, maxval):
        try:
            #see docs for other options
            self.__widgets = [msg, progressbar.Percentage(), \
                            ' ', progressbar.Bar(marker='#', \
                            left='[',right=']'),
                            ' ', progressbar.ETA(), ' ', \
                            progressbar.FileTransferSpeed()]

            self.__bar = progressbar.ProgressBar(maxval=maxval, widgets=self.__widgets)
        except Exception as e:
            raise Exception("Failed to init progressbar: " + str(e)) 

def dump_rom(self, slot, output, silent=False, progress=None):
        """Dump rom from sdcard to file

        This opens the rom position header at the specified slot, seeks to
        the start point on sdcard, and just starts dumping data to the output-
        file until the whole rom has been dumped. After dumping sky3ds specific
        data (0x1400 - 0x1600) gets removed from the romfile.

        Keyword Arguments:
        slot -- rom position header slot
        output -- output rom file"""

        self.fail_on_non_sky3ds()

        start = self.rom_list[slot][1]
        rom_size = self.rom_list[slot][2]

        self.diskfp.seek(start)

        outputfp = open(output, "wb")

        # read rom
        try:
            if not silent and not progress:
                progress = ProgressBar(widgets=[Percentage(), Bar(), FileTransferSpeed()], maxval=rom_size).start()
        except:
            pass
        written = 0
        while written < rom_size:
            chunk = self.diskfp.read(1024*1024)

            outputfp.write(chunk)
            os.fsync(outputfp)

            written = written + len(chunk)
            try:
                if not silent:
                    progress.update(written)
            except:
                pass
        try:
            if not silent:
                progress.finish()
        except:
            pass

        # remove sky3ds specific data from
        outputfp.seek(0x1400)
        outputfp.write(bytearray([0xff]*0x200))

        # cleanup
        os.fsync(outputfp)
        outputfp.close()

    # delete rom from sdcard 

def download_file(self, browser, url):
        print('downloading {}\n'.format(url))
        local_filename = url.split('/')[-1]
        headers = {}
        done = False
        file_size = 0
        content_length = int(
            browser.request('head', url).headers.get('Content-Length')
        )

        bar = progressbar.ProgressBar()
        widgets = [local_filename, ' ', progressbar.Percentage(), ' ',
                   progressbar.Bar(marker='#'), ' ',
                   progressbar.ETA(), ' ', progressbar.FileTransferSpeed()]

        if os.path.isfile(local_filename):
            file_size = os.path.getsize(local_filename)
            if file_size < content_length:
                headers['Range'] = 'bytes={}-'.format(file_size)
            else:
                done = True

        finished_bytes = file_size

        if file_size == content_length:
            print('{} already downloaded !'.format(local_filename))
            return
        elif file_size > content_length:
            print('Something wrong here, Incorrect file !')
            return
        else:
            bar = progressbar.ProgressBar(widgets=widgets,
                                          maxval=content_length).start()
            bar.update(finished_bytes)

        if not done:
            stream = browser.get(url, stream=True, headers=headers)

            if not self.is_downloadable(stream):
                warning = (
                    'Warning: '
                    'download url for file {} resolves to an html document '
                    'rather than a downloadable file. \n'
                    'Is it possible you have not '
                    'accepted the competition\'s rules on the kaggle website?'
                    .format(local_filename)
                )
                print('\n\n{}\n'.format(warning))
                return False

            with open(local_filename, 'ab') as f:
                for chunk in stream.iter_content(chunk_size=1024):
                    if chunk:  # filter out keep-alive new chunks
                        f.write(chunk)
                        finished_bytes += len(chunk)
                        bar.update(finished_bytes)

            bar.finish()
            print('')

        return True 

def _download_file(self, browser, url, destination_path):
        local_filename = url.split('/')[-1]
        headers = {}
        done = False
        file_size = 0
        content_length = int(
            browser.request('head', url).headers.get('Content-Length')
        )

        widgets = [local_filename, ' ', progressbar.Percentage(), ' ',
                   progressbar.Bar(marker='#'), ' ',
                   progressbar.ETA(), ' ', progressbar.FileTransferSpeed()]

        local_filename = destination_path + local_filename
        print('downloading {} to {}\n'.format(url, local_filename))
        if os.path.isfile(local_filename):
            file_size = os.path.getsize(local_filename)
            if file_size < content_length:
                headers['Range'] = 'bytes={}-'.format(file_size)
            else:
                done = True

        finished_bytes = file_size

        if file_size == content_length:
            print('{} already downloaded !'.format(local_filename))
            return local_filename
        elif file_size > content_length:
            raise Exception('Something wrong here, Incorrect file !')
        else:
            bar = progressbar.ProgressBar(widgets=widgets,
                                          maxval=content_length).start()
            bar.update(finished_bytes)

        if not done:
            stream = browser.get(url, stream=True, headers=headers)
            if not self.is_downloadable(stream):
                warning = (
                    'Warning:'
                    'download url for file {} resolves to an html document'
                    'rather than a downloadable file. \n'
                    'See the downloaded file for details.'
                    'Is it possible you have not'
                    'accepted the competition\'s rules on the kaggle website?'.format(local_filename)
                )
                raise Exception('{}\n'.format(warning))
            os.makedirs(os.path.dirname(local_filename), exist_ok=True)
            with open(local_filename, 'ab') as f:
                for chunk in stream.iter_content(chunk_size=1024):
                    if chunk:  # filter out keep-alive new chunks
                        f.write(chunk)
                        finished_bytes += len(chunk)
                        bar.update(finished_bytes)
            bar.finish()
        return local_filename 

def __call__(self, count, blocksize, totalsize):

        # In case we don't know the size of the file. zget < 0.9 did not
        # report file sizes via HTTP.
        if totalsize <= 0:
            if self.pbar is None:
                self.pbar = progressbar.ProgressBar(
                    widgets=[
                        self.filename,
                        ' ',
                        progressbar.BouncingBar(),
                        ' ',
                        progressbar.FileTransferSpeed(),
                    ],
                    maxval=progressbar.UnknownLength
                )
                self.pbar.start()

            # Make sure we have at least 1, otherwise the bar does not show
            # 100% for small transfers
            self.pbar.update(max(count * blocksize, 1))

        # zget >= 0.9 does report file sizes and enables percentage and ETA
        # display.
        else:
            if self.pbar is None:
                self.pbar = progressbar.ProgressBar(
                    widgets=[
                        self.filename,
                        ' ',
                        progressbar.Percentage(),
                        ' ',
                        progressbar.Bar(),
                        ' ',
                        progressbar.ETA(),
                        ' ',
                        progressbar.FileTransferSpeed(),
                    ],
                    # Make sure we have at least 1, otherwise the bar does
                    # not show 100% for small transfers
                    maxval=max(totalsize, 1)
                )
                self.pbar.start()

            # Make sure we have at least 1, otherwise the bar does not show
            # 100% for small transfers
            self.pbar.update(max(min(count * blocksize, totalsize), 1)) 

def download_file(cls, url, local_file_name=None, force=False, chunk_size=1024):
        """
        Download file from a given url
        """

        local_file_name = local_file_name if local_file_name else url.split('/')[-1]
        filepath = os.path.join(cls.data_path, local_file_name)

        if not os.path.exists(filepath) or force:
            try:
                headers = requests.head(url, allow_redirects=True).headers
                length = headers.get('Content-Length')

                logger.info("Starting download of {} file with {} bytes ...".format(url, length))

                widgets = [
                    'Downloading file please wait...', progressbar.Percentage(),
                    ' ', progressbar.Bar(),
                    ' ', progressbar.ETA(),
                    ' ', progressbar.FileTransferSpeed(),
                ]
                bar = progressbar.ProgressBar(widgets=widgets, max_value=int(length) + chunk_size).start()

                r = requests.get(url, stream=True)

                with open(filepath, 'wb') as f:
                    total_chunk = 0

                    for chunk in r.iter_content(chunk_size):
                        if chunk:
                            f.write(chunk)
                            total_chunk += chunk_size
                            bar.update(total_chunk)

                bar.finish()

            except:
                if os.path.exists(filepath):
                    os.remove(filepath)

                raise

        return filepath