Python progressbar.ProgressBar() Examples

def download_to_file(path, ext):
    # Is it worth to have the same name?
    # ppath = urlparse(path).path.split("/")
    # ppath = ppath[-1] if len(ppath) > 0 else None
    _, file_name = tempfile.mkstemp("." + ext)
    r = requests.get(path, stream=True)
    total_length = r.headers.get('content-length')
    dl = 0
    with open(file_name, 'wb') as f:
        if total_length is None:
            f.write(r.content)
        else:
            total_length = int(total_length)
            pbar = ProgressBar(maxval=total_length).start()
            for chunk in r.iter_content(chunk_size=1024):
                if chunk:  # filter out keep-alive new chunks
                    pbar.update(dl * len(chunk))
                    dl += 1
                    f.write(chunk)
                    f.flush()
            pbar.finish()
    return file_name 

def do_import_licenses(file_data: List[dict]):
    imported = set()
    print("Importing licenses ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(file_data)) as bar:
        for idx, p in enumerate(file_data):
            info = p.get('info')
            license_text = detect_license(info.get('license'))

            if license_text and license_text not in imported:
                imported.add(license_text)
                session: Session = DbSession.factory()

                package_license = License()
                package_license.id = license_text
                package_license.description = info.get('license')

                session.add(package_license)
                session.commit()

            bar.update(idx)

    sys.stderr.flush()
    sys.stdout.flush() 

def do_import_licenses(file_data: List[dict]):
    imported = set()
    print("Importing licenses ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(file_data)) as bar:
        for idx, p in enumerate(file_data):
            info = p.get('info')
            license_text = detect_license(info.get('license'))

            if license_text and license_text not in imported:
                imported.add(license_text)
                session: Session = DbSession.factory()

                package_license = License()
                package_license.id = license_text
                package_license.description = info.get('license')

                session.add(package_license)
                session.commit()

            bar.update(idx)

    sys.stderr.flush()
    sys.stdout.flush() 

def do_user_import(user_lookup: Dict[str, str]) -> Dict[str, User]:
    print("Importing users ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(user_lookup)) as bar:
        for idx, (email, name) in enumerate(user_lookup.items()):
            session: Session = DbSession.factory()
            session.expire_on_commit = False

            user = User()
            user.email = email
            user.name = name
            session.add(user)

            session.commit()
            bar.update(idx)

    print()
    sys.stderr.flush()
    sys.stdout.flush()

    session: Session = DbSession.factory()
    return {u.email: u for u in session.query(User)} 

def do_import_licenses(file_data: List[dict]):
    imported = set()
    print("Importing licenses ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(file_data)) as bar:
        for idx, p in enumerate(file_data):
            info = p.get('info')
            license_text = detect_license(info.get('license'))

            if license_text and license_text not in imported:
                imported.add(license_text)
                session: Session = DbSession.factory()

                package_license = License()
                package_license.id = license_text
                package_license.description = info.get('license')

                session.add(package_license)
                session.commit()

            bar.update(idx)

    sys.stderr.flush()
    sys.stdout.flush() 

def capture_on_interface(interface, name, timeout=60):
    """
    :param interface: The name of the interface on which to capture traffic
    :param name: The name of the capture file
    :param timeout: A limit in seconds specifying how long to capture traffic
    """

    if timeout < 15:
        logger.error("Timeout must be over 15 seconds.")
        return
    if not sys.warnoptions:
        warnings.simplefilter("ignore")
    start = time.time()
    widgets = [
        progressbar.Bar(marker=progressbar.RotatingMarker()),
        ' ',
        progressbar.FormatLabel('Packets Captured: %(value)d'),
        ' ',
        progressbar.Timer(),
    ]
    progress = progressbar.ProgressBar(widgets=widgets)
    capture = pyshark.LiveCapture(interface=interface, output_file=os.path.join('tmp', name))
    pcap_size = 0
    for i, packet in enumerate(capture.sniff_continuously()):
        progress.update(i)
        if os.path.getsize(os.path.join('tmp', name)) != pcap_size:
            pcap_size = os.path.getsize(os.path.join('tmp', name))
        if not isinstance(packet, pyshark.packet.packet.Packet):
            continue
        if time.time() - start > timeout:
            break
        if pcap_size > const.PT_MAX_BYTES:
            break
    capture.clear()
    capture.close()
    return pcap_size 

def do_import_packages(file_data: List[dict], user_lookup: Dict[str, User]):
    errored_packages = []
    print("Importing packages and releases ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(file_data)) as bar:
        for idx, p in enumerate(file_data):
            try:
                load_package(p, user_lookup)
                bar.update(idx)
            except Exception as x:
                errored_packages.append((p, " *** Errored out for package {}, {}".format(p.get('package_name'), x)))
                raise
    sys.stderr.flush()
    sys.stdout.flush()
    print()
    print("Completed packages with {} errors.".format(len(errored_packages)))
    for (p, txt) in errored_packages:
        print(txt) 

def do_load_files() -> List[dict]:
    data_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../data/pypi-top-100'))
    print("Loading files from {}".format(data_path))
    files = get_file_names(data_path)
    print("Found {:,} files, loading ...".format(len(files)), flush=True)
    time.sleep(.1)

    file_data = []
    with progressbar.ProgressBar(max_value=len(files)) as bar:
        for idx, f in enumerate(files):
            file_data.append(load_file_data(f))
            bar.update(idx)

    sys.stderr.flush()
    sys.stdout.flush()
    print()
    return file_data 

def do_import_licenses(file_data: List[dict]):
    imported = set()
    print("Importing licenses ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(file_data)) as bar:
        for idx, p in enumerate(file_data):
            info = p.get('info')
            license_text = detect_license(info.get('license'))

            if license_text and license_text not in imported:
                imported.add(license_text)
                session: Session = DbSession.factory()

                package_license = License()
                package_license.id = license_text
                package_license.description = info.get('license')

                session.add(package_license)
                session.commit()

            bar.update(idx)

    sys.stderr.flush()
    sys.stdout.flush() 

def do_user_import(user_lookup: Dict[str, str]) -> Dict[str, User]:
    print("Importing users ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(user_lookup)) as bar:
        for idx, (email, name) in enumerate(user_lookup.items()):
            session: Session = DbSession.factory()
            session.expire_on_commit = False

            user = User()
            user.email = email
            user.name = name
            session.add(user)

            session.commit()
            bar.update(idx)

    print()
    sys.stderr.flush()
    sys.stdout.flush()

    session: Session = DbSession.factory()
    return {u.email: u for u in session.query(User)} 

def do_import_packages(file_data: List[dict], user_lookup: Dict[str, User]):
    errored_packages = []
    print("Importing packages and releases ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(file_data)) as bar:
        for idx, p in enumerate(file_data):
            try:
                load_package(p, user_lookup)
                bar.update(idx)
            except Exception as x:
                errored_packages.append((p, " *** Errored out for package {}, {}".format(p.get('package_name'), x)))
                raise
    sys.stderr.flush()
    sys.stdout.flush()
    print()
    print("Completed packages with {} errors.".format(len(errored_packages)))
    for (p, txt) in errored_packages:
        print(txt) 

def do_load_files() -> List[dict]:
    data_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../data/pypi-top-100'))
    print("Loading files from {}".format(data_path))
    files = get_file_names(data_path)
    print("Found {:,} files, loading ...".format(len(files)), flush=True)
    time.sleep(.1)

    file_data = []
    with progressbar.ProgressBar(max_value=len(files)) as bar:
        for idx, f in enumerate(files):
            file_data.append(load_file_data(f))
            bar.update(idx)

    sys.stderr.flush()
    sys.stdout.flush()
    print()
    return file_data 

def do_import_licenses(file_data: List[dict]):
    imported = set()
    print("Importing licenses ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(file_data)) as bar:
        for idx, p in enumerate(file_data):
            info = p.get('info')
            license_text = detect_license(info.get('license'))

            if license_text and license_text not in imported:
                imported.add(license_text)
                session: Session = DbSession.factory()

                package_license = License()
                package_license.id = license_text
                package_license.description = info.get('license')

                session.add(package_license)
                session.commit()

            bar.update(idx)

    sys.stderr.flush()
    sys.stdout.flush() 

def do_user_import(user_lookup: Dict[str, str]) -> Dict[str, User]:
    print("Importing users ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(user_lookup)) as bar:
        for idx, (email, name) in enumerate(user_lookup.items()):
            session: Session = DbSession.factory()
            session.expire_on_commit = False

            user = User()
            user.email = email
            user.name = name
            session.add(user)

            session.commit()
            bar.update(idx)

    print()
    sys.stderr.flush()
    sys.stdout.flush()

    session: Session = DbSession.factory()
    return {u.email: u for u in session.query(User)} 

def do_import_packages(file_data: List[dict], user_lookup: Dict[str, User]):
    errored_packages = []
    print("Importing packages and releases ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(file_data)) as bar:
        for idx, p in enumerate(file_data):
            try:
                load_package(p, user_lookup)
                bar.update(idx)
            except Exception as x:
                errored_packages.append((p, " *** Errored out for package {}, {}".format(p.get('package_name'), x)))
                raise
    sys.stderr.flush()
    sys.stdout.flush()
    print()
    print("Completed packages with {} errors.".format(len(errored_packages)))
    for (p, txt) in errored_packages:
        print(txt) 

def do_user_import(user_lookup: Dict[str, str]) -> Dict[str, User]:
    print("Importing users ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(user_lookup)) as bar:
        for idx, (email, name) in enumerate(user_lookup.items()):
            session: Session = DbSession.factory()
            session.expire_on_commit = False

            user = User()
            user.email = email
            user.name = name
            session.add(user)

            session.commit()
            bar.update(idx)

    print()
    sys.stderr.flush()
    sys.stdout.flush()

    session: Session = DbSession.factory()
    return {u.email: u for u in session.query(User)} 

def do_load_files() -> List[dict]:
    data_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../data/pypi-top-100'))
    print("Loading files from {}".format(data_path))
    files = get_file_names(data_path)
    print("Found {:,} files, loading ...".format(len(files)), flush=True)
    time.sleep(.1)

    file_data = []
    with progressbar.ProgressBar(max_value=len(files)) as bar:
        for idx, f in enumerate(files):
            file_data.append(load_file_data(f))
            bar.update(idx)

    sys.stderr.flush()
    sys.stdout.flush()
    print()
    return file_data 

def do_load_files() -> List[dict]:
    data_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../data/pypi-top-100'))
    print("Loading files from {}".format(data_path))
    files = get_file_names(data_path)
    print("Found {:,} files, loading ...".format(len(files)), flush=True)
    time.sleep(.1)

    file_data = []
    with progressbar.ProgressBar(max_value=len(files)) as bar:
        for idx, f in enumerate(files):
            file_data.append(load_file_data(f))
            bar.update(idx)

    sys.stderr.flush()
    sys.stdout.flush()
    print()
    return file_data 

def do_import_packages(file_data: List[dict], user_lookup: Dict[str, User]):
    errored_packages = []
    print("Importing packages and releases ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(file_data)) as bar:
        for idx, p in enumerate(file_data):
            try:
                load_package(p, user_lookup)
                bar.update(idx)
            except Exception as x:
                errored_packages.append((p, " *** Errored out for package {}, {}".format(p.get('package_name'), x)))
                raise
    sys.stderr.flush()
    sys.stdout.flush()
    print()
    print("Completed packages with {} errors.".format(len(errored_packages)))
    for (p, txt) in errored_packages:
        print(txt) 

def do_user_import(user_lookup: Dict[str, str]) -> Dict[str, User]:
    print("Importing users ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(user_lookup)) as bar:
        for idx, (email, name) in enumerate(user_lookup.items()):
            session: Session = DbSession.factory()
            session.expire_on_commit = False

            user = User()
            user.email = email
            user.name = name
            session.add(user)

            session.commit()
            bar.update(idx)

    print()
    sys.stderr.flush()
    sys.stdout.flush()

    session: Session = DbSession.factory()
    return {u.email: u for u in session.query(User)} 

def do_import_licenses(file_data: List[dict]):
    imported = set()
    print("Importing licenses ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(file_data)) as bar:
        for idx, p in enumerate(file_data):
            info = p.get('info')
            license_text = detect_license(info.get('license'))

            if license_text and license_text not in imported:
                imported.add(license_text)
                session: Session = DbSession.factory()

                package_license = License()
                package_license.id = license_text
                package_license.description = info.get('license')

                session.add(package_license)
                session.commit()

            bar.update(idx)

    sys.stderr.flush()
    sys.stdout.flush() 

def do_load_files() -> List[dict]:
    data_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../data/pypi-top-100'))
    print("Loading files from {}".format(data_path))
    files = get_file_names(data_path)
    print("Found {:,} files, loading ...".format(len(files)), flush=True)
    time.sleep(.1)

    file_data = []
    with progressbar.ProgressBar(max_value=len(files)) as bar:
        for idx, f in enumerate(files):
            file_data.append(load_file_data(f))
            bar.update(idx)

    sys.stderr.flush()
    sys.stdout.flush()
    print()
    return file_data 

def do_import_packages(file_data: List[dict], user_lookup: Dict[str, User]):
    errored_packages = []
    print("Importing packages and releases ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(file_data)) as bar:
        for idx, p in enumerate(file_data):
            try:
                load_package(p, user_lookup)
                bar.update(idx)
            except Exception as x:
                errored_packages.append((p, " *** Errored out for package {}, {}".format(p.get('package_name'), x)))
                raise
    sys.stderr.flush()
    sys.stdout.flush()
    print()
    print("Completed packages with {} errors.".format(len(errored_packages)))
    for (p, txt) in errored_packages:
        print(txt) 

def do_user_import(user_lookup: Dict[str, str]) -> Dict[str, User]:
    print("Importing users ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(user_lookup)) as bar:
        for idx, (email, name) in enumerate(user_lookup.items()):
            session: Session = DbSession.factory()
            session.expire_on_commit = False

            user = User()
            user.email = email
            user.name = name
            session.add(user)

            session.commit()
            bar.update(idx)

    print()
    sys.stderr.flush()
    sys.stdout.flush()

    session: Session = DbSession.factory()
    return {u.email: u for u in session.query(User)} 

def do_import_licenses(file_data: List[dict]):
    imported = set()
    print("Importing licenses ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(file_data)) as bar:
        for idx, p in enumerate(file_data):
            info = p.get('info')
            license_text = detect_license(info.get('license'))

            if license_text and license_text not in imported:
                imported.add(license_text)
                session: Session = DbSession.factory()

                package_license = License()
                package_license.id = license_text
                package_license.description = info.get('license')

                session.add(package_license)
                session.commit()

            bar.update(idx)

    sys.stderr.flush()
    sys.stdout.flush() 

def do_load_files() -> List[dict]:
    data_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../data/pypi-top-100'))
    print("Loading files from {}".format(data_path))
    files = get_file_names(data_path)
    print("Found {:,} files, loading ...".format(len(files)), flush=True)
    time.sleep(.1)

    file_data = []
    with progressbar.ProgressBar(max_value=len(files)) as bar:
        for idx, f in enumerate(files):
            file_data.append(load_file_data(f))
            bar.update(idx)

    sys.stderr.flush()
    sys.stdout.flush()
    print()
    return file_data 

def do_import_packages(file_data: List[dict], user_lookup: Dict[str, User]):
    errored_packages = []
    print("Importing packages and releases ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(file_data)) as bar:
        for idx, p in enumerate(file_data):
            try:
                load_package(p, user_lookup)
                bar.update(idx)
            except Exception as x:
                errored_packages.append((p, " *** Errored out for package {}, {}".format(p.get('package_name'), x)))
                raise
    sys.stderr.flush()
    sys.stdout.flush()
    print()
    print("Completed packages with {} errors.".format(len(errored_packages)))
    for (p, txt) in errored_packages:
        print(txt) 

def do_user_import(user_lookup: Dict[str, str]) -> Dict[str, User]:
    print("Importing users ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(user_lookup)) as bar:
        for idx, (email, name) in enumerate(user_lookup.items()):
            session: Session = DbSession.factory()
            session.expire_on_commit = False

            user = User()
            user.email = email
            user.name = name
            session.add(user)

            session.commit()
            bar.update(idx)

    print()
    sys.stderr.flush()
    sys.stdout.flush()

    session: Session = DbSession.factory()
    return {u.email: u for u in session.query(User)} 

def do_import_licenses(file_data: List[dict]):
    imported = set()
    print("Importing licenses ... ", flush=True)
    with progressbar.ProgressBar(max_value=len(file_data)) as bar:
        for idx, p in enumerate(file_data):
            info = p.get('info')
            license_text = detect_license(info.get('license'))

            if license_text and license_text not in imported:
                imported.add(license_text)
                session: Session = DbSession.factory()

                package_license = License()
                package_license.id = license_text
                package_license.description = info.get('license')

                session.add(package_license)
                session.commit()

            bar.update(idx)

    sys.stderr.flush()
    sys.stdout.flush() 

def killProcTree(pid=None, including_parent=True):
    if not pid:
        # kill the process itself
        pid = os.getpid()
    parent = psutil.Process(pid)
    children = parent.children(recursive=True)
    for child in children:
        child.kill()
    psutil.wait_procs(children, timeout=5)
    if including_parent:
        parent.kill()
        parent.wait(5)


###########################################################
# ProgressBar
###########################################################