Python shutil.disk_usage() Examples

def get_free_space_in_dir(path):
    """
    Given a path to a directory, returns the amount of free space (in
    bytes) on that filesystem.

    Parameters
    ----------
    path : str
        The path to a directory

    Returns
    -------
    bytes : int
        The amount of free space on the partition that the directory
        is on.
    """
    if not os.path.isdir(path):
        raise OSError(
            "Can only determine free space associated with directories, "
            "not files.")
        # Actually you can on Linux but I want to avoid code that fails
        # on Windows only.
    return shutil.disk_usage(path).free 

def available_bytes_at_path(path):
    """Get the available disk space in a given directory.

    Args:
      path (str): Directory path to check.

    Returns:
      int: Available space, in bytes

    Raises:
      OSError: if the specified path does not exist or is not readable.
    """
    if not os.path.exists(path):
        raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), path)
    if not os.path.isdir(path):
        raise OSError(errno.ENOTDIR, os.strerror(errno.ENOTDIR), path)
    available = disk_usage(path).free
    logger.debug("There appears to be %s available at %s",
                 pretty_bytes(available), path)
    return available 

def test_disk_usage(self):
        usage = psutil.disk_usage(os.getcwd())
        self.assertEqual(usage._fields, ('total', 'used', 'free', 'percent'))

        assert usage.total > 0, usage
        assert usage.used > 0, usage
        assert usage.free > 0, usage
        assert usage.total > usage.used, usage
        assert usage.total > usage.free, usage
        assert 0 <= usage.percent <= 100, usage.percent
        if hasattr(shutil, 'disk_usage'):
            # py >= 3.3, see: http://bugs.python.org/issue12442
            shutil_usage = shutil.disk_usage(os.getcwd())
            tolerance = 5 * 1024 * 1024  # 5MB
            self.assertEqual(usage.total, shutil_usage.total)
            self.assertAlmostEqual(usage.free, shutil_usage.free,
                                   delta=tolerance)
            self.assertAlmostEqual(usage.used, shutil_usage.used,
                                   delta=tolerance)

        # if path does not exist OSError ENOENT is expected across
        # all platforms
        fname = self.get_testfn()
        with self.assertRaises(FileNotFoundError):
            psutil.disk_usage(fname) 

def __init__(self, preprocessing=None, postprocessing=None, detections_path=None, max_detections=100,
                 sort_by_prob=False, load_in_tmp=True):
        self.max_detections = max_detections
        self.detections_path = detections_path
        self.sort_by_prob = sort_by_prob

        tmp_detections_path = os.path.join('/tmp', os.path.basename(detections_path))

        if load_in_tmp:
            if not os.path.isfile(tmp_detections_path):
                if shutil.disk_usage("/tmp")[-1] < os.path.getsize(detections_path):
                    warnings.warn('Loading from %s, because /tmp has no enough space.' % detections_path)
                else:
                    warnings.warn("Copying detection file to /tmp")
                    shutil.copyfile(detections_path, tmp_detections_path)
                    warnings.warn("Done.")
                    self.detections_path = tmp_detections_path
            else:
                self.detections_path = tmp_detections_path

        super(ImageDetectionsField, self).__init__(preprocessing, postprocessing) 

def _get_message_if_space_insufficient(cls, paths_to_copy):
        INSUFFIENT_SPACE_ERROR_FMT = "There is insufficient space remaining on this instance to " + \
        "containerize this notebook. Containerization would require {} of additional space."

        files_to_copy_bytes = cls._get_total_size_of_files(paths_to_copy)
        _, _, free_space_bytes = shutil.disk_usage("/")

        required_bytes = int(cls.REQUIRED_SPACE_PER_FILES_SPACE * files_to_copy_bytes
            ) + cls.SPACE_REQUIREMENT_FUDGE_BYTES

        if required_bytes > free_space_bytes:
            cls.logger.info("Insufficient space to containerize. Has {} bytes, requires {} bytes, " +
                "with fudge space requires {} bytes.".format(
                    free_space_bytes, files_to_copy_bytes, required_bytes))

            additional_required_bytes = required_bytes - free_space_bytes
            human_readable_additional_space_required = size(required_bytes - free_space_bytes)
            return INSUFFIENT_SPACE_ERROR_FMT.format("{} bytes ({})".format(
                additional_required_bytes, human_readable_additional_space_required)) 

def _check_tmp_free_space() -> CheckLevels:
    """
    Warn if there is not enough free space in the default temporary directory.
    """
    free_space = shutil.disk_usage(gettempdir()).free
    free_space_gb = free_space / 1024 / 1024 / 1024

    low_space_message = (
        'The default temporary directory ("{tmp_prefix}") has '
        '{free_space:.1f} GB of free space available. '
        'Creating a cluster typically takes approximately 2 GB of temporary '
        'storage. '
        'If you encounter problems with disk space usage, set the ``TMPDIR`` '
        'environment variable to a suitable temporary directory or use the '
        '``--workspace-dir`` option on the ``minidcos docker create`` command.'
    ).format(
        tmp_prefix=Path('/') / gettempprefix(),
        free_space=free_space_gb,
    )

    if free_space_gb < 5:
        warn(message=low_space_message)
        return CheckLevels.WARNING

    return CheckLevels.NONE 

def opt_check_disk_space(warnlimit_mb=200):
    if disk_usage('.').free < warnlimit_mb*1024*2:
        log.warning("Less than %sMB of free space remains on this device" % warnlimit_mb)


################################################# 

def disk_usage(mount_path):
    """
    Return disk usage statistics about the given path as a (total, used, free)
    namedtuple.  Values are expressed in bytes.
    """
    # Author: Giampaolo Rodola' <g.rodola [AT] gmail [DOT] com>
    # License: MIT
    _ntuple_diskusage = collections.namedtuple('usage', 'total used free')

    if platform.system() == "Linux":
        st = os.statvfs(mount_path)
        free = st.f_bavail * st.f_frsize
        total = st.f_blocks * st.f_frsize
        used = (st.f_blocks - st.f_bfree) * st.f_frsize

        return _ntuple_diskusage(total, used, free)

    elif platform.system() == "Windows":

        _, total, free = ctypes.c_ulonglong(), ctypes.c_ulonglong(), \
                         ctypes.c_ulonglong()
        if sys.version_info >= (3,) or isinstance(mount_path, unicode):
            fun = ctypes.windll.kernel32.GetDiskFreeSpaceExW
        else:
            fun = ctypes.windll.kernel32.GetDiskFreeSpaceExA
        ret = fun(mount_path, ctypes.byref(_), ctypes.byref(total), ctypes.byref(free))
        if ret == 0:
            raise ctypes.WinError()
        used = total.value - free.value

        return _ntuple_diskusage(total.value, used, free.value)
    else:
        raise NotImplementedError("Platform not supported.") 

def used_space(self):
        x = shutil.disk_usage('/')
        return format(x.used / 1024 / 1024 / 1024, '.2f') 

def _free_space() -> int:
        """
        Get space free of disk that this script is installed to.

        :return: Integer free space
        """
        return disk_usage(realpath('/')).free 

def get_available_disk_space(path_to_check):
    disk_usage = shutil.disk_usage(path_to_check)
    available = (disk_usage.free * 100) / disk_usage.total
    return available 

def storage_node(self, name, path, parent, attr=None):
        '''create a new node representing a storage'''
        path = os.path.abspath(path)
        free = shutil.disk_usage(path).free
        total = shutil.disk_usage(path).total
        epoch = int(time.time())
        return anytree.AnyNode(name=name, type=self.TYPE_STORAGE, free=free,
                               total=total, parent=parent, attr=attr, ts=epoch) 

def disk_get_free_space(path, bytes_flag=False):

    """Can be called by anything.

    Returns the size of the disk on which a specified file/directory exists,
    minus the used space on that disk.

    Args:

        path (str): Path to a file/directory on the disk, typically Tartube's
            data directory

        bytes_flag (bool): True to return an integer value in MB, false to
            return a value in bytes

    Returns:

        The free space in MB (or in bytes, if the flag is specified), or 0 if
            the size can't be calculated for any reason

    """

    try:
        total_bytes, used_bytes, free_bytes = shutil.disk_usage(
            os.path.realpath(path),
        )

        if not bytes_flag:
            return int(free_bytes / 1000000)
        else:
            return free_bytes

    except:
        return 0 

def disk_get_total_space(path, bytes_flag=False):

    """Can be called by anything.

    Returns the size of the disk on which a specified file/directory exists.

    Args:

        path (str): Path to a file/directory on the disk, typically Tartube's
            data directory

        bytes_flag (bool): True to return an integer value in MB, false to
            return a value in bytes

    Returns:

        The total size in MB (or in bytes, if the flag is specified)

    """

    total_bytes, used_bytes, free_bytes = shutil.disk_usage(
        os.path.realpath(path),
    )

    if not bytes_flag:
        return int(total_bytes / 1000000)
    else:
        return total_bytes 

def disk_get_used_space(path, bytes_flag=False):

    """Can be called by anything.

    Returns the size of the disk on which a specified file/directory exists,
    minus the free space on that disk.

    Args:

        path (str): Path to a file/directory on the disk, typically Tartube's
            data directory

        bytes_flag (bool): True to return an integer value in MB, false to
            return a value in bytes

    Returns:

        The used space in MB (or in bytes, if the flag is specified)

    """

    total_bytes, used_bytes, free_bytes = shutil.disk_usage(
        os.path.realpath(path),
    )

    if not bytes_flag:
        return int(used_bytes / 1000000)
    else:
        return used_bytes 

def test_disk_usage(self):
        usage = psutil.disk_usage(os.getcwd())
        self.assertEqual(usage._fields, ('total', 'used', 'free', 'percent'))

        assert usage.total > 0, usage
        assert usage.used > 0, usage
        assert usage.free > 0, usage
        assert usage.total > usage.used, usage
        assert usage.total > usage.free, usage
        assert 0 <= usage.percent <= 100, usage.percent
        if hasattr(shutil, 'disk_usage'):
            # py >= 3.3, see: http://bugs.python.org/issue12442
            shutil_usage = shutil.disk_usage(os.getcwd())
            tolerance = 5 * 1024 * 1024  # 5MB
            self.assertEqual(usage.total, shutil_usage.total)
            self.assertAlmostEqual(usage.free, shutil_usage.free,
                                   delta=tolerance)
            self.assertAlmostEqual(usage.used, shutil_usage.used,
                                   delta=tolerance)

        # if path does not exist OSError ENOENT is expected across
        # all platforms
        fname = tempfile.mktemp()
        with self.assertRaises(OSError) as exc:
            psutil.disk_usage(fname)
        self.assertEqual(exc.exception.errno, errno.ENOENT) 

def opt_check_disk_space(warnlimit_mb=200):
    if disk_usage('.').free < warnlimit_mb*1024*2:
        log.warning("Less than %sMB of free space remains on this device" % warnlimit_mb)


################################################# 

def test_disk_usage(self):
        usage = shutil.disk_usage(os.getcwd())
        self.assertGreater(usage.total, 0)
        self.assertGreater(usage.used, 0)
        self.assertGreaterEqual(usage.free, 0)
        self.assertGreaterEqual(usage.total, usage.used)
        self.assertGreater(usage.total, usage.free) 

def test_module_all_attribute(self):
        self.assertTrue(hasattr(shutil, '__all__'))
        target_api = ['copyfileobj', 'copyfile', 'copymode', 'copystat',
                      'copy', 'copy2', 'copytree', 'move', 'rmtree', 'Error',
                      'SpecialFileError', 'ExecError', 'make_archive',
                      'get_archive_formats', 'register_archive_format',
                      'unregister_archive_format', 'get_unpack_formats',
                      'register_unpack_format', 'unregister_unpack_format',
                      'unpack_archive', 'ignore_patterns', 'chown', 'which',
                      'get_terminal_size', 'SameFileError']
        if hasattr(os, 'statvfs') or os.name == 'nt':
            target_api.append('disk_usage')
        self.assertEqual(set(shutil.__all__), set(target_api)) 

def stats(update, context):
    currentTime = get_readable_time((time.time() - botStartTime))
    total, used, free = shutil.disk_usage('.')
    total = get_readable_file_size(total)
    used = get_readable_file_size(used)
    free = get_readable_file_size(free)
    stats = f'Bot Uptime: {currentTime}\n' \
            f'Total disk space: {total}\n' \
            f'Used: {used}\n' \
            f'Free: {free}'
    sendMessage(stats, context.bot, update) 

def test_disk_usage_bytes(self):
        psutil.disk_usage(b'.') 

def request_fs_info(self, path):
        if path == "":
            self.notify_missing_selection()
        else:
            if not os.path.isdir(path):
                path = os.path.dirname(path)

            import shutil

            self.present_fs_info(shutil.disk_usage(path)._asdict()) 

def test_disk_usage_unicode(self):
        # See: https://github.com/giampaolo/psutil/issues/416
        if ASCII_FS:
            with self.assertRaises(UnicodeEncodeError):
                psutil.disk_usage(TESTFN_UNICODE) 

def get_disk_free(path):
    '''
    @type path: str

    @rtype tuple
    '''
    return (path, floor(float(shutil.disk_usage(path).free) / MB)) 

def __init__(self, preprocessing=None, postprocessing=None, detections_path=None, max_detections=100,
                 sort_by_prob=False, load_in_tmp=True):
        self.max_detections = max_detections
        self.detections_path = detections_path
        self.sort_by_prob = sort_by_prob

        tmp_detections_path = os.path.join('/tmp', os.path.basename(detections_path))

        if load_in_tmp:
            if not os.path.isfile(tmp_detections_path):
                if shutil.disk_usage("/tmp")[-1] < os.path.getsize(detections_path):
                    warnings.warn('Loading from %s, because /tmp has no enough space.' % detections_path)
                else:
                    warnings.warn("Copying detection file to /tmp")
                    shutil.copyfile(detections_path, tmp_detections_path)
                    warnings.warn("Done.")
                    self.detections_path = tmp_detections_path
            else:
                self.detections_path = tmp_detections_path

        super(ImageDetectionsField, self).__init__(preprocessing, postprocessing) 

def splitCopy(filepath, output_dir=""):
    fileSize = os.path.getsize(filepath)
    info = shutil.disk_usage(os.path.dirname(os.path.abspath(filepath)))
    if info.free < fileSize*2:
        print('Not enough free space to run. Will require twice the space as the NSP file\n')
        return
    print('Calculating number of splits...\n')
    splitNum = int(fileSize/splitSize)
    if splitNum == 0:
        print('This NSP is under 4GiB and does not need to be split.\n')
        return
    
    print('Splitting NSP into {0} parts...\n'.format(splitNum + 1))

    # Create directory, delete if already exists
    if output_dir == "":
        dir = filepath[:-4] + '_split.nsp'
    else:
        if output_dir[-4:] != '.nsp':
            output_dir+= ".nsp"
        dir = output_dir
    if os.path.exists(dir):
        shutil.rmtree(dir)
    os.makedirs(dir)

    remainingSize = fileSize

    # Open source file and begin writing to output files stoping at splitSize
    with open(filepath, 'rb') as nspFile:
        for i in range(splitNum + 1):
            partSize = 0
            print('Starting part {:02}'.format(i))
            outFile = os.path.join(dir, '{:02}'.format(i))
            with open(outFile, 'wb') as splitFile: 
                if remainingSize > splitSize:
                    while partSize < splitSize:
                        splitFile.write(nspFile.read(chunkSize))
                        partSize += chunkSize
                    remainingSize -= splitSize
                else:
                    while partSize < remainingSize:
                        splitFile.write(nspFile.read(chunkSize))
                        partSize += chunkSize
            print('Part {:02} complete'.format(i))
    print('\nNSP successfully split!\n') 

def info():
	
	# Verify that Docker is installed
	if DockerUtils.installed() == False:
		print('Error: could not detect Docker version. Please ensure Docker is installed.', file=sys.stderr)
		sys.exit(1)
	
	# Gather our information about the Docker daemon
	dockerInfo = DockerUtils.info()
	nvidiaDocker = platform.system() == 'Linux' and 'nvidia' in dockerInfo['Runtimes']
	maxSize = DockerUtils.maxsize()
	rootDir = dockerInfo['DockerRootDir']
	
	# If we are communicating with a Linux Docker daemon under Windows or macOS then we can't query the available disk space
	canQueryDisk = dockerInfo['OSType'].lower() == platform.system().lower()
	
	# Gather our information about the host system
	diskSpace = _formatSize(shutil.disk_usage(rootDir).free) if canQueryDisk == True else 'Unknown (typically means the Docker daemon is running in a Moby VM, e.g. Docker Desktop)'
	memPhysical = psutil.virtual_memory().total
	memVirtual = psutil.swap_memory().total
	cpuPhysical = psutil.cpu_count(False)
	cpuLogical = psutil.cpu_count()
	
	# Attempt to query PyPI to determine the latest version of ue4-docker
	# (We ignore any errors here to ensure the `ue4-docker info` command still works without network access)
	try:
		latestVersion = GlobalConfiguration.getLatestVersion()
	except:
		latestVersion = None
	
	# Prepare our report items
	items = [
		('ue4-docker version', '{}{}'.format(__version__, '' if latestVersion is None else ' (latest available version is {})'.format(latestVersion))),
		('Operating system', _osName(dockerInfo)),
		('Docker daemon version', dockerInfo['ServerVersion']),
		('NVIDIA Docker supported', 'Yes' if nvidiaDocker == True else 'No'),
		('Maximum image size', '{:.0f}GB'.format(maxSize) if maxSize != -1 else 'No limit detected'),
		('Available disk space', diskSpace),
		('Total system memory', '{} physical, {} virtual'.format(_formatSize(memPhysical), _formatSize(memVirtual))),
		('Number of processors', '{} physical, {} logical'.format(cpuPhysical, cpuLogical))
	]
	
	# Determine the longest item name so we can format our list in nice columns
	longestName = max([len(i[0]) for i in items])
	minSpaces = 4
	
	# Print our report
	for item in items:
		print('{}:{}{}'.format(
			item[0],
			' ' * ((longestName + minSpaces) - len(item[0])),
			item[1]
		)) 

def run(self):
		'''
		The resource monitor loop itself
		'''
		
		# Determine which filesystem the Docker daemon uses for storing its data directory
		dockerInfo = DockerUtils.info()
		rootDir = dockerInfo['DockerRootDir']
		
		# If we cannot access the Docker data directory (e.g. when the daemon is in a Moby VM), don't report disk space
		reportDisk = os.path.exists(rootDir)
		
		# Sample the CPU usage using an interval of 1 second the first time to prime the system
		# (See: <https://psutil.readthedocs.io/en/latest/#psutil.cpu_percent>)
		psutil.cpu_percent(1.0)
		
		# Loop until asked to stop
		while True:
			
			# Check that the thread has not been asked to stop
			with self._lock:
				if self._shouldStop == True:
					return
			
			# Format the timestamp for the current time in ISO 8601 format (albeit without the "T" separator)
			isoTime = datetime.datetime.now().replace(microsecond=0).isoformat(' ')
			
			# We format data sizes using binary units (KiB, MiB, GiB, etc.)
			formatSize = lambda size: humanfriendly.format_size(size, binary=True, keep_width=True)
			
			# Format the current quantity of available disk space on the Docker data directory's filesystem
			diskSpace = formatSize(shutil.disk_usage(rootDir).free) if reportDisk == True else 'Unknown'
			
			# Format the current quantity of available system memory
			physicalMemory = formatSize(psutil.virtual_memory().free)
			virtualMemory = formatSize(psutil.swap_memory().free)
			
			# Format the current CPU usage levels
			cpu = psutil.cpu_percent()
			
			# Report the current levels of our available resources
			self._logger.info('[{}] [Available disk: {}] [Available memory: {} physical, {} virtual] [CPU usage: {:.2f}%]'.format(
				isoTime,
				diskSpace,
				physicalMemory,
				virtualMemory,
				cpu
			), False)
			
			# Sleep until the next sampling interval
			time.sleep(self._interval) 

def test_disk_partitions(self):
        # all = False
        ls = psutil.disk_partitions(all=False)
        # on travis we get:
        #     self.assertEqual(p.cpu_affinity(), [n])
        # AssertionError: Lists differ: [0, 1, 2, 3, 4, 5, 6, 7,... != [0]
        self.assertTrue(ls, msg=ls)
        for disk in ls:
            self.assertIsInstance(disk.device, str)
            self.assertIsInstance(disk.mountpoint, str)
            self.assertIsInstance(disk.fstype, str)
            self.assertIsInstance(disk.opts, str)
            if WINDOWS and 'cdrom' in disk.opts:
                continue
            if not POSIX:
                assert os.path.exists(disk.device), disk
            else:
                # we cannot make any assumption about this, see:
                # http://goo.gl/p9c43
                disk.device
            if SUNOS or TRAVIS:
                # on solaris apparently mount points can also be files
                assert os.path.exists(disk.mountpoint), disk
            else:
                assert os.path.isdir(disk.mountpoint), disk
            assert disk.fstype, disk

        # all = True
        ls = psutil.disk_partitions(all=True)
        self.assertTrue(ls, msg=ls)
        for disk in psutil.disk_partitions(all=True):
            if not WINDOWS:
                try:
                    os.stat(disk.mountpoint)
                except OSError as err:
                    if TRAVIS and OSX and err.errno == errno.EIO:
                        continue
                    # http://mail.python.org/pipermail/python-dev/
                    #     2012-June/120787.html
                    if err.errno not in (errno.EPERM, errno.EACCES):
                        raise
                else:
                    if SUNOS or TRAVIS:
                        # on solaris apparently mount points can also be files
                        assert os.path.exists(disk.mountpoint), disk
                    else:
                        assert os.path.isdir(disk.mountpoint), disk
            self.assertIsInstance(disk.fstype, str)
            self.assertIsInstance(disk.opts, str)

        def find_mount_point(path):
            path = os.path.abspath(path)
            while not os.path.ismount(path):
                path = os.path.dirname(path)
            return path.lower()

        mount = find_mount_point(__file__)
        mounts = [x.mountpoint.lower() for x in
                  psutil.disk_partitions(all=True)]
        self.assertIn(mount, mounts)
        psutil.disk_usage(mount) 

def test_disk_usage_bytes(self):
        psutil.disk_usage(b'.')