Python pkg_resources.resource_listdir() Examples

def _fuzzdb_get_strings(max_len=0):
    'Helper to get all the strings from fuzzdb'

    ignored = ['integer-overflow']

    for subdir in pkg_resources.resource_listdir('protofuzz', BASE_PATH):
        if subdir in ignored:
            continue

        path = '{}/{}'.format(BASE_PATH, subdir)
        listing = pkg_resources.resource_listdir('protofuzz', path)
        for filename in listing:
            if not filename.endswith('.txt'):
                continue

            path = '{}/{}/{}'.format(BASE_PATH, subdir, filename)
            source = _open_fuzzdb_file(path)
            for line in source:
                string = line.decode('utf-8').strip()
                if not string or string.startswith('#'):
                    continue
                if max_len != 0 and len(line) > max_len:
                    continue

                yield string 

def __get_resource_list(*path_parts):
    """ Return the list of elements in a directory based on whether its frozen or not.

    Paths parts given should be relative to the pympress package dir.

    Args:
        name (`tuple` of `str`): The directories that constitute the path to the resource,
        relative to the pympress distribution

    Returns:
        `list` of `str`: The paths to the resources in the directory
    """
    if getattr(sys, 'frozen', False):
        return os.listdir(os.path.join(os.path.dirname(sys.executable), *path_parts))
    else:
        req = pkg_resources.Requirement.parse('pympress')
        return pkg_resources.resource_listdir(req, '/'.join(('pympress',) + path_parts)) 

def image_resources(package=None, directory='resources'):
    """
    Returns all images under the directory relative to a package path. If no directory or package is specified then the
    resources module of the calling package will be used. Images are recursively discovered.

    :param package: package name in dotted format.
    :param directory: path relative to package path of the resources directory.
    :return: a list of images under the specified resources path.
    """
    if not package:
        package = calling_package()
    package_dir = '.'.join([package, directory])
    images = []
    for i in resource_listdir(package, directory):
        if i.startswith('__') or i.endswith('.egg-info'):
            continue
        fname = resource_filename(package_dir, i)
        if resource_isdir(package_dir, i):
            images.extend(image_resources(package_dir, i))
        elif what(fname):
            images.append(fname)
    return images


# etc 

def _unpack_assets(output_dir, module, asset_root, execute, current_path):
  """
    The internal helper function for unpack_assets(...) recursion.
    :param current_path: Records the current
  """
  for asset in pkg_resources.resource_listdir(module, current_path):
    asset_target = os.path.join(os.path.relpath(current_path, asset_root), asset)
    if pkg_resources.resource_isdir(module, os.path.join(current_path, asset)):
      safe_mkdir(os.path.join(output_dir, asset_target))
      _unpack_assets(output_dir, module, asset_root, execute, os.path.join(current_path, asset))
    else:
      output_file = os.path.join(output_dir, asset_target)
      with open(output_file, 'wb') as fp:
        fp.write(pkg_resources.resource_string(
            module, os.path.join(asset_root, asset_target)))
        execute(output_file) 

def get(name):
    if not has(name):
        data = None
        dirname = name + "_registry"
        for fname in sorted(resource_listdir(__name__, dirname)):
            if os.path.splitext(fname)[1] != ".json":
                continue
            fname = resource_filename(__name__, os.path.join(dirname, fname))
            with open(fname, "r") as fp:
                chunk = json.load(fp)
                if data is None:
                    data = chunk
                elif isinstance(data, list):
                    data.extend(chunk)
                elif isinstance(data, dict):
                    data.updated(chunk)
        if data is None:
            raise ValueError("Failed to load registry {}".format(name))
        save(name, data)
    return _registry[name] 

def fetch_schema_cityobjects(self, folder_schemas=None):
        if folder_schemas is None:
            #-- fetch proper schema from the stored ones 
            tmp = resource_listdir(__name__, '/schemas/')
            tmp.sort()
            v = tmp[-1]
            try:
                schema = resource_filename(__name__, '/schemas/%s/cityjson.schema.json' % (v))
            except:
                return (False, None)
        else:
            schema = os.path.join(folder_schemas, 'cityjson.schema.json')  
        abs_path = os.path.abspath(os.path.dirname(schema))
        sco_path = abs_path + '/cityobjects.schema.json'
        #-- because Windows uses \ and not /        
        if platform == "darwin" or platform == "linux" or platform == "linux2":
            base_uri = 'file://{}/'.format(abs_path)
        else:
            base_uri = 'file:///{}/'.format(abs_path.replace('\\', '/'))
        jsco = jsonref.loads(open(sco_path).read(), jsonschema=True, base_uri=base_uri)
        # jsco = json.loads(open(sco_path).read())
        return (True, jsco) 

def get_migrations():
    """Return a sorted list of versions and migration modules."""
    migrations = []
    for file_ in pkg_resources.resource_listdir(
        'renku.core.management', 'migrations'
    ):
        match = re.search(r'm_([0-9]{4})__[a-zA-Z0-9_-]*.py', file_)

        if match is None:  # migration files match m_0000__[name].py format
            continue

        version = int(match.groups()[0])
        path = 'renku.core.management.migrations.{}'.format(Path(file_).stem)
        migrations.append((version, path))

    migrations = sorted(migrations, key=lambda v: v[1].lower())
    return migrations 

def manpages_helper(self):
        """Verify or install COT's manual pages.

        Returns:
          tuple: (result, message)
        """
        try:
            resource_listdir("COT", "docs/man")
        except OSError as exc:
            return False, "UNABLE TO FIND PAGES: " + str(exc)

        man_dir = guess_manpath()

        if self.verify_only:
            return verify_manpages(man_dir)
        else:
            return install_manpages(man_dir) 

def _stage_transperf_src(tmp=None):
    """Stages transperf src in tmpdir before copy to destination node."""
    listfn = lambda: pkg_resources.resource_listdir('transperf', '')
    readfn = lambda fname: pkg_resources.resource_string('transperf', fname)

    to_sync = []
    if tmp is None:
        tmp = os.path.join(tempfile.gettempdir(), 'transperf')
        if not os.path.exists(tmp):
            os.mkdir(tmp)

    for f in listfn():
        _, ext = os.path.splitext(f)
        if ext != '.py':
            continue
        content = readfn(f)
        path = os.path.join(tmp, f)
        tempf = open(path, 'w')
        tempf.write(content)
        tempf.close()
        to_sync.append(path)
    return tmp, to_sync 

def _get_standard_message_registry_collection(self):
        """Load packaged standard message registries

        :returns: list of MessageRegistry
        """

        message_registries = []
        resource_package_name = __name__
        for json_file in pkg_resources.resource_listdir(
                resource_package_name, STANDARD_REGISTRY_PATH):
            # Not using path.join according to pkg_resources docs
            mes_reg = message_registry.MessageRegistry(
                None, STANDARD_REGISTRY_PATH + json_file,
                reader=base.JsonPackagedFileReader(
                    resource_package_name))
            message_registries.append(mes_reg)

        return message_registries 

def __init__(self, package_paths=None):
        # Array of gpkg names to SingleMPackage objects
        self._hmms_and_positions = {}

        if package_paths:
            self.singlem_packages = [SingleMPackage.acquire(path) for path in package_paths]
            logging.info("Loaded %i SingleM packages" % len(self.singlem_packages))
        else:
            # Prefer production DB directory
            pkg_resources_db_directory = 'data'

            pkg_paths = pkg_resources.resource_listdir('singlem',pkg_resources_db_directory)
            basedir = pkg_resources.resource_filename('singlem',pkg_resources_db_directory)
            logging.debug("Searching for SingleM packages via pkg_resources in %s .." % basedir)
            pkg_paths = [os.path.join(basedir,d) for d in pkg_paths if d[-5:]=='.spkg']
            if len(pkg_paths) == 0:
                raise Exception("Unable to find any SingleM packages using pkg_resources")

            logging.debug("Found %i SingleM packages: %s" % (len(pkg_paths),
                                                        ', '.join(pkg_paths)))
            self.singlem_packages = [SingleMPackage.acquire(path) for path in pkg_paths]

        for pkg in self.singlem_packages:
            self._hmms_and_positions[pkg.base_directory()] = pkg 

def scan_subpackages(cls, package: str) -> Sequence[str]:
        """Return a list of sub-packages defined under a named package."""
        # FIXME use importlib.resources in python 3.7
        if "." in package:
            package, sub_pkg = package.split(".", 1)
        else:
            sub_pkg = "."
        if not pkg_resources.resource_isdir(package, sub_pkg):
            raise ModuleLoadError(f"Undefined package {package}")
        found = []
        joiner = "" if sub_pkg == "." else f"{sub_pkg}."
        for sub_path in pkg_resources.resource_listdir(package, sub_pkg):
            if pkg_resources.resource_exists(
                package, f"{sub_pkg}/{sub_path}/__init__.py"
            ):
                found.append(f"{package}.{joiner}{sub_path}")
        return found 

def __init__(self, root):
        """
        :param root: watchdog base directory
        :param reboot_script: reboot script
        """
        self.root = root
        self.cpu_count = multiprocessing.cpu_count()
        self.pid_file = '/var/run/watchdog.pid'
        self.config_file = os.path.join(self.root, 'watchdog.conf')
        self.script_directory = os.path.join(self.root, 'script')
        self.test_directory = os.path.join(self.root, 'watchdog.d')
        self.tests = {}
        utf8_reader = codecs.getreader('utf8')
        test_names = pkg_resources.resource_listdir(
            'treadmill', '/etc/kernel_watchdog_tests'
        )
        for test_name in test_names:
            self.tests[test_name] = utf8_reader(
                pkg_resources.resource_stream(
                    'treadmill',
                    '/etc/kernel_watchdog_tests/{}'.format(test_name)
                )
            ).read()
        self.start_command = ['watchdog', '-c', self.config_file, '-b'] 

def _package_classpath():
	jars = []
	resources = pkg_resources.resource_listdir("sklearn2pmml.resources", "")
	for resource in resources:
		if resource.endswith(".jar"):
			jars.append(pkg_resources.resource_filename("sklearn2pmml.resources", resource))
	return jars 

def _get_settings(self):
        """
        Return any settings defined by the user, as well as any pre-defined
        settings files that exist for the image modalities to be registered.
        """
        # If user-defined settings exist...
        if isdefined(self.inputs.settings):
            # Note this in the log and return those settings.
            NIWORKFLOWS_LOG.info("User-defined settings, overriding defaults")
            return self.inputs.settings

        # Define a prefix for output files based on the modality of the moving image.
        filestart = "{}-mni_registration_{}_".format(
            self.inputs.moving.lower(), self.inputs.flavor
        )

        # Get a list of settings files that match the flavor.
        filenames = [
            i
            for i in pkgr.resource_listdir("niworkflows", "data")
            if i.startswith(filestart) and i.endswith(".json")
        ]
        # Return the settings files.
        return [
            pkgr.resource_filename("niworkflows.data", f) for f in sorted(filenames)
        ] 

def _get_global_builders():
    """Find builders defined globally
    """
    res = {}

    for name in pkg_resources.resource_listdir('pkgpanda', DOCKERFILE_DIR):
        res[name] = pkg_resources.resource_filename('pkgpanda',
                                                    DOCKERFILE_DIR + name)
    return res 

def get_effective_genome_length(genome, read_length):
    # type: (str, int) -> float

    genome_names = pkg_resources.resource_listdir("epic",
                                                  "scripts/effective_sizes")
    name_dict = {n.split("_")[0]: "".join(n.split("_")[:-1])
                 for n in genome_names}

    try:
        genome_exact = name_dict[genome.lower()]
        egf = pkg_resources.resource_string( # type: ignore
            "epic", "scripts/effective_sizes/{}_{}.txt".format(
                genome_exact, read_length)).split()[-1].decode()
    except KeyError:
        genome_list = "\n".join(list(name_dict.keys()))
        logging.error(
            "Genome " + genome +
            " not found.\n These are the available genomes: " + genome_list +
            "\nIf yours is not there, please request it at github.com/endrebak/epic .")

    genome_length = sum(create_genome_size_dict(genome).values())

    logging.info("Using an effective genome fraction of {}.".format(egf))

    assert float(
        egf) < 1, "Something wrong happened, effective genome fraction over 1!"

    egs = float(egf) * genome_length

    return egs 

def get_all_test_configs():
    return resource_listdir(__name__, 'test_configs') 

def resources_for_test_config(test_config):
    resources = {}
    for key in [constants.CONFIG_BUNDLES_KEY, 'apps', 'libs', 'services']:
        key_path = 'test_configs/{}/{}'.format(test_config, key)
        if resource_isdir(__name__, key_path):
            resources[key] = {resource_name: resource_string(__name__, '{}/{}'.format(key_path, resource_name))
                              for resource_name in resource_listdir(__name__, key_path)}
    return resources 

def get_genome_size_file(genome):
    # type: (str) -> str

    genome_names = pkg_resources.resource_listdir("epic", "scripts/chromsizes")
    name_dict = {n.lower().replace(".chromsizes", ""): n for n in genome_names}

    # No try/except here, because get_egs would already have failed if genome
    # did not exist
    genome_exact = name_dict[genome.lower()]

    return pkg_resources.resource_filename(
        "epic", "scripts/chromsizes/{}".format(genome_exact)) 

def list_api_descriptions():
    """List API description documents.

    They're searched for in the same directory as this file, and their name
    must match "apiXX.json" where "XX" denotes the major and minor version
    number of the API.
    """
    for filename in resource_listdir(__name__, "."):
        match = re.match(r"api(\d)(\d)[.]json", filename)
        if match is not None:
            version = tuple(map(int, match.groups()))
            path = resource_filename(__name__, filename)
            name = "%d.%d" % version
            yield name, version, Path(path) 

def list_locales():
    """Return a list of available locales in xclim."""
    locale_list = pkg_resources.resource_listdir("xclim.locales", "")
    return [locale.split(".")[0] for locale in locale_list if locale.endswith(".json")] 

def get_robot_tests(cat):
    tests = []
    dir = os.path.dirname(os. path.abspath(__file__))
    path = "{}/{}".format(dir, cat)
    for item in os.listdir(path):
        if os.path.isdir('{}/{}'.format(path, item)):
             tests.extend(f for f in resource_listdir("baobab.lims.tests.{}".format(cat), item)
                if f.endswith(".robot"))            
    return tests 

def find_components(package, base_class):
    """Find components which are subclass of a given base class.
    """
    for filename in resource_listdir(package, ''):
        basename, extension = os.path.splitext(filename)
        if extension != '.py' or basename.startswith('.'):
            continue
        module_name = "{}.{}".format(package, basename)
        __import__(module_name, fromlist='*')
        module = sys.modules[module_name]
        if not hasattr(module, '__all__'):
            continue
        yield from scan_module(module, base_class) 

def list_logging_conf():
    """List all defined logging configurations."""
    import pkg_resources

    configs = set()
    for plugin in plugin_manager.load_all(__name__):
        configs.update({
            cfg for cfg in pkg_resources.resource_listdir(__name__, '.')
            if cfg.endswith('.json')
        })

    return configs 

def list_main(argv_unused):  # pylint: disable=unused-argument
    """
    list
        List the FILENAMEs that edx_lint can provide.
    """
    print("edx_lint knows about these files:")
    for filename in pkg_resources.resource_listdir("edx_lint", "files"):
        print(filename)

    return 0 

def load_logging_conf(name):
    """load plugin log conf from various modules
    """
    # Shortcut - check if logging already exists.
    logconf_path = os.path.join(
        os.environ.get('TREADMILL_APPROOT', ''),
        'logging',
        name
    )

    if os.path.exists(logconf_path):
        with io.open(logconf_path) as f:
            return json.loads(f.read())

    # load core logging config first
    conf = _load_logging_file(__name__, name)
    # get 'treadmill' default log configure
    default_conf = conf['loggers'].get(_package_root(__name__), {})

    # then load plugin component config
    import pkg_resources

    for plugin in plugin_manager.load_all(__name__):

        contents = pkg_resources.resource_listdir(__name__, '.')
        try:
            plugin_conf = _load_logging_file(plugin.__name__, name)

            # TODO: deep merge conf
            conf['loggers'].update(plugin_conf.get('loggers', {}))
        except FileNotFoundError as _e:
            # it is possible for module to be lack of specific log file
            # e.g. some module does not have daemon logging configuration
            # we use default configure for it
            plugin_package_root_name = _package_root(plugin.__name__)
            conf['loggers'][plugin_package_root_name] = default_conf

    return conf 

def builtin_list():
    """Show a listing of all our builtin templates"""
    for template in resource_listdir(__name__, "templates"):
        builtin, ext = os.path.splitext(os.path.basename(abspath(template)))
        if ext == '.yml':
            continue

        help_obj = load_template_help(builtin)
        if 'name' in help_obj:
            print("%-*s %s" % (20, builtin, help_obj['name']))
        else:
            print("%s" % builtin) 

def get_all_ini_files(base_ini_dir):
    file_names = []
    ini_files = pkg_resources.resource_listdir(__name__, base_ini_dir)
    for f in ini_files:
        file_path = pkg_resources.resource_filename(__name__, base_ini_dir + '/' + f)
        if file_path[-4:] == '.ini':
            file_names.append(file_path)
    return file_names 

def adjust_settings_paths(self, paths):
        base = resource_filename('avocado_vt', 'conf.d')
        for path in [os.path.join(base, conf)
                     for conf in resource_listdir('avocado_vt', 'conf.d')
                     if conf.endswith('.conf')]:
            paths.insert(0, path)