Python pkgutil.walk_packages() Examples

def pkg_submodules(package, recursive=True):
    """ Return a list of all submodules in a given package, recursively by default """

    if isinstance(package, str):
        try:
            package = importlib.import_module(package)
        except ImportError:
            return []

    submodules = []
    for _loader, name, is_pkg in pkgutil.walk_packages(package.__path__):
        full_name = package.__name__ + "." + name

        try:
            submodules.append(importlib.import_module(full_name))
        except ImportError:
            continue

        if recursive and is_pkg:
            submodules += pkg_submodules(full_name)

    return submodules 

def load(path):
    # Path needs to be a list in order for the following calls to work.
    if type(path) != list: path = [path]
    # Walk the directory.
    for loader, name, ispkg in pkgutil.walk_packages(path):
        module = loader.find_module(name).load_module(name)

        # Try to find the module's defined class, and add it to the
        # list.
        qname = name + "Question"
        try:
            if qname in dir(module):
                categories[qname.lower()] = module.__dict__[qname]
            else:
                raise NoQuestionInModuleException("No class %s" % qname)

        except Exception as e:
            print("Skipping module {}: {}".format(name, e))

# Load all of the questions in this path. 

def load(self, container):
        await Module.load(self, container)
        loadmodules = []
        for p in self.autoloadpackages:
            try:
                def onerror(name):
                    self._logger.warning("Autoload package %r on package %r failed", name, p)
                pkg = __import__(p, fromlist = ('dummy',))
                for _, name, ispkg in pkgutil.walk_packages(pkg.__path__, p + '.', onerror):
                    if not ispkg:
                        try:
                            pymod = __import__(name, fromlist = ('dummy',))
                            for m in vars(pymod).values():
                                if isinstance(m, type) and issubclass(m, Module) and getattr(m, '__module__', '') == name:
                                    loadmodules.append(m)
                        except Exception:
                            self._logger.warning('Autoload module %r failed', name, exc_info = True)
            except Exception:
                self._logger.warning('Autoload package %r failed', p, exc_info = True)
        if loadmodules:
            await container.execute_all([self.server.moduleloader.loadmodule(m) for m in loadmodules])
        await self.changestate(ModuleLoadStateChanged.SUCCEEDED, container) 

def get_package_libraries(pkg):
    """
    Recursively yield template tag libraries defined in submodules of a
    package.
    """
    for entry in walk_packages(pkg.__path__, pkg.__name__ + '.'):
        try:
            module = import_module(entry[1])
        except ImportError as e:
            raise InvalidTemplateLibrary(
                "Invalid template library specified. ImportError raised when "
                "trying to load '%s': %s" % (entry[1], e)
            )

        if hasattr(module, 'register'):
            yield entry[1] 

def load_tests(loader, tests, *args):
    for _, name, _ in pkgutil.walk_packages(n6sdk._ABS_PATH):
        try:
            mod_suite = doctest.DocTestSuite(name)
        except ValueError as exc:
            try:
                msg = getattr(exc, 'args', ())[1]
            except (IndexError, TypeError):
                msg = None
            if msg != 'has no tests':
                raise
        else:
            tests.addTests(mod_suite)
    return tests


# dissuade nose from using that function
# (note that nose has its own ways to discover doctests) 

def test_init_docstring_empty(self):
        errors = []
        root = chainer.__path__
        for loader, modname, ispkg in pkgutil.walk_packages(root, 'chainer.'):
            # Skip modules generated by protobuf.
            if '_pb2' in modname:
                continue

            try:
                mod = importlib.import_module(modname)
            except ImportError:
                continue

            self.check_init_docstring(mod, errors)

        if errors:
            msg = ''
            for mod, value, init_doc in errors:
                msg += '{}.{} has __init__.__doc__:\n{}\n\n'.format(
                    mod.__name__, value, init_doc)
            self.fail(msg) 

def import_submodules(package, recursive=True):
    """ Import all submodules of a module, recursively, including subpackages
    :param package: package (name or actual module)
    :type package: str | module
    :rtype: dict[str, types.ModuleType]
    """

    if isinstance(package, str):
        package = importlib.import_module(package)
    results = {}
    for loader, name, is_pkg in pkgutil.walk_packages(package.__path__):
        full_name = package.__name__ + '.' + name
        try:
            results[full_name] = importlib.import_module(full_name)
        except:
            pass
        if recursive and is_pkg:
            results.update(import_submodules(full_name))
    return results 

def import_submodules(package):
    """Import all submodules of a module, recursively, including subpackages.

    :param package: package (name or actual module)
    :type package: str | module
    :rtype: dict[str, types.ModuleType]
    """
    if isinstance(package, str):
        package = importlib.import_module(package)
    results = {}
    for loader, name, is_pkg in pkgutil.walk_packages(package.__path__):
        full_name = package.__name__ + '.' + name
        try:
            results[full_name] = importlib.import_module(full_name)
            if is_pkg:
                results.update(import_submodules(full_name))
        except ModuleNotFoundError as mnfe:
            print("module not found: {}".format(full_name))
            capture_exception(mnfe)
        except Exception as general_exception:
            print(general_exception)
            capture_exception(general_exception)
    return results 

def prepare_mainsubstituter():
    """Prepare and return a |Substituter| object for the main `__init__`
    file of *HydPy*."""
    substituter = Substituter()
    for module in (builtins, numpy, datetime, unittest, doctest, inspect, io,
                   os, sys, time, collections, itertools, subprocess, scipy,
                   typing, platform, math, mimetypes):
        substituter.add_module(module)
    for subpackage in (auxs, core, cythons, exe):
        for _, name, _ in pkgutil.walk_packages(subpackage.__path__):
            full_name = subpackage.__name__ + '.' + name
            substituter.add_module(importlib.import_module(full_name))
    substituter.add_module(examples)
    substituter.add_modules(models)
    for cymodule in (annutils, smoothutils, pointerutils):
        substituter.add_module(cymodule, cython=True)
    substituter.short2long['|pub|'] = ':mod:`~hydpy.pub`'
    substituter.short2long['|config|'] = ':mod:`~hydpy.config`'
    return substituter 

def test_tabs():
    """Test that there are no tabs in our source files."""
    ignore = _tab_ignores[:]

    for importer, modname, ispkg in walk_packages(celer.__path__,
                                                  prefix='celer.'):
        if not ispkg and modname not in ignore:
            # mod = importlib.import_module(modname)  # not py26 compatible!
            try:
                with warnings.catch_warnings(record=True):  # traits
                    __import__(modname)
            except Exception:  # can't import properly
                continue
            mod = sys.modules[modname]
            try:
                source = getsource(mod)
            except IOError:  # user probably should have run "make clean"
                continue
            assert '\t' not in source, ('"%s" has tabs, please remove them '
                                        'or add it to the ignore list'
                                        % modname) 

def test_all_tests_are_importable():
    # Ensure that for each contentful subpackage, there is a test directory
    # within it that is also a subpackage (i.e. a directory with __init__.py)

    HAS_TESTS_EXCEPTIONS = re.compile(r'''(?x)
                                      \.externals(\.|$)|
                                      \.tests(\.|$)|
                                      \._
                                      ''')
    lookup = {name: ispkg
              for _, name, ispkg
              in pkgutil.walk_packages(sklearn.__path__, prefix='sklearn.')}
    missing_tests = [name for name, ispkg in lookup.items()
                     if ispkg
                     and not HAS_TESTS_EXCEPTIONS.search(name)
                     and name + '.tests' not in lookup]
    assert_equal(missing_tests, [],
                 '{0} do not have `tests` subpackages. Perhaps they require '
                 '__init__.py or an add_subpackage directive in the parent '
                 'setup.py'.format(missing_tests)) 

def sub_modules(settings_path=None):
    """
    In the current PYTHONPATH, we can traverse all modules and determine if they
    have a settings.py or directory with a subs.py module. If either one of
    those exists, we import it, and return the settings module, and
    paths to the subs file.

    If a settings module is not found, we return None.

    :return: (settings module, List[string: subs module paths])
    """
    module_paths = []
    for f, package, is_package in pkgutil.walk_packages(path=["."]):
        if package == "settings":
            settings_path = package
        if is_package and module_has_submodule(package, "subs"):
            module = package + ".subs"
            module_paths.append(module)
            print(" * Discovered subs module: %r" % module)

    settings = _import_settings_from_path(settings_path)
    return settings, module_paths 

def test_import_all_consistency():
    # Smoke test to check that any name in a __all__ list is actually defined
    # in the namespace of the module or package.
    pkgs = pkgutil.walk_packages(path=sklearn.__path__, prefix='sklearn.',
                                 onerror=lambda _: None)
    submods = [modname for _, modname, _ in pkgs]
    for modname in submods + ['sklearn']:
        if ".tests." in modname:
            continue
        if IS_PYPY and ('_svmlight_format' in modname or
                        'feature_extraction._hashing' in modname):
            continue
        package = __import__(modname, fromlist="dummy")
        for name in getattr(package, '__all__', ()):
            if getattr(package, name, None) is None:
                raise AttributeError(
                    "Module '{0}' has no attribute '{1}'".format(
                        modname, name)) 

def main():
  results = {'codes': {0: 0, 1: 0}, 'errors': 0}
  for _, test, _ in pkgutil.walk_packages(glazier.__path__,
                                          glazier.__name__ + '.'):
    if '_test' in test:
      print('**** %s ****\n' % test)
      proc = subprocess.Popen(['python', '-m', test], stderr=subprocess.PIPE)
      _, err = proc.communicate()
      err = err.decode()
      print(err)
      failed = FAILED_RE.search(err)
      if failed:
        results['errors'] += int(failed.group(1))
      results['codes'][proc.returncode] = results['codes'].setdefault(
          proc.returncode, 0) + 1

  print('Success: %s' % results['codes'][0])
  print('Failure: %s' % results['codes'][1])
  sys.exit(results['codes'][1]) 

def list_proxy(root_package = 'vlcp'):
    '''
    Walk through all the sub modules, find subclasses of vlcp.server.module._ProxyModule,
    list their default values
    '''
    proxy_dict = OrderedDict()
    pkg = __import__(root_package, fromlist=['_'])
    for imp, module, _ in walk_packages(pkg.__path__, root_package + '.'):
        m = __import__(module, fromlist = ['_'])
        for _, v in vars(m).items():
            if v is not None and isinstance(v, type) and issubclass(v, _ProxyModule) \
                    and v is not _ProxyModule \
                    and v.__module__ == module \
                    and hasattr(v, '_default'):
                name = v.__name__.lower()
                if name not in proxy_dict:
                    proxy_dict[name] = {'defaultmodule': v._default.__name__.lower(),
                                        'class': repr(v._default.__module__ + '.' + v._default.__name__)}
    return proxy_dict 

def setUp(self):

        self.dev_null = open(os.devnull, 'w')

        # self.spec = {"modules": {"PlanetPhysicalModel": "PlanetPhysicalModel"}}
        self.script = resource_path('test-scripts/template_minimal.json')
        with open(self.script) as f:
            self.spec = json.loads(f.read())

        modtype = getattr(EXOSIMS.Prototypes.Observatory.Observatory, '_modtype')
        pkg = EXOSIMS.Observatory
        self.allmods = [get_module(modtype)]
        for loader, module_name, is_pkg in pkgutil.walk_packages(pkg.__path__, pkg.__name__ + '.'):
            if not is_pkg:
                mod = get_module(module_name.split('.')[-1], modtype)
                self.assertTrue(mod._modtype is modtype, '_modtype mismatch for %s' % mod.__name__)
                self.allmods.append(mod) 

def load_tests(loader, tests, ignore):
    # This function loads doctests from all submodules and runs them
    # with the __future__ imports necessary for Python 2
    for _, module, _ in pkgutil.walk_packages(path=fuel.__path__,
                                              prefix=fuel.__name__ + '.'):
        try:
            tests.addTests(doctest.DocTestSuite(
                module=importlib.import_module(module), setUp=setup,
                optionflags=doctest.IGNORE_EXCEPTION_DETAIL,
                checker=Py23DocChecker()))
        except:
            pass

    # This part loads the doctests from the documentation
    docs = []
    for root, _, filenames in os.walk(os.path.join(fuel.__path__[0],
                                                   '../docs')):
        for doc in fnmatch.filter(filenames, '*.rst'):
            docs.append(os.path.abspath(os.path.join(root, doc)))
    tests.addTests(doctest.DocFileSuite(
        *docs, module_relative=False, setUp=setup,
        optionflags=doctest.IGNORE_EXCEPTION_DETAIL,
        checker=Py23DocChecker()))

    return tests 

def setUp(self):

        self.dev_null = open(os.devnull, 'w')
        self.script = resource_path('test-scripts/template_minimal.json')
        with open(self.script) as f:
            self.spec = json.loads(f.read())
        
        with RedirectStreams(stdout=self.dev_null):
            self.TL = TargetList(ntargs=10,**copy.deepcopy(self.spec))
        self.TL.dist = np.random.uniform(low=0,high=100,size=self.TL.nStars)*u.pc
        
        modtype = getattr(Completeness,'_modtype')
        pkg = EXOSIMS.Completeness
        self.allmods = [get_module(modtype)]
        for loader, module_name, is_pkg in pkgutil.walk_packages(pkg.__path__, pkg.__name__+'.'):
            if (not 'starkAYO' in module_name) and not is_pkg:
                mod = get_module(module_name.split('.')[-1],modtype)
                self.assertTrue(mod._modtype is modtype,'_modtype mismatch for %s'%mod.__name__)
                self.allmods.append(mod) 

def setUp(self):
        self.dev_null = open(os.devnull, 'w')
        self.script = resource_path('test-scripts/template_prototype_testing.json')
        with open(self.script) as f:
            self.spec = json.loads(f.read())

        with RedirectStreams(stdout=self.dev_null):
            self.sim = MissionSim.MissionSim(self.script)
        self.TL = self.sim.TargetList
        self.nStars = self.TL.nStars
        self.star_index = np.array(range(0, self.nStars))
        self.Obs = self.sim.Observatory
        self.mode = self.sim.OpticalSystem.observingModes[0]
        self.TK = self.sim.TimeKeeping
        assert self.nStars > 10, "Need at least 10 stars in the target list for the unit test."
        self.unit = 1./u.arcsec**2

        modtype = getattr(EXOSIMS.Prototypes.ZodiacalLight.ZodiacalLight, '_modtype')
        pkg = EXOSIMS.ZodiacalLight
        self.allmods = [get_module(modtype)]
        for loader, module_name, is_pkg in pkgutil.walk_packages(pkg.__path__, pkg.__name__ + '.'):
            if not is_pkg:
                mod = get_module(module_name.split('.')[-1], modtype)
                self.assertTrue(mod._modtype is modtype, '_modtype mismatch for %s' % mod.__name__)
                self.allmods.append(mod) 

def setUp(self):

        self.dev_null = open(os.devnull, 'w')
        self.script = resource_path('test-scripts/template_minimal.json')
        with open(self.script) as f:
            self.spec = json.loads(f.read())
        
        with RedirectStreams(stdout=self.dev_null):
            self.TL = TargetList(ntargs=10,**copy.deepcopy(self.spec))
        self.TL.dist = np.random.uniform(low=0,high=100,size=self.TL.nStars)*u.pc
        
        modtype = getattr(SimulatedUniverse,'_modtype')
        pkg = EXOSIMS.SimulatedUniverse
        self.allmods = [get_module(modtype)]
        for loader, module_name, is_pkg in pkgutil.walk_packages(pkg.__path__, pkg.__name__+'.'):
            if not is_pkg:
                mod = get_module(module_name.split('.')[-1],modtype)
                self.assertTrue(mod._modtype is modtype,'_modtype mismatch for %s'%mod.__name__)
                self.allmods.append(mod) 

def setUp(self):

        self.dev_null = open(os.devnull, 'w')
        self.script = resource_path('test-scripts/template_minimal.json')
        with open(self.script) as f:
            self.spec = json.loads(f.read())
        
        with RedirectStreams(stdout=self.dev_null):
            self.TL = TargetList(ntargs=10,**copy.deepcopy(self.spec))
        
        modtype = getattr(OpticalSystem,'_modtype')
        pkg = EXOSIMS.OpticalSystem
        self.allmods = [get_module(modtype)]
        for loader, module_name, is_pkg in pkgutil.walk_packages(pkg.__path__, pkg.__name__+'.'):
            if not is_pkg:
                mod = get_module(module_name.split('.')[-1],modtype)
                self.assertTrue(mod._modtype is modtype,'_modtype mismatch for %s'%mod.__name__)
                self.allmods.append(mod) 

def load_tests(loader, tests, ignore):
    # This function loads doctests from all submodules and runs them
    # with the __future__ imports necessary for Python 2
    for _, module, _ in pkgutil.walk_packages(path=blocks.__path__,
                                              prefix=blocks.__name__ + '.'):
        try:
            tests.addTests(doctest.DocTestSuite(
                module=importlib.import_module(module), setUp=setup,
                optionflags=doctest.IGNORE_EXCEPTION_DETAIL))
        except:
            pass

    # This part loads the doctests from the documentation
    docs = []
    for root, _, filenames in os.walk(os.path.join(blocks.__path__[0],
                                                   '../docs')):
        for doc in fnmatch.filter(filenames, '*.rst'):
            docs.append(os.path.abspath(os.path.join(root, doc)))
    tests.addTests(doctest.DocFileSuite(
        *docs, module_relative=False, setUp=setup,
        optionflags=doctest.IGNORE_EXCEPTION_DETAIL))

    return tests 

def setUp(self):
        self.dev_null = open(os.devnull, 'w')
        modtype = getattr(EXOSIMS.Prototypes.BackgroundSources.BackgroundSources,'_modtype')
        pkg = EXOSIMS.BackgroundSources
        self.allmods = [get_module(modtype)]
        for loader, module_name, is_pkg in pkgutil.walk_packages(pkg.__path__, pkg.__name__+'.'):
            if not is_pkg:
                mod = get_module(module_name.split('.')[-1],modtype)
                self.assertTrue(mod._modtype is modtype,'_modtype mismatch for %s'%mod.__name__)
                self.allmods.append(mod)
        # need a TargetList object for testing
        script = resource_path('test-scripts/template_prototype_testing.json')
        with open(script) as f:
            spec = json.loads(f.read())
        with RedirectStreams(stdout=self.dev_null):
            self.TL = TargetList(**spec) 

def test_tabs():
    # Test that there are no tabs in our source files
    for importer, modname, ispkg in walk_packages(sklearn.__path__,
                                                  prefix='sklearn.'):

        if IS_PYPY and ('_svmlight_format' in modname or
                        'feature_extraction._hashing' in modname):
            continue

        # because we don't import
        mod = importlib.import_module(modname)
        try:
            source = getsource(mod)
        except IOError:  # user probably should have run "make clean"
            continue
        assert '\t' not in source, ('"%s" has tabs, please remove them ',
                                    'or add it to theignore list'
                                    % modname) 

def find_packages(path):
    # This method returns packages and subpackages as well.
    return [name for _, name, is_pkg in walk_packages([path]) if is_pkg] 

def test_root_import_all_completeness():
    EXCEPTIONS = ('utils', 'tests', 'base', 'setup', 'conftest')
    for _, modname, _ in pkgutil.walk_packages(path=sklearn.__path__,
                                               onerror=lambda _: None):
        if '.' in modname or modname.startswith('_') or modname in EXCEPTIONS:
            continue
        assert_in(modname, sklearn.__all__) 

def walk_modules():
    root = chainer.__path__
    for loader, modname, ispkg in pkgutil.walk_packages(root, module_prefix):
        # Skip modules generated by protobuf.
        if '_pb2' in modname:
            continue

        try:
            mod = importlib.import_module(modname)
        except ImportError:
            continue

        yield mod 

def test_unreadable_dir_on_syspath(self):
        # issue7367 - walk_packages failed if unreadable dir on sys.path
        package_name = "unreadable_package"
        d = os.path.join(self.dirname, package_name)
        # this does not appear to create an unreadable dir on Windows
        #   but the test should not fail anyway
        os.mkdir(d, 0)
        self.addCleanup(os.rmdir, d)
        for t in pkgutil.walk_packages(path=[self.dirname]):
            self.fail("unexpected package found") 

def _load_modules(self):
        self.loaded_extensions = []
        if extensions:
            for module_loader, name, ispkg in pkgutil.walk_packages(extensions.__path__,
                                                                    extensions.__name__ + '.'):
                mod_name = name.split('.')[-1]
                if not ispkg and mod_name.startswith('pydevd_plugin'):
                    try:
                        __import__(name)
                        module = sys.modules[name]
                        self.loaded_extensions.append(module)
                    except ImportError:
                        pydev_log.critical('Unable to load extension: %s', name) 

def create_test_suite():
    suite = unittest.TestSuite()
    for imp, modname, _ in pkgutil.walk_packages(__path__):
        #if modname in ('test_achievement_integration_tests',):
            mod = imp.find_module(modname).load_module(modname)
            for test in unittest.defaultTestLoader.loadTestsFromModule(mod):
                suite.addTests(test)
    return suite