Python modulefinder.ModuleFinder() Examples

def __init__(self, *args, **kwargs):
        # In the Python 2 stdlib, ModuleFinder is an old-style class that
        # does not inherit from object, therefore we cannot use super().
        modulefinder.ModuleFinder.__init__(self, *args, **kwargs)
        logger.Loggable.__init__(self)  # Enable logging via self.logger

        # Mapping of module object to a set of its dependencies. We store the
        # dependencies in a list instead of a set to preserve ordering. While
        # not strictly necessary, it makes testing easier when the ordering
        # of nodes in the graph can be relied on. However, we must ensure that
        # each dependency is unique, so we check before adding each new
        # dependency below. This should be a significant performance issue,
        # as we only expect each module to have a relatively small number
        # of dependencies (around 10s) so O(n) adding is acceptable.
        self._module_deps = collections.defaultdict(list)
        self._curr_caller = None
        self._module_nodes = {} 

def import_hook(self, name, caller=None, *args, **kwargs):
        """
        Hook called when ``caller`` imports module ``name``. Store off the
        caller so we can use it later.

        :param name: Name of the module being imported.
        :type name: ``str``
        :param caller: Module object that is importing ``name``.
        :type caller: ``modulefinder.Module``
        :param args: Other args are passed down to
            ``modulefinder.ModuleFinder.import_hook``
        :param kwargs: Other kwargs are passed down to
            ``modulefinder.ModuleFinder.import_hook``
        :return: Return value from ``modulefinder.ModuleFinder.import_hook``
        """
        self._curr_caller = caller
        return modulefinder.ModuleFinder.import_hook(
            self, name, caller, *args, **kwargs
        ) 

def _get_modules(fn):
    finder = modulefinder.ModuleFinder()
    finder.run_script(fn)
    all = []
    for m in finder.modules.values():
        if not isinstance(m, modulefinder.Module):
            continue
        if not m.__file__:
            continue
        # skip shared object files
        if m.__file__.endswith('.so'):
            continue
        # skip mac system stuff...
        # FIXME: would need to augment with  other OS's system stuff
        if m.__file__.startswith('/Library/Frameworks'):
            continue
        all.append(m)
    return all 

def add_dependencies(module_dir, filename, modules):
    finder = modulefinder.ModuleFinder()
    finder.run_script(os.path.join(module_dir, filename))
    for module in finder.modules.values():
        # if the module comes from the module_dir then process it to get
        # its dependencies.
        if os.path.dirname(str(module.__file__)) == module_dir:
            mod_filename = os.path.basename(module.__file__)
            if mod_filename == filename:
                continue
            # if this module has not already been added then add it.
            if (mod_filename.endswith('_types.py') or
                    (mod_filename.find('_types_v') != -1 and
                     mod_filename.endswith('.py'))):
                if mod_filename not in modules:
                    # add the modules that this module is dependent on
                    add_dependencies(module_dir, mod_filename, modules)
                    modules.append(mod_filename) 

def _(
    module_name=each("test_apples", "apples"), rv=each(True, False),
):
    module = ModuleInfo(ModuleFinder(), module_name, False)
    assert is_test_module(module) == rv 

def _do_test(self, info, report=False):
        import_this, modules, missing, maybe_missing, source = info
        create_package(source)
        try:
            mf = modulefinder.ModuleFinder(path=TEST_PATH)
            mf.import_hook(import_this)
            if report:
                mf.report()
##                # This wouldn't work in general when executed several times:
##                opath = sys.path[:]
##                sys.path = TEST_PATH
##                try:
##                    __import__(import_this)
##                except:
##                    import traceback; traceback.print_exc()
##                sys.path = opath
##                return
            modules = set(modules)
            found = set(mf.modules.keys())
            more = list(found - modules)
            less = list(modules - found)
            # check if we found what we expected, not more, not less
            self.assertEqual((more, less), ([], []))

            # check for missing and maybe missing modules
            bad, maybe = mf.any_missing_maybe()
            self.assertEqual(bad, missing)
            self.assertEqual(maybe, maybe_missing)
        finally:
            distutils.dir_util.remove_tree(TEST_DIR) 

def _do_test(self, info, report=False, debug=0, replace_paths=[]):
        import_this, modules, missing, maybe_missing, source = info
        create_package(source)
        try:
            mf = modulefinder.ModuleFinder(path=TEST_PATH, debug=debug,
                                           replace_paths=replace_paths)
            mf.import_hook(import_this)
            if report:
                mf.report()
##                # This wouldn't work in general when executed several times:
##                opath = sys.path[:]
##                sys.path = TEST_PATH
##                try:
##                    __import__(import_this)
##                except:
##                    import traceback; traceback.print_exc()
##                sys.path = opath
##                return
            modules = sorted(set(modules))
            found = sorted(mf.modules)
            # check if we found what we expected, not more, not less
            self.assertEqual(found, modules)

            # check for missing and maybe missing modules
            bad, maybe = mf.any_missing_maybe()
            self.assertEqual(bad, missing)
            self.assertEqual(maybe, maybe_missing)
        finally:
            shutil.rmtree(TEST_DIR) 

def find_imported_packages(filename, executable_path, debug=0, exclude_stdlib=True):
    """
    Find all imported top-level packages for a given Python file.

    We cannot assume that the version of Python being used to run Sumatra is the
    same as that used to run the simulation/analysis. Therefore we need to run
    all the dependency finding and version checking in a subprocess with the
    correct version of Python.
    """
    # Actually, we could check whether executable_path matches sys.executable, and
    # then do it in this process. On the other hand, the dependency finding
    # could run in parallel with the simulation (good for multicore): we could
    # move setting of dependencies to after the simulation, rather than having it
    # in record.register()

    # HACK Added excludes=['jinja2.asyncsupport']
    script = textwrap.dedent("""
        from modulefinder import ModuleFinder
        import sys, os
        import distutils.sysconfig
        stdlib_path = distutils.sysconfig.get_python_lib(standard_lib=True)
        stdlib_paths = (stdlib_path,
                        os.path.join(stdlib_path, "plat-mac"),
                        os.path.join(stdlib_path, "plat-mac", "lib-scriptpackages"))
        exclude_stdlib = %s
        finder = ModuleFinder(path=sys.path, debug=%d, excludes=['jinja2.asyncsupport'])
        try:
            finder.run_script("%s")
        except Exception as ex:
            sys.stdout.write("Determining dependencies failed for some Python modules.")
        top_level_packages = {}
        for name, module in finder.modules.items():
            if module.__path__ and "." not in name:
                if not(exclude_stdlib and os.path.dirname(module.__path__[0]) in stdlib_paths):
                    top_level_packages[name] = module
        sys.stdout.write("%s" + str(top_level_packages))""" % (exclude_stdlib, int(debug), filename, SENTINEL))
    return run_script(executable_path, script) 

def _do_test(self, info, report=False):
        import_this, modules, missing, maybe_missing, source = info
        create_package(source)
        try:
            mf = modulefinder.ModuleFinder(path=TEST_PATH)
            mf.import_hook(import_this)
            if report:
                mf.report()
##                # This wouldn't work in general when executed several times:
##                opath = sys.path[:]
##                sys.path = TEST_PATH
##                try:
##                    __import__(import_this)
##                except:
##                    import traceback; traceback.print_exc()
##                sys.path = opath
##                return
            modules = set(modules)
            found = set(mf.modules.keys())
            more = list(found - modules)
            less = list(modules - found)
            # check if we found what we expected, not more, not less
            self.assertEqual((more, less), ([], []))

            # check for missing and maybe missing modules
            bad, maybe = mf.any_missing_maybe()
            self.assertEqual(bad, missing)
            self.assertEqual(maybe, maybe_missing)
        finally:
            distutils.dir_util.remove_tree(TEST_DIR) 

def import_module(self, partname, fqname, parent):
        m = modulefinder.ModuleFinder.import_module(self, partname, fqname,
                                                    parent)
        if self.caller and m and m.__file__:
            self.deps.setdefault(self.caller, set()).add(m.__file__)
        return m 

def scan_code(self, co, m):
        # caller is the module doing the importing; remember it for when
        # import_module is called further down the stack.
        oldCaller = self.caller
        self.caller = m.__file__
        try:
            return modulefinder.ModuleFinder.scan_code(self, co, m)
        finally:
            self.caller = oldCaller 

def __init__(self, baseDir, *args, **kw):
        self.caller = None
        self.deps = {}
        self.missing = {}
        self.baseDir = baseDir
        modulefinder.ModuleFinder.__init__(self, *args, **kw) 

def patched(self, project_dir, spec, sack):
        """ Find python dependencies """
        for item in list(project_dir.glob('**/*.py')):
            try:
                mod = ModuleFinder()
                mod.run_script(str(item))
                for _, mod in mod.modules.items():
                    if mod.__file__ and mod.__file__.startswith("/usr/lib"):
                        spec.required_files.add(mod.__file__)
            except ImportError as ie:
                logging.warn("Exception was raised by ModuleFinder:\n" +
                             str(ie) + "\nOn file: " + str(item)) 

def _do_test(self, info, report=False):
        import_this, modules, missing, maybe_missing, source = info
        create_package(source)
        try:
            mf = modulefinder.ModuleFinder(path=TEST_PATH)
            mf.import_hook(import_this)
            if report:
                mf.report()
##                # This wouldn't work in general when executed several times:
##                opath = sys.path[:]
##                sys.path = TEST_PATH
##                try:
##                    __import__(import_this)
##                except:
##                    import traceback; traceback.print_exc()
##                sys.path = opath
##                return
            modules = set(modules)
            found = set(mf.modules.keys())
            more = list(found - modules)
            less = list(modules - found)
            # check if we found what we expected, not more, not less
            self.assertEqual((more, less), ([], []))

            # check for missing and maybe missing modules
            bad, maybe = mf.any_missing_maybe()
            self.assertEqual(bad, missing)
            self.assertEqual(maybe, maybe_missing)
        finally:
            distutils.dir_util.remove_tree(TEST_DIR) 

def _do_test(self, info, report=False, debug=0, replace_paths=[]):
        import_this, modules, missing, maybe_missing, source = info
        create_package(source)
        try:
            mf = modulefinder.ModuleFinder(path=TEST_PATH, debug=debug,
                                           replace_paths=replace_paths)
            mf.import_hook(import_this)
            if report:
                mf.report()
##                # This wouldn't work in general when executed several times:
##                opath = sys.path[:]
##                sys.path = TEST_PATH
##                try:
##                    __import__(import_this)
##                except:
##                    import traceback; traceback.print_exc()
##                sys.path = opath
##                return
            modules = sorted(set(modules))
            found = sorted(mf.modules)
            # check if we found what we expected, not more, not less
            self.assertEqual(found, modules)

            # check for missing and maybe missing modules
            bad, maybe = mf.any_missing_maybe()
            self.assertEqual(bad, missing)
            self.assertEqual(maybe, maybe_missing)
        finally:
            shutil.rmtree(TEST_DIR) 

def _do_test(self, info, report=False):
        import_this, modules, missing, maybe_missing, source = info
        create_package(source)
        try:
            mf = modulefinder.ModuleFinder(path=TEST_PATH)
            mf.import_hook(import_this)
            if report:
                mf.report()
##                # This wouldn't work in general when executed several times:
##                opath = sys.path[:]
##                sys.path = TEST_PATH
##                try:
##                    __import__(import_this)
##                except:
##                    import traceback; traceback.print_exc()
##                sys.path = opath
##                return
            modules = set(modules)
            found = set(mf.modules.keys())
            more = list(found - modules)
            less = list(modules - found)
            # check if we found what we expected, not more, not less
            self.assertEqual((more, less), ([], []))

            # check for missing and maybe missing modules
            bad, maybe = mf.any_missing_maybe()
            self.assertEqual(bad, missing)
            self.assertEqual(maybe, maybe_missing)
        finally:
            distutils.dir_util.remove_tree(TEST_DIR) 

def _do_test(self, info, report=False, debug=0, replace_paths=[]):
        import_this, modules, missing, maybe_missing, source = info
        create_package(source)
        try:
            mf = modulefinder.ModuleFinder(path=TEST_PATH, debug=debug,
                                           replace_paths=replace_paths)
            mf.import_hook(import_this)
            if report:
                mf.report()
##                # This wouldn't work in general when executed several times:
##                opath = sys.path[:]
##                sys.path = TEST_PATH
##                try:
##                    __import__(import_this)
##                except:
##                    import traceback; traceback.print_exc()
##                sys.path = opath
##                return
            modules = sorted(set(modules))
            found = sorted(mf.modules)
            # check if we found what we expected, not more, not less
            self.assertEqual(found, modules)

            # check for missing and maybe missing modules
            bad, maybe = mf.any_missing_maybe()
            self.assertEqual(bad, missing)
            self.assertEqual(maybe, maybe_missing)
        finally:
            shutil.rmtree(TEST_DIR) 

def _do_test(self, info, report=False, debug=0, replace_paths=[]):
        import_this, modules, missing, maybe_missing, source = info
        create_package(source)
        try:
            mf = modulefinder.ModuleFinder(path=TEST_PATH, debug=debug,
                                           replace_paths=replace_paths)
            mf.import_hook(import_this)
            if report:
                mf.report()
##                # This wouldn't work in general when executed several times:
##                opath = sys.path[:]
##                sys.path = TEST_PATH
##                try:
##                    __import__(import_this)
##                except:
##                    import traceback; traceback.print_exc()
##                sys.path = opath
##                return
            modules = sorted(set(modules))
            found = sorted(mf.modules)
            # check if we found what we expected, not more, not less
            self.assertEqual(found, modules)

            # check for missing and maybe missing modules
            bad, maybe = mf.any_missing_maybe()
            self.assertEqual(bad, missing)
            self.assertEqual(maybe, maybe_missing)
        finally:
            shutil.rmtree(TEST_DIR) 

def add_mocks(filename):
    gp = mock.patch('ansible.module_utils.basic.get_platform').start()
    gp.return_value = 'linux'

    module_mock = mock.MagicMock()
    mocks = []
    for module_class in MODULE_CLASSES:
        mocks.append(
            mock.patch('ansible.module_utils.basic.AnsibleModule',
                       new=module_mock)
        )
    for m in mocks:
        p = m.start()
        p.side_effect = AnsibleModuleCallError()

    finder = ModuleFinder()
    try:
        finder.run_script(filename)
    except:
        pass

    sys_mock = mock.MagicMock()
    sys_mock.__version__ = '0.0.0'
    sys_mocks = []
    for module, sources in finder.badmodules.items():
        if module in sys.modules:
            continue
        if [s for s in sources if s[:7] in ['ansible', '__main_']]:
            parts = module.split('.')
            for i in range(len(parts)):
                dotted = '.'.join(parts[:i+1])
                sys.modules[dotted] = sys_mock
                sys_mocks.append(dotted)

    return module_mock, mocks, sys_mocks 

def mock_reload_env():
    """Set up the mock environment for unit-testing the reloader module."""
    # For unmodified files, just return a fake stat result with everything set
    # to 0.
    unmodified_stat_result = os.stat_result(0 for _ in range(10))

    # For modified files, set the st_mtime to 1 day in the future. It doesn't
    # matter that the modification time is in the future - just so long as it
    # is greater than the current time. The reloader only checks st_mtime,
    # which is the value at the 8th index passed to os.stat_result.
    one_day_hence = time.time() + 24 * 60 * 60
    modified_stat_result = os.stat_result(
        0 if i != 8 else one_day_hence for i in range(10)
    )

    def mock_stat(filepath):
        if filepath in mock_stat.modified_files:
            return modified_stat_result
        else:
            return unmodified_stat_result

    mock_stat.modified_files = set()

    reloader_patch = "testplan.runnable.interactive.reloader.reload_module"
    with mock.patch("modulefinder.ModuleFinder", new=MockModuleFinder), (
        mock.patch(reloader_patch, side_effect=lambda module: module)
    ) as mock_reload, (mock.patch("os.stat", new=mock_stat)), (
        mock.patch("sys.modules", new=MOCK_SYSMODULES)
    ):

        # Despite mocking modulefinder.ModuleFinder above, we also need to
        # swap out the real ModuleFinder with our mock one in the list of
        # bases for the GraphModuleFinder.
        reloader._GraphModuleFinder.__bases__ = (
            MockModuleFinder,
            logger.Loggable,
        )
        reload_obj = reloader.ModuleReloader()

        yield reload_obj, mock_reload, mock_stat 

def import_module(self, *args, **kwargs):
        """
        Called a bit further down the stack when a module is imported. Update
        the internal mapping of module dependencies.

        :param args: all args are passed down to
            ``modulefinder.ModuleFinder.import_module``
        :param kwargs: all kwargs are passed down to
            ``modulefinder.ModuleFinder.import_module``
        :return: Imported module object.
        :rtype: ``modulefinder.Module``
        """
        caller = self._curr_caller
        self._curr_caller = None
        mod = modulefinder.ModuleFinder.import_module(self, *args, **kwargs)

        if (
            (caller is not None)
            and (mod is not None)
            and _has_file(mod)
            and mod not in self._module_deps[caller]
        ):
            self.logger.debug(
                "Adding %(mod)s to %(caller)s's dependencies",
                {"mod": mod.__name__, "caller": caller.__name__},
            )
            self._module_deps[caller].append(mod)

        return mod 

def _do_test(self, info, report=False):
        import_this, modules, missing, maybe_missing, source = info
        create_package(source)
        try:
            mf = modulefinder.ModuleFinder(path=TEST_PATH)
            mf.import_hook(import_this)
            if report:
                mf.report()
##                # This wouldn't work in general when executed several times:
##                opath = sys.path[:]
##                sys.path = TEST_PATH
##                try:
##                    __import__(import_this)
##                except:
##                    import traceback; traceback.print_exc()
##                sys.path = opath
##                return
            modules = set(modules)
            found = set(mf.modules.keys())
            more = list(found - modules)
            less = list(modules - found)
            # check if we found what we expected, not more, not less
            self.assertEqual((more, less), ([], []))

            # check for missing and maybe missing modules
            bad, maybe = mf.any_missing_maybe()
            self.assertEqual(bad, missing)
            self.assertEqual(maybe, maybe_missing)
        finally:
            distutils.dir_util.remove_tree(TEST_DIR) 

def _do_test(self, info, report=False):
        import_this, modules, missing, maybe_missing, source = info
        create_package(source)
        try:
            mf = modulefinder.ModuleFinder(path=TEST_PATH)
            mf.import_hook(import_this)
            if report:
                mf.report()
##                # This wouldn't work in general when executed several times:
##                opath = sys.path[:]
##                sys.path = TEST_PATH
##                try:
##                    __import__(import_this)
##                except:
##                    import traceback; traceback.print_exc()
##                sys.path = opath
##                return
            modules = set(modules)
            found = set(mf.modules.keys())
            more = list(found - modules)
            less = list(modules - found)
            # check if we found what we expected, not more, not less
            self.assertEqual((more, less), ([], []))

            # check for missing and maybe missing modules
            bad, maybe = mf.any_missing_maybe()
            self.assertEqual(bad, missing)
            self.assertEqual(maybe, maybe_missing)
        finally:
            distutils.dir_util.remove_tree(TEST_DIR) 

def zip_std_lib(src_module, dst_file):
    """Compiles the Python standard library modules used by the source module
    and outputs to zip file."""

    finder = ModuleFinder()
    finder.run_script(src_module)

    modules = set()

    print('Writing dependencies to "%s"...' % DEP_OUT)

    with open(DEP_OUT, 'w') as f:
        for name, mod in finder.modules.items():
            print('%s: ' % name, end='', file=f)
            print(mod.__file__, file=f)

            if mod.__file__ is None:
                continue

            path = os.path.realpath(mod.__file__)

            if not path.startswith(os.path.normpath(STD_LIB)):
                continue

            while(os.path.dirname(path) != os.path.normpath(STD_LIB)):
                path = os.path.dirname(path)

            if os.path.isfile(path):
                modules.add(path)
            elif os.path.isdir(path):
                for root, dirs, files in os.walk(path):
                    for i in files:
                        modules.add(os.path.join(root, i))

        print('-' * 50, file=f)
        print('### Modules NOT imported ###', file=f)
        print('\n'.join(finder.badmodules.keys()), file=f)

    modules = sorted([i for i in modules if i.endswith(('.py','.pyc')) and not os.path.dirname(i).endswith('__pycache__')])

    print('Writing standard library to "%s"...' % dst_file)

    with zipfile.ZipFile(dst_file, 'w', compression=zipfile.ZIP_DEFLATED) as z:
        for i in modules:
            root, ext = os.path.splitext(i)

            if ext == '.py':
                arcname = os.path.relpath(root, STD_LIB) + '.pyc'
                pyc = create_pyc(i)
            else:
                arcname = os.path.relpath(i, STD_LIB)
                with open(i, 'rb') as f:
                    pyc = f.read()

            z.writestr(arcname, pyc) 

def find_modules(self, scriptname, excludes=[], verbose=False):
        """ Find the modules we'd want to include in the 
            distribution.
        """
        path = sys.path[:]
        path.insert(0, os.path.dirname(scriptname))
        
        mf = ModuleFinder(path=path, excludes=excludes)
        mf.run_script(scriptname)
        
        modulenames = mf.modules.keys()
        modulenames.sort()
        
        self.log("Looking for modules used by '%s'...\n" % scriptname)
        log_format = "%-2s %-30s %s"
        self.log(log_format % ('', 'Module name', 'Module location'))
        self.log(log_format % ('--', '-' * 30, '-' * 30))
        modules = []
        
        for name in modulenames: 
            m = mf.modules[name]
            
            # builtin
            #
            if not m.__file__: continue 
            
            mpath = os.path.normcase(m.__file__)
            
            # Modules in Python distribution.
            # Pass on only those that live in site-packages
            #
            if mpath.startswith(self.site_packages): 
                pass
            elif mpath.startswith(self.sys_prefix):
                continue
            
            type = "P" if m.__path__ else "m"
            modules.append((name, type, m.__file__))
            
            self.log(log_format % (type, name, m.__file__))

        self.log("")
        return modules 

def find_modules(self, scriptname, excludes=[], verbose=False):
        """ Find the modules we'd want to include in the 
            distribution.
        """
        path = sys.path[:]
        path.insert(0, os.path.dirname(scriptname))
        
        mf = ModuleFinder(path=path, excludes=excludes)
        mf.run_script(scriptname)
        
        modulenames = mf.modules.keys()
        modulenames.sort()
        
        self.log("Looking for modules used by '%s'...\n" % scriptname)
        log_format = "%-2s %-30s %s"
        self.log(log_format % ('', 'Module name', 'Module location'))
        self.log(log_format % ('--', '-' * 30, '-' * 30))
        modules = []
        
        for name in modulenames: 
            m = mf.modules[name]
            
            # builtin
            #
            if not m.__file__: continue 
            
            mpath = os.path.normcase(m.__file__)
            
            # Modules in Python distribution.
            # Pass on only those that live in site-packages
            #
            if mpath.startswith(self.site_packages): 
                pass
            elif mpath.startswith(self.sys_prefix):
                continue
            
            type = "P" if m.__path__ else "m"
            modules.append((name, type, m.__file__))
            
            self.log(log_format % (type, name, m.__file__))

        self.log("")
        return modules