Python sys.path_importer_cache() Examples

def get_importer(path_item):
    """Retrieve a PEP 302 importer for the given path item

    The returned importer is cached in sys.path_importer_cache
    if it was newly created by a path hook.

    The cache (or part of it) can be cleared manually if a
    rescan of sys.path_hooks is necessary.
    """
    try:
        importer = sys.path_importer_cache[path_item]
    except KeyError:
        for path_hook in sys.path_hooks:
            try:
                importer = path_hook(path_item)
                sys.path_importer_cache.setdefault(path_item, importer)
                break
            except ImportError:
                pass
        else:
            importer = None
    return importer 

def test_parallel_path_hooks(self):
        # Here the Finder instance is only used to check concurrent calls
        # to path_hook().
        finder = Finder()
        # In order for our path hook to be called at each import, we need
        # to flush the path_importer_cache, which we do by registering a
        # dedicated meta_path entry.
        flushing_finder = FlushingFinder()
        def path_hook(path):
            finder.find_spec('')
            raise ImportError
        sys.path_hooks.insert(0, path_hook)
        sys.meta_path.append(flushing_finder)
        try:
            # Flush the cache a first time
            flushing_finder.find_spec('')
            numtests = self.check_parallel_module_init()
            self.assertGreater(finder.numcalls, 0)
            self.assertEqual(finder.x, finder.numcalls)
        finally:
            sys.meta_path.remove(flushing_finder)
            sys.path_hooks.remove(path_hook) 

def _install_pypy_zipimporter_workaround(cls, pex_file):
    # The pypy zipimporter implementation always freshly loads a module instead of re-importing
    # when the module already exists in sys.modules. This breaks the PEP-302 importer protocol and
    # violates pkg_resources assumptions based on that protocol in its handling of namespace
    # packages. See: https://bitbucket.org/pypy/pypy/issues/1686

    def pypy_zipimporter_workaround(path):
      import os

      if not path.startswith(pex_file) or '.' in os.path.relpath(path, pex_file):
        # We only need to claim the pex zipfile root modules.
        #
        # The protocol is to raise if we don't want to hook the given path.
        # See: https://www.python.org/dev/peps/pep-0302/#specification-part-2-registering-hooks
        raise ImportError()

      return cls._CachingZipImporter(path)

    for path in list(sys.path_importer_cache):
      if path.startswith(pex_file):
        sys.path_importer_cache.pop(path)

    sys.path_hooks.insert(0, pypy_zipimporter_workaround) 

def test_method_called(self):
        # If defined the method should be called.
        class InvalidatingNullFinder:
            def __init__(self, *ignored):
                self.called = False
            def find_module(self, *args):
                return None
            def invalidate_caches(self):
                self.called = True

        key = 'gobledeegook'
        meta_ins = InvalidatingNullFinder()
        path_ins = InvalidatingNullFinder()
        sys.meta_path.insert(0, meta_ins)
        self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key))
        sys.path_importer_cache[key] = path_ins
        self.addCleanup(lambda: sys.meta_path.remove(meta_ins))
        self.init.invalidate_caches()
        self.assertTrue(meta_ins.called)
        self.assertTrue(path_ins.called) 

def minimum_sys(cls, inherit_path):
    """Return the minimum sys necessary to run this interpreter, a la python -S.

    :returns: (sys.path, sys.path_importer_cache, sys.modules) tuple of a
      bare python installation.
    """
    site_libs = set(cls.site_libs())
    for site_lib in site_libs:
      TRACER.log('Found site-library: %s' % site_lib)
    for extras_path in cls._extras_paths():
      TRACER.log('Found site extra: %s' % extras_path)
      site_libs.add(extras_path)
    site_libs = set(os.path.normpath(path) for path in site_libs)

    sys_path, sys_path_importer_cache = cls.minimum_sys_path(site_libs, inherit_path)
    sys_modules = cls.minimum_sys_modules(site_libs)

    return sys_path, sys_path_importer_cache, sys_modules 

def patch_sys(self, inherit_path):
    """Patch sys with all site scrubbed."""
    def patch_dict(old_value, new_value):
      old_value.clear()
      old_value.update(new_value)

    def patch_all(path, path_importer_cache, modules):
      sys.path[:] = path
      patch_dict(sys.path_importer_cache, path_importer_cache)
      patch_dict(sys.modules, modules)

    new_sys_path, new_sys_path_importer_cache, new_sys_modules = self.minimum_sys(inherit_path)

    if self._vars.PEX_EXTRA_SYS_PATH:
      TRACER.log('Adding %s to sys.path' % self._vars.PEX_EXTRA_SYS_PATH)
      new_sys_path.extend(self._vars.PEX_EXTRA_SYS_PATH.split(':'))
    TRACER.log('New sys.path: %s' % new_sys_path)

    patch_all(new_sys_path, new_sys_path_importer_cache, new_sys_modules) 

def load_model_from_file(path, as_builder=False):
  """Loads a model from a configuration file.

  Args:
    path: The relative path to the configuration file.
    as_builder: If ``True``, return a callable building the model on call.

  Returns:
    A :class:`opennmt.models.Model` instance or a callable returning such
    instance.
  """
  module = load_model_module(path)
  model = module.model
  if not as_builder:
    model = model()
  del sys.path_importer_cache[os.path.dirname(module.__file__)]
  del sys.modules[module.__name__]
  return model 

def tearDownClass(cls):
        try:
            sys.path.remove(cls._zip_path)
        except ValueError:
            pass

        try:
            del sys.path_importer_cache[cls._zip_path]
            del sys.modules[cls.data.__name__]
        except KeyError:
            pass

        try:
            del cls.data
            del cls._zip_path
        except AttributeError:
            pass 

def test_method_called(self):
        # If defined the method should be called.
        class InvalidatingNullFinder:
            def __init__(self, *ignored):
                self.called = False
            def find_module(self, *args):
                return None
            def invalidate_caches(self):
                self.called = True

        key = 'gobledeegook'
        meta_ins = InvalidatingNullFinder()
        path_ins = InvalidatingNullFinder()
        sys.meta_path.insert(0, meta_ins)
        self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key))
        sys.path_importer_cache[key] = path_ins
        self.addCleanup(lambda: sys.meta_path.remove(meta_ins))
        self.init.invalidate_caches()
        self.assertTrue(meta_ins.called)
        self.assertTrue(path_ins.called) 

def get_importer(path_item):
    """Retrieve a finder for the given path item

    The returned finder is cached in sys.path_importer_cache
    if it was newly created by a path hook.

    The cache (or part of it) can be cleared manually if a
    rescan of sys.path_hooks is necessary.
    """
    try:
        importer = sys.path_importer_cache[path_item]
    except KeyError:
        for path_hook in sys.path_hooks:
            try:
                importer = path_hook(path_item)
                sys.path_importer_cache.setdefault(path_item, importer)
                break
            except ImportError:
                pass
        else:
            importer = None
    return importer 

def _precache_zipimporters(path=None):
    pic = sys.path_importer_cache

    # When measured, despite having the same complexity (O(n)),
    # converting to tuples and then caching the conversion to sets
    # and the set difference is faster than converting to sets
    # and then only caching the set difference.

    req_paths = tuple(path or sys.path)
    cached_paths = tuple(pic)
    new_paths = _cached_set_diff(req_paths, cached_paths)
    for entry_path in new_paths:
        try:
            pic[entry_path] = zipimport.zipimporter(entry_path)
        except zipimport.ZipImportError:
            continue
    return pic 

def test_parallel_path_hooks(self):
        # Here the Finder instance is only used to check concurrent calls
        # to path_hook().
        finder = Finder()
        # In order for our path hook to be called at each import, we need
        # to flush the path_importer_cache, which we do by registering a
        # dedicated meta_path entry.
        flushing_finder = FlushingFinder()
        def path_hook(path):
            finder.find_spec('')
            raise ImportError
        sys.path_hooks.insert(0, path_hook)
        sys.meta_path.append(flushing_finder)
        try:
            # Flush the cache a first time
            flushing_finder.find_spec('')
            numtests = self.check_parallel_module_init()
            self.assertGreater(finder.numcalls, 0)
            self.assertEqual(finder.x, finder.numcalls)
        finally:
            sys.meta_path.remove(flushing_finder)
            sys.path_hooks.remove(path_hook) 

def test_path_hooks(self):
        import toimport
        def prepare(f):
            sys.path_importer_cache = {}
            sys.path_hooks = [f]
            if 'toimport' in sys.modules: del sys.modules['toimport']
        
        def hook(*args):  raise Exception('hello')
        prepare(hook)
        def f(): import toimport
        self.assertRaisesMessage(Exception, 'hello', f)

        # ImportError shouldn't propagate out
        def hook(*args):  raise ImportError('foo')
        prepare(hook)
        f()

        # returning none should be ok
        def hook(*args): pass
        prepare(hook)
        f()
        
        sys.path_hooks = [] 

def test_method_called(self):
        # If defined the method should be called.
        class InvalidatingNullFinder:
            def __init__(self, *ignored):
                self.called = False
            def find_module(self, *args):
                return None
            def invalidate_caches(self):
                self.called = True

        key = 'gobledeegook'
        meta_ins = InvalidatingNullFinder()
        path_ins = InvalidatingNullFinder()
        sys.meta_path.insert(0, meta_ins)
        self.addCleanup(lambda: sys.path_importer_cache.__delitem__(key))
        sys.path_importer_cache[key] = path_ins
        self.addCleanup(lambda: sys.meta_path.remove(meta_ins))
        self.init.invalidate_caches()
        self.assertTrue(meta_ins.called)
        self.assertTrue(path_ins.called) 

def test_parallel_path_hooks(self):
        # Here the Finder instance is only used to check concurrent calls
        # to path_hook().
        finder = Finder()
        # In order for our path hook to be called at each import, we need
        # to flush the path_importer_cache, which we do by registering a
        # dedicated meta_path entry.
        flushing_finder = FlushingFinder()
        def path_hook(path):
            finder.find_spec('')
            raise ImportError
        sys.path_hooks.insert(0, path_hook)
        sys.meta_path.append(flushing_finder)
        try:
            # Flush the cache a first time
            flushing_finder.find_spec('')
            numtests = self.check_parallel_module_init()
            self.assertGreater(finder.numcalls, 0)
            self.assertEqual(finder.x, finder.numcalls)
        finally:
            sys.meta_path.remove(flushing_finder)
            sys.path_hooks.remove(path_hook) 

def uncache_zipdir(path):
    """
    Remove any globally cached zip file related data for `path`

    Stale zipimport.zipimporter objects need to be removed when a zip file is
    replaced as they contain cached zip file directory information. If they are
    asked to get data from their zip file, they will use that cached
    information to calculate the data location in the zip file. This calculated
    location may be incorrect for the replaced zip file, which may in turn
    cause the read operation to either fail or return incorrect data.

    Note we have no way to clear any local caches from here. That is left up to
    whomever is in charge of maintaining that cache.

    """
    normalized_path = normalize_path(path)
    _uncache(normalized_path, zipimport._zip_directory_cache)
    _uncache(normalized_path, sys.path_importer_cache) 

def test_None_on_sys_path(self):
        # Putting None in sys.path[0] caused an import regression from Python
        # 3.2: http://bugs.python.org/issue16514
        new_path = sys.path[:]
        new_path.insert(0, None)
        new_path_importer_cache = sys.path_importer_cache.copy()
        new_path_importer_cache.pop(None, None)
        new_path_hooks = [zipimport.zipimporter,
                          self.machinery.FileFinder.path_hook(
                              *self.importlib._bootstrap._get_supported_file_loaders())]
        missing = object()
        email = sys.modules.pop('email', missing)
        try:
            with util.import_state(meta_path=sys.meta_path[:],
                                   path=new_path,
                                   path_importer_cache=new_path_importer_cache,
                                   path_hooks=new_path_hooks):
                module = self.importlib.import_module('email')
                self.assertIsInstance(module, ModuleType)
        finally:
            if email is not missing:
                sys.modules['email'] = email 

def _precache_zipimporters(path=None):
    pic = sys.path_importer_cache

    # When measured, despite having the same complexity (O(n)),
    # converting to tuples and then caching the conversion to sets
    # and the set difference is faster than converting to sets
    # and then only caching the set difference.

    req_paths = tuple(path or sys.path)
    cached_paths = tuple(pic)
    new_paths = _cached_set_diff(req_paths, cached_paths)
    for entry_path in new_paths:
        try:
            pic[entry_path] = zipimport.zipimporter(entry_path)
        except zipimport.ZipImportError:
            continue
    return pic 

def _precache_zipimporters(path=None):
    pic = sys.path_importer_cache

    # When measured, despite having the same complexity (O(n)),
    # converting to tuples and then caching the conversion to sets
    # and the set difference is faster than converting to sets
    # and then only caching the set difference.

    req_paths = tuple(path or sys.path)
    cached_paths = tuple(pic)
    new_paths = _cached_set_diff(req_paths, cached_paths)
    for entry_path in new_paths:
        try:
            pic[entry_path] = zipimport.zipimporter(entry_path)
        except zipimport.ZipImportError:
            continue
    return pic 

def _precache_zipimporters(path=None):
    pic = sys.path_importer_cache

    # When measured, despite having the same complexity (O(n)),
    # converting to tuples and then caching the conversion to sets
    # and the set difference is faster than converting to sets
    # and then only caching the set difference.

    req_paths = tuple(path or sys.path)
    cached_paths = tuple(pic)
    new_paths = _cached_set_diff(req_paths, cached_paths)
    for entry_path in new_paths:
        try:
            pic[entry_path] = zipimport.zipimporter(entry_path)
        except zipimport.ZipImportError:
            continue
    return pic 

def get_importer(path_item):
    """Retrieve a finder for the given path item

    The returned finder is cached in sys.path_importer_cache
    if it was newly created by a path hook.

    The cache (or part of it) can be cleared manually if a
    rescan of sys.path_hooks is necessary.
    """
    try:
        importer = sys.path_importer_cache[path_item]
    except KeyError:
        for path_hook in sys.path_hooks:
            try:
                importer = path_hook(path_item)
                sys.path_importer_cache.setdefault(path_item, importer)
                break
            except ImportError:
                pass
        else:
            importer = None
    return importer 

def get_importer(path_item):
    """Retrieve a PEP 302 importer for the given path item

    The returned importer is cached in sys.path_importer_cache
    if it was newly created by a path hook.

    The cache (or part of it) can be cleared manually if a
    rescan of sys.path_hooks is necessary.
    """
    try:
        importer = sys.path_importer_cache[path_item]
    except KeyError:
        for path_hook in sys.path_hooks:
            try:
                importer = path_hook(path_item)
                sys.path_importer_cache.setdefault(path_item, importer)
                break
            except ImportError:
                pass
        else:
            importer = None
    return importer 

def test_package___file__(self):
        try:
            m = __import__('pep3147')
        except ImportError:
            pass
        else:
            self.fail("pep3147 module already exists: %r" % (m,))
        # Test that a package's __file__ points to the right source directory.
        os.mkdir('pep3147')
        sys.path.insert(0, os.curdir)
        def cleanup():
            if sys.path[0] == os.curdir:
                del sys.path[0]
            shutil.rmtree('pep3147')
        self.addCleanup(cleanup)
        # Touch the __init__.py file.
        support.create_empty_file('pep3147/__init__.py')
        importlib.invalidate_caches()
        expected___file__ = os.sep.join(('.', 'pep3147', '__init__.py'))
        m = __import__('pep3147')
        self.assertEqual(m.__file__, expected___file__, (m.__file__, m.__path__, sys.path, sys.path_importer_cache))
        # Ensure we load the pyc file.
        support.unload('pep3147')
        m = __import__('pep3147')
        support.unload('pep3147')
        self.assertEqual(m.__file__, expected___file__, (m.__file__, m.__path__, sys.path, sys.path_importer_cache)) 

def test_path(self):
        # Test that 'path' is used when set.
        # Implicitly tests that sys.path_importer_cache is used.
        module = '<test module>'
        path = '<test path>'
        importer = util.mock_spec(module)
        with util.import_state(path_importer_cache={path: importer}):
            loader = self.machinery.PathFinder.find_module(module, [path])
            self.assertIs(loader, importer) 

def test_path_hooks(self):
        # Test that sys.path_hooks is used.
        # Test that sys.path_importer_cache is set.
        module = '<test module>'
        path = '<test path>'
        importer = util.mock_spec(module)
        hook = import_util.mock_path_hook(path, importer=importer)
        with util.import_state(path_hooks=[hook]):
            loader = self.machinery.PathFinder.find_module(module, [path])
            self.assertIs(loader, importer)
            self.assertIn(path, sys.path_importer_cache)
            self.assertIs(sys.path_importer_cache[path], importer) 

def test_path_hooks(self):
        import toimport
        def prepare(f):
            sys.path_importer_cache = {}
            sys.path_hooks = [f] + old_path_hooks
            if 'toimport' in sys.modules: del sys.modules['toimport']

        old_path_hooks = sys.path_hooks[:]
        try:
            def hook(*args):  raise Exception('hello')
            prepare(hook)
            def f(): import toimport
            self.assertRaisesMessage(Exception, 'hello', f)

            # ImportError shouldn't propagate out
            def hook(*args):  raise ImportError('foo')
            prepare(hook)
            f()

            # returning None
            def hook(*args): pass
            prepare(hook)
            if is_cli:
                f()
            else:
                self.assertRaises(ImportError, f)
        finally:
            sys.path_hooks = old_path_hooks 

def find_spec(self, name, path=None, target=None):
        sys.path_importer_cache.clear() 

def get_importer(path_item):
    """Retrieve a PEP 302 "importer" for the given path item

    If there is no importer, this returns a wrapper around the builtin import
    machinery.  The returned importer is only cached if it was created by a
    path hook.
    """
    try:
        importer = sys.path_importer_cache[path_item]
    except KeyError:
        for hook in sys.path_hooks:
            try:
                importer = hook(path_item)
            except ImportError:
                pass
            else:
                break
        else:
            importer = None

    sys.path_importer_cache.setdefault(path_item,importer)
    if importer is None:
        try:
            importer = ImpWrapper(path_item)
        except ImportError:
            pass
    return importer 

def test_sys_path(self):
        # Test that sys.path is used when 'path' is None.
        # Implicitly tests that sys.path_importer_cache is used.
        module = '<test module>'
        path = '<test path>'
        importer = util.mock_spec(module)
        with util.import_state(path_importer_cache={path: importer},
                               path=[path]):
            loader = self.machinery.PathFinder.find_module(module)
            self.assertIs(loader, importer) 

def test_empty_path_hooks(self):
        # Test that if sys.path_hooks is empty a warning is raised,
        # sys.path_importer_cache gets None set, and PathFinder returns None.
        path_entry = 'bogus_path'
        with util.import_state(path_importer_cache={}, path_hooks=[],
                               path=[path_entry]):
            with warnings.catch_warnings(record=True) as w:
                warnings.simplefilter('always')
                self.assertIsNone(self.machinery.PathFinder.find_module('os'))
                self.assertIsNone(sys.path_importer_cache[path_entry])
                self.assertEqual(len(w), 1)
                self.assertTrue(issubclass(w[-1].category, ImportWarning))