Python gc.get_referrers() Examples

def _verify_object_death(wref: ReferenceType) -> None:
    obj = wref()
    if obj is None:
        return

    try:
        name = type(obj).__name__
    except Exception:
        print(f'Note: unable to get type name for {obj}')
        name = 'object'

    print(f'{Clr.RED}Error: {name} not dying'
          f' when expected to: {Clr.BLD}{obj}{Clr.RST}')
    refs = list(gc.get_referrers(obj))
    print(f'{Clr.YLW}Active References:{Clr.RST}')
    i = 1
    for ref in refs:
        print(f'{Clr.YLW}  reference {i}:{Clr.BLU} {ref}{Clr.RST}')
        i += 1 

def load(self):
        import oncall
        importlib.reload(oncall.utils)

        import oncall.app
        app = oncall.app.get_wsgi_app()

        if not self.skip_build_assets:
            for r in gc.get_referrers(self):
                if isinstance(r, dict) and '_num_workers' in r:
                    gunicorn_arbiter = r

            # only build assets on one worker to avoid race conditions
            if gunicorn_arbiter['worker_age'] % self.options['workers'] == 0:
                import oncall.ui
                oncall.ui.build_assets()

        return app 

def load(self):
        import iris
        imp.reload(iris)
        imp.reload(iris.config)
        config = iris.config.load_config(sys.argv[1])

        import iris.api
        app = iris.api.get_api(config)

        if not self.skip_build_assets:
            for r in gc.get_referrers(self):
                if isinstance(r, dict) and '_num_workers' in r:
                    gunicorn_arbiter = r

            # only build assets on one worker to avoid race conditions
            if gunicorn_arbiter['worker_age'] % self.options['workers'] == 0:
                import iris.ui
                iris.ui.build_assets()

        return app 

def test_it(collectible):

    dd()
    dd('======= ', ('collectible' if collectible else 'uncollectible'), ' object =======')
    dd()

    gc.collect()
    dd('*** init,     nr of referrers: ', len(gc.get_referrers(One)))
    dd('              garbage:         ', gc.garbage)

    one = One(collectible)
    dd('              created:         ', one.typ, ': ', one)
    dd('              nr of referrers: ', len(gc.get_referrers(One)))

    dd('              delete:')
    del one

    gc.collect()

    dd('*** after gc, nr of referrers: ', len(gc.get_referrers(One)))
    dd('              garbage:         ', gc.garbage) 

def print_referrers(obj, dbg=False):
    import gc
    for referrer in gc.get_referrers(obj):
        if isinstance(referrer, dict):
            for field, value in referrer.items():
                if value is obj:
                    print referrer, field
        elif hasattr(referrer, '__dict__'):
            for field,value in referrer.__dict__.items():
                if value is obj:
                    print referrer, field
        elif hasattr(referrer, 'remove'):
            print referrer
        elif type(referrer) == tuple:
            clear_referrers(referrer, dbg=True)
        else:
            #import pprint
            #pp = pprint.PrettyPrinter(indent=4)
            #pp.pprint(referrer)
            pass 

def obj_referrers(klass):
    find_obj = False
    for obj in gc.get_objects():
        # closures are evil !
        if isinstance(obj, types.FunctionType) and obj.__closure__ is not None:
            for c in obj.__closure__:
                try:
                    if isinstance(c.cell_contents, klass):
                        print('!!!', obj, c.cell_contents)
                except ValueError:
                    print("Cell is empty...")
        if isinstance(obj, klass):
            find_obj = True
            rs = gc.get_referrers(obj)
            print("---------------------------referrers of %s" % klass.__name__)
            for ob in rs:
                print(type(ob), ob.__name__ if type(ob) is type else repr(ob)[:140])
                rs1 = gc.get_referrers(ob)
                for ob1 in rs1:
                    print('    ', type(ob1), ob1.__name__ if type(ob1) is type else repr(ob1)[:140])
            print("---------------------------")
    if not find_obj:
        print("Nothing refrences %s" % klass.__name__) 

def find_backref_chain(obj, predicate, max_depth=20, extra_ignore=()):
    """Find a shortest chain of references leading to obj.

    The start of the chain will be some object that matches your predicate.

    ``predicate`` is a function taking one argument and returning a boolean.

    ``max_depth`` limits the search depth.

    ``extra_ignore`` can be a list of object IDs to exclude those objects from
    your search.

    Example:

        >>> find_backref_chain(obj, inspect.ismodule)
        [<module ...>, ..., obj]

    Returns ``[obj]`` if such a chain could not be found.

    .. versionchanged:: 1.5
       Returns ``obj`` instead of ``None`` when a chain could not be found.

    """
    return find_chain(obj, predicate, gc.get_referrers,
                      max_depth=max_depth, extra_ignore=extra_ignore) 

def step(self):
        bgan = self.gan
        self.gan.step()
        if bgan.destroy:
            self.sampler=None
            self.gan = self.gan.newgan
            gc.collect()
            refs = gc.get_referrers(bgan)
            d = bgan.trainer._delegate
            bgan.trainer=None
            gc.collect()
            del bgan
            tf.reset_default_graph()

            gc.collect()

        if(self.steps % self.sample_every == 0):
            sample_list = self.sample()

        self.steps+=1 

def disable_splitbrain():
    """Disables (deactivates) the Splitbrain machinery"""
    global _enabled
    if not _enabled:
        return
    _enabled = False
    for funcname, origfunc in _prev_builtins.items():
        setattr(builtins, funcname, origfunc)
    for modname, mod in sys.modules.items():
        if isinstance(mod, RoutedModule):
            sys.modules[modname] = mod.__realmod__
            for ref in gc.get_referrers(mod):
                if isinstance(ref, dict) and "__name__" in ref and ref.get("__file__") is not None:
                    for k, v in ref.items():
                        if v is mod:
                            ref[k] = mod.__realmod__
    sys.modules["sys"] = sys
    builtins.__import__ = _orig_import 

def disable_splitbrain():
    """Disables (deactivates) the Splitbrain machinery"""
    global _enabled
    if not _enabled:
        return
    _enabled = False
    for funcname, origfunc in _prev_builtins.items():
        setattr(builtins, funcname, origfunc)
    for modname, mod in sys.modules.items():
        if isinstance(mod, RoutedModule):
            sys.modules[modname] = mod.__realmod__
            for ref in gc.get_referrers(mod):
                if isinstance(ref, dict) and "__name__" in ref and ref.get("__file__") is not None:
                    for k, v in ref.items():
                        if v is mod:
                            ref[k] = mod.__realmod__
    sys.modules["sys"] = sys
    builtins.__import__ = _orig_import 

def test_methods_garbage_collection(self):

        class Randomy(object):
            @memoize
            def randomy(self, do_random):
                if do_random:
                    return random.random()
                else:
                    return "constant"

        r = Randomy()
        rand1 = r.randomy(True)

        for gc_ref in gc.get_referrers(r):
            if inspect.isframe(gc_ref):
                continue
            else:
                raise AssertionError('Unexpected reference to `r` instance: {!r}\n'
                                     '@memoize probably made a reference to it and has created a circular reference loop'.format(gc_ref)) 

def children(obj, objtype, depth=1, ignore=()): #XXX: objtype=object ?
    """Find the chain of referrers for obj. Chain will start with obj.

    objtype: an object type or tuple of types to search for
    depth: search depth (e.g. depth=2 is 'grandchildren')
    ignore: an object or tuple of objects to ignore in the search

    NOTE: a common thing to ignore is all globals, 'ignore=(globals(),)'

    NOTE: repeated calls may yield different results, as python stores
    the last value in the special variable '_'; thus, it is often good
    to execute something to replace '_' (e.g. >>> 1+1).
    """
    edge_func = gc.get_referrers # looking for back_refs, not refs
    predicate = lambda x: isinstance(x, objtype) # looking for child type
   #if objtype is None: predicate = lambda x: True #XXX: in obj.mro() ?
    ignore = (ignore,) if not hasattr(ignore, '__len__') else ignore
    ignore = (id(obj) for obj in ignore)
    chain = find_chain(obj, predicate, edge_func, depth, ignore)
    #XXX: should pop off obj... ?
    return chain


# more generic helper function (cut-n-paste from objgraph)
# Source at http://mg.pov.lt/objgraph/
# Copyright (c) 2008-2010 Marius Gedminas <marius@pov.lt>
# Copyright (c) 2010 Stefano Rivera <stefano@rivera.za.net>
# Released under the MIT licence (see objgraph/objgrah.py) 

def file_module_function_of(self, frame):
        code = frame.f_code
        filename = code.co_filename
        if filename:
            modulename = _modname(filename)
        else:
            modulename = None

        funcname = code.co_name
        clsname = None
        if code in self._caller_cache:
            if self._caller_cache[code] is not None:
                clsname = self._caller_cache[code]
        else:
            self._caller_cache[code] = None
            ## use of gc.get_referrers() was suggested by Michael Hudson
            # all functions which refer to this code object
            funcs = [f for f in gc.get_referrers(code)
                         if inspect.isfunction(f)]
            # require len(func) == 1 to avoid ambiguity caused by calls to
            # new.function(): "In the face of ambiguity, refuse the
            # temptation to guess."
            if len(funcs) == 1:
                dicts = [d for d in gc.get_referrers(funcs[0])
                             if isinstance(d, dict)]
                if len(dicts) == 1:
                    classes = [c for c in gc.get_referrers(dicts[0])
                                   if hasattr(c, "__bases__")]
                    if len(classes) == 1:
                        # ditto for new.classobj()
                        clsname = classes[0].__name__
                        # cache the result - assumption is that new.* is
                        # not called later to disturb this relationship
                        # _caller_cache could be flushed if functions in
                        # the new module get called.
                        self._caller_cache[code] = clsname
        if clsname is not None:
            funcname = "%s.%s" % (clsname, funcname)

        return filename, modulename, funcname 

def referrednested(func, recurse=True): #XXX: return dict of {__name__: obj} ?
    """get functions defined inside of func (e.g. inner functions in a closure)

    NOTE: results may differ if the function has been executed or not.
    If len(nestedcode(func)) > len(referrednested(func)), try calling func().
    If possible, python builds code objects, but delays building functions
    until func() is called.
    """
    if PY3:
        att1 = '__code__'
        att0 = '__func__'
    else:
        att1 = 'func_code' # functions
        att0 = 'im_func'   # methods

    import gc
    funcs = set()
    # get the code objects, and try to track down by referrence
    for co in nestedcode(func, recurse):
        # look for function objects that refer to the code object
        for obj in gc.get_referrers(co):
            # get methods
            _ = getattr(obj, att0, None) # ismethod
            if getattr(_, att1, None) is co: funcs.add(obj)
            # get functions
            elif getattr(obj, att1, None) is co: funcs.add(obj)
            # get frame objects
            elif getattr(obj, 'f_code', None) is co: funcs.add(obj)
            # get code objects
            elif hasattr(obj, 'co_code') and obj is co: funcs.add(obj)
#     frameobjs => func.func_code.co_varnames not in func.func_code.co_cellvars
#     funcobjs => func.func_code.co_cellvars not in func.func_code.co_varnames
#     frameobjs are not found, however funcobjs are...
#     (see: test_mixins.quad ... and test_mixins.wtf)
#     after execution, code objects get compiled, and then may be found by gc
    return list(funcs) 

def _check_activity_death(cls, activity_ref: ReferenceType[Activity],
                              counter: List[int]) -> None:
        """Sanity check to make sure an Activity was destroyed properly.

        Receives a weakref to a ba.Activity which should have torn itself
        down due to no longer being referenced anywhere. Will complain
        and/or print debugging info if the Activity still exists.
        """
        try:
            import gc
            import types
            activity = activity_ref()
            print('ERROR: Activity is not dying when expected:', activity,
                  '(warning ' + str(counter[0] + 1) + ')')
            print('This means something is still strong-referencing it.')
            counter[0] += 1

            # FIXME: Running the code below shows us references but winds up
            #  keeping the object alive; need to figure out why.
            #  For now we just print refs if the count gets to 3, and then we
            #  kill the app at 4 so it doesn't matter anyway.
            if counter[0] == 3:
                print('Activity references for', activity, ':')
                refs = list(gc.get_referrers(activity))
                i = 1
                for ref in refs:
                    if isinstance(ref, types.FrameType):
                        continue
                    print('  reference', i, ':', ref)
                    i += 1
            if counter[0] == 4:
                print('Killing app due to stuck activity... :-(')
                _ba.quit()

        except Exception:
            print_exception('Error on _check_activity_death/') 

def print_refs(obj: Any) -> None:
    """Print a list of known live references to an object."""

    # Hmmm; I just noticed that calling this on an object
    # seems to keep it alive. Should figure out why.
    print('REFERENCES FOR', obj, ':')
    refs = list(gc.get_referrers(obj))
    i = 1
    for ref in refs:
        print('     ref', i, ':', ref)
        i += 1 

def describeObj(obj, depth=4, path=None, ignore=None):
    """
    Trace all reference paths backward, printing a list of different ways this object can be accessed.
    Attempts to answer the question "who has a reference to this object"
    """
    if path is None:
        path = [obj]
    if ignore is None:
        ignore = {}   ## holds IDs of objects used within the function.
    ignore[id(sys._getframe())] = None
    ignore[id(path)] = None
    gc.collect()
    refs = gc.get_referrers(obj)
    ignore[id(refs)] = None
    printed=False
    for ref in refs:
        if id(ref) in ignore:
            continue
        if id(ref) in list(map(id, path)):
            print("Cyclic reference: " + refPathString([ref]+path))
            printed = True
            continue
        newPath = [ref]+path
        if len(newPath) >= depth:
            refStr = refPathString(newPath)
            if '[_]' not in refStr:           ## ignore '_' references generated by the interactive shell
                print(refStr)
            printed = True
        else:
            describeObj(ref, depth, newPath, ignore)
            printed = True
    if not printed:
        print("Dead end: " + refPathString(path)) 

def get_routers():
    return dict(
        (_hex(id(router)), router)
        for klass in get_subclasses(mitogen.core.Router)
        for router in gc.get_referrers(klass)
        if isinstance(router, mitogen.core.Router)
    ) 

def children(obj, objtype, depth=1, ignore=()): #XXX: objtype=object ?
    """Find the chain of referrers for obj. Chain will start with obj.

    objtype: an object type or tuple of types to search for
    depth: search depth (e.g. depth=2 is 'grandchildren')
    ignore: an object or tuple of objects to ignore in the search

    NOTE: a common thing to ignore is all globals, 'ignore=(globals(),)'

    NOTE: repeated calls may yield different results, as python stores
    the last value in the special variable '_'; thus, it is often good
    to execute something to replace '_' (e.g. >>> 1+1).
    """
    edge_func = gc.get_referrers # looking for back_refs, not refs
    predicate = lambda x: isinstance(x, objtype) # looking for child type
   #if objtype is None: predicate = lambda x: True #XXX: in obj.mro() ?
    ignore = (ignore,) if not hasattr(ignore, '__len__') else ignore
    ignore = (id(obj) for obj in ignore)
    chain = find_chain(obj, predicate, edge_func, depth, ignore)
    #XXX: should pop off obj... ?
    return chain


# more generic helper function (cut-n-paste from objgraph)
# Source at http://mg.pov.lt/objgraph/
# Copyright (c) 2008-2010 Marius Gedminas <marius@pov.lt>
# Copyright (c) 2010 Stefano Rivera <stefano@rivera.za.net>
# Released under the MIT licence (see objgraph/objgrah.py) 

def ascend(self, obj, depth=1):
        """Return a nested list containing referrers of the given object."""
        depth += 1
        parents = []

        # Gather all referrers in one step to minimize
        # cascading references due to repr() logic.
        refs = gc.get_referrers(obj)
        self.ignore.append(refs)
        if len(refs) > self.maxparents:
            return [('[%s referrers]' % len(refs), [])]

        try:
            ascendcode = self.ascend.__code__
        except AttributeError:
            ascendcode = self.ascend.im_func.func_code
        for parent in refs:
            if inspect.isframe(parent) and parent.f_code is ascendcode:
                continue
            if parent in self.ignore:
                continue
            if depth <= self.maxdepth:
                parents.append((parent, self.ascend(parent, depth)))
            else:
                parents.append((parent, []))

        return parents 

def g():
    marker = object()
    yield marker
    # now the marker is in the tuple being constructed
    [tup] = [x for x in gc.get_referrers(marker) if type(x) is tuple]
    print tup
    print tup[1] 

def ascend(self, obj, depth=1):
        """Return a nested list containing referrers of the given object."""
        depth += 1
        parents = []

        # Gather all referrers in one step to minimize
        # cascading references due to repr() logic.
        refs = gc.get_referrers(obj)
        self.ignore.append(refs)
        if len(refs) > self.maxparents:
            return [("[%s referrers]" % len(refs), [])]

        try:
            ascendcode = self.ascend.__code__
        except AttributeError:
            ascendcode = self.ascend.im_func.func_code
        for parent in refs:
            if inspect.isframe(parent) and parent.f_code is ascendcode:
                continue
            if parent in self.ignore:
                continue
            if depth <= self.maxdepth:
                parents.append((parent, self.ascend(parent, depth)))
            else:
                parents.append((parent, []))

        return parents 

def allInstancesOf(cls):
    """
    Use L{gc.get_referrers} to retrieve all instances of a given class.
    """
    for o in gc.get_referrers(cls):
        if isinstance(o, cls):
            yield o 

def test_get_referrers(self):
        if is_cli:
            self.assertRaises(NotImplementedError, gc.get_referrers,1,"hello",True)
            self.assertRaises(NotImplementedError, gc.get_referrers)
        else:
            gc.get_referrers(1,"hello",True)
            gc.get_referrers()

            class TempClass: pass
            tc = TempClass()
            self.assertEqual(gc.get_referrers(TempClass).count(tc), 1) 

def g():
    marker = object()
    yield marker
    # now the marker is in the tuple being constructed
    [tup] = [x for x in gc.get_referrers(marker) if type(x) is tuple]
    print(tup)
    print(tup[1]) 

def file_module_function_of(self, frame):
        code = frame.f_code
        filename = code.co_filename
        if filename:
            modulename = _modname(filename)
        else:
            modulename = None

        funcname = code.co_name
        clsname = None
        if code in self._caller_cache:
            if self._caller_cache[code] is not None:
                clsname = self._caller_cache[code]
        else:
            self._caller_cache[code] = None
            ## use of gc.get_referrers() was suggested by Michael Hudson
            # all functions which refer to this code object
            funcs = [f for f in gc.get_referrers(code)
                         if inspect.isfunction(f)]
            # require len(func) == 1 to avoid ambiguity caused by calls to
            # new.function(): "In the face of ambiguity, refuse the
            # temptation to guess."
            if len(funcs) == 1:
                dicts = [d for d in gc.get_referrers(funcs[0])
                             if isinstance(d, dict)]
                if len(dicts) == 1:
                    classes = [c for c in gc.get_referrers(dicts[0])
                                   if hasattr(c, "__bases__")]
                    if len(classes) == 1:
                        # ditto for new.classobj()
                        clsname = classes[0].__name__
                        # cache the result - assumption is that new.* is
                        # not called later to disturb this relationship
                        # _caller_cache could be flushed if functions in
                        # the new module get called.
                        self._caller_cache[code] = clsname
        if clsname is not None:
            funcname = "%s.%s" % (clsname, funcname)

        return filename, modulename, funcname 

def __getstate__(self):
        log.msg( "WARNING: serializing ephemeral %s" % self )
        if not _PYPY:
            import gc
            if getattr(gc, 'get_referrers', None):
                for r in gc.get_referrers(self):
                    log.msg( " referred to by %s" % (r,))
        return None 

def file_module_function_of(self, frame):
        code = frame.f_code
        filename = code.co_filename
        if filename:
            modulename = _modname(filename)
        else:
            modulename = None

        funcname = code.co_name
        clsname = None
        if code in self._caller_cache:
            if self._caller_cache[code] is not None:
                clsname = self._caller_cache[code]
        else:
            self._caller_cache[code] = None
            ## use of gc.get_referrers() was suggested by Michael Hudson
            # all functions which refer to this code object
            funcs = [f for f in gc.get_referrers(code)
                         if inspect.isfunction(f)]
            # require len(func) == 1 to avoid ambiguity caused by calls to
            # new.function(): "In the face of ambiguity, refuse the
            # temptation to guess."
            if len(funcs) == 1:
                dicts = [d for d in gc.get_referrers(funcs[0])
                             if isinstance(d, dict)]
                if len(dicts) == 1:
                    classes = [c for c in gc.get_referrers(dicts[0])
                                   if hasattr(c, "__bases__")]
                    if len(classes) == 1:
                        # ditto for new.classobj()
                        clsname = classes[0].__name__
                        # cache the result - assumption is that new.* is
                        # not called later to disturb this relationship
                        # _caller_cache could be flushed if functions in
                        # the new module get called.
                        self._caller_cache[code] = clsname
        if clsname is not None:
            funcname = "%s.%s" % (clsname, funcname)

        return filename, modulename, funcname 

def _get_tree(self, root, maxdepth):
        """Workhorse of the get_tree implementation.

        This is an recursive method which is why we have a wrapper method.
        root is the current root object of the tree which should be returned.
        Note that root is not of the type _Node.
        maxdepth defines how much further down the from the root the tree
        should be build.

        """
        self.ignore.append(inspect.currentframe())
        res = _Node(root, self.str_func) #PYCHOK use root parameter
        self.already_included.add(id(root)) #PYCHOK use root parameter
        if maxdepth == 0:
            return res
        objects = gc.get_referrers(root) #PYCHOK use root parameter
        self.ignore.append(objects)
        for o in objects:
            # XXX: find a better way to ignore dict of _Node objects
            if isinstance(o, dict):
                sampleNode = _Node(1)
                if list(sampleNode.__dict__.keys()) == list(o.keys()):
                    continue
            _id = id(o)
            if not self.repeat and (_id in self.already_included):
                s = self.str_func(o)
                res.children.append("%s (already included, id %s)" %\
                                    (s, _id))
                continue
            if (not isinstance(o, _Node)) and (o not in self.ignore):
                res.children.append(self._get_tree(o, maxdepth-1))
        return res 

def g():
    marker = object()
    yield marker
    # now the marker is in the tuple being constructed
    [tup] = [x for x in gc.get_referrers(marker) if type(x) is tuple]
    print(tup)
    print(tup[1])