Python gc.get_referents() Examples

def _getr(slist, olist, first=True):
    i = 0 
    for e in slist:
        
        oid = id(e)
        typ = type(e)
        if oid in olist or typ is int:    ## or e in olist:     ## since we're excluding all ints, there is no longer a need to check for olist keys
            continue
        olist[oid] = e
        if first and (i%1000) == 0:
            gc.collect()
        tl = gc.get_referents(e)
        if tl:
            _getr(tl, olist, first=False)
        i += 1        
# The public function. 

def test_get_referents(self):
        alist = [1, 3, 5]
        got = gc.get_referents(alist)
        got.sort()
        self.assertEqual(got, alist)

        atuple = tuple(alist)
        got = gc.get_referents(atuple)
        got.sort()
        self.assertEqual(got, alist)

        adict = {1: 3, 5: 7}
        expected = [1, 3, 5, 7]
        got = gc.get_referents(adict)
        got.sort()
        self.assertEqual(got, expected)

        got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
        got.sort()
        self.assertEqual(got, [0, 0] + range(5))

        self.assertEqual(gc.get_referents(1, 'a', 4j), []) 

def CollectGarbage():
    import gc
    #gc.set_debug(gc.DEBUG_SAVEALL)
    #gc.set_debug(gc.DEBUG_UNCOLLECTABLE)
    from pprint import pprint
    print "threshold:", gc.get_threshold()
    print "unreachable object count:", gc.collect()
    garbageList = gc.garbage[:]
    for i, obj in enumerate(garbageList):
        print "Object Num %d:" % i
        pprint(obj)
        #print "Referrers:"
        #print(gc.get_referrers(o))
        #print "Referents:"
        #print(gc.get_referents(o))
    print "Done."
    #print "unreachable object count:", gc.collect()
    #from pprint import pprint
    #pprint(gc.garbage) 

def _getr(slist, olist, first=True):
    i = 0 
    for e in slist:
        
        oid = id(e)
        typ = type(e)
        if oid in olist or typ is int:    ## or e in olist:     ## since we're excluding all ints, there is no longer a need to check for olist keys
            continue
        olist[oid] = e
        if first and (i%1000) == 0:
            gc.collect()
        tl = gc.get_referents(e)
        if tl:
            _getr(tl, olist, first=False)
        i += 1        
# The public function. 

def test_get_referents(self):
        alist = [1, 3, 5]
        got = gc.get_referents(alist)
        got.sort()
        self.assertEqual(got, alist)

        atuple = tuple(alist)
        got = gc.get_referents(atuple)
        got.sort()
        self.assertEqual(got, alist)

        adict = {1: 3, 5: 7}
        expected = [1, 3, 5, 7]
        got = gc.get_referents(adict)
        got.sort()
        self.assertEqual(got, expected)

        got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
        got.sort()
        self.assertEqual(got, [0, 0] + range(5))

        self.assertEqual(gc.get_referents(1, 'a', 4j), []) 

def _getr(slist, olist, first=True):
    i = 0 
    for e in slist:
        
        oid = id(e)
        typ = type(e)
        if oid in olist or typ is int:    ## or e in olist:     ## since we're excluding all ints, there is no longer a need to check for olist keys
            continue
        olist[oid] = e
        if first and (i%1000) == 0:
            gc.collect()
        tl = gc.get_referents(e)
        if tl:
            _getr(tl, olist, first=False)
        i += 1        
# The public function. 

def getsize(obj):
    """Sum size of object & members."""
    if isinstance(obj, BLACKLIST):
        raise TypeError('getsize() does not take argument of type: ' +
                        str(type(obj)))
    seen_ids = set()
    size = 0
    objects = [obj]
    while objects:
        need_referents = []
        for obj in objects:
            if not isinstance(obj, BLACKLIST) and id(obj) not in seen_ids:
                seen_ids.add(id(obj))
                size += sys.getsizeof(obj)
                need_referents.append(obj)
        objects = get_referents(*need_referents)
    return size 

def test_get_referents(self):
        alist = [1, 3, 5]
        got = gc.get_referents(alist)
        got.sort()
        self.assertEqual(got, alist)

        atuple = tuple(alist)
        got = gc.get_referents(atuple)
        got.sort()
        self.assertEqual(got, alist)

        adict = {1: 3, 5: 7}
        expected = [1, 3, 5, 7]
        got = gc.get_referents(adict)
        got.sort()
        self.assertEqual(got, expected)

        got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
        got.sort()
        self.assertEqual(got, [0, 0] + list(range(5)))

        self.assertEqual(gc.get_referents(1, 'a', 4j), []) 

def test_get_referents(self):
        alist = [1, 3, 5]
        got = gc.get_referents(alist)
        got.sort()
        self.assertEqual(got, alist)

        atuple = tuple(alist)
        got = gc.get_referents(atuple)
        got.sort()
        self.assertEqual(got, alist)

        adict = {1: 3, 5: 7}
        expected = [1, 3, 5, 7]
        got = gc.get_referents(adict)
        got.sort()
        self.assertEqual(got, expected)

        got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
        got.sort()
        self.assertEqual(got, [0, 0] + range(5))

        self.assertEqual(gc.get_referents(1, 'a', 4j), []) 

def getsize(obj):
    """sum size of object & members."""
    if isinstance(obj, BLACKLIST):
        raise TypeError('getsize() does not take argument of type: '+ str(type(obj)))
    seen_ids = set()
    size = 0
    objects = [obj]
    while objects:
        need_referents = []
        for obj in objects:
            if not isinstance(obj, BLACKLIST) and id(obj) not in seen_ids:
                seen_ids.add(id(obj))
                size += sys.getsizeof(obj)
                need_referents.append(obj)
                print(size, type(obj), obj)
        objects = get_referents(*need_referents)
    return size 

def test_get_referents(self):
        alist = [1, 3, 5]
        got = gc.get_referents(alist)
        got.sort()
        self.assertEqual(got, alist)

        atuple = tuple(alist)
        got = gc.get_referents(atuple)
        got.sort()
        self.assertEqual(got, alist)

        adict = {1: 3, 5: 7}
        expected = [1, 3, 5, 7]
        got = gc.get_referents(adict)
        got.sort()
        self.assertEqual(got, expected)

        got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
        got.sort()
        self.assertEqual(got, [0, 0] + list(range(5)))

        self.assertEqual(gc.get_referents(1, 'a', 4j), []) 

def _get_edges(self):
        """
        Compute the edges for the reference graph.
        The function returns a set of tuples (id(a), id(b), ref) if a
        references b with the referent 'ref'.
        """
        idset = set([id(x) for x in self.objects])
        self.edges = set([])
        for n in self.objects:
            refset = set([id(x) for x in get_referents(n)])
            for ref in refset.intersection(idset):
                label = ''
                members = None
                if isinstance(n, dict):
                    members = n.items()
                if not members:
                    members = named_refs(n)
                for (k, v) in members:
                    if id(v) == ref:
                        label = k
                        break
                self.edges.add(_Edge(id(n), ref, label)) 

def test_get_referents(self):
        alist = [1, 3, 5]
        got = gc.get_referents(alist)
        got.sort()
        self.assertEqual(got, alist)

        atuple = tuple(alist)
        got = gc.get_referents(atuple)
        got.sort()
        self.assertEqual(got, alist)

        adict = {1: 3, 5: 7}
        expected = [1, 3, 5, 7]
        got = gc.get_referents(adict)
        got.sort()
        self.assertEqual(got, expected)

        got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
        got.sort()
        self.assertEqual(got, [0, 0] + list(range(5)))

        self.assertEqual(gc.get_referents(1, 'a', 4j), []) 

def get_leaking_objects(objects=None):
    """Return objects that do not have any referents.

    These could indicate reference-counting bugs in C code.  Or they could
    be legitimate.

    Note that the GC does not track simple objects like int or str.

    .. versionadded:: 1.7
    """
    if objects is None:
        gc.collect()
        objects = gc.get_objects()
    try:
        ids = set(id(i) for i in objects)
        for i in objects:
            ids.difference_update(id(j) for j in gc.get_referents(i))
        # this then is our set of objects without referrers
        return [i for i in objects if id(i) in ids]
    finally:
        objects = i = j = None # clear cyclic references to frame 

def find_ref_chain(obj, predicate, max_depth=20, extra_ignore=()):
    """Find a shortest chain of references leading from obj.

    The end of the chain will be some object that matches your predicate.

    ``predicate`` is a function taking one argument and returning a boolean.

    ``max_depth`` limits the search depth.

    ``extra_ignore`` can be a list of object IDs to exclude those objects from
    your search.

    Example:

        >>> find_chain(obj, lambda x: isinstance(x, MyClass))
        [obj, ..., <MyClass object at ...>]

    Returns ``[obj]`` if such a chain could not be found.

    .. versionadded:: 1.7
    """
    return find_chain(obj, predicate, gc.get_referents,
                      max_depth=max_depth, extra_ignore=extra_ignore)[::-1] 

def test_get_referents(self):
        alist = [1, 3, 5]
        got = gc.get_referents(alist)
        got.sort()
        self.assertEqual(got, alist)

        atuple = tuple(alist)
        got = gc.get_referents(atuple)
        got.sort()
        self.assertEqual(got, alist)

        adict = {1: 3, 5: 7}
        expected = [1, 3, 5, 7]
        got = gc.get_referents(adict)
        got.sort()
        self.assertEqual(got, expected)

        got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
        got.sort()
        self.assertEqual(got, [0, 0] + range(5))

        self.assertEqual(gc.get_referents(1, 'a', 4j), []) 

def test_get_referents(self):
        alist = [1, 3, 5]
        got = gc.get_referents(alist)
        got.sort()
        self.assertEqual(got, alist)

        atuple = tuple(alist)
        got = gc.get_referents(atuple)
        got.sort()
        self.assertEqual(got, alist)

        adict = {1: 3, 5: 7}
        expected = [1, 3, 5, 7]
        got = gc.get_referents(adict)
        got.sort()
        self.assertEqual(got, expected)

        got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
        got.sort()
        self.assertEqual(got, [0, 0] + range(5))

        self.assertEqual(gc.get_referents(1, 'a', 4j), []) 

def test_get_referents(self):
        alist = [1, 3, 5]
        got = gc.get_referents(alist)
        got.sort()
        self.assertEqual(got, alist)

        atuple = tuple(alist)
        got = gc.get_referents(atuple)
        got.sort()
        self.assertEqual(got, alist)

        adict = {1: 3, 5: 7}
        expected = [1, 3, 5, 7]
        got = gc.get_referents(adict)
        got.sort()
        self.assertEqual(got, expected)

        got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
        got.sort()
        self.assertEqual(got, [0, 0] + range(5))

        self.assertEqual(gc.get_referents(1, 'a', 4j), []) 

def get_size(obj):
    """Sum size of object and its members."""
    size = 0
    processed_ids = set()
    objects = [obj]
    while objects:
        need_refer = []
        for obj in objects:
            if id(obj) in processed_ids:
                continue
            processed_ids.add(id(obj))
            need_refer.append(obj)
            size += sys.getsizeof(obj)
        objects = gc.get_referents(*need_refer)
    return size 

def show_refs(objs, max_depth=3, extra_ignore=(), filter=None, too_many=10,
              highlight=None):
    """Generate an object reference graph starting at ``objs``

    The graph will show you what objects are reachable from ``objs``, directly
    and indirectly.

    ``objs`` can be a single object, or it can be a list of objects.

    Produces a Graphviz .dot file and spawns a viewer (xdot) if one is
    installed, otherwise converts the graph to a .png image.

    Use ``max_depth`` and ``too_many`` to limit the depth and breadth of the
    graph.

    Use ``filter`` (a predicate) and ``extra_ignore`` (a list of object IDs) to
    remove undesired objects from the graph.

    Use ``highlight`` (a predicate) to highlight certain graph nodes in blue.

    Examples:

        >>> show_refs(obj)
        >>> show_refs([obj1, obj2])
        >>> show_refs(obj, max_depth=5)
        >>> show_refs(obj, filter=lambda x: not inspect.isclass(x))
        >>> show_refs(obj, highlight=inspect.isclass)
        >>> show_refs(obj, extra_ignore=[id(locals())])

    """
    show_graph(objs, max_depth=max_depth, extra_ignore=extra_ignore,
               filter=filter, too_many=too_many, highlight=highlight,
               edge_func=gc.get_referents, swap_source_target=True)

#
# Internal helpers
# 

def _GetModuleCodeObjects(module):
  """Gets all code objects defined in the specified module.

  There are two BFS traversals involved. One in this function and the other in
  _FindCodeObjectsReferents. Only the BFS in _FindCodeObjectsReferents has
  a depth limit. This function does not. The motivation is that this function
  explores code object of the module and they can have any arbitrary nesting
  level. _FindCodeObjectsReferents, on the other hand, traverses through class
  definitions and random references. It's much more expensive and will likely
  go into unrelated objects.

  There is also a limit on how many total objects are going to be traversed in
  all. This limit makes sure that if something goes wrong, the lookup doesn't
  hang.

  Args:
    module: module to explore.

  Returns:
    Set of code objects defined in module.
  """

  visit_recorder = _VisitRecorder()
  current = [module]
  code_objects = set()
  while current:
    current = _FindCodeObjectsReferents(module, current, visit_recorder)
    code_objects |= current

    # Unfortunately Python code objects don't implement tp_traverse, so this
    # type can't be used with gc.get_referents. The workaround is to get the
    # relevant objects explicitly here.
    current = [code_object.co_consts for code_object in current]

  return code_objects 

def parents(obj, objtype, depth=1, ignore=()): #XXX: objtype=object ?
    """Find the chain of referents for obj. Chain will end with obj.

    objtype: an object type or tuple of types to search for
    depth: search depth (e.g. depth=2 is 'grandparents')
    ignore: an object or tuple of objects to ignore in the search
    """
    edge_func = gc.get_referents # looking for refs, not back_refs
    predicate = lambda x: isinstance(x, objtype) # looking for parent type
   #if objtype is None: predicate = lambda x: True #XXX: in obj.mro() ?
    ignore = (ignore,) if not hasattr(ignore, '__len__') else ignore
    ignore = (id(obj) for obj in ignore)
    chain = find_chain(obj, predicate, edge_func, depth)[::-1]
    #XXX: should pop off obj... ?
    return chain 

def parents(obj, objtype, depth=1, ignore=()): #XXX: objtype=object ?
    """Find the chain of referents for obj. Chain will end with obj.

    objtype: an object type or tuple of types to search for
    depth: search depth (e.g. depth=2 is 'grandparents')
    ignore: an object or tuple of objects to ignore in the search
    """
    edge_func = gc.get_referents # looking for refs, not back_refs
    predicate = lambda x: isinstance(x, objtype) # looking for parent type
   #if objtype is None: predicate = lambda x: True #XXX: in obj.mro() ?
    ignore = (ignore,) if not hasattr(ignore, '__len__') else ignore
    ignore = (id(obj) for obj in ignore)
    chain = find_chain(obj, predicate, edge_func, depth)[::-1]
    #XXX: should pop off obj... ?
    return chain 

def test_all():
    gc.collect() # Delete 2nd generation garbage
    run_test("lists", test_list)
    run_test("dicts", test_dict)
    run_test("tuples", test_tuple)
    run_test("classes", test_class)
    run_test("new style classes", test_newstyleclass)
    run_test("instances", test_instance)
    run_test("new instances", test_newinstance)
    run_test("methods", test_method)
    run_test("functions", test_function)
    run_test("frames", test_frame)
    run_test("finalizers", test_finalizer)
    run_test("finalizers (new class)", test_finalizer_newclass)
    run_test("__del__", test_del)
    run_test("__del__ (new class)", test_del_newclass)
    run_test("get_count()", test_get_count)
    run_test("collect(n)", test_collect_generations)
    run_test("saveall", test_saveall)
    run_test("trashcan", test_trashcan)
    run_test("boom", test_boom)
    run_test("boom2", test_boom2)
    run_test("boom_new", test_boom_new)
    run_test("boom2_new", test_boom2_new)
    run_test("get_referents", test_get_referents)
    run_test("bug1055820b", test_bug1055820b)

    gc.enable()
    try:
        run_test("bug1055820c", test_bug1055820c)
    finally:
        gc.disable()

    gc.enable()
    try:
        run_test("bug1055820d", test_bug1055820d)
    finally:
        gc.disable() 

def test_get_referents():
    alist = [1, 3, 5]
    got = gc.get_referents(alist)
    got.sort()
    expect(got, alist, "get_referents")

    atuple = tuple(alist)
    got = gc.get_referents(atuple)
    got.sort()
    expect(got, alist, "get_referents")

    adict = {1: 3, 5: 7}
    expected = [1, 3, 5, 7]
    got = gc.get_referents(adict)
    got.sort()
    expect(got, expected, "get_referents")

    got = gc.get_referents([1, 2], {3: 4}, (0, 0, 0))
    got.sort()
    expect(got, [0, 0] + range(5), "get_referents")

    expect(gc.get_referents(1, 'a', 4j), [], "get_referents")

# Bug 1055820 has several tests of longstanding bugs involving weakrefs and
# cyclic gc.

# An instance of C1055820 has a self-loop, so becomes cyclic trash when
# unreachable. 

def test_get_referents(self):
        if is_cli:
            self.assertRaises(NotImplementedError, gc.get_referents,1,"hello",True)
            self.assertRaises(NotImplementedError, gc.get_referents)
        else:
            gc.get_referents(1,"hello",True)
            gc.get_referents()
            
            class TempClass: pass
            self.assertEqual(gc.get_referents(TempClass).count('TempClass'), 1) 

def __init__(self, obj):
        self.obj = obj
        self.referents = [id(ref) for ref in gc.get_referents(obj)]
        self.referrers = set() 

def format_stack(coro):
    # Stop when reaching trio modules
    if hasattr(coro, "cr_code"):
        module = inspect.getmodule(coro.cr_code)
        if module and module.__name__.startswith("trio."):
            return

    # Work around https://bugs.python.org/issue32810
    if hasattr(coro, "__class__") and coro.__class__.__name__ in (
        "async_generator_asend",
        "async_generator_athrow",
    ):
        coro, *_ = gc.get_referents(coro)
        if not inspect.isasyncgen(coro):
            return

    # Follow a generator
    if getattr(coro, "ag_frame", None):
        yield from traceback.format_stack(coro.ag_frame)
        yield from format_stack(coro.ag_await)
        return

    # Follow a coroutine
    if getattr(coro, "cr_frame", None):
        yield from traceback.format_stack(coro.cr_frame)
        yield from format_stack(coro.cr_await)
        return

    # Follow a decorated coroutine
    if getattr(coro, "gi_frame", None):
        yield from traceback.format_stack(coro.gi_frame)
        yield from format_stack(coro.gi_yieldfrom)
        return 

def test_get_referents(self):
        if is_cli:
            self.assertRaises(NotImplementedError, gc.get_referents,1,"hello",True)
            self.assertRaises(NotImplementedError, gc.get_referents)
        else:
            gc.get_referents(1,"hello",True)
            gc.get_referents()

            class TempClass: pass
            self.assertEqual(gc.get_referents(TempClass).count((TempClass, object)), 1) 

def _check_setitem_copy(self, stacklevel=4, t='setting'):
        """ validate if we are doing a settitem on a chained copy.

        If you call this function, be sure to set the stacklevel such that the
        user will see the error *at the level of setting*"""
        if self.is_copy:

            value = config.get_option('mode.chained_assignment')
            if value is None:
                return

            # see if the copy is not actually refererd; if so, then disolve
            # the copy weakref
            try:
                gc.collect(2)
                if not gc.get_referents(self.is_copy()):
                    self.is_copy = None
                    return
            except:
                pass

            if t == 'referant':
                t = ("A value is trying to be set on a copy of a slice from a "
                     "DataFrame")
            else:
                t = ("A value is trying to be set on a copy of a slice from a "
                     "DataFrame.\nTry using .loc[row_index,col_indexer] = value "
                     "instead")
            if value == 'raise':
                raise SettingWithCopyError(t)
            elif value == 'warn':
                warnings.warn(t, SettingWithCopyWarning, stacklevel=stacklevel)