Python gc.collect() Examples

def _wrapper(method, check_steps_end):
        if hasattr(method, '_already_decorated') and method._already_decorated:
            return method

        @wraps(method)
        def wrapped(*args, **kwargs):
            MolerTest._steps_start()
            caught_exception = None
            try:
                result = method(*args, **kwargs)
            except Exception as exc:
                caught_exception = exc
            finally:
                MolerTest._check_exceptions_occured(caught_exception)
                if check_steps_end:
                    MolerTest._check_steps_end()
            gc.collect()
            return result

        wrapped._already_decorated = True
        return wrapped 

def test_close_auto_created_session(self, mocker):
        # Setup
        import asyncio
        import gc
        import aiohttp

        mock_session = mocker.Mock(spec=aiohttp.ClientSession)
        session_cls_mock = mocker.patch("aiohttp.ClientSession")
        session_cls_mock.return_value = mock_session

        positionals = [1]
        keywords = {"keyword": 2}

        # Run: Create client
        client = aiohttp_.AiohttpClient.create(*positionals, **keywords)

        # Run: Get session
        loop = asyncio.get_event_loop()
        loop.run_until_complete(asyncio.ensure_future(client.session()))

        # Verify: session created with args
        session_cls_mock.assert_called_with(*positionals, **keywords)
        del client
        gc.collect()
        session_cls_mock.return_value.close.assert_called_with() 

def test_close_auto_generated_session(self, mocker):
        # Setup
        import requests
        import gc

        session_mock = mocker.Mock(spec=requests.Session)
        session_mock.request.return_value = "response"
        session_cls_mock = mocker.patch("requests.Session")
        session_cls_mock.return_value = session_mock

        # Run
        client = requests_.RequestsClient()
        client.send(("method", "url", {}))
        del client
        gc.collect()

        assert session_mock.close.call_count == 1 

def test_structured_object_take_and_repeat(self, dt, pat, count, singleton):
        """Structured object reference counting for specialized functions.
        The older functions such as take and repeat use different code paths
        then item setting (when writing this).
        """
        indices = [0, 1]

        arr = np.array([pat] * 3, dt)
        gc.collect()
        before = sys.getrefcount(singleton)
        res = arr.take(indices)
        after = sys.getrefcount(singleton)
        assert after - before == count * 2
        new = res.repeat(10)
        gc.collect()
        after_repeat = sys.getrefcount(singleton)
        assert after_repeat - after == count * 2 * 10 

def test_leak_in_structured_dtype_comparison(self):
        # gh-6250
        recordtype = np.dtype([('a', np.float64),
                               ('b', np.int32),
                               ('d', (str, 5))])

        # Simple case
        a = np.zeros(2, dtype=recordtype)
        for i in range(100):
            a == a
        assert_(sys.getrefcount(a) < 10)

        # The case in the bug report.
        before = sys.getrefcount(a)
        u, v = a[0], a[1]
        u == v
        del u, v
        gc.collect()
        after = sys.getrefcount(a)
        assert_equal(before, after) 

def _place_electrodes(self, fix_th=True):
        """ Add the defined electrodes to a mesh

        Parameters:
        ------------
        fn_out: str
            name of output file
        """
        w_elec = copy.deepcopy(self.mesh)
        w_elec.fix_tr_node_ordering()
        electrode_surfaces = [None for i in range(len(self.electrode))]
        for i, el in enumerate(self.electrode):
            logger.info('Placing Electrode:\n{0}'.format(str(el)))
            w_elec, n = el.add_electrode_to_mesh(w_elec)
            electrode_surfaces[i] = n

        w_elec.fix_th_node_ordering()
        w_elec.fix_tr_node_ordering()
        if fix_th:
            logger.info('Improving mesh quality')
            w_elec.fix_thin_tetrahedra()

        gc.collect()
        return w_elec, electrode_surfaces 

def test_structured_object_indexing(self, shape, index, items_changed,
                                        dt, pat, count, singleton):
        """Structured object reference counting for advanced indexing."""
        zero = 0
        one = 1

        arr = np.zeros(shape, dt)

        gc.collect()
        before_zero = sys.getrefcount(zero)
        before_one = sys.getrefcount(one)
        # Test item getting:
        part = arr[index]
        after_zero = sys.getrefcount(zero)
        assert after_zero - before_zero == count * items_changed
        del part
        # Test item setting:
        arr[index] = one
        gc.collect()
        after_zero = sys.getrefcount(zero)
        after_one = sys.getrefcount(one)
        assert before_zero - after_zero == count * items_changed
        assert after_one - before_one == count * items_changed 

def fix_tr_node_ordering(self):
        ''' Fixes the node ordering of the triangles in-place '''
        corresponding = self.find_corresponding_tetrahedra()
        triangles = np.where(self.elm.elm_type == 2)[0]

        triangles = triangles[corresponding != -1]
        corresponding = corresponding[corresponding != -1]

        normals = self.triangle_normals().value[triangles]
        baricenters = self.elements_baricenters().value
        pos_bar = baricenters[corresponding - 1] - baricenters[triangles]

        dotp = np.einsum('ij, ij -> i', normals, pos_bar)
        switch = triangles[dotp > 0]

        tmp = np.copy(self.elm.node_number_list[switch, 1])
        self.elm.node_number_list[switch, 1] = self.elm.node_number_list[switch, 0]
        self.elm.node_number_list[switch, 0] = tmp
        del tmp
        gc.collect() 

def unload_group(chat_id):
    global gcache, gc_counter
    try:
        with open("markov/chat_" + str(chat_id) + ".dat", "wb") as f:
            pickle.dump(groups[chat_id], f)
            groups[chat_id] = None
            del groups[chat_id]
        gcache.remove(chat_id)
        gc_counter -= 1
        if gc_counter < 1:
            gc_counter = gc_every_unload
            gc.collect()
    except KeyboardInterrupt as e:
        raise e
    except:
        pass 

def test_env_dirty(self):
        self.odoo.config['auto_commit'] = False

        def test_record_garbarge_collected():
            user_ids = self.odoo.env['res.users'].search([('id', '!=', 1)])
            user = self.user_obj.browse(user_ids[0])
            self.assertNotIn(user, self.odoo.env.dirty)
            self.assertNotIn(user, user.env.dirty)
            user.name = "Joe"
            self.assertIn(user, self.odoo.env.dirty)
            self.assertIn(user, user.env.dirty)

        test_record_garbarge_collected()
        # Ensure the record has been garbage collected for the next test
        import gc

        gc.collect()
        self.assertEqual(list(self.odoo.env.dirty), []) 

def clear(self):
        self.user = None
        self._users = weakref.WeakValueDictionary()
        self._emojis = {}
        self._calls = {}
        self._guilds = {}
        self._voice_clients = {}

        # LRU of max size 128
        self._private_channels = OrderedDict()
        # extra dict to look up private channels by user id
        self._private_channels_by_user = {}
        self._messages = self.max_messages and deque(maxlen=self.max_messages)

        # In cases of large deallocations the GC should be called explicitly
        # To free the memory more immediately, especially true when it comes
        # to reconnect loops which cause mass allocations and deallocations.
        gc.collect() 

def test_garbage_collected_subscriber_is_not_notified():
    from moler.threaded_moler_connection import ThreadedMolerConnection

    moler_conn = ThreadedMolerConnection()
    received_data = []

    class Subscriber(object):
        def __call__(self, data, time_recv):
            received_data.append(data)

    subscr1 = Subscriber()
    subscr2 = Subscriber()
    moler_conn.subscribe(observer=subscr1, connection_closed_handler=do_nothing_func)
    moler_conn.subscribe(observer=subscr2, connection_closed_handler=do_nothing_func)

    del subscr1
    gc.collect()

    moler_conn.data_received("data", datetime.datetime.now())
    MolerTest.sleep(1, True)  # Processing in separate thread so have to wait.
    assert len(received_data) == 1

# --------------------------- resources --------------------------- 

def test_leaks(self):
        def _get_items():
            items, stops = [], []
            for creator in _CREATORS:
                item, stop = creator()
                items.append(item)
                stops.append(stop)
            [stop() for stop in stops]
            return weakref.WeakSet(items)

        items = _get_items()
        _pause()
        # If this next line were uncommented, it would work without all the
        # weakrefs in the code.
        # gc.collect()
        self.assertEqual(list(items), []) 

def test_subscription_doesnt_block_subscriber_to_be_garbage_collected():
    from moler.threaded_moler_connection import ThreadedMolerConnection

    moler_conn = ThreadedMolerConnection()
    garbage_collected_subscribers = []

    class Subscriber(object):
        def __del__(self):
            garbage_collected_subscribers.append('Subscriber')

    class CloseSubscriber(object):
        pass

    subscr = Subscriber()
    close_subscr = CloseSubscriber()
    moler_conn.subscribe(subscr, close_subscr)

    del subscr
    gc.collect()

    assert 'Subscriber' in garbage_collected_subscribers 

def test_garbage_collected_subscriber_is_not_notified():
    from moler.publisher import Publisher

    notifier = Publisher()
    received_data = []

    class GcSubscriber(object):
        def __call__(self, data):
            received_data.append(data)

    subscr1 = GcSubscriber()
    subscr2 = GcSubscriber()
    notifier.subscribe(subscriber=subscr1)
    notifier.subscribe(subscriber=subscr2)

    del subscr1
    gc.collect()

    notifier.notify_subscribers("data")
    assert len(received_data) == 1 

def loadW2V(self,emb_path, type="bin"):
        print("Loading W2V data...")
        num_keys = 0
        if type=="textgz":
            # this seems faster than gensim non-binary load
            for line in gzip.open(emb_path):
                l = line.strip().split()
                st=l[0].lower()
                self.pre_emb[st]=np.asarray(l[1:])
            num_keys=len(self.pre_emb)
        if type=="text":
            # this seems faster than gensim non-binary load
            for line in open(emb_path):
                l = line.strip().split()
                st=l[0].lower()
                self.pre_emb[st]=np.asarray(l[1:])
            num_keys=len(self.pre_emb)
        else:
            self.pre_emb = Word2Vec.load_word2vec_format(emb_path,binary=True)
            self.pre_emb.init_sims(replace=True)
            num_keys=len(self.pre_emb.vocab)
        print("loaded word2vec len ", num_keys)
        gc.collect() 

def test_unclosed_client_session_issue_645_in_async_mode(self):
        def exception_handler(_, context):
            nonlocal session_unclosed
            if context["message"] == "Unclosed client session":
                session_unclosed = True

        async def issue_645():
            client = WebClient(base_url="http://localhost:8888", timeout=1, run_async=True)
            try:
                await client.users_list(token="xoxb-timeout")
            except asyncio.TimeoutError:
                pass

        session_unclosed = False
        loop = asyncio.get_event_loop()
        loop.set_exception_handler(exception_handler)
        loop.run_until_complete(issue_645())
        gc.collect()  # force Python to gc unclosed client session
        self.assertFalse(session_unclosed, "Unclosed client session") 

def test_subscription_doesnt_block_subscriber_to_be_garbage_collected():
    from moler.publisher import Publisher

    notifier = Publisher()
    garbage_collected_subscribers = []

    class GcSubscriber(object):
        def __del__(self):
            garbage_collected_subscribers.append('Subscriber')

    subscr = GcSubscriber()
    notifier.subscribe(subscr)

    del subscr
    gc.collect()

    assert 'Subscriber' in garbage_collected_subscribers 

def test_write_del():

    for i in range(N):
        pids0 = get_ffmpeg_pids()
        w = imageio_ffmpeg.write_frames(test_file2, (64, 64))
        pids1 = get_ffmpeg_pids().difference(pids0)  # generator has not started
        w.send(None)
        w.send(b"x" * 64 * 64 * 3)
        pids2 = get_ffmpeg_pids().difference(pids0)  # now ffmpeg is running
        del w
        gc.collect()
        pids3 = get_ffmpeg_pids().difference(pids0)  # now its not

        assert len(pids1) == 0
        assert len(pids2) == 1
        assert len(pids3) == 0 

def test_threading():
    # See issue #20

    num_threads = 16
    num_frames = 5

    def make_iterator(q, n):
        for i in range(n):
            gen = imageio_ffmpeg.read_frames(test_file1)
            gen.__next__()  # meta data
            q.put(gen.__next__())  # first frame

    q = queue.Queue()
    threads = []
    for i in range(num_threads):
        t = threading.Thread(target=make_iterator, args=(q, num_frames))
        t.daemon = True
        t.start()
        threads.append(t)

    for i in range(num_threads * num_frames):
        print(i, end=" ")
        q.get()
        gc.collect()  # this seems to help invoke the segfault earlier 

def __getitem__(self, item):
        if isinstance(item, (tuple, list)):
            time_args = item[0]
            filters = item[1]
        else:
            time_args = item
            filters = None

        if isinstance(time_args, slice):
            return self.collect(
                time_args.start, time_args.stop, filters=filters,
            )
        elif isinstance(time_args, (datetime, str)):
            filename = self.find_closest(time_args, filters=filters)
            if filename is None:
                return None

            return self.read(filename) 

def debug_cycles(self):
        """Debugging support"""
        gc.set_debug(gc.DEBUG_SAVEALL)
        gc.collect()
        for obj in gc.garbage:
            logging.debug(repr(obj) + " " + str(type(obj))) 

def check(self):
        """Called by the QTimer periodically in the GUI thread"""
        # return self.debug_cycles() # uncomment to just debug cycles
        lvl0, lvl1, lvl2 = gc.get_count()
        logging.debug("gc_check called: %d, %d, %d", lvl0, lvl1, lvl2)
        if lvl0 > self.threshold[0]:
            num = gc.collect(0)
            logging.debug("collecting gen 0, found: %d unreachable", num)
            if lvl1 > self.threshold[1]:
                num = gc.collect(1)
                logging.debug("collecting gen 1, found: %d unreachable", num)
                if lvl2 > self.threshold[2]:
                    num = gc.collect(2)
                    logging.debug("collecting gen 2, found: %d unreachable",
                                  num) 

def __iter__(self):
    records = []
    size = 0
    for record in input_readers.RecordsReader.__iter__(self):
      records.append(record)
      size += len(record)
      if size > self.BATCH_SIZE:
        yield records
        size = 0
        records = []
        gc.collect()
    if records:
      yield records
      records = []
      gc.collect() 

def test_memory_leak(self):
        """ Check that every plot type gets properly collected. """
        import weakref
        import gc

        results = {}
        for kind in plotting._core._plot_klass.keys():
            if not _ok_for_gaussian_kde(kind):
                continue
            args = {}
            if kind in ['hexbin', 'scatter', 'pie']:
                df = self.hexbin_df
                args = {'x': 'A', 'y': 'B'}
            elif kind == 'area':
                df = self.tdf.abs()
            else:
                df = self.tdf

            # Use a weakref so we can see if the object gets collected without
            # also preventing it from being collected
            results[kind] = weakref.proxy(df.plot(kind=kind, **args))

        # have matplotlib delete all the figures
        tm.close()
        # force a garbage collection
        gc.collect()
        for key in results:
            # check that every plot was collected
            with pytest.raises(ReferenceError):
                # need to actually access something to get an error
                results[key].lines 

def test_structured_object_create_delete(self, dt, pat, count, singleton,
                                             creation_func, creation_obj):
        """Structured object reference counting in creation and deletion"""
        # The test assumes that 0, 1, and None are singletons.
        gc.collect()
        before = sys.getrefcount(creation_obj)
        arr = creation_func(3, dt)

        now = sys.getrefcount(creation_obj)
        assert now - before == count * 3
        del arr
        now = sys.getrefcount(creation_obj)
        assert now == before 

def test_refcount(self):
        # make sure we do not hold references to the array due to a recursive
        # closure (gh-10620)
        gc.disable()
        a = np.arange(2)
        r1 = sys.getrefcount(a)
        np.array2string(a)
        np.array2string(a)
        r2 = sys.getrefcount(a)
        gc.collect()
        gc.enable()
        assert_(r1 == r2) 

def test_shuffle_of_array_of_objects(self):
        # Test that permuting an array of objects will not cause
        # a segfault on garbage collection.
        # See gh-7719
        np.random.seed(1234)
        a = np.array([np.arange(1), np.arange(4)])

        for _ in range(1000):
            np.random.shuffle(a)

        # Force Garbage Collection - should not segfault.
        import gc
        gc.collect() 

def test_shuffle_of_array_of_different_length_strings(self):
        # Test that permuting an array of different length strings
        # will not cause a segfault on garbage collection
        # Tests gh-7710
        np.random.seed(1234)

        a = np.array(['a', 'a' * 1000])

        for _ in range(100):
            np.random.shuffle(a)

        # Force Garbage Collection - should not segfault.
        import gc
        gc.collect() 

def _release_all(self):
        """Release all connections.

        Note that this can lead to some code fail badly when trying to use
        an already closed connection. If you call .release_all() make sure
        your code can deal with it.
        """
        # Make sure that all connections lying about are collected before we go on.
        try:
            gc.collect()
        except (TypeError, AttributeError):
            # We've detected that we're being called in an incomplete
            # finalization state, we just bail out, leaving the connections
            # to take care of themselves.
            return

        if self._disposed:
            raise PoolError('Connection pool is disposed')

        if not self._disable_pooling:
            close_list = self._pool + list(self._used.values())
            self._log('Closing %d connection(s)' % len(close_list))

            for conn in close_list:
                try:
                    conn.close()
                except:
                    pass

        self._disposed = True
        self._pool = []
        self._used = {}