Python resource.getrusage() Examples

def print_used_time(func):
        ''' 打印运行时间

        :param func: 运行的函数名称
        :return:
        '''

        @wraps(func)
        def wrapper(*args, **kwargs):
            start_time, start_resources = timestamp(), resource_usage(RUSAGE_SELF)
            func(*args, **kwargs)
            end_resources, end_time = resource_usage(RUSAGE_SELF), timestamp()
            print({'消耗时间': {'real': end_time - start_time,
                            'sys': end_resources.ru_stime - start_resources.ru_stime,
                            'user': end_resources.ru_utime - start_resources.ru_utime}})
            return True
        return wrapper 

def _get_internal_metrics(tasks: List[Task]) -> List[Metric]:
    """Internal wca metrics e.g. memory usage, profiling information."""

    # Memory usage.
    memory_usage_rss_self = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
    memory_usage_rss_children = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss
    memory_usage_rss = memory_usage_rss_self + memory_usage_rss_children

    metrics = [
        Metric.create_metric_with_metadata(MetricName.WCA_UP, value=time.time()),
        Metric.create_metric_with_metadata(MetricName.WCA_TASKS, value=len(tasks)),
        Metric.create_metric_with_metadata(MetricName.WCA_MEM_USAGE_BYTES,
                                           value=int(memory_usage_rss * 1024)),
    ]

    return metrics 

def update(self):
            """
            Get memory metrics of current process through `getrusage`.  Only
            available on Unix, on Linux most of the fields are not set,
            and on BSD units are used that are not very helpful, see:

            http://www.perlmonks.org/?node_id=626693

            Furthermore, getrusage only provides accumulated statistics (e.g.
            max rss vs current rss).
            """
            usage = getrusage(RUSAGE_SELF)
            self.rss = usage.ru_maxrss * 1024
            self.data_segment = usage.ru_idrss * 1024 # TODO: ticks?
            self.shared_segment = usage.ru_ixrss * 1024 # TODO: ticks?
            self.stack_segment = usage.ru_isrss * 1024 # TODO: ticks?
            self.vsz = self.data_segment + self.shared_segment + \
                       self.stack_segment

            self.pagefaults = usage.ru_majflt
            return self.rss != 0 

def load_dev_data2(self,data_path:str, start:int, stop:int):
        pref = self.parameters.pref

        dev_data = []

        logging.info('dev_data')
        logging.info('Memory usage: %s', resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
        for x in pad_generator(batch_generator(
                generate_arrays_from_file(data_path, self.unique_features_dict,
                                          pref['feature_name'], pref['label_name'],
                                          start=start,
                                          stop=stop, keep_infinity=False,
                                          keep_unaligned=True),
                batch_size=pref['batch_size'])):
            dev_data.append(x)

        self.dev_data = DataList(dev_data) 

def x(self):
        pref = self.parameters.pref

        self.load_test_data()
        self.load_dev_data()


        logging.info('train_data')
        logging.info('Memory usage: %s', resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
        train_data = DataGenerator(self.data_path, self.unique_features_dict, pref,
                                   range(pref['train_data_size']))
        self.train_data = train_data




        logging.info('Data created') 

def start(self, verbose=False, payload=None):
        if verbose:
            self.process = subprocess.Popen(filter(None, self.cmd),
                                            stdin=None,
                                            stdout=None,
                                            stderr=None)
        else:
            self.process = subprocess.Popen(filter(None, self.cmd),
                                            stdin=subprocess.PIPE,
                                            stdout=subprocess.PIPE,
                                            stderr=None)

        self.stat_fd = open("/proc/" + str(self.process.pid) + "/stat")
        self.init()
        try:
            self.set_init_state(payload=payload)
        except:
            return False
        self.initial_mem_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
        # time.sleep(1)
        self.kafl_shm.seek(0x0)
        self.kafl_shm.write(self.virgin_bitmap)
        # self.kafl_shm.flush()
        return True 

def get_used_memory():
        """ Return the used memory in MB """
        if platform.system() == 'Linux':
            for line in open('/proc/self/status'):
                if line.startswith('VmRSS:'):
                    return int(line.split()[1]) >> 10

        else:
            warnings.warn("Please install psutil to have better "
                          "support with spilling")
            if platform.system() == "Darwin":
                import resource
                rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
                return rss >> 20
            # TODO: support windows

        return 0 

def peak_memory_mb()         :
    u"""
    Get peak memory usage for this process, as measured by
    max-resident-set size:

    https://unix.stackexchange.com/questions/30940/getrusage-system-call-what-is-maximum-resident-set-size

    Only works on OSX and Linux, returns 0.0 otherwise.
    """
    if resource is None or sys.platform not in (u'linux', u'darwin'):
        return 0.0

    # TODO(joelgrus): For whatever, our pinned version 0.521 of mypy does not like
    # next line, but later versions (e.g. 0.530) are fine with it. Once we get that
    # figured out, remove the type: ignore.
    peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss  # type: ignore

    if sys.platform == u'darwin':
        # On OSX the result is in bytes.
        return peak / 1000000

    else:
        # On Linux the result is in kilobytes.
        return peak / 1000 

def get_usage():
    # This function only works if you have a signal handler for the
    # signal.SIGCHLD signal.
    raw_usage = resource.getrusage(resource.RUSAGE_CHILDREN)
    # 0   ru_utime    time in user mode (float)
    # 1   ru_stime    time in system mode (float)
    # 2   ru_maxrss   maximum resident set size
    #
    # These fields are always zero on Linux
    # 3   ru_ixrss    shared memory size
    # 4   ru_idrss    unshared memory size
    # 5   ru_isrss    unshared stack size
    return "User:{}s System:{}s".format(
        int(raw_usage.ru_utime),
        int(raw_usage.ru_stime),
    ) 

def get_usage():
    # This function only works if you have a signal handler for the
    # signal.SIGCHLD signal.
    raw_usage = resource.getrusage(resource.RUSAGE_CHILDREN)
    # 0   ru_utime    time in user mode (float)
    # 1   ru_stime    time in system mode (float)
    # 2   ru_maxrss   maximum resident set size
    #
    # These fields are always zero on Linux
    # 3   ru_ixrss    shared memory size
    # 4   ru_idrss    unshared memory size
    # 5   ru_isrss    unshared stack size
    return "User:{}s System:{}s".format(
        int(raw_usage.ru_utime),
        int(raw_usage.ru_stime),
    ) 

def start(self, verbose=False):
        if verbose:
            self.process = subprocess.Popen(filter(None, self.cmd.split(" ")),
                                            stdin=None,
                                            stdout=None,
                                            stderr=None)
        else:
            self.process = subprocess.Popen(filter(None, self.cmd.split(" ")),
                                            stdin=subprocess.PIPE,
                                            stdout=subprocess.PIPE,
                                            stderr=subprocess.PIPE)

        self.stat_fd = open("/proc/" + str(self.process.pid) + "/stat")
        self.init()
        try:
            self.set_init_state()
        except:
            return False
        self.initial_mem_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
        #time.sleep(1)
        self.kafl_shm.seek(0x0)
        self.kafl_shm.write(self.virgin_bitmap)
        self.kafl_shm.flush()
        return True 

def _test_get_metadata_memory_usage(self, ec_driver):
        # 1. Prepare the expected memory allocation
        encoded = ec_driver.encode(b'aaa')
        ec_driver.get_metadata(encoded[0], formatted=True)
        loop_range = range(400000)

        # 2. Get current memory usage
        baseline_usage = resource.getrusage(resource.RUSAGE_SELF)[2]

        # 3. Loop to call get_metadata
        for x in loop_range:
            ec_driver.get_metadata(encoded[0], formatted=True)

        # 4. memory usage shouldn't increase
        new_usage = resource.getrusage(resource.RUSAGE_SELF)[2]
        self.assertEqual(baseline_usage, new_usage,
                         'Memory usage is increased unexpectedly %s -> %s' %
                         (baseline_usage, new_usage)) 

def _test_get_segment_info_memory_usage(self, ec_driver):
        # 1. Preapre the expected memory allocation
        ec_driver.get_segment_info(1024 * 1024, 1024 * 1024)
        loop_range = range(1000)

        # 2. Get current memory usage
        usage = resource.getrusage(resource.RUSAGE_SELF)[2]

        # 3. Loop to call get_segment_info
        for x in loop_range:
            ec_driver.get_segment_info(1024 * 1024, 1024 * 1024)

        # 4. memory usage shoudln't be increased
        self.assertEqual(usage, resource.getrusage(resource.RUSAGE_SELF)[2],
                         'Memory usage is increased unexpectedly %s - %s' %
                         (usage, resource.getrusage(resource.RUSAGE_SELF)[2])) 

def execute(self, verbose=False):
        if verbose:
            note('executing: %s' % ' '.join("'%s'" % arg
                                            for arg in self.command))

        start_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
        start_time = time.time()

        p = subprocess.Popen(self.command,
                             stdout=open(self.stdout_path, 'w'),
                             stderr=open(self.stderr_path, 'w'),
                             env=self.env)
        self.result = p.wait() == 0

        end_time = time.time()
        end_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
        self.metrics["user_time"] = end_rusage.ru_utime - start_rusage.ru_utime
        self.metrics["sys_time"] = end_rusage.ru_stime - start_rusage.ru_stime
        self.metrics["wall_time"] = end_time - start_time

        if verbose:
            note("command executed in -- "
                 "user: %.4fs, wall: %.4fs, sys: %.4fs" % (
                    self.metrics["user_time"], self.metrics["wall_time"],
                    self.metrics["sys_time"])) 

def memory_usage_resource():
	import resource
	rusage_denom = 1024.
	if sys.platform == 'darwin':
		# ... it seems that in OSX the output is different units ...
		rusage_denom = rusage_denom * rusage_denom
	mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rusage_denom
	return round(mem,0) 

def get_usage_info(self):
        """Returns CPU and memory usage information.

        It returns the results in a tuple, with the first element being the number of
        CPU seconds spent in user land, the second is the number of CPU seconds spent in system land,
        and the third is the current resident size of the process in bytes."""

        usage_info = resource.getrusage(resource.RUSAGE_SELF)
        user_cpu = usage_info[0]
        system_cpu = usage_info[1]
        rss_size = usage_info[2]

        return user_cpu, system_cpu, rss_size 

def peakmem() -> int:
    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss 

def peakmem() -> int:
    return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss 

def memusage( point = "") :
    usage = resource.getrusage( resource.RUSAGE_SELF) 
    return '''%s: usertime = %s systime = %s mem = %s mb
           '''%( point, usage[ 0 ], usage[ 1 ], 
                ( usage[ 2 ]*resource.getpagesize( ) ) /1000000.0 ) 

def memory_usage_resource():
	import resource
	rusage_denom = 1024.
	if sys.platform == 'darwin':
		# ... it seems that in OSX the output is different units ...
		rusage_denom = rusage_denom * rusage_denom
	mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rusage_denom
	return round(mem, 0) 

def memory_usage_resource():
	import resource
	rusage_denom = 1024.
	if sys.platform == 'darwin':
		# ... it seems that in OSX the output is different units ...
		rusage_denom = rusage_denom * rusage_denom
	mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rusage_denom
	return round(mem,0) 

def _get_mem_usage(self):
        """
        Gets the RUSAGE memory usage, returns in K. Encapsulates the difference
        between macOS and Linux rss reporting

        :returns: memory usage in kilobytes
        """

        from resource import getrusage, RUSAGE_SELF
        mem = getrusage(RUSAGE_SELF).ru_maxrss
        if sys.platform == 'darwin':
            # man 2 getrusage:
            #     ru_maxrss
            # This is the maximum resident set size utilized (in bytes).
            return mem / 1024  # Kb
        else:
            # linux
            # man 2 getrusage
            #        ru_maxrss (since Linux 2.6.32)
            #  This is the maximum resident set size used (in kilobytes).
            return mem  # Kb 

def memory_usage_resource():
	import resource
	rusage_denom = 1024.
	if sys.platform == 'darwin':
		# ... it seems that in OSX the output is different units ...
		rusage_denom = rusage_denom * rusage_denom
	mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rusage_denom
	return round(mem,0) 

def test_no_leak_many_call_lazy():
        # Verify no memory leaks when calling a function a lot of times

        # This isn't really a unit test, you have to run it and look at top to
        # see if there's a leak

        def build_graph(x, depth=5):
            z = x
            for d in range(depth):
                z = ifelse(z.mean() > 0.5, -z, z)
            return z

        def time_linker(name, linker):
            steps_a = 10
            x = tensor.dvector()
            a = build_graph(x, steps_a)

            f_a = function([x], a,
                           mode=Mode(optimizer=None,
                                     linker=linker()))
            inp = numpy.random.rand(1000000)
            for i in xrange(100):
                f_a(inp)
            if 0:  # this doesn't seem to work, prints 0 for everything
                import resource
                pre = resource.getrusage(resource.RUSAGE_SELF)
                post = resource.getrusage(resource.RUSAGE_SELF)
                print(pre.ru_ixrss, post.ru_ixrss)
                print(pre.ru_idrss, post.ru_idrss)
                print(pre.ru_maxrss, post.ru_maxrss)
        print(1)
        time_linker('vmLinker_C',
                    lambda: vm.VM_Linker(allow_gc=False, use_cloop=True))
        print(2)
        time_linker('vmLinker',
                    lambda: vm.VM_Linker(allow_gc=False, use_cloop=False)) 

def init_objects_and_adjacency_records(self):
        print("Initializing the segmenter...")
        print("Max mem: {} GB".format(resource.getrusage(
            resource.RUSAGE_SELF).ru_maxrss / 1024 / 1024))
        obj_id = 0
        for row in range(self.img_height):
            for col in range(self.img_width):
                pixels = set([(row, col)])
                obj = Object(pixels, obj_id, self)
                self.objects[obj_id] = obj
                self.pixel2obj[(row, col)] = obj
                obj_id += 1

        for row in range(self.img_height):
            for col in range(self.img_width):
                obj1 = self.pixel2obj[(row, col)]
                for o, idx in zip(self.offsets, range(len(self.offsets))):
                    (i, j) = o
                    if (0 <= row + i < self.img_height and
                            0 <= col + j < self.img_width):
                        obj2 = self.pixel2obj[(row + i, col + j)]
                        arec = AdjacencyRecord(obj1, obj2, self, (row, col), idx)
                        self.adjacency_records[arec] = arec
                        obj1.adjacency_list[arec] = arec
                        obj2.adjacency_list[arec] = arec
                        if arec.merge_priority >= 0:
                            heappush(self.queue, (-arec.merge_priority, arec)) 

def init_objects_and_adjacency_records(self):
        print("Initializing the segmenter...")
        print("Max mem: {} GB".format(resource.getrusage(
            resource.RUSAGE_SELF).ru_maxrss / 1024 / 1024))
        obj_id = 0
        for row in range(self.img_height):
            for col in range(self.img_width):
                pixels = set([(row, col)])
                obj = Object(pixels, obj_id, self)
                self.objects[obj_id] = obj
                self.pixel2obj[(row, col)] = obj
                obj_id += 1

        for row in range(self.img_height):
            for col in range(self.img_width):
                obj1 = self.pixel2obj[(row, col)]
                for o, idx in zip(self.offsets, range(len(self.offsets))):
                    (i, j) = o
                    if (0 <= row + i < self.img_height and
                            0 <= col + j < self.img_width):
                        obj2 = self.pixel2obj[(row + i, col + j)]
                        arec = AdjacencyRecord(obj1, obj2, self, (row, col), idx)
                        self.adjacency_records[arec] = arec
                        obj1.adjacency_list[arec] = arec
                        obj2.adjacency_list[arec] = arec
                        if arec.merge_priority >= 0:
                            heappush(self.queue, (-arec.merge_priority, arec)) 

def init_objects_and_adjacency_records(self):
        print("Initializing the segmenter...")
        print("Max mem: {} GB".format(resource.getrusage(
            resource.RUSAGE_SELF).ru_maxrss / 1024 / 1024))
        obj_id = 0
        for row in range(self.img_height):
            for col in range(self.img_width):
                pixels = set([(row, col)])
                obj = Object(pixels, obj_id, self)
                self.objects[obj_id] = obj
                self.pixel2obj[(row, col)] = obj
                obj_id += 1

        for row in range(self.img_height):
            for col in range(self.img_width):
                obj1 = self.pixel2obj[(row, col)]
                for o, idx in zip(self.offsets, range(len(self.offsets))):
                    (i, j) = o
                    if (0 <= row + i < self.img_height and
                            0 <= col + j < self.img_width):
                        obj2 = self.pixel2obj[(row + i, col + j)]
                        arec = AdjacencyRecord(obj1, obj2, self, (row, col), idx)
                        self.adjacency_records[arec] = arec
                        obj1.adjacency_list[arec] = arec
                        obj2.adjacency_list[arec] = arec
                        if arec.merge_priority >= 0:
                            heappush(self.queue, (-arec.merge_priority, arec)) 

def init_objects_and_adjacency_records(self):
        print("Initializing the segmenter...")
        print("Max mem: {} GB".format(resource.getrusage(
            resource.RUSAGE_SELF).ru_maxrss / 1024 / 1024))
        obj_id = 0
        for row in range(self.img_height):
            for col in range(self.img_width):
                pixels = set([(row, col)])
                obj = Object(pixels, obj_id, self)
                self.objects[obj_id] = obj
                self.pixel2obj[(row, col)] = obj
                obj_id += 1

        for row in range(self.img_height):
            for col in range(self.img_width):
                obj1 = self.pixel2obj[(row, col)]
                for o, idx in zip(self.offsets, range(len(self.offsets))):
                    (i, j) = o
                    if (0 <= row + i < self.img_height and
                            0 <= col + j < self.img_width):
                        obj2 = self.pixel2obj[(row + i, col + j)]
                        arec = AdjacencyRecord(obj1, obj2, self, (row, col), idx)
                        self.adjacency_records[arec] = arec
                        obj1.adjacency_list[arec] = arec
                        obj2.adjacency_list[arec] = arec
                        if arec.merge_priority >= 0:
                            heappush(self.queue, (-arec.merge_priority, arec)) 

def _get_mem_usage(self):
        from resource import getpagesize, getrusage, RUSAGE_SELF
        mem = getrusage(RUSAGE_SELF).ru_maxrss
        return mem * getpagesize() / 1024 / 1024 

def get_rusage():
        usage = resource.getrusage(resource.RUSAGE_SELF)
        return {
            "utime": usage.ru_utime,
            "stime": usage.ru_stime,
            "rss": usage.ru_maxrss,
        }