Python cProfile.Profile() Examples

def profileit(func):
    """
    Decorator straight up stolen from stackoverflow
    """
    def wrapper(*args, **kwargs):
        datafn = func.__name__ + ".profile" # Name the data file sensibly
        prof = cProfile.Profile()
        prof.enable()
        retval = prof.runcall(func, *args, **kwargs)
        prof.disable()
        stats = pstats.Stats(prof)
        stats.sort_stats('tottime').print_stats(20)
        print()
        print()
        stats.sort_stats('cumtime').print_stats(20)
        return retval

    return wrapper 

def run(self, reactor):
        """
        Run reactor under the standard profiler.
        """
        try:
            import profile
        except ImportError as e:
            self._reportImportError("profile", e)

        p = profile.Profile()
        p.runcall(reactor.run)
        if self.saveStats:
            p.dump_stats(self.profileOutput)
        else:
            tmp, sys.stdout = sys.stdout, open(self.profileOutput, 'a')
            try:
                p.print_stats()
            finally:
                sys.stdout, tmp = tmp, sys.stdout
                tmp.close() 

def test_time_ipfix(self):
        """
        Profile function calls and CPU time.
        TODO: this does not work with threading in the collector, yet
        :return:
        """
        profile = cProfile.Profile()
        profile.enable(subcalls=True, builtins=True)
        pkts, t1, t2 = send_recv_packets(generate_packets(NUM_PACKETS_PERFORMANCE, 10), delay=0, store_packets=500)
        self.assertEqual(len(pkts), NUM_PACKETS_PERFORMANCE)
        profile.disable()

        for sort_by in ['cumulative', 'calls']:
            s = io.StringIO()
            ps = pstats.Stats(profile, stream=s)
            ps.sort_stats(sort_by).print_stats("netflow")
            ps.sort_stats(sort_by).print_callees(.5)
            print(s.getvalue()) 

def profile(filename=None, comm=MPI.COMM_WORLD):
    def prof_decorator(f):
        def wrap_f(*args, **kwargs):
            pr = cProfile.Profile()
            pr.enable()
            result = f(*args, **kwargs)
            pr.disable()

            if filename is None:
                pr.print_stats()
            else:
                filename_r = filename  # + ".{}".format(comm.rank)
                pr.dump_stats(filename_r)

            return result
        return wrap_f
    return prof_decorator 

def __call__(self, *args, **kw):
        """Profile a singe call to the function."""
        fn = self.fn
        timer = self.timer
        self.ncalls += 1
        start = timer()
        try:
            return fn(*args, **kw)
        finally:
            duration = timer() - start
            self.totaltime += duration
            if self.immediate:
                funcname = fn.__name__
                filename = fn.__code__.co_filename
                lineno = fn.__code__.co_firstlineno
                message = "%s (%s:%s):\n    %.3f seconds\n\n" % (
                    funcname, filename, lineno, duration,
                )
                if self.logger:
                    self.logger.log(self.log_level, message)
                else:
                    sys.stderr.write("\n  " + message)
                    sys.stderr.flush() 

def __call__(self, *args, **kw):
            """Profile a singe call to the function."""
            self.ncalls += 1
            if self.skip > 0:
                self.skip -= 1
                self.skipped += 1
                return self.fn(*args, **kw)
            if HotShotFuncProfile.in_profiler:
                # handle recursive calls
                return self.fn(*args, **kw)
            if self.profiler is None:
                self.profiler = hotshot.Profile(self.logfilename)
            try:
                HotShotFuncProfile.in_profiler = True
                return self.profiler.runcall(self.fn, *args, **kw)
            finally:
                HotShotFuncProfile.in_profiler = False
                if self.immediate:
                    self.print_stats()
                    self.reset_stats() 

def __call__(self, *args, **kw):
        """Profile a singe call to the function."""
        self.ncalls += 1
        if self.skip > 0:
            self.skip -= 1
            self.skipped += 1
            return self.fn(*args, **kw)
        if FuncProfile.in_profiler:
            # handle recursive calls
            return self.fn(*args, **kw)
        # You cannot reuse the same profiler for many calls and accumulate
        # stats that way.  :-/
        profiler = self.Profile()
        try:
            FuncProfile.in_profiler = True
            return profiler.runcall(self.fn, *args, **kw)
        finally:
            FuncProfile.in_profiler = False
            self.stats.add(profiler)
            if self.immediate:
                self.print_stats()
                self.reset_stats() 

def profile(fnc):
    """
    Profiles any function in following class just by adding @profile above function
    """
    import cProfile, pstats, io
    def inner (*args, **kwargs):
        pr = cProfile.Profile()
        pr.enable()
        retval = fnc (*args, **kwargs)
        pr.disable()
        s = io.StringIO()
        sortby = 'cumulative'   #Ordered
        ps = pstats.Stats(pr,stream=s).strip_dirs().sort_stats(sortby)
        n=20                    #reduced the list to be monitored
        ps.print_stats(n)
        #ps.dump_stats("profile.prof")
        print(s.getvalue())
        return retval
    return inner 

def main():
    print('create y...')
    y = np.random.randint(2, size=N_OBS)
    print('create X...')
    row = np.random.randint(N_OBS, size=N_VALUE)
    col = np.random.randint(N_FEATURE, size=N_VALUE)
    data = np.ones(N_VALUE)
    X = sparse.csr_matrix((data, (row, col)), dtype=np.int8)

    print('train...')
    profiler = cProfile.Profile(subcalls=True, builtins=True, timeunit=0.001,)
    clf = FTRL(interaction=False)
    profiler.enable()
    clf.fit(X, y)
    profiler.disable()
    profiler.print_stats()

    p = clf.predict(X)
    print('AUC: {:.4f}'.format(auc(y, p)))

    assert auc(y, p) > .5 

def run_test(layers):
    print("Running perf test")
    i = 0
    profiler = cProfile.Profile()
    for layer in layers:
        layer_description = {
            'features' : layer,
            'name': 'bar'
        }
        profiler.enable()
        res = encode(layer_description, on_invalid_geometry=on_invalid_geometry_ignore, round_fn=round)
        profiler.disable()
        if i % 100 == 0:
            print("{} tiles produced".format(i))
        i += 1

    print ("Perf result :")
    profiler.print_stats() 

def __call__(self, environ, start_response):
        query = environ['QUERY_STRING']
        if ('state=' in query) and (('code=' in query) or ('error=' in query)):
            request = webob.Request(environ)
            environ['QUERY_STRING'] += ('&' + request.params['state'])
            environ['REQUEST_METHOD'] = 'POST'
        if self.debug:
            perf = profile.Profile()
            start = time.time()
            ret = perf.runcall(super(WSGIApp, self).__call__, environ, start_response)
            if time.time() - start > 1:
                stats = pstats.Stats(perf)
                stats.sort_stats('cumtime')
                stats.print_stats(60)
            return ret
        else:
            return super(WSGIApp, self).__call__(environ, start_response) 

def __init__(self, messageProcessorFactory, enableProfiling=False):
        self.messageProcessor = messageProcessorFactory()
        self.events = {}
        self.reactorThread = None
        self.nextMessageId = 0
        self.messageHandlers = {}
        self.testMessageVisitor = None

        self.lock = threading.Lock()
        self.messageQueue = Queue.Queue()

        if enableProfiling:
            self.profiler = cProfile.Profile()
        else:
            self.profiler = None

        with self.lock:
            self.reactorThread = threading.Thread(target=self.reactorThreadLoop)
            self.reactorThread.start() 

def _profile_package(self):
        """Runs cProfile on a package."""
        prof = cProfile.Profile()
        prof.enable()
        try:
            runpy.run_path(self._run_object, run_name='__main__')
        except SystemExit:
            pass
        prof.disable()
        prof_stats = pstats.Stats(prof)
        prof_stats.calc_callees()
        return {
            'objectName': self._object_name,
            'callStats': self._transform_stats(prof_stats),
            'totalTime': prof_stats.total_tt,
            'primitiveCalls': prof_stats.prim_calls,
            'totalCalls': prof_stats.total_calls,
            'timestamp': int(time.time())
        } 

def _profile_module(self):
        """Runs cProfile on a module."""
        prof = cProfile.Profile()
        try:
            with open(self._run_object, 'rb') as srcfile:
                code = compile(srcfile.read(), self._run_object, 'exec')
            prof.runctx(code, self._globs, None)
        except SystemExit:
            pass
        prof_stats = pstats.Stats(prof)
        prof_stats.calc_callees()
        return {
            'objectName': self._object_name,
            'callStats': self._transform_stats(prof_stats),
            'totalTime': prof_stats.total_tt,
            'primitiveCalls': prof_stats.prim_calls,
            'totalCalls': prof_stats.total_calls,
            'timestamp': int(time.time())
        } 

def profile_function(self):
        """Runs cProfile on a function."""
        prof = cProfile.Profile()
        prof.enable()
        result = self._run_object(*self._run_args, **self._run_kwargs)
        prof.disable()
        prof_stats = pstats.Stats(prof)
        prof_stats.calc_callees()
        return {
            'objectName': self._object_name,
            'callStats': self._transform_stats(prof_stats),
            'totalTime': prof_stats.total_tt,
            'primitiveCalls': prof_stats.prim_calls,
            'totalCalls': prof_stats.total_calls,
            'result': result,
            'timestamp': int(time.time())
        } 

def do_loop(self, callbacks: List[Callable]):
        if self.env.arguments.enable_profiler:
            profile = cProfile.Profile()
            profile.enable()

        try:
            self.loop.run_until_complete(self.shutdown_future)
        except KeyboardInterrupt:
            pass
        finally:
            for callback in callbacks:
                if callable(callback):
                    callback()

        if self.env.arguments.enable_profiler:
            profile.disable()
            profile.print_stats("time") 

def test_load_env(self):
        """
        Just executes the SetUp and tearDown functions.
        :return:
        """
        if DEBUG:
            if PROFILE_CODE:
                cp = cProfile.Profile()
                cp.enable()
            import pandapower as pp
            nb_powerflow = 5000
            beg_ = time.time()
            for i in range(nb_powerflow):
                pp.runpp(self.backend._grid)
            end_ = time.time()
            print("Time to compute {} powerflows: {:.2f}".format(nb_powerflow, end_-beg_))
            if PROFILE_CODE:
                cp.disable()
                cp.print_stats(sort="tottime")
        pass 

def test_load_env(self):
        """
        Just executes the SetUp and tearDown functions.
        :return:
        """
        if DEBUG:
            if PROFILE_CODE:
                cp = cProfile.Profile()
                cp.enable()
            import pandapower as pp
            nb_powerflow = 5000
            beg_ = time.time()
            for i in range(nb_powerflow):
                pp.runpp(self.backend._grid)
            end_ = time.time()
            print("Time to compute {} powerflows: {:.2f}".format(nb_powerflow, end_-beg_))
            if PROFILE_CODE:
                cp.disable()
                cp.print_stats(sort="tottime")
        pass 

def main(max_ts, name, use_lightsim=False, test_env=True):
    param = Parameters()
    if use_lightsim:
        if light_sim_avail:
            backend = LightSimBackend()
        else:
            raise RuntimeError("LightSimBackend not available")
    else:
        backend = PandaPowerBackend()

    param.init_from_dict({"NO_OVERFLOW_DISCONNECTION": True})

    env_klu = make(name, backend=backend, param=param, gamerules_class=AlwaysLegal, test=test_env)
    agent = TestAgent(action_space=env_klu.action_space, env_name=name)

    cp = cProfile.Profile()
    cp.enable()
    nb_ts_klu, time_klu, aor_klu, gen_p_klu, gen_q_klu = run_env(env_klu, max_ts, agent)
    cp.disable()
    nm_f, ext = os.path.splitext(__file__)
    nm_out = "{}_{}_{}.prof".format(nm_f, "lightsim" if use_ls else "pp", name)
    cp.dump_stats(nm_out)
    print("You can view profiling results with:\n\tsnakeviz {}".format(nm_out)) 

def flocker_container_agent_main():
    """
    Implementation of the ``flocker-container-agent`` command line script.

    This starts a Docker-based container convergence agent.
    """
    def deployer_factory(cluster_uuid, **kwargs):
        return ApplicationNodeDeployer(**kwargs)
    service_factory = AgentServiceFactory(
        deployer_factory=deployer_factory
    ).get_service
    agent_script = AgentScript(service_factory=service_factory)

    # Use CPU time instead of wallclock time.
    pr = cProfile.Profile(clock)

    signal.signal(signal.SIGUSR1, partial(enable_profiling, pr))
    signal.signal(signal.SIGUSR2, partial(disable_profiling, pr, 'container'))

    return FlockerScriptRunner(
        script=agent_script,
        options=ContainerAgentOptions()
    ).main() 

def flocker_dataset_agent_main():
    """
    Implementation of the ``flocker-dataset-agent`` command line script.

    This starts a dataset convergence agent.  It currently supports only a
    small number of hard-coded storage drivers.  Later it will be capable of
    starting a dataset agent using any Flocker-supplied storage driver any
    third-party drivers via plugins.
    """
    service_factory = DatasetServiceFactory()
    agent_script = AgentScript(service_factory=service_factory.get_service)
    options = DatasetAgentOptions()

    # Use CPU time instead of wallclock time.
    pr = cProfile.Profile(clock)

    signal.signal(signal.SIGUSR1, partial(enable_profiling, pr))
    signal.signal(signal.SIGUSR2, partial(disable_profiling, pr, 'dataset'))

    return FlockerScriptRunner(
        script=agent_script,
        options=options,
    ).main() 

def __init__(
        self,
        *,
        prefix: str = PREFIX,
        suffix: str = PROF_SUFFIX,
        dir: Optional[str] = None,
        save_every_n_calls: int = 1,
    ):
        """Create the decorator.

        If `save_every_n_calls` is greater than 1, the profiler will not
        dump data to files on every call to the profiled function.  This speeds
        up the running program but risks incomplete data if the process is
        terminated non-gracefully.

        `dir`, `prefix`, and `suffix` after `tempfile.mkstemp`.
        """
        self.prefix = prefix
        self.suffix = suffix
        self.save_every_n_calls = save_every_n_calls
        self.n_calls = 0
        self._dir: Union[str, pathlib.Path, None] = dir
        self._profiler: Optional[cProfile.Profile] = None
        self._dump_file_path: Optional[str] = None 

def profiled(print_callers=False):
    """
    This decorator provide deterministic profiling. It uses ``cProfile`` internally.  It generates statistic
    and print on the screen.
    """
    pr = cProfile.Profile()
    pr.enable()
    try:
        yield
    finally:
        pr.disable()
        s = io.StringIO()
        ps = pstats.Stats(pr, stream=s).sort_stats("cumulative")
        if print_callers:
            ps.print_callers()
        else:
            ps.print_stats()
        print(s.getvalue()) 

def profileMain(args, config):  # pragma: no cover
    """This is the main function for profiling
    http://code.google.com/appengine/kb/commontasks.html#profiling
    """
    import cProfile
    import pstats

    eyed3.log.debug("driver profileMain")
    prof = cProfile.Profile()
    prof = prof.runctx("main(args)", globals(), locals())

    stream = StringIO()
    stats = pstats.Stats(prof, stream=stream)
    stats.sort_stats("time")  # Or cumulative
    stats.print_stats(100)  # 80 = how many to print

    # The rest is optional.
    stats.print_callees()
    stats.print_callers()
    sys.stderr.write("Profile data:\n%s\n" % stream.getvalue())

    return 0 

def symm_inner_product_array(
    num_states, num_vecs, max_vecs_per_node, verbosity=1):
    """
    Computes symmetric inner product array from known vecs (as in POD).
    """
    vec_handles = [mr.VecHandlePickle(join(data_dir, row_vec_name%row_num))
        for row_num in mr.range(num_vecs)]

    generate_vecs(data_dir, num_states, vec_handles)

    my_VS = mr.VectorSpaceHandles(
        inner_product=np.vdot, max_vecs_per_node=max_vecs_per_node,
        verbosity=verbosity)

    prof = cProfile.Profile()
    start_time = time.time()
    prof.runcall(my_VS.compute_symm_inner_product_array, vec_handles)
    total_time = time.time() - start_time
    prof.dump_stats('IP_symm_array_r%d.prof'%mr.parallel.get_rank())

    return total_time 

def _wrap_profiling(self, runner, *args):
    if not self._vars.PEX_PROFILE and self._vars.PEX_PROFILE_FILENAME is None:
      return runner(*args)

    pex_profile_filename = self._vars.PEX_PROFILE_FILENAME
    pex_profile_sort = self._vars.PEX_PROFILE_SORT
    try:
      import cProfile as profile
    except ImportError:
      import profile

    profiler = profile.Profile()

    try:
      return profiler.runcall(runner, *args)
    finally:
      if pex_profile_filename is not None:
        profiler.dump_stats(pex_profile_filename)
      else:
        profiler.print_stats(sort=pex_profile_sort) 

def run(self, reactor):
        """
        Run reactor under the cProfile profiler.
        """
        try:
            import cProfile
            import pstats
        except ImportError as e:
            self._reportImportError("cProfile", e)

        p = cProfile.Profile()
        p.runcall(reactor.run)
        if self.saveStats:
            p.dump_stats(self.profileOutput)
        else:
            with open(self.profileOutput, 'w') as stream:
                s = pstats.Stats(p, stream=stream)
                s.strip_dirs()
                s.sort_stats(-1)
                s.print_stats() 

def main():
    from quarkchain.cluster.jsonrpc import JSONRPCWebsocketServer

    os.chdir(os.path.dirname(os.path.abspath(__file__)))
    env = parse_args()

    if env.arguments.enable_profiler:
        profile = cProfile.Profile()
        profile.enable()
    slave_server = SlaveServer(env)
    slave_server.start()

    callbacks = []
    if env.slave_config.WEBSOCKET_JSON_RPC_PORT is not None:
        json_rpc_websocket_server = JSONRPCWebsocketServer.start_websocket_server(
            env, slave_server
        )
        callbacks.append(json_rpc_websocket_server.shutdown)

    slave_server.do_loop()
    if env.arguments.enable_profiler:
        profile.disable()
        profile.print_stats("time")

    Logger.info("Slave server is shutdown") 

def main(_):
    sys.excepthook = ultratb.FormattedTB(color_scheme='Linux', call_pdb=1)
    pr = cProfile.Profile()
    pr.enable()
    run()
    pr.disable()
    pr.dump_stats('%s.profile' %  os.path.basename(__file__)) 

def lin_combine(
    num_states, num_bases, num_products, max_vecs_per_node, verbosity=1):
    """
    Computes linear combination of vecs from saved vecs and random coeffs

    num_bases is number of vecs to be linearly combined
    num_products is the resulting number of vecs
    """

    basis_handles = [mr.VecHandlePickle(join(data_dir, basis_name%basis_num))
        for basis_num in mr.range(num_bases)]
    product_handles = [mr.VecHandlePickle(join(data_dir,
        product_name%product_num))
        for product_num in mr.range(num_products)]

    generate_vecs(data_dir, num_states, basis_handles)
    my_VS = mr.VectorSpaceHandles(
        inner_product=np.vdot, max_vecs_per_node=max_vecs_per_node,
        verbosity=verbosity)
    coeff_array = np.random.random((num_bases, num_products))
    mr.parallel.barrier()

    prof = cProfile.Profile()
    start_time = time.time()
    prof.runcall(my_VS.lin_combine, *(product_handles, basis_handles,
        coeff_array))
    total_time = time.time() - start_time
    prof.dump_stats('lincomb_r%d.prof'%mr.parallel.get_rank())
    return total_time