codekingpro/portable-devtools
114k
1"""Class for printing reports on profiled python code."""2 3# Written by James Roskind4# Based on prior profile module by Sjoerd Mullender...5# which was hacked somewhat by: Guido van Rossum6 7# Copyright Disney Enterprises, Inc. All Rights Reserved.8# Licensed to PSF under a Contributor Agreement9#10# Licensed under the Apache License, Version 2.0 (the "License");11# you may not use this file except in compliance with the License.12# You may obtain a copy of the License at13#14# http://www.apache.org/licenses/LICENSE-2.015#16# Unless required by applicable law or agreed to in writing, software17# distributed under the License is distributed on an "AS IS" BASIS,18# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,19# either express or implied. See the License for the specific language20# governing permissions and limitations under the License.21 22 23import sys24import os25import time26import marshal27import re28 29from enum import StrEnum, _simple_enum30from functools import cmp_to_key31from dataclasses import dataclass32 33__all__ = ["Stats", "SortKey", "FunctionProfile", "StatsProfile"]34 35@_simple_enum(StrEnum)36class SortKey:37 CALLS = 'calls', 'ncalls'38 CUMULATIVE = 'cumulative', 'cumtime'39 FILENAME = 'filename', 'module'40 LINE = 'line'41 NAME = 'name'42 NFL = 'nfl'43 PCALLS = 'pcalls'44 STDNAME = 'stdname'45 TIME = 'time', 'tottime'46 47 def __new__(cls, *values):48 value = values[0]49 obj = str.__new__(cls, value)50 obj._value_ = value51 for other_value in values[1:]:52 cls._value2member_map_[other_value] = obj53 obj._all_values = values54 return obj55 56 57@dataclass(unsafe_hash=True)58class FunctionProfile:59 ncalls: str60 tottime: float61 percall_tottime: float62 cumtime: float63 percall_cumtime: float64 file_name: str65 line_number: int66 67@dataclass(unsafe_hash=True)68class StatsProfile:69 '''Class for keeping track of an item in inventory.'''70 total_tt: float71 func_profiles: dict[str, FunctionProfile]72 73class Stats:74 """This class is used for creating reports from data generated by the75 Profile class. It is a "friend" of that class, and imports data either76 by direct access to members of Profile class, or by reading in a dictionary77 that was emitted (via marshal) from the Profile class.78 79 The big change from the previous Profiler (in terms of raw functionality)80 is that an "add()" method has been provided to combine Stats from81 several distinct profile runs. Both the constructor and the add()82 method now take arbitrarily many file names as arguments.83 84 All the print methods now take an argument that indicates how many lines85 to print. If the arg is a floating-point number between 0 and 1.0, then86 it is taken as a decimal percentage of the available lines to be printed87 (e.g., .1 means print 10% of all available lines). If it is an integer,88 it is taken to mean the number of lines of data that you wish to have89 printed.90 91 The sort_stats() method now processes some additional options (i.e., in92 addition to the old -1, 0, 1, or 2 that are respectively interpreted as93 'stdname', 'calls', 'time', and 'cumulative'). It takes either an94 arbitrary number of quoted strings or SortKey enum to select the sort95 order.96 97 For example sort_stats('time', 'name') or sort_stats(SortKey.TIME,98 SortKey.NAME) sorts on the major key of 'internal function time', and on99 the minor key of 'the name of the function'. Look at the two tables in100 sort_stats() and get_sort_arg_defs(self) for more examples.101 102 All methods return self, so you can string together commands like:103 Stats('foo', 'goo').strip_dirs().sort_stats('calls').\104 print_stats(5).print_callers(5)105 """106 107 def __init__(self, *args, stream=None):108 self.stream = stream or sys.stdout109 if not len(args):110 arg = None111 else:112 arg = args[0]113 args = args[1:]114 self.init(arg)115 self.add(*args)116 117 def init(self, arg):118 self.all_callees = None # calc only if needed119 self.files = []120 self.fcn_list = None121 self.total_tt = 0122 self.total_calls = 0123 self.prim_calls = 0124 self.max_name_len = 0125 self.top_level = set()126 self.stats = {}127 self.sort_arg_dict = {}128 self.load_stats(arg)129 try:130 self.get_top_level_stats()131 except Exception:132 print("Invalid timing data %s" %133 (self.files[-1] if self.files else ''), file=self.stream)134 raise135 136 def load_stats(self, arg):137 if arg is None:138 self.stats = {}139 return140 elif isinstance(arg, str):141 with open(arg, 'rb') as f:142 self.stats = marshal.load(f)143 try:144 file_stats = os.stat(arg)145 arg = time.ctime(file_stats.st_mtime) + " " + arg146 except: # in case this is not unix147 pass148 self.files = [arg]149 elif hasattr(arg, 'create_stats'):150 arg.create_stats()151 self.stats = arg.stats152 arg.stats = {}153 if not self.stats:154 raise TypeError("Cannot create or construct a %r object from %r"155 % (self.__class__, arg))156 return157 158 def get_top_level_stats(self):159 for func, (cc, nc, tt, ct, callers) in self.stats.items():160 self.total_calls += nc161 self.prim_calls += cc162 self.total_tt += tt163 if ("jprofile", 0, "profiler") in callers:164 self.top_level.add(func)165 if len(func_std_string(func)) > self.max_name_len:166 self.max_name_len = len(func_std_string(func))167 168 def add(self, *arg_list):169 if not arg_list:170 return self171 for item in reversed(arg_list):172 if type(self) != type(item):173 item = Stats(item)174 self.files += item.files175 self.total_calls += item.total_calls176 self.prim_calls += item.prim_calls177 self.total_tt += item.total_tt178 for func in item.top_level:179 self.top_level.add(func)180 181 if self.max_name_len < item.max_name_len:182 self.max_name_len = item.max_name_len183 184 self.fcn_list = None185 186 for func, stat in item.stats.items():187 if func in self.stats:188 old_func_stat = self.stats[func]189 else:190 old_func_stat = (0, 0, 0, 0, {},)191 self.stats[func] = add_func_stats(old_func_stat, stat)192 return self193 194 def dump_stats(self, filename):195 """Write the profile data to a file we know how to load back."""196 with open(filename, 'wb') as f:197 marshal.dump(self.stats, f)198 199 # list the tuple indices and directions for sorting,200 # along with some printable description201 sort_arg_dict_default = {202 "calls" : (((1,-1), ), "call count"),203 "ncalls" : (((1,-1), ), "call count"),204 "cumtime" : (((3,-1), ), "cumulative time"),205 "cumulative": (((3,-1), ), "cumulative time"),206 "filename" : (((4, 1), ), "file name"),207 "line" : (((5, 1), ), "line number"),208 "module" : (((4, 1), ), "file name"),209 "name" : (((6, 1), ), "function name"),210 "nfl" : (((6, 1),(4, 1),(5, 1),), "name/file/line"),211 "pcalls" : (((0,-1), ), "primitive call count"),212 "stdname" : (((7, 1), ), "standard name"),213 "time" : (((2,-1), ), "internal time"),214 "tottime" : (((2,-1), ), "internal time"),215 }216 217 def get_sort_arg_defs(self):218 """Expand all abbreviations that are unique."""219 if not self.sort_arg_dict:220 self.sort_arg_dict = dict = {}221 bad_list = {}222 for word, tup in self.sort_arg_dict_default.items():223 fragment = word224 while fragment:225 if fragment in dict:226 bad_list[fragment] = 0227 break228 dict[fragment] = tup229 fragment = fragment[:-1]230 for word in bad_list:231 del dict[word]232 return self.sort_arg_dict233 234 def sort_stats(self, *field):235 if not field:236 self.fcn_list = 0237 return self238 if len(field) == 1 and isinstance(field[0], int):239 # Be compatible with old profiler240 field = [ {-1: "stdname",241 0: "calls",242 1: "time",243 2: "cumulative"}[field[0]] ]244 elif len(field) >= 2:245 for arg in field[1:]:246 if type(arg) != type(field[0]):247 raise TypeError("Can't have mixed argument type")248 249 sort_arg_defs = self.get_sort_arg_defs()250 251 sort_tuple = ()252 self.sort_type = ""253 connector = ""254 for word in field:255 if isinstance(word, SortKey):256 word = word.value257 sort_tuple = sort_tuple + sort_arg_defs[word][0]258 self.sort_type += connector + sort_arg_defs[word][1]259 connector = ", "260 261 stats_list = []262 for func, (cc, nc, tt, ct, callers) in self.stats.items():263 stats_list.append((cc, nc, tt, ct) + func +264 (func_std_string(func), func))265 266 stats_list.sort(key=cmp_to_key(TupleComp(sort_tuple).compare))267 268 self.fcn_list = fcn_list = []269 for tuple in stats_list:270 fcn_list.append(tuple[-1])271 return self272 273 def reverse_order(self):274 if self.fcn_list:275 self.fcn_list.reverse()276 return self277 278 def strip_dirs(self):279 oldstats = self.stats280 self.stats = newstats = {}281 max_name_len = 0282 for func, (cc, nc, tt, ct, callers) in oldstats.items():283 newfunc = func_strip_path(func)284 if len(func_std_string(newfunc)) > max_name_len:285 max_name_len = len(func_std_string(newfunc))286 newcallers = {}287 for func2, caller in callers.items():288 newcallers[func_strip_path(func2)] = caller289 290 if newfunc in newstats:291 newstats[newfunc] = add_func_stats(292 newstats[newfunc],293 (cc, nc, tt, ct, newcallers))294 else:295 newstats[newfunc] = (cc, nc, tt, ct, newcallers)296 old_top = self.top_level297 self.top_level = new_top = set()298 for func in old_top:299 new_top.add(func_strip_path(func))300 301 self.max_name_len = max_name_len302 303 self.fcn_list = None304 self.all_callees = None305 return self306 307 def calc_callees(self):308 if self.all_callees:309 return310 self.all_callees = all_callees = {}311 for func, (cc, nc, tt, ct, callers) in self.stats.items():312 if not func in all_callees:313 all_callees[func] = {}314 for func2, caller in callers.items():315 if not func2 in all_callees:316 all_callees[func2] = {}317 all_callees[func2][func] = caller318 return319 320 #******************************************************************321 # The following functions support actual printing of reports322 #******************************************************************323 324 # Optional "amount" is either a line count, or a percentage of lines.325 326 def eval_print_amount(self, sel, list, msg):327 new_list = list328 if isinstance(sel, str):329 try:330 rex = re.compile(sel)331 except re.PatternError:332 msg += " <Invalid regular expression %r>\n" % sel333 return new_list, msg334 new_list = []335 for func in list:336 if rex.search(func_std_string(func)):337 new_list.append(func)338 else:339 count = len(list)340 if isinstance(sel, float) and 0.0 <= sel < 1.0:341 count = int(count * sel + .5)342 new_list = list[:count]343 elif isinstance(sel, int) and 0 <= sel < count:344 count = sel345 new_list = list[:count]346 if len(list) != len(new_list):347 msg += " List reduced from %r to %r due to restriction <%r>\n" % (348 len(list), len(new_list), sel)349 350 return new_list, msg351 352 def get_stats_profile(self):353 """This method returns an instance of StatsProfile, which contains a mapping354 of function names to instances of FunctionProfile. Each FunctionProfile355 instance holds information related to the function's profile such as how356 long the function took to run, how many times it was called, etc...357 """358 func_list = self.fcn_list[:] if self.fcn_list else list(self.stats.keys())359 if not func_list:360 return StatsProfile(0, {})361 362 total_tt = float(f8(self.total_tt))363 func_profiles = {}364 stats_profile = StatsProfile(total_tt, func_profiles)365 366 for func in func_list:367 cc, nc, tt, ct, callers = self.stats[func]368 file_name, line_number, func_name = func369 ncalls = str(nc) if nc == cc else (str(nc) + '/' + str(cc))370 tottime = float(f8(tt))371 percall_tottime = -1 if nc == 0 else float(f8(tt/nc))372 cumtime = float(f8(ct))373 percall_cumtime = -1 if cc == 0 else float(f8(ct/cc))374 func_profile = FunctionProfile(375 ncalls,376 tottime, # time spent in this function alone377 percall_tottime,378 cumtime, # time spent in the function plus all functions that this function called,379 percall_cumtime,380 file_name,381 line_number382 )383 func_profiles[func_name] = func_profile384 385 return stats_profile386 387 def get_print_list(self, sel_list):388 width = self.max_name_len389 if self.fcn_list:390 stat_list = self.fcn_list[:]391 msg = " Ordered by: " + self.sort_type + '\n'392 else:393 stat_list = list(self.stats.keys())394 msg = " Random listing order was used\n"395 396 for selection in sel_list:397 stat_list, msg = self.eval_print_amount(selection, stat_list, msg)398 399 count = len(stat_list)400 401 if not stat_list:402 return 0, stat_list403 print(msg, file=self.stream)404 if count < len(self.stats):405 width = 0406 for func in stat_list:407 if len(func_std_string(func)) > width:408 width = len(func_std_string(func))409 return width+2, stat_list410 411 def print_stats(self, *amount):412 for filename in self.files:413 print(filename, file=self.stream)414 if self.files:415 print(file=self.stream)416 indent = ' ' * 8417 for func in self.top_level:418 print(indent, func_get_function_name(func), file=self.stream)419 420 print(indent, self.total_calls, "function calls", end=' ', file=self.stream)421 if self.total_calls != self.prim_calls:422 print("(%d primitive calls)" % self.prim_calls, end=' ', file=self.stream)423 print("in %.3f seconds" % self.total_tt, file=self.stream)424 print(file=self.stream)425 width, list = self.get_print_list(amount)426 if list:427 self.print_title()428 for func in list:429 self.print_line(func)430 print(file=self.stream)431 print(file=self.stream)432 return self433 434 def print_callees(self, *amount):435 width, list = self.get_print_list(amount)436 if list:437 self.calc_callees()438 439 self.print_call_heading(width, "called...")440 for func in list:441 if func in self.all_callees:442 self.print_call_line(width, func, self.all_callees[func])443 else:444 self.print_call_line(width, func, {})445 print(file=self.stream)446 print(file=self.stream)447 return self448 449 def print_callers(self, *amount):450 width, list = self.get_print_list(amount)451 if list:452 self.print_call_heading(width, "was called by...")453 for func in list:454 cc, nc, tt, ct, callers = self.stats[func]455 self.print_call_line(width, func, callers, "<-")456 print(file=self.stream)457 print(file=self.stream)458 return self459 460 def print_call_heading(self, name_size, column_title):461 print("Function ".ljust(name_size) + column_title, file=self.stream)462 # print sub-header only if we have new-style callers463 subheader = False464 for cc, nc, tt, ct, callers in self.stats.values():465 if callers:466 value = next(iter(callers.values()))467 subheader = isinstance(value, tuple)468 break469 if subheader:470 print(" "*name_size + " ncalls tottime cumtime", file=self.stream)471 472 def print_call_line(self, name_size, source, call_dict, arrow="->"):473 print(func_std_string(source).ljust(name_size) + arrow, end=' ', file=self.stream)474 if not call_dict:475 print(file=self.stream)476 return477 clist = sorted(call_dict.keys())478 indent = ""479 for func in clist:480 name = func_std_string(func)481 value = call_dict[func]482 if isinstance(value, tuple):483 nc, cc, tt, ct = value484 if nc != cc:485 substats = '%d/%d' % (nc, cc)486 else:487 substats = '%d' % (nc,)488 substats = '%s %s %s %s' % (substats.rjust(7+2*len(indent)),489 f8(tt), f8(ct), name)490 left_width = name_size + 1491 else:492 substats = '%s(%r) %s' % (name, value, f8(self.stats[func][3]))493 left_width = name_size + 3494 print(indent*left_width + substats, file=self.stream)495 indent = " "496 497 def print_title(self):498 print(' ncalls tottime percall cumtime percall', end=' ', file=self.stream)499 print('filename:lineno(function)', file=self.stream)500 501 def print_line(self, func): # hack: should print percentages502 cc, nc, tt, ct, callers = self.stats[func]503 c = str(nc)504 if nc != cc:505 c = c + '/' + str(cc)506 print(c.rjust(9), end=' ', file=self.stream)507 print(f8(tt), end=' ', file=self.stream)508 if nc == 0:509 print(' '*8, end=' ', file=self.stream)510 else:511 print(f8(tt/nc), end=' ', file=self.stream)512 print(f8(ct), end=' ', file=self.stream)513 if cc == 0:514 print(' '*8, end=' ', file=self.stream)515 else:516 print(f8(ct/cc), end=' ', file=self.stream)517 print(func_std_string(func), file=self.stream)518 519class TupleComp:520 """This class provides a generic function for comparing any two tuples.521 Each instance records a list of tuple-indices (from most significant522 to least significant), and sort direction (ascending or descending) for523 each tuple-index. The compare functions can then be used as the function524 argument to the system sort() function when a list of tuples need to be525 sorted in the instances order."""526 527 def __init__(self, comp_select_list):528 self.comp_select_list = comp_select_list529 530 def compare (self, left, right):531 for index, direction in self.comp_select_list:532 l = left[index]533 r = right[index]534 if l < r:535 return -direction536 if l > r:537 return direction538 return 0539 540 541#**************************************************************************542# func_name is a triple (file:string, line:int, name:string)543 544def func_strip_path(func_name):545 filename, line, name = func_name546 return os.path.basename(filename), line, name547 548def func_get_function_name(func):549 return func[2]550 551def func_std_string(func_name): # match what old profile produced552 if func_name[:2] == ('~', 0):553 # special case for built-in functions554 name = func_name[2]555 if name.startswith('<') and name.endswith('>'):556 return '{%s}' % name[1:-1]557 else:558 return name559 else:560 return "%s:%d(%s)" % func_name561 562#**************************************************************************563# The following functions combine statistics for pairs functions.564# The bulk of the processing involves correctly handling "call" lists,565# such as callers and callees.566#**************************************************************************567 568def add_func_stats(target, source):569 """Add together all the stats for two profile entries."""570 cc, nc, tt, ct, callers = source571 t_cc, t_nc, t_tt, t_ct, t_callers = target572 return (cc+t_cc, nc+t_nc, tt+t_tt, ct+t_ct,573 add_callers(t_callers, callers))574 575def add_callers(target, source):576 """Combine two caller lists in a single list."""577 new_callers = {}578 for func, caller in target.items():579 new_callers[func] = caller580 for func, caller in source.items():581 if func in new_callers:582 if isinstance(caller, tuple):583 # format used by cProfile584 new_callers[func] = tuple(i + j for i, j in zip(caller, new_callers[func]))585 else:586 # format used by profile587 new_callers[func] += caller588 else:589 new_callers[func] = caller590 return new_callers591 592def count_calls(callers):593 """Sum the caller statistics to get total number of calls received."""594 nc = 0595 for calls in callers.values():596 nc += calls597 return nc598 599#**************************************************************************600# The following functions support printing of reports601#**************************************************************************602 603def f8(x):604 return "%8.3f" % x605 606#**************************************************************************607# Statistics browser added by ESR, April 2001608#**************************************************************************609 610if __name__ == '__main__':611 import cmd612 try:613 import readline # noqa: F401614 except ImportError:615 pass616 617 class ProfileBrowser(cmd.Cmd):618 def __init__(self, profile=None):619 cmd.Cmd.__init__(self)620 self.prompt = "% "621 self.stats = None622 self.stream = sys.stdout623 if profile is not None:624 self.do_read(profile)625 626 def generic(self, fn, line):627 args = line.split()628 processed = []629 for term in args:630 try:631 processed.append(int(term))632 continue633 except ValueError:634 pass635 try:636 frac = float(term)637 if frac > 1 or frac < 0:638 print("Fraction argument must be in [0, 1]", file=self.stream)639 continue640 processed.append(frac)641 continue642 except ValueError:643 pass644 processed.append(term)645 if self.stats:646 getattr(self.stats, fn)(*processed)647 else:648 print("No statistics object is loaded.", file=self.stream)649 return 0650 def generic_help(self):651 print("Arguments may be:", file=self.stream)652 print("* An integer maximum number of entries to print.", file=self.stream)653 print("* A decimal fractional number between 0 and 1, controlling", file=self.stream)654 print(" what fraction of selected entries to print.", file=self.stream)655 print("* A regular expression; only entries with function names", file=self.stream)656 print(" that match it are printed.", file=self.stream)657 658 def do_add(self, line):659 if self.stats:660 try:661 self.stats.add(line)662 except OSError as e:663 print("Failed to load statistics for %s: %s" % (line, e), file=self.stream)664 else:665 print("No statistics object is loaded.", file=self.stream)666 return 0667 def help_add(self):668 print("Add profile info from given file to current statistics object.", file=self.stream)669 670 def do_callees(self, line):671 return self.generic('print_callees', line)672 def help_callees(self):673 print("Print callees statistics from the current stat object.", file=self.stream)674 self.generic_help()675 676 def do_callers(self, line):677 return self.generic('print_callers', line)678 def help_callers(self):679 print("Print callers statistics from the current stat object.", file=self.stream)680 self.generic_help()681 682 def do_EOF(self, line):683 print("", file=self.stream)684 return 1685 def help_EOF(self):686 print("Leave the profile browser.", file=self.stream)687 688 def do_quit(self, line):689 return 1690 def help_quit(self):691 print("Leave the profile browser.", file=self.stream)692 693 def do_read(self, line):694 if line:695 try:696 self.stats = Stats(line)697 except OSError as err:698 print(err.args[1], file=self.stream)699 return700 except Exception as err:701 print(err.__class__.__name__ + ':', err, file=self.stream)702 return703 self.prompt = line + "% "704 elif len(self.prompt) > 2:705 line = self.prompt[:-2]706 self.do_read(line)707 else:708 print("No statistics object is current -- cannot reload.", file=self.stream)709 return 0710 def help_read(self):711 print("Read in profile data from a specified file.", file=self.stream)712 print("Without argument, reload the current file.", file=self.stream)713 714 def do_reverse(self, line):715 if self.stats:716 self.stats.reverse_order()717 else:718 print("No statistics object is loaded.", file=self.stream)719 return 0720 def help_reverse(self):721 print("Reverse the sort order of the profiling report.", file=self.stream)722 723 def do_sort(self, line):724 if not self.stats:725 print("No statistics object is loaded.", file=self.stream)726 return727 abbrevs = self.stats.get_sort_arg_defs()728 if line and all((x in abbrevs) for x in line.split()):729 self.stats.sort_stats(*line.split())730 else:731 print("Valid sort keys (unique prefixes are accepted):", file=self.stream)732 for (key, value) in Stats.sort_arg_dict_default.items():733 print("%s -- %s" % (key, value[1]), file=self.stream)734 return 0735 def help_sort(self):736 print("Sort profile data according to specified keys.", file=self.stream)737 print("(Typing `sort' without arguments lists valid keys.)", file=self.stream)738 def complete_sort(self, text, *args):739 return [a for a in Stats.sort_arg_dict_default if a.startswith(text)]740 741 def do_stats(self, line):742 return self.generic('print_stats', line)743 def help_stats(self):744 print("Print statistics from the current stat object.", file=self.stream)745 self.generic_help()746 747 def do_strip(self, line):748 if self.stats:749 self.stats.strip_dirs()750 else:751 print("No statistics object is loaded.", file=self.stream)752 def help_strip(self):753 print("Strip leading path information from filenames in the report.", file=self.stream)754 755 def help_help(self):756 print("Show help for a given command.", file=self.stream)757 758 def postcmd(self, stop, line):759 if stop:760 return stop761 return None762 763 if len(sys.argv) > 1:764 initprofile = sys.argv[1]765 else:766 initprofile = None767 try:768 browser = ProfileBrowser(initprofile)769 for profile in sys.argv[2:]:770 browser.do_add(profile)771 print("Welcome to the profile statistics browser.", file=browser.stream)772 browser.cmdloop()773 print("Goodbye.", file=browser.stream)774 except KeyboardInterrupt:775 pass776 777# That's all, folks.778