codekingpro/portable-devtools
115k
1"""2Customisable progressbar decorator for iterators.3Includes a default `range` iterator printing to `stderr`.4 5Usage:6>>> from tqdm import trange, tqdm7>>> for i in trange(10):8... ...9"""10import sys11from collections import OrderedDict, defaultdict12from contextlib import contextmanager13from datetime import datetime, timedelta, timezone14from numbers import Number15from time import time16from warnings import warn17from weakref import WeakSet18 19from ._monitor import TMonitor20from .utils import (21 CallbackIOWrapper, Comparable, DisableOnWriteError, FormatReplace, SimpleTextIOWrapper,22 _is_ascii, _screen_shape_wrapper, _supports_unicode, _term_move_up, disp_len, disp_trim,23 envwrap)24 25__author__ = "https://github.com/tqdm/tqdm#contributions"26__all__ = ['tqdm', 'trange',27 'TqdmTypeError', 'TqdmKeyError', 'TqdmWarning',28 'TqdmExperimentalWarning', 'TqdmDeprecationWarning',29 'TqdmMonitorWarning']30 31 32class TqdmTypeError(TypeError):33 pass34 35 36class TqdmKeyError(KeyError):37 pass38 39 40class TqdmWarning(Warning):41 """base class for all tqdm warnings.42 43 Used for non-external-code-breaking errors, such as garbled printing.44 """45 def __init__(self, msg, fp_write=None): # noqa: B04246 if fp_write is not None:47 fp_write("\n" + self.__class__.__name__ + ": " + str(msg).rstrip() + '\n')48 else:49 super().__init__(msg)50 51 52class TqdmExperimentalWarning(TqdmWarning, FutureWarning):53 """beta feature, unstable API and behaviour"""54 pass55 56 57class TqdmDeprecationWarning(TqdmWarning, DeprecationWarning):58 # not suppressed if raised59 pass60 61 62class TqdmMonitorWarning(TqdmWarning, RuntimeWarning):63 """tqdm monitor errors which do not affect external functionality"""64 pass65 66 67def TRLock(*args, **kwargs):68 """threading RLock"""69 try:70 from threading import RLock71 return RLock(*args, **kwargs)72 except (ImportError, OSError): # pragma: no cover73 pass74 75 76class TqdmDefaultWriteLock:77 """78 Provide a default write lock for thread and multiprocessing safety.79 Works only on platforms supporting `fork` (so Windows is excluded).80 You must initialise a `tqdm` or `TqdmDefaultWriteLock` instance81 before forking in order for the write lock to work.82 On Windows, you need to supply the lock from the parent to the children as83 an argument to joblib or the parallelism lib you use.84 """85 # global thread lock so no setup required for multithreading.86 # NB: Do not create multiprocessing lock as it sets the multiprocessing87 # context, disallowing `spawn()`/`forkserver()`88 th_lock = TRLock()89 90 def __init__(self):91 # Create global parallelism locks to avoid racing issues with parallel92 # bars works only if fork available (Linux/MacOSX, but not Windows)93 cls = type(self)94 root_lock = cls.th_lock95 if root_lock is not None:96 root_lock.acquire()97 cls.create_mp_lock()98 self.locks = [lk for lk in [cls.mp_lock, cls.th_lock] if lk is not None]99 if root_lock is not None:100 root_lock.release()101 102 def acquire(self, *a, **k):103 for lock in self.locks:104 lock.acquire(*a, **k)105 106 def release(self):107 for lock in self.locks[::-1]: # Release in inverse order of acquisition108 lock.release()109 110 def __enter__(self):111 self.acquire()112 113 def __exit__(self, *exc):114 self.release()115 116 @classmethod117 def create_mp_lock(cls):118 if not hasattr(cls, 'mp_lock'):119 try:120 from multiprocessing import RLock121 cls.mp_lock = RLock()122 except (ImportError, OSError): # pragma: no cover123 cls.mp_lock = None124 125 @classmethod126 def create_th_lock(cls):127 assert hasattr(cls, 'th_lock')128 warn("create_th_lock not needed anymore", TqdmDeprecationWarning, stacklevel=2)129 130 131class Bar:132 """133 `str.format`-able bar with format specifiers: `[width][type]`134 135 - `width`136 + unspecified (default): use `self.default_len`137 + `int >= 0`: overrides `self.default_len`138 + `int < 0`: subtract from `self.default_len`139 - `type`140 + `a`: ascii (`charset=self.ASCII` override)141 + `u`: unicode (`charset=self.UTF` override)142 + `b`: blank (`charset=" "` override)143 """144 ASCII = " 123456789#"145 UTF = " " + ''.join(map(chr, range(0x258F, 0x2587, -1)))146 BLANK = " "147 COLOUR_RESET = '\x1b[0m'148 COLOUR_RGB = '\x1b[38;2;%d;%d;%dm'149 COLOURS = {'BLACK': '\x1b[30m', 'RED': '\x1b[31m', 'GREEN': '\x1b[32m',150 'YELLOW': '\x1b[33m', 'BLUE': '\x1b[34m', 'MAGENTA': '\x1b[35m',151 'CYAN': '\x1b[36m', 'WHITE': '\x1b[37m'}152 153 def __init__(self, frac, default_len=10, charset=UTF, colour=None):154 if not 0 <= frac <= 1:155 warn("clamping frac to range [0, 1]", TqdmWarning, stacklevel=2)156 frac = max(0, min(1, frac))157 assert default_len > 0158 self.frac = frac159 self.default_len = default_len160 self.charset = charset161 self.colour = colour162 163 @property164 def colour(self):165 return self._colour166 167 @colour.setter168 def colour(self, value):169 if not value:170 self._colour = None171 return172 try:173 if value.upper() in self.COLOURS:174 self._colour = self.COLOURS[value.upper()]175 elif value[0] == '#' and len(value) == 7:176 self._colour = self.COLOUR_RGB % tuple(177 int(i, 16) for i in (value[1:3], value[3:5], value[5:7]))178 else:179 raise KeyError180 except (KeyError, AttributeError):181 warn(f"Unknown colour ({value}); valid choices:"182 f" [hex (#00ff00), {', '.join(self.COLOURS)}]", TqdmWarning, stacklevel=2)183 self._colour = None184 185 def __format__(self, format_spec):186 if format_spec:187 _type = format_spec[-1].lower()188 try:189 charset = {'a': self.ASCII, 'u': self.UTF, 'b': self.BLANK}[_type]190 except KeyError:191 charset = self.charset192 else:193 format_spec = format_spec[:-1]194 if format_spec:195 N_BARS = int(format_spec)196 if N_BARS < 0:197 N_BARS += self.default_len198 else:199 N_BARS = self.default_len200 else:201 charset = self.charset202 N_BARS = self.default_len203 204 nsyms = len(charset) - 1205 bar_length, frac_bar_length = divmod(int(self.frac * N_BARS * nsyms), nsyms)206 207 res = charset[-1] * bar_length208 if bar_length < N_BARS: # whitespace padding209 res = res + charset[frac_bar_length] + charset[0] * (N_BARS - bar_length - 1)210 return self.colour + res + self.COLOUR_RESET if self.colour else res211 212 213class EMA:214 """215 Exponential moving average: smoothing to give progressively lower216 weights to older values.217 218 Parameters219 ----------220 smoothing : float, optional221 Smoothing factor in range [0, 1], [default: 0.3].222 Increase to give more weight to recent values.223 Ranges from 0 (yields old value) to 1 (yields new value).224 """225 def __init__(self, smoothing=0.3):226 self.alpha = smoothing227 self.last = 0228 self.calls = 0229 230 def __call__(self, x=None):231 """232 Parameters233 ----------234 x : float235 New value to include in EMA.236 """237 beta = 1 - self.alpha238 if x is not None:239 self.last = self.alpha * x + beta * self.last240 self.calls += 1241 return self.last / (1 - beta ** self.calls) if self.calls else self.last242 243 244class tqdm(Comparable):245 """246 Decorate an iterable object, returning an iterator which acts exactly247 like the original iterable, but prints a dynamically updating248 progressbar every time a value is requested.249 250 Parameters251 ----------252 iterable : iterable, optional253 Iterable to decorate with a progressbar.254 Leave blank to manually manage the updates.255 desc : str, optional256 Prefix for the progressbar.257 total : int or float, optional258 The number of expected iterations. If unspecified,259 len(iterable) is used if possible. If float("inf") or as a last260 resort, only basic progress statistics are displayed261 (no ETA, no progressbar).262 If `gui` is True and this parameter needs subsequent updating,263 specify an initial arbitrary large positive number,264 e.g. 9e9.265 leave : bool, optional266 If [default: True], keeps all traces of the progressbar267 upon termination of iteration.268 If `None`, will leave only if `position` is `0`.269 file : `io.TextIOWrapper` or `io.StringIO`, optional270 Specifies where to output the progress messages271 (default: sys.stderr). Uses `file.write(str)` and `file.flush()`272 methods. For encoding, see `write_bytes`.273 ncols : int, optional274 The width of the entire output message. If specified,275 dynamically resizes the progressbar to stay within this bound.276 If unspecified, attempts to use environment width. The277 fallback is a meter width of 10 and no limit for the counter and278 statistics. If 0, will not print any meter (only stats).279 mininterval : float, optional280 Minimum progress display update interval [default: 0.1] seconds.281 maxinterval : float, optional282 Maximum progress display update interval [default: 10] seconds.283 Automatically adjusts `miniters` to correspond to `mininterval`284 after long display update lag. Only works if `dynamic_miniters`285 or monitor thread is enabled.286 miniters : int or float, optional287 Minimum progress display update interval, in iterations.288 If 0 and `dynamic_miniters`, will automatically adjust to equal289 `mininterval` (more CPU efficient, good for tight loops).290 If > 0, will skip display of specified number of iterations.291 Tweak this and `mininterval` to get very efficient loops.292 If your progress is erratic with both fast and slow iterations293 (network, skipping items, etc) you should set miniters=1.294 ascii : bool or str, optional295 If unspecified or False, use unicode (smooth blocks) to fill296 the meter. The fallback is to use ASCII characters " 123456789#".297 disable : bool, optional298 Whether to disable the entire progressbar wrapper299 [default: False]. If set to None, disable on non-TTY.300 unit : str, optional301 String that will be used to define the unit of each iteration302 [default: it].303 unit_scale : bool or int or float, optional304 If 1 or True, the number of iterations will be reduced/scaled305 automatically and a metric prefix following the306 International System of Units standard will be added307 (kilo, mega, etc.) [default: False]. If any other non-zero308 number, will scale `total` and `n`.309 dynamic_ncols : bool, optional310 If set, constantly alters `ncols` and `nrows` to the311 environment (allowing for window resizes) [default: False].312 smoothing : float, optional313 Exponential moving average smoothing factor for speed estimates314 (ignored in GUI mode). Ranges from 0 (average speed) to 1315 (current/instantaneous speed) [default: 0.3].316 bar_format : str, optional317 Specify a custom bar string formatting. May impact performance.318 [default: '{l_bar}{bar}{r_bar}'], where319 l_bar='{desc}: {percentage:3.0f}%|' and320 r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, '321 '{rate_fmt}{postfix}]'322 Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt,323 percentage, elapsed, elapsed_s, ncols, nrows, desc, unit,324 rate, rate_fmt, rate_noinv, rate_noinv_fmt,325 rate_inv, rate_inv_fmt, postfix, unit_divisor,326 remaining, remaining_s, eta.327 Note that a trailing ": " is automatically removed after {desc}328 if the latter is empty.329 initial : int or float, optional330 The initial counter value. Useful when restarting a progress331 bar [default: 0]. If using float, consider specifying `{n:.3f}`332 or similar in `bar_format`, or specifying `unit_scale`.333 position : int, optional334 Specify the line offset to print this bar (starting from 0)335 Automatic if unspecified.336 Useful to manage multiple bars at once (eg, from threads).337 postfix : dict or *, optional338 Specify additional stats to display at the end of the bar.339 Calls `set_postfix(**postfix)` if possible (dict).340 unit_divisor : float, optional341 [default: 1000], ignored unless `unit_scale` is True.342 write_bytes : bool, optional343 Whether to write bytes. If (default: False) will write unicode.344 lock_args : tuple, optional345 Passed to `refresh` for intermediate output346 (initialisation, iterating, and updating).347 nrows : int, optional348 The screen height. If specified, hides nested bars outside this349 bound. If unspecified, attempts to use environment height.350 The fallback is 20.351 colour : str, optional352 Bar colour (e.g. 'green', '#00ff00').353 delay : float, optional354 Don't display until [default: 0] seconds have elapsed.355 gui : bool, optional356 WARNING: internal parameter - do not use.357 Use tqdm.gui.tqdm(...) instead. If set, will attempt to use358 matplotlib animations for a graphical output [default: False].359 360 Returns361 -------362 out : decorated iterator.363 """364 365 monitor_interval = 10 # set to 0 to disable the thread366 monitor = None367 _instances = WeakSet()368 369 @staticmethod370 def format_sizeof(num, suffix='', divisor=1000):371 """372 Formats a number (greater than unity) with SI Order of Magnitude373 prefixes.374 375 Parameters376 ----------377 num : float378 Number ( >= 1) to format.379 suffix : str, optional380 Post-postfix [default: ''].381 divisor : float, optional382 Divisor between prefixes [default: 1000].383 384 Returns385 -------386 out : str387 Number with Order of Magnitude SI unit postfix.388 """389 for unit in ['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z']:390 if abs(num) < 999.5:391 if abs(num) < 99.95:392 if abs(num) < 9.995:393 return f'{num:1.2f}{unit}{suffix}'394 return f'{num:2.1f}{unit}{suffix}'395 return f'{num:3.0f}{unit}{suffix}'396 num /= divisor397 return f'{num:3.1f}Y{suffix}'398 399 @staticmethod400 def format_interval(t):401 """402 Formats a number of seconds as a clock time, [H:]MM:SS403 404 Parameters405 ----------406 t : int407 Number of seconds.408 409 Returns410 -------411 out : str412 [H:]MM:SS413 """414 sign = '-' if t < 0 else ''415 mins, s = divmod(abs(int(t)), 60)416 h, m = divmod(mins, 60)417 return f'{sign}{h:d}:{m:02d}:{s:02d}' if h else f'{sign}{m:02d}:{s:02d}'418 419 @staticmethod420 def format_num(n):421 """422 Intelligent scientific notation (.3g).423 424 Parameters425 ----------426 n : int or float or Numeric427 A Number.428 429 Returns430 -------431 out : str432 Formatted number.433 """434 f = f'{n:.3g}'.replace('e+0', 'e+').replace('e-0', 'e-')435 n = str(n)436 return f if len(f) < len(n) else n437 438 @staticmethod439 def status_printer(file):440 """441 Manage the printing and in-place updating of a line of characters.442 Note that if the string is longer than a line, then in-place443 updating may not work (it will print a new line at each refresh).444 """445 fp = file446 fp_flush = getattr(fp, 'flush', lambda: None) # pragma: no cover447 if fp in (sys.stderr, sys.stdout):448 getattr(sys.stderr, 'flush', lambda: None)()449 getattr(sys.stdout, 'flush', lambda: None)()450 451 def fp_write(s):452 fp.write(str(s))453 fp_flush()454 455 last_len = [0]456 457 def print_status(s):458 len_s = disp_len(s)459 fp_write('\r' + s + (' ' * max(last_len[0] - len_s, 0)))460 last_len[0] = len_s461 462 return print_status463 464 @staticmethod465 def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, unit='it',466 unit_scale=False, rate=None, bar_format=None, postfix=None,467 unit_divisor=1000, initial=0, colour=None, **extra_kwargs):468 """469 Return a string-based progress bar given some parameters470 471 Parameters472 ----------473 n : int or float474 Number of finished iterations.475 total : int or float476 The expected total number of iterations. If meaningless (None),477 only basic progress statistics are displayed (no ETA).478 elapsed : float479 Number of seconds passed since start.480 ncols : int, optional481 The width of the entire output message. If specified,482 dynamically resizes `{bar}` to stay within this bound483 [default: None]. If `0`, will not print any bar (only stats).484 The fallback is `{bar:10}`.485 prefix : str, optional486 Prefix message (included in total width) [default: ''].487 Use as {desc} in bar_format string.488 ascii : bool, optional or str, optional489 If not set, use unicode (smooth blocks) to fill the meter490 [default: False]. The fallback is to use ASCII characters491 " 123456789#".492 unit : str, optional493 The iteration unit [default: 'it'].494 unit_scale : bool or int or float, optional495 If 1 or True, the number of iterations will be printed with an496 appropriate SI metric prefix (k = 10^3, M = 10^6, etc.)497 [default: False]. If any other non-zero number, will scale498 `total` and `n`.499 rate : float, optional500 Manual override for iteration rate.501 If [default: None], uses n/elapsed.502 bar_format : str, optional503 Specify a custom bar string formatting. May impact performance.504 [default: '{l_bar}{bar}{r_bar}'], where505 l_bar='{desc}: {percentage:3.0f}%|' and506 r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, '507 '{rate_fmt}{postfix}]'508 Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt,509 percentage, elapsed, elapsed_s, ncols, nrows, desc, unit,510 rate, rate_fmt, rate_noinv, rate_noinv_fmt,511 rate_inv, rate_inv_fmt, postfix, unit_divisor,512 remaining, remaining_s, eta.513 Note that a trailing ": " is automatically removed after {desc}514 if the latter is empty.515 postfix : *, optional516 Similar to `prefix`, but placed at the end517 (e.g. for additional stats).518 Note: postfix is usually a string (not a dict) for this method,519 and will if possible be set to postfix = ', ' + postfix.520 However other types are supported (#382).521 unit_divisor : float, optional522 [default: 1000], ignored unless `unit_scale` is True.523 initial : int or float, optional524 The initial counter value [default: 0].525 colour : str, optional526 Bar colour (e.g. 'green', '#00ff00').527 528 Returns529 -------530 out : Formatted meter and stats, ready to display.531 """532 533 # sanity check: total534 if total and n >= (total + 0.5): # allow float imprecision (#849)535 total = None536 537 # apply custom scale if necessary538 if unit_scale and unit_scale not in (True, 1):539 if total:540 total *= unit_scale541 n *= unit_scale542 if rate:543 rate *= unit_scale # by default rate = self.avg_dn / self.avg_dt544 unit_scale = False545 546 elapsed_str = tqdm.format_interval(elapsed)547 548 # if unspecified, attempt to use rate = average speed549 # (we allow manual override since predicting time is an arcane art)550 if rate is None and elapsed:551 rate = (n - initial) / elapsed552 inv_rate = 1 / rate if rate else None553 format_sizeof = tqdm.format_sizeof554 rate_noinv_fmt = ((format_sizeof(rate) if unit_scale else f'{rate:5.2f}')555 if rate else '?') + unit + '/s'556 rate_inv_fmt = (557 (format_sizeof(inv_rate) if unit_scale else f'{inv_rate:5.2f}')558 if inv_rate else '?') + 's/' + unit559 rate_fmt = rate_inv_fmt if inv_rate and inv_rate > 1 else rate_noinv_fmt560 561 if unit_scale:562 n_fmt = format_sizeof(n, divisor=unit_divisor)563 total_fmt = format_sizeof(total, divisor=unit_divisor) if total is not None else '?'564 else:565 n_fmt = str(n)566 total_fmt = str(total) if total is not None else '?'567 568 try:569 postfix = ', ' + postfix if postfix else ''570 except TypeError:571 pass572 573 remaining = (total - n) / rate if rate and total else 0574 remaining_str = tqdm.format_interval(remaining) if rate else '?'575 try:576 eta_dt = (datetime.now() + timedelta(seconds=remaining)577 if rate and total else datetime.fromtimestamp(0, timezone.utc))578 except OverflowError:579 eta_dt = datetime.max580 581 # format the stats displayed to the left and right sides of the bar582 if prefix:583 # old prefix setup work around584 bool_prefix_colon_already = (prefix[-2:] == ": ")585 l_bar = prefix if bool_prefix_colon_already else prefix + ": "586 else:587 l_bar = ''588 589 r_bar = f'| {n_fmt}/{total_fmt} [{elapsed_str}<{remaining_str}, {rate_fmt}{postfix}]'590 591 # Custom bar formatting592 # Populate a dict with all available progress indicators593 format_dict = {594 # slight extension of self.format_dict595 'n': n, 'n_fmt': n_fmt, 'total': total, 'total_fmt': total_fmt,596 'elapsed': elapsed_str, 'elapsed_s': elapsed,597 'ncols': ncols, 'desc': prefix or '', 'unit': unit,598 'rate': inv_rate if inv_rate and inv_rate > 1 else rate,599 'rate_fmt': rate_fmt, 'rate_noinv': rate,600 'rate_noinv_fmt': rate_noinv_fmt, 'rate_inv': inv_rate,601 'rate_inv_fmt': rate_inv_fmt,602 'postfix': postfix, 'unit_divisor': unit_divisor,603 'colour': colour,604 # plus more useful definitions605 'remaining': remaining_str, 'remaining_s': remaining,606 'l_bar': l_bar, 'r_bar': r_bar, 'eta': eta_dt,607 **extra_kwargs}608 609 # total is known: we can predict some stats610 if total:611 # fractional and percentage progress612 frac = n / total613 percentage = frac * 100614 615 l_bar += f'{percentage:3.0f}%|'616 617 if ncols == 0:618 return l_bar[:-1] + r_bar[1:]619 620 format_dict.update(l_bar=l_bar)621 if bar_format:622 format_dict.update(percentage=percentage)623 624 # auto-remove colon for empty `{desc}`625 if not prefix:626 bar_format = bar_format.replace("{desc}: ", '')627 else:628 bar_format = "{l_bar}{bar}{r_bar}"629 630 full_bar = FormatReplace()631 nobar = bar_format.format(bar=full_bar, **format_dict)632 if not full_bar.format_called:633 return nobar # no `{bar}`; nothing else to do634 635 # Formatting progress bar space available for bar's display636 full_bar = Bar(frac,637 max(1, ncols - disp_len(nobar)) if ncols else 10,638 charset=Bar.ASCII if ascii is True else ascii or Bar.UTF,639 colour=colour)640 if not _is_ascii(full_bar.charset) and _is_ascii(bar_format):641 bar_format = str(bar_format)642 res = bar_format.format(bar=full_bar, **format_dict)643 return disp_trim(res, ncols) if ncols else res644 645 elif bar_format:646 # user-specified bar_format but no total647 l_bar += '|'648 format_dict.update(l_bar=l_bar, percentage=0)649 full_bar = FormatReplace()650 nobar = bar_format.format(bar=full_bar, **format_dict)651 if not full_bar.format_called:652 return nobar653 full_bar = Bar(0,654 max(1, ncols - disp_len(nobar)) if ncols else 10,655 charset=Bar.BLANK, colour=colour)656 res = bar_format.format(bar=full_bar, **format_dict)657 return disp_trim(res, ncols) if ncols else res658 else:659 # no total: no progressbar, ETA, just progress stats660 return (f'{(prefix + ": ") if prefix else ""}'661 f'{n_fmt}{unit} [{elapsed_str}, {rate_fmt}{postfix}]')662 663 def __new__(cls, *_, **__):664 instance = object.__new__(cls)665 with cls.get_lock(): # also constructs lock if non-existent666 cls._instances.add(instance)667 # create monitoring thread668 if cls.monitor_interval and (cls.monitor is None669 or not cls.monitor.report()):670 try:671 cls.monitor = TMonitor(cls, cls.monitor_interval)672 except Exception as e: # pragma: nocover673 warn("tqdm:disabling monitor support"674 " (monitor_interval = 0) due to:\n" + str(e),675 TqdmMonitorWarning, stacklevel=2)676 cls.monitor_interval = 0677 return instance678 679 @classmethod680 def _get_free_pos(cls, instance=None):681 """Skips specified instance."""682 positions = {abs(inst.pos) for inst in cls._instances683 if inst is not instance and hasattr(inst, "pos")}684 return min(set(range(len(positions) + 1)).difference(positions))685 686 @classmethod687 def _decr_instances(cls, instance):688 """689 Remove from list and reposition another unfixed bar690 to fill the new gap.691 692 This means that by default (where all nested bars are unfixed),693 order is not maintained but screen flicker/blank space is minimised.694 (tqdm<=4.44.1 moved ALL subsequent unfixed bars up.)695 """696 with cls._lock:697 try:698 cls._instances.remove(instance)699 except KeyError:700 # if not instance.gui: # pragma: no cover701 # raise702 pass # py2: maybe magically removed already703 # else:704 if not instance.gui:705 last = (instance.nrows or 20) - 1706 # find unfixed (`pos >= 0`) overflow (`pos >= nrows - 1`)707 instances = list(filter(708 lambda i: hasattr(i, "pos") and last <= i.pos,709 cls._instances))710 # set first found to current `pos`711 if instances:712 inst = min(instances, key=lambda i: i.pos)713 inst.clear(nolock=True)714 inst.pos = abs(instance.pos)715 716 @classmethod717 def write(cls, s, file=None, end="\n", nolock=False):718 """Print a message via tqdm (without overlap with bars)."""719 fp = file if file is not None else sys.stdout720 with cls.external_write_mode(file=file, nolock=nolock):721 # Write the message722 fp.write(s)723 fp.write(end)724 725 @classmethod726 @contextmanager727 def external_write_mode(cls, file=None, nolock=False):728 """729 Disable tqdm within context and refresh tqdm when exits.730 Useful when writing to standard output stream731 """732 fp = file if file is not None else sys.stdout733 734 try:735 if not nolock:736 cls.get_lock().acquire()737 # Clear all bars738 inst_cleared = []739 for inst in getattr(cls, '_instances', []):740 # Clear instance if in the target output file741 # or if write output + tqdm output are both either742 # sys.stdout or sys.stderr (because both are mixed in terminal)743 if hasattr(inst, "start_t") and (inst.fp == fp or all(744 f in (sys.stdout, sys.stderr) for f in (fp, inst.fp))):745 inst.clear(nolock=True)746 inst_cleared.append(inst)747 yield748 # Force refresh display of bars we cleared749 for inst in inst_cleared:750 inst.refresh(nolock=True)751 finally:752 if not nolock:753 cls._lock.release()754 755 @classmethod756 def set_lock(cls, lock):757 """Set the global lock."""758 cls._lock = lock759 760 @classmethod761 def get_lock(cls):762 """Get the global lock. Construct it if it does not exist."""763 if not hasattr(cls, '_lock'):764 cls._lock = TqdmDefaultWriteLock()765 return cls._lock766 767 @classmethod768 def pandas(cls, **tqdm_kwargs):769 """770 Registers the current `tqdm` class with771 pandas.core.772 ( frame.DataFrame773 | series.Series774 | groupby.(generic.)DataFrameGroupBy775 | groupby.(generic.)SeriesGroupBy776 ).progress_apply777 778 A new instance will be created every time `progress_apply` is called,779 and each instance will automatically `close()` upon completion.780 781 Parameters782 ----------783 tqdm_kwargs : arguments for the tqdm instance784 785 Examples786 --------787 >>> import pandas as pd788 >>> import numpy as np789 >>> from tqdm import tqdm790 >>> from tqdm.gui import tqdm as tqdm_gui791 >>>792 >>> df = pd.DataFrame(np.random.randint(0, 100, (100000, 6)))793 >>> tqdm.pandas(ncols=50) # can use tqdm_gui, optional kwargs, etc794 >>> # Now you can use `progress_apply` instead of `apply`795 >>> df.groupby(0).progress_apply(lambda x: x**2)796 797 References798 ----------799 <https://stackoverflow.com/questions/18603270/\800 progress-indicator-during-pandas-operations-python>801 """802 from warnings import catch_warnings, simplefilter803 804 from pandas.core.frame import DataFrame805 from pandas.core.series import Series806 try:807 with catch_warnings():808 simplefilter("ignore", category=FutureWarning)809 from pandas import Panel810 except ImportError: # pandas>=1.2.0811 Panel = None812 Rolling, Expanding = None, None813 try: # pandas>=1.0.0814 from pandas.core.window.rolling import _Rolling_and_Expanding815 except ImportError:816 try: # pandas>=0.18.0817 from pandas.core.window import _Rolling_and_Expanding818 except ImportError: # pandas>=1.2.0819 try: # pandas>=1.2.0820 from pandas.core.window.expanding import Expanding821 from pandas.core.window.rolling import Rolling822 _Rolling_and_Expanding = Rolling, Expanding823 except ImportError: # pragma: no cover824 _Rolling_and_Expanding = None825 try: # pandas>=0.25.0826 from pandas.core.groupby.generic import SeriesGroupBy # , NDFrameGroupBy827 from pandas.core.groupby.generic import DataFrameGroupBy828 except ImportError: # pragma: no cover829 try: # pandas>=0.23.0830 from pandas.core.groupby.groupby import DataFrameGroupBy, SeriesGroupBy831 except ImportError:832 from pandas.core.groupby import DataFrameGroupBy, SeriesGroupBy833 try: # pandas>=0.23.0834 from pandas.core.groupby.groupby import GroupBy835 except ImportError: # pragma: no cover836 from pandas.core.groupby import GroupBy837 838 try: # pandas>=0.23.0839 from pandas.core.groupby.groupby import PanelGroupBy840 except ImportError:841 try:842 from pandas.core.groupby import PanelGroupBy843 except ImportError: # pandas>=0.25.0844 PanelGroupBy = None845 846 tqdm_kwargs = tqdm_kwargs.copy()847 deprecated_t = [tqdm_kwargs.pop('deprecated_t', None)]848 849 def inner_generator(df_function='apply'):850 def inner(df, func, *args, **kwargs):851 """852 Parameters853 ----------854 df : (DataFrame|Series)[GroupBy]855 Data (may be grouped).856 func : function857 To be applied on the (grouped) data.858 **kwargs : optional859 Transmitted to `df.apply()`.860 """861 862 # Precompute total iterations863 total = tqdm_kwargs.pop("total", getattr(df, 'ngroups', None))864 if total is None: # not grouped865 if df_function == 'applymap':866 total = df.size867 elif isinstance(df, Series):868 total = len(df)869 elif (_Rolling_and_Expanding is None or870 not isinstance(df, _Rolling_and_Expanding)):871 # DataFrame or Panel872 axis = kwargs.get('axis', 0)873 if axis == 'index':874 axis = 0875 elif axis == 'columns':876 axis = 1877 # when axis=0, total is shape[axis1]878 total = df.size // df.shape[axis]879 880 # Init bar881 if deprecated_t[0] is not None:882 t = deprecated_t[0]883 deprecated_t[0] = None884 else:885 t = cls(total=total, **tqdm_kwargs)886 887 if len(args) > 0:888 # *args intentionally not supported (see #244, #299)889 TqdmDeprecationWarning(890 "Except func, normal arguments are intentionally" +891 " not supported by" +892 " `(DataFrame|Series|GroupBy).progress_apply`." +893 " Use keyword arguments instead.",894 fp_write=getattr(t.fp, 'write', sys.stderr.write))895 896 try: # pandas>=1.3.0,<3.0897 from pandas.core.common import is_builtin_func898 except ImportError: # pandas<1.3.0899 is_builtin_func = getattr(df, '_is_builtin_func', lambda f: f)900 try:901 func = is_builtin_func(func)902 except TypeError:903 pass904 905 # Define bar updating wrapper906 def wrapper(*args, **kwargs):907 # update tbar correctly908 # it seems `pandas apply` calls `func` twice909 # on the first column/row to decide whether it can910 # take a fast or slow code path; so stop when t.total==t.n911 t.update(n=1 if not t.total or t.n < t.total else 0)912 return func(*args, **kwargs)913 914 # Apply the provided function (in **kwargs)915 # on the df using our wrapper (which provides bar updating)916 try:917 return getattr(df, df_function)(wrapper, **kwargs)918 finally:919 t.close()920 921 return inner922 923 # Monkeypatch pandas to provide easy methods924 # Enable custom tqdm progress in pandas!925 Series.progress_apply = inner_generator()926 SeriesGroupBy.progress_apply = inner_generator()927 Series.progress_map = inner_generator('map')928 SeriesGroupBy.progress_map = inner_generator('map')929 930 DataFrame.progress_apply = inner_generator()931 DataFrameGroupBy.progress_apply = inner_generator()932 DataFrame.progress_applymap = inner_generator('applymap')933 DataFrame.progress_map = inner_generator('map')934 DataFrameGroupBy.progress_map = inner_generator('map')935 936 if Panel is not None:937 Panel.progress_apply = inner_generator()938 if PanelGroupBy is not None:939 PanelGroupBy.progress_apply = inner_generator()940 941 GroupBy.progress_apply = inner_generator()942 GroupBy.progress_aggregate = inner_generator('aggregate')943 GroupBy.progress_transform = inner_generator('transform')944 945 if Rolling is not None and Expanding is not None:946 Rolling.progress_apply = inner_generator()947 Expanding.progress_apply = inner_generator()948 elif _Rolling_and_Expanding is not None:949 _Rolling_and_Expanding.progress_apply = inner_generator()950 951 # override defaults via env vars952 @envwrap("TQDM_", is_method=True, types={'total': float, 'ncols': int, 'miniters': float,953 'position': int, 'nrows': int})954 def __init__(self, iterable=None, desc=None, total=None, leave=True, file=None,955 ncols=None, mininterval=0.1, maxinterval=10.0, miniters=None,956 ascii=None, disable=False, unit='it', unit_scale=False,957 dynamic_ncols=False, smoothing=0.3, bar_format=None, initial=0,958 position=None, postfix=None, unit_divisor=1000, write_bytes=False,959 lock_args=None, nrows=None, colour=None, delay=0.0, gui=False,960 **kwargs):961 """see tqdm.tqdm for arguments"""962 if file is None:963 file = sys.stderr964 965 if write_bytes:966 # Despite coercing unicode into bytes, py2 sys.std* streams967 # should have bytes written to them.968 file = SimpleTextIOWrapper(969 file, encoding=getattr(file, 'encoding', None) or 'utf-8')970 971 file = DisableOnWriteError(file, tqdm_instance=self)972 973 if disable is None and hasattr(file, "isatty") and not file.isatty():974 disable = True975 976 if total is None and iterable is not None:977 try:978 total = len(iterable)979 except (TypeError, AttributeError):980 total = None981 if total == float("inf"):982 # Infinite iterations, behave same as unknown983 total = None984 985 if disable:986 self.iterable = iterable987 self.disable = disable988 with self._lock:989 self.pos = self._get_free_pos(self)990 self._instances.remove(self)991 self.n = initial992 self.total = total993 self.leave = leave994 return995 996 if kwargs:997 self.disable = True998 with self._lock:999 self.pos = self._get_free_pos(self)1000 self._instances.remove(self)1001 raise (1002 TqdmDeprecationWarning(1003 "`nested` is deprecated and automated.\n"1004 "Use `position` instead for manual control.\n",1005 fp_write=getattr(file, 'write', sys.stderr.write))1006 if "nested" in kwargs else1007 TqdmKeyError("Unknown argument(s): " + str(kwargs)))1008 1009 # Preprocess the arguments1010 if (1011 (ncols is None or nrows is None) and (file in (sys.stderr, sys.stdout))1012 ) or dynamic_ncols: # pragma: no cover1013 if dynamic_ncols:1014 dynamic_ncols = _screen_shape_wrapper()1015 if dynamic_ncols:1016 ncols, nrows = dynamic_ncols(file)1017 else:1018 _dynamic_ncols = _screen_shape_wrapper()1019 if _dynamic_ncols:1020 _ncols, _nrows = _dynamic_ncols(file)1021 if ncols is None:1022 ncols = _ncols1023 if nrows is None:1024 nrows = _nrows1025 1026 if miniters is None:1027 miniters = 01028 dynamic_miniters = True1029 else:1030 dynamic_miniters = False1031 1032 if mininterval is None:1033 mininterval = 01034 1035 if maxinterval is None:1036 maxinterval = 01037 1038 if ascii is None:1039 ascii = not _supports_unicode(file)1040 1041 if bar_format and ascii is not True and not _is_ascii(ascii):1042 # Convert bar format into unicode since terminal uses unicode1043 bar_format = str(bar_format)1044 1045 if smoothing is None:1046 smoothing = 01047 1048 # Store the arguments1049 self.iterable = iterable1050 self.desc = desc or ''1051 self.total = total1052 self.leave = leave1053 self.fp = file1054 self.ncols = ncols1055 self.nrows = nrows1056 self.mininterval = mininterval1057 self.maxinterval = maxinterval1058 self.miniters = miniters1059 self.dynamic_miniters = dynamic_miniters1060 self.ascii = ascii1061 self.disable = disable1062 self.unit = unit1063 self.unit_scale = unit_scale1064 self.unit_divisor = unit_divisor1065 self.initial = initial1066 self.lock_args = lock_args1067 self.delay = delay1068 self.gui = gui1069 self.dynamic_ncols = dynamic_ncols1070 self.smoothing = smoothing1071 self._ema_dn = EMA(smoothing)1072 self._ema_dt = EMA(smoothing)1073 self._ema_miniters = EMA(smoothing)1074 self.bar_format = bar_format1075 self.postfix = None1076 self.colour = colour1077 self._time = time1078 if postfix:1079 try:1080 self.set_postfix(refresh=False, **postfix)1081 except TypeError:1082 self.postfix = postfix1083 1084 # Init the iterations counters1085 self.last_print_n = initial1086 self.n = initial1087 1088 # if nested, at initial sp() call we replace '\r' by '\n' to1089 # not overwrite the outer progress bar1090 with self._lock:1091 # mark fixed positions as negative1092 self.pos = self._get_free_pos(self) if position is None else -position1093 1094 if not gui:1095 # Initialize the screen printer1096 self.sp = self.status_printer(self.fp)1097 if delay <= 0:1098 self.refresh(lock_args=self.lock_args)1099 1100 # Init the time counter1101 self.last_print_t = self._time()1102 # NB: Avoid race conditions by setting start_t at the very end of init1103 self.start_t = self.last_print_t1104 1105 def __bool__(self):1106 if self.total is not None:1107 return self.total > 01108 if self.iterable is None:1109 raise TypeError('bool() undefined when iterable == total == None')1110 return bool(self.iterable)1111 1112 def __len__(self):1113 return (1114 self.total if self.iterable is None1115 else self.iterable.shape[0] if hasattr(self.iterable, "shape")1116 else len(self.iterable) if hasattr(self.iterable, "__len__")1117 else self.iterable.__length_hint__() if hasattr(self.iterable, "__length_hint__")1118 else getattr(self, "total", None))1119 1120 def __reversed__(self):1121 try:1122 orig = self.iterable1123 except AttributeError:1124 raise TypeError("'tqdm' object is not reversible")1125 else:1126 self.iterable = reversed(self.iterable)1127 return self.__iter__()1128 finally:1129 self.iterable = orig1130 1131 def __contains__(self, item):1132 contains = getattr(self.iterable, '__contains__', None)1133 return contains(item) if contains is not None else item in self.__iter__()1134 1135 def __enter__(self):1136 return self1137 1138 def __exit__(self, exc_type, exc_value, traceback):1139 try:1140 self.close()1141 except AttributeError:1142 # maybe eager thread cleanup upon external error1143 if (exc_type, exc_value, traceback) == (None, None, None):1144 raise1145 warn("AttributeError ignored", TqdmWarning, stacklevel=2)1146 1147 def __del__(self):1148 self.close()1149 1150 def __str__(self):1151 return self.format_meter(**self.format_dict)1152 1153 @property1154 def _comparable(self):1155 return abs(getattr(self, "pos", 1 << 31))1156 1157 def __hash__(self):1158 return id(self)1159 1160 def __iter__(self):1161 """Backward-compatibility to use: for x in tqdm(iterable)"""1162 1163 # Inlining instance variables as locals (speed optimisation)1164 iterable = self.iterable1165 1166 # If the bar is disabled, then just walk the iterable1167 # (note: keep this check outside the loop for performance)1168 if self.disable:1169 for obj in iterable:1170 yield obj1171 return1172 1173 mininterval = self.mininterval1174 last_print_t = self.last_print_t1175 last_print_n = self.last_print_n1176 min_start_t = self.start_t + self.delay1177 n = self.n1178 time = self._time1179 1180 try:1181 for obj in iterable:1182 yield obj1183 # Update and possibly print the progressbar.1184 # Note: does not call self.update(1) for speed optimisation.1185 n += 11186 1187 if n - last_print_n >= self.miniters:1188 cur_t = time()1189 dt = cur_t - last_print_t1190 if dt >= mininterval and cur_t >= min_start_t:1191 self.update(n - last_print_n)1192 last_print_n = self.last_print_n1193 last_print_t = self.last_print_t1194 finally:1195 self.n = n1196 self.close()1197 1198 def update(self, n=1):1199 """1200 Manually update the progress bar, useful for streams