codekingpro/portable-devtools
114k
1"""Module/script to byte-compile all .py files to .pyc files.2 3When called as a script with arguments, this compiles the directories4given as arguments recursively; the -l option prevents it from5recursing into directories.6 7Without arguments, it compiles all modules on sys.path, without8recursing into subdirectories. (Even though it should do so for9packages -- for now, you'll have to deal with packages separately.)10 11See module py_compile for details of the actual byte-compilation.12"""13import os14import sys15import importlib.util16import py_compile17import struct18import filecmp19 20from functools import partial21from pathlib import Path22 23__all__ = ["compile_dir","compile_file","compile_path"]24 25def _walk_dir(dir, maxlevels, quiet=0):26 if quiet < 2 and isinstance(dir, os.PathLike):27 dir = os.fspath(dir)28 if not quiet:29 print('Listing {!r}...'.format(dir))30 try:31 names = os.listdir(dir)32 except OSError:33 if quiet < 2:34 print("Can't list {!r}".format(dir))35 names = []36 names.sort()37 for name in names:38 if name == '__pycache__':39 continue40 fullname = os.path.join(dir, name)41 if not os.path.isdir(fullname):42 yield fullname43 elif (maxlevels > 0 and name != os.curdir and name != os.pardir and44 os.path.isdir(fullname) and not os.path.islink(fullname)):45 yield from _walk_dir(fullname, maxlevels=maxlevels - 1,46 quiet=quiet)47 48def compile_dir(dir, maxlevels=None, ddir=None, force=False,49 rx=None, quiet=0, legacy=False, optimize=-1, workers=1,50 invalidation_mode=None, *, stripdir=None,51 prependdir=None, limit_sl_dest=None, hardlink_dupes=False):52 """Byte-compile all modules in the given directory tree.53 54 Arguments (only dir is required):55 56 dir: the directory to byte-compile57 maxlevels: maximum recursion level (default `sys.getrecursionlimit()`)58 ddir: the directory that will be prepended to the path to the59 file as it is compiled into each byte-code file.60 force: if True, force compilation, even if timestamps are up-to-date61 quiet: full output with False or 0, errors only with 1,62 no output with 263 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths64 optimize: int or list of optimization levels or -1 for level of65 the interpreter. Multiple levels leads to multiple compiled66 files each with one optimization level.67 workers: maximum number of parallel workers68 invalidation_mode: how the up-to-dateness of the pyc will be checked69 stripdir: part of path to left-strip from source file path70 prependdir: path to prepend to beginning of original file path, applied71 after stripdir72 limit_sl_dest: ignore symlinks if they are pointing outside of73 the defined path74 hardlink_dupes: hardlink duplicated pyc files75 """76 ProcessPoolExecutor = None77 if ddir is not None and (stripdir is not None or prependdir is not None):78 raise ValueError(("Destination dir (ddir) cannot be used "79 "in combination with stripdir or prependdir"))80 if ddir is not None:81 stripdir = dir82 prependdir = ddir83 ddir = None84 if workers < 0:85 raise ValueError('workers must be greater or equal to 0')86 if workers != 1:87 # Check if this is a system where ProcessPoolExecutor can function.88 from concurrent.futures.process import _check_system_limits89 try:90 _check_system_limits()91 except NotImplementedError:92 workers = 193 else:94 from concurrent.futures import ProcessPoolExecutor95 if maxlevels is None:96 maxlevels = sys.getrecursionlimit()97 files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)98 success = True99 if workers != 1 and ProcessPoolExecutor is not None:100 import multiprocessing101 if multiprocessing.get_start_method() == 'fork':102 mp_context = multiprocessing.get_context('forkserver')103 else:104 mp_context = None105 # If workers == 0, let ProcessPoolExecutor choose106 workers = workers or None107 with ProcessPoolExecutor(max_workers=workers,108 mp_context=mp_context) as executor:109 results = executor.map(partial(compile_file,110 ddir=ddir, force=force,111 rx=rx, quiet=quiet,112 legacy=legacy,113 optimize=optimize,114 invalidation_mode=invalidation_mode,115 stripdir=stripdir,116 prependdir=prependdir,117 limit_sl_dest=limit_sl_dest,118 hardlink_dupes=hardlink_dupes),119 files,120 chunksize=4)121 success = min(results, default=True)122 else:123 for file in files:124 if not compile_file(file, ddir, force, rx, quiet,125 legacy, optimize, invalidation_mode,126 stripdir=stripdir, prependdir=prependdir,127 limit_sl_dest=limit_sl_dest,128 hardlink_dupes=hardlink_dupes):129 success = False130 return success131 132def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,133 legacy=False, optimize=-1,134 invalidation_mode=None, *, stripdir=None, prependdir=None,135 limit_sl_dest=None, hardlink_dupes=False):136 """Byte-compile one file.137 138 Arguments (only fullname is required):139 140 fullname: the file to byte-compile141 ddir: if given, the directory name compiled in to the142 byte-code file.143 force: if True, force compilation, even if timestamps are up-to-date144 quiet: full output with False or 0, errors only with 1,145 no output with 2146 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths147 optimize: int or list of optimization levels or -1 for level of148 the interpreter. Multiple levels leads to multiple compiled149 files each with one optimization level.150 invalidation_mode: how the up-to-dateness of the pyc will be checked151 stripdir: part of path to left-strip from source file path152 prependdir: path to prepend to beginning of original file path, applied153 after stripdir154 limit_sl_dest: ignore symlinks if they are pointing outside of155 the defined path.156 hardlink_dupes: hardlink duplicated pyc files157 """158 159 if ddir is not None and (stripdir is not None or prependdir is not None):160 raise ValueError(("Destination dir (ddir) cannot be used "161 "in combination with stripdir or prependdir"))162 163 success = True164 fullname = os.fspath(fullname)165 stripdir = os.fspath(stripdir) if stripdir is not None else None166 name = os.path.basename(fullname)167 168 dfile = None169 170 if ddir is not None:171 dfile = os.path.join(ddir, name)172 173 if stripdir is not None:174 fullname_parts = fullname.split(os.path.sep)175 stripdir_parts = stripdir.split(os.path.sep)176 177 if stripdir_parts != fullname_parts[:len(stripdir_parts)]:178 if quiet < 2:179 print("The stripdir path {!r} is not a valid prefix for "180 "source path {!r}; ignoring".format(stripdir, fullname))181 else:182 dfile = os.path.join(*fullname_parts[len(stripdir_parts):])183 184 if prependdir is not None:185 if dfile is None:186 dfile = os.path.join(prependdir, fullname)187 else:188 dfile = os.path.join(prependdir, dfile)189 190 if isinstance(optimize, int):191 optimize = [optimize]192 193 # Use set() to remove duplicates.194 # Use sorted() to create pyc files in a deterministic order.195 optimize = sorted(set(optimize))196 197 if hardlink_dupes and len(optimize) < 2:198 raise ValueError("Hardlinking of duplicated bytecode makes sense "199 "only for more than one optimization level")200 201 if rx is not None:202 mo = rx.search(fullname)203 if mo:204 return success205 206 if limit_sl_dest is not None and os.path.islink(fullname):207 if Path(limit_sl_dest).resolve() not in Path(fullname).resolve().parents:208 return success209 210 opt_cfiles = {}211 212 if os.path.isfile(fullname):213 for opt_level in optimize:214 if legacy:215 opt_cfiles[opt_level] = fullname + 'c'216 else:217 if opt_level >= 0:218 opt = opt_level if opt_level >= 1 else ''219 cfile = (importlib.util.cache_from_source(220 fullname, optimization=opt))221 opt_cfiles[opt_level] = cfile222 else:223 cfile = importlib.util.cache_from_source(fullname)224 opt_cfiles[opt_level] = cfile225 226 head, tail = name[:-3], name[-3:]227 if tail == '.py':228 if not force:229 try:230 mtime = int(os.stat(fullname).st_mtime)231 expect = struct.pack('<4sLL', importlib.util.MAGIC_NUMBER,232 0, mtime & 0xFFFF_FFFF)233 for cfile in opt_cfiles.values():234 with open(cfile, 'rb') as chandle:235 actual = chandle.read(12)236 if expect != actual:237 break238 else:239 return success240 except OSError:241 pass242 if not quiet:243 print('Compiling {!r}...'.format(fullname))244 try:245 for index, opt_level in enumerate(optimize):246 cfile = opt_cfiles[opt_level]247 ok = py_compile.compile(fullname, cfile, dfile, True,248 optimize=opt_level,249 invalidation_mode=invalidation_mode)250 if index > 0 and hardlink_dupes:251 previous_cfile = opt_cfiles[optimize[index - 1]]252 if filecmp.cmp(cfile, previous_cfile, shallow=False):253 os.unlink(cfile)254 os.link(previous_cfile, cfile)255 except py_compile.PyCompileError as err:256 success = False257 if quiet >= 2:258 return success259 elif quiet:260 print('*** Error compiling {!r}...'.format(fullname))261 else:262 print('*** ', end='')263 # escape non-printable characters in msg264 encoding = sys.stdout.encoding or sys.getdefaultencoding()265 msg = err.msg.encode(encoding, errors='backslashreplace').decode(encoding)266 print(msg)267 except (SyntaxError, UnicodeError, OSError) as e:268 success = False269 if quiet >= 2:270 return success271 elif quiet:272 print('*** Error compiling {!r}...'.format(fullname))273 else:274 print('*** ', end='')275 print(e.__class__.__name__ + ':', e)276 else:277 if ok == 0:278 success = False279 return success280 281def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,282 legacy=False, optimize=-1,283 invalidation_mode=None):284 """Byte-compile all module on sys.path.285 286 Arguments (all optional):287 288 skip_curdir: if true, skip current directory (default True)289 maxlevels: max recursion level (default 0)290 force: as for compile_dir() (default False)291 quiet: as for compile_dir() (default 0)292 legacy: as for compile_dir() (default False)293 optimize: as for compile_dir() (default -1)294 invalidation_mode: as for compiler_dir()295 """296 success = True297 for dir in sys.path:298 if (not dir or dir == os.curdir) and skip_curdir:299 if quiet < 2:300 print('Skipping current directory')301 else:302 success = success and compile_dir(303 dir,304 maxlevels,305 None,306 force,307 quiet=quiet,308 legacy=legacy,309 optimize=optimize,310 invalidation_mode=invalidation_mode,311 )312 return success313 314 315def main():316 """Script main program."""317 import argparse318 319 parser = argparse.ArgumentParser(320 description='Utilities to support installing Python libraries.',321 color=True,322 )323 parser.add_argument('-l', action='store_const', const=0,324 default=None, dest='maxlevels',325 help="don't recurse into subdirectories")326 parser.add_argument('-r', type=int, dest='recursion',327 help=('control the maximum recursion level. '328 'if `-l` and `-r` options are specified, '329 'then `-r` takes precedence.'))330 parser.add_argument('-f', action='store_true', dest='force',331 help='force rebuild even if timestamps are up to date')332 parser.add_argument('-q', action='count', dest='quiet', default=0,333 help='output only error messages; -qq will suppress '334 'the error messages as well.')335 parser.add_argument('-b', action='store_true', dest='legacy',336 help='use legacy (pre-PEP3147) compiled file locations')337 parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None,338 help=('directory to prepend to file paths for use in '339 'compile-time tracebacks and in runtime '340 'tracebacks in cases where the source file is '341 'unavailable'))342 parser.add_argument('-s', metavar='STRIPDIR', dest='stripdir',343 default=None,344 help=('part of path to left-strip from path '345 'to source file - for example buildroot. '346 '`-d` and `-s` options cannot be '347 'specified together.'))348 parser.add_argument('-p', metavar='PREPENDDIR', dest='prependdir',349 default=None,350 help=('path to add as prefix to path '351 'to source file - for example / to make '352 'it absolute when some part is removed '353 'by `-s` option. '354 '`-d` and `-p` options cannot be '355 'specified together.'))356 parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,357 help=('skip files matching the regular expression; '358 'the regexp is searched for in the full path '359 'of each file considered for compilation'))360 parser.add_argument('-i', metavar='FILE', dest='flist',361 help=('add all the files and directories listed in '362 'FILE to the list considered for compilation; '363 'if "-", names are read from stdin'))364 parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*',365 help=('zero or more file and directory names '366 'to compile; if no arguments given, defaults '367 'to the equivalent of -l sys.path'))368 parser.add_argument('-j', '--workers', default=1,369 type=int, help='Run compileall concurrently')370 invalidation_modes = [mode.name.lower().replace('_', '-')371 for mode in py_compile.PycInvalidationMode]372 parser.add_argument('--invalidation-mode',373 choices=sorted(invalidation_modes),374 help=('set .pyc invalidation mode; defaults to '375 '"checked-hash" if the SOURCE_DATE_EPOCH '376 'environment variable is set, and '377 '"timestamp" otherwise.'))378 parser.add_argument('-o', action='append', type=int, dest='opt_levels',379 help=('Optimization levels to run compilation with. '380 'Default is -1 which uses the optimization level '381 'of the Python interpreter itself (see -O).'))382 parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest',383 help='Ignore symlinks pointing outsite of the DIR')384 parser.add_argument('--hardlink-dupes', action='store_true',385 dest='hardlink_dupes',386 help='Hardlink duplicated pyc files')387 388 args = parser.parse_args()389 compile_dests = args.compile_dest390 391 if args.rx:392 import re393 args.rx = re.compile(args.rx)394 395 if args.limit_sl_dest == "":396 args.limit_sl_dest = None397 398 if args.recursion is not None:399 maxlevels = args.recursion400 else:401 maxlevels = args.maxlevels402 403 if args.opt_levels is None:404 args.opt_levels = [-1]405 406 if len(args.opt_levels) == 1 and args.hardlink_dupes:407 parser.error(("Hardlinking of duplicated bytecode makes sense "408 "only for more than one optimization level."))409 410 if args.ddir is not None and (411 args.stripdir is not None or args.prependdir is not None412 ):413 parser.error("-d cannot be used in combination with -s or -p")414 415 # if flist is provided then load it416 if args.flist:417 try:418 with (sys.stdin if args.flist=='-' else419 open(args.flist, encoding="utf-8")) as f:420 for line in f:421 compile_dests.append(line.strip())422 except OSError:423 if args.quiet < 2:424 print("Error reading file list {}".format(args.flist))425 return False426 427 if args.invalidation_mode:428 ivl_mode = args.invalidation_mode.replace('-', '_').upper()429 invalidation_mode = py_compile.PycInvalidationMode[ivl_mode]430 else:431 invalidation_mode = None432 433 success = True434 try:435 if compile_dests:436 for dest in compile_dests:437 if os.path.isfile(dest):438 if not compile_file(dest, args.ddir, args.force, args.rx,439 args.quiet, args.legacy,440 invalidation_mode=invalidation_mode,441 stripdir=args.stripdir,442 prependdir=args.prependdir,443 optimize=args.opt_levels,444 limit_sl_dest=args.limit_sl_dest,445 hardlink_dupes=args.hardlink_dupes):446 success = False447 else:448 if not compile_dir(dest, maxlevels, args.ddir,449 args.force, args.rx, args.quiet,450 args.legacy, workers=args.workers,451 invalidation_mode=invalidation_mode,452 stripdir=args.stripdir,453 prependdir=args.prependdir,454 optimize=args.opt_levels,455 limit_sl_dest=args.limit_sl_dest,456 hardlink_dupes=args.hardlink_dupes):457 success = False458 return success459 else:460 return compile_path(legacy=args.legacy, force=args.force,461 quiet=args.quiet,462 invalidation_mode=invalidation_mode)463 except KeyboardInterrupt:464 if args.quiet < 2:465 print("\n[interrupted]")466 return False467 return True468 469 470if __name__ == '__main__':471 exit_status = int(not main())472 sys.exit(exit_status)473