codekingpro/portable-devtools

1"""distutils.util2 3Miscellaneous utility functions -- anything that doesn't fit into4one of the other *util.py modules.5"""6 7import functools8import importlib.util9import os10import re11import string12import subprocess13import sys14import sysconfig15 16from ._log import log17from ._modified import newer18from .errors import DistutilsByteCompileError, DistutilsPlatformError19from .spawn import spawn20 21 22def get_host_platform():23    """24    Return a string that identifies the current platform. Use this25    function to distinguish platform-specific build directories and26    platform-specific built distributions.27    """28 29    # This function initially exposed platforms as defined in Python 3.930    # even with older Python versions when distutils was split out.31    # Now it delegates to stdlib sysconfig, but maintains compatibility.32 33    if sys.version_info < (3, 9):34        if os.name == "posix" and hasattr(os, 'uname'):35            osname, host, release, version, machine = os.uname()36            if osname[:3] == "aix":37                from .py38compat import aix_platform38 39                return aix_platform(osname, version, release)40 41    return sysconfig.get_platform()42 43 44def get_platform():45    if os.name == 'nt':46        TARGET_TO_PLAT = {47            'x86': 'win32',48            'x64': 'win-amd64',49            'arm': 'win-arm32',50            'arm64': 'win-arm64',51        }52        target = os.environ.get('VSCMD_ARG_TGT_ARCH')53        return TARGET_TO_PLAT.get(target) or get_host_platform()54    return get_host_platform()55 56 57if sys.platform == 'darwin':58    _syscfg_macosx_ver = None  # cache the version pulled from sysconfig59MACOSX_VERSION_VAR = 'MACOSX_DEPLOYMENT_TARGET'60 61 62def _clear_cached_macosx_ver():63    """For testing only. Do not call."""64    global _syscfg_macosx_ver65    _syscfg_macosx_ver = None66 67 68def get_macosx_target_ver_from_syscfg():69    """Get the version of macOS latched in the Python interpreter configuration.70    Returns the version as a string or None if can't obtain one. Cached."""71    global _syscfg_macosx_ver72    if _syscfg_macosx_ver is None:73        from distutils import sysconfig74 75        ver = sysconfig.get_config_var(MACOSX_VERSION_VAR) or ''76        if ver:77            _syscfg_macosx_ver = ver78    return _syscfg_macosx_ver79 80 81def get_macosx_target_ver():82    """Return the version of macOS for which we are building.83 84    The target version defaults to the version in sysconfig latched at time85    the Python interpreter was built, unless overridden by an environment86    variable. If neither source has a value, then None is returned"""87 88    syscfg_ver = get_macosx_target_ver_from_syscfg()89    env_ver = os.environ.get(MACOSX_VERSION_VAR)90 91    if env_ver:92        # Validate overridden version against sysconfig version, if have both.93        # Ensure that the deployment target of the build process is not less94        # than 10.3 if the interpreter was built for 10.3 or later.  This95        # ensures extension modules are built with correct compatibility96        # values, specifically LDSHARED which can use97        # '-undefined dynamic_lookup' which only works on >= 10.3.98        if (99            syscfg_ver100            and split_version(syscfg_ver) >= [10, 3]101            and split_version(env_ver) < [10, 3]102        ):103            my_msg = (104                '$' + MACOSX_VERSION_VAR + ' mismatch: '105                f'now "{env_ver}" but "{syscfg_ver}" during configure; '106                'must use 10.3 or later'107            )108            raise DistutilsPlatformError(my_msg)109        return env_ver110    return syscfg_ver111 112 113def split_version(s):114    """Convert a dot-separated string into a list of numbers for comparisons"""115    return [int(n) for n in s.split('.')]116 117 118def convert_path(pathname):119    """Return 'pathname' as a name that will work on the native filesystem,120    i.e. split it on '/' and put it back together again using the current121    directory separator.  Needed because filenames in the setup script are122    always supplied in Unix style, and have to be converted to the local123    convention before we can actually use them in the filesystem.  Raises124    ValueError on non-Unix-ish systems if 'pathname' either starts or125    ends with a slash.126    """127    if os.sep == '/':128        return pathname129    if not pathname:130        return pathname131    if pathname[0] == '/':132        raise ValueError("path '%s' cannot be absolute" % pathname)133    if pathname[-1] == '/':134        raise ValueError("path '%s' cannot end with '/'" % pathname)135 136    paths = pathname.split('/')137    while '.' in paths:138        paths.remove('.')139    if not paths:140        return os.curdir141    return os.path.join(*paths)142 143 144# convert_path ()145 146 147def change_root(new_root, pathname):148    """Return 'pathname' with 'new_root' prepended.  If 'pathname' is149    relative, this is equivalent to "os.path.join(new_root,pathname)".150    Otherwise, it requires making 'pathname' relative and then joining the151    two, which is tricky on DOS/Windows and Mac OS.152    """153    if os.name == 'posix':154        if not os.path.isabs(pathname):155            return os.path.join(new_root, pathname)156        else:157            return os.path.join(new_root, pathname[1:])158 159    elif os.name == 'nt':160        (drive, path) = os.path.splitdrive(pathname)161        if path[0] == '\\':162            path = path[1:]163        return os.path.join(new_root, path)164 165    raise DistutilsPlatformError(f"nothing known about platform '{os.name}'")166 167 168@functools.lru_cache169def check_environ():170    """Ensure that 'os.environ' has all the environment variables we171    guarantee that users can use in config files, command-line options,172    etc.  Currently this includes:173      HOME - user's home directory (Unix only)174      PLAT - description of the current platform, including hardware175             and OS (see 'get_platform()')176    """177    if os.name == 'posix' and 'HOME' not in os.environ:178        try:179            import pwd180 181            os.environ['HOME'] = pwd.getpwuid(os.getuid())[5]182        except (ImportError, KeyError):183            # bpo-10496: if the current user identifier doesn't exist in the184            # password database, do nothing185            pass186 187    if 'PLAT' not in os.environ:188        os.environ['PLAT'] = get_platform()189 190 191def subst_vars(s, local_vars):192    """193    Perform variable substitution on 'string'.194    Variables are indicated by format-style braces ("{var}").195    Variable is substituted by the value found in the 'local_vars'196    dictionary or in 'os.environ' if it's not in 'local_vars'.197    'os.environ' is first checked/augmented to guarantee that it contains198    certain values: see 'check_environ()'.  Raise ValueError for any199    variables not found in either 'local_vars' or 'os.environ'.200    """201    check_environ()202    lookup = dict(os.environ)203    lookup.update((name, str(value)) for name, value in local_vars.items())204    try:205        return _subst_compat(s).format_map(lookup)206    except KeyError as var:207        raise ValueError(f"invalid variable {var}")208 209 210def _subst_compat(s):211    """212    Replace shell/Perl-style variable substitution with213    format-style. For compatibility.214    """215 216    def _subst(match):217        return f'{{{match.group(1)}}}'218 219    repl = re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s)220    if repl != s:221        import warnings222 223        warnings.warn(224            "shell/Perl-style substitutions are deprecated",225            DeprecationWarning,226        )227    return repl228 229 230def grok_environment_error(exc, prefix="error: "):231    # Function kept for backward compatibility.232    # Used to try clever things with EnvironmentErrors,233    # but nowadays str(exception) produces good messages.234    return prefix + str(exc)235 236 237# Needed by 'split_quoted()'238_wordchars_re = _squote_re = _dquote_re = None239 240 241def _init_regex():242    global _wordchars_re, _squote_re, _dquote_re243    _wordchars_re = re.compile(r'[^\\\'\"%s ]*' % string.whitespace)244    _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'")245    _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"')246 247 248def split_quoted(s):249    """Split a string up according to Unix shell-like rules for quotes and250    backslashes.  In short: words are delimited by spaces, as long as those251    spaces are not escaped by a backslash, or inside a quoted string.252    Single and double quotes are equivalent, and the quote characters can253    be backslash-escaped.  The backslash is stripped from any two-character254    escape sequence, leaving only the escaped character.  The quote255    characters are stripped from any quoted string.  Returns a list of256    words.257    """258 259    # This is a nice algorithm for splitting up a single string, since it260    # doesn't require character-by-character examination.  It was a little261    # bit of a brain-bender to get it working right, though...262    if _wordchars_re is None:263        _init_regex()264 265    s = s.strip()266    words = []267    pos = 0268 269    while s:270        m = _wordchars_re.match(s, pos)271        end = m.end()272        if end == len(s):273            words.append(s[:end])274            break275 276        if s[end] in string.whitespace:277            # unescaped, unquoted whitespace: now278            # we definitely have a word delimiter279            words.append(s[:end])280            s = s[end:].lstrip()281            pos = 0282 283        elif s[end] == '\\':284            # preserve whatever is being escaped;285            # will become part of the current word286            s = s[:end] + s[end + 1 :]287            pos = end + 1288 289        else:290            if s[end] == "'":  # slurp singly-quoted string291                m = _squote_re.match(s, end)292            elif s[end] == '"':  # slurp doubly-quoted string293                m = _dquote_re.match(s, end)294            else:295                raise RuntimeError("this can't happen (bad char '%c')" % s[end])296 297            if m is None:298                raise ValueError("bad string (mismatched %s quotes?)" % s[end])299 300            (beg, end) = m.span()301            s = s[:beg] + s[beg + 1 : end - 1] + s[end:]302            pos = m.end() - 2303 304        if pos >= len(s):305            words.append(s)306            break307 308    return words309 310 311# split_quoted ()312 313 314def execute(func, args, msg=None, verbose=0, dry_run=0):315    """Perform some action that affects the outside world (eg.  by316    writing to the filesystem).  Such actions are special because they317    are disabled by the 'dry_run' flag.  This method takes care of all318    that bureaucracy for you; all you have to do is supply the319    function to call and an argument tuple for it (to embody the320    "external action" being performed), and an optional message to321    print.322    """323    if msg is None:324        msg = f"{func.__name__}{args!r}"325        if msg[-2:] == ',)':  # correct for singleton tuple326            msg = msg[0:-2] + ')'327 328    log.info(msg)329    if not dry_run:330        func(*args)331 332 333def strtobool(val):334    """Convert a string representation of truth to true (1) or false (0).335 336    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values337    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if338    'val' is anything else.339    """340    val = val.lower()341    if val in ('y', 'yes', 't', 'true', 'on', '1'):342        return 1343    elif val in ('n', 'no', 'f', 'false', 'off', '0'):344        return 0345    else:346        raise ValueError(f"invalid truth value {val!r}")347 348 349def byte_compile(  # noqa: C901350    py_files,351    optimize=0,352    force=0,353    prefix=None,354    base_dir=None,355    verbose=1,356    dry_run=0,357    direct=None,358):359    """Byte-compile a collection of Python source files to .pyc360    files in a __pycache__ subdirectory.  'py_files' is a list361    of files to compile; any files that don't end in ".py" are silently362    skipped.  'optimize' must be one of the following:363      0 - don't optimize364      1 - normal optimization (like "python -O")365      2 - extra optimization (like "python -OO")366    If 'force' is true, all files are recompiled regardless of367    timestamps.368 369    The source filename encoded in each bytecode file defaults to the370    filenames listed in 'py_files'; you can modify these with 'prefix' and371    'basedir'.  'prefix' is a string that will be stripped off of each372    source filename, and 'base_dir' is a directory name that will be373    prepended (after 'prefix' is stripped).  You can supply either or both374    (or neither) of 'prefix' and 'base_dir', as you wish.375 376    If 'dry_run' is true, doesn't actually do anything that would377    affect the filesystem.378 379    Byte-compilation is either done directly in this interpreter process380    with the standard py_compile module, or indirectly by writing a381    temporary script and executing it.  Normally, you should let382    'byte_compile()' figure out to use direct compilation or not (see383    the source for details).  The 'direct' flag is used by the script384    generated in indirect mode; unless you know what you're doing, leave385    it set to None.386    """387 388    # nothing is done if sys.dont_write_bytecode is True389    if sys.dont_write_bytecode:390        raise DistutilsByteCompileError('byte-compiling is disabled.')391 392    # First, if the caller didn't force us into direct or indirect mode,393    # figure out which mode we should be in.  We take a conservative394    # approach: choose direct mode *only* if the current interpreter is395    # in debug mode and optimize is 0.  If we're not in debug mode (-O396    # or -OO), we don't know which level of optimization this397    # interpreter is running with, so we can't do direct398    # byte-compilation and be certain that it's the right thing.  Thus,399    # always compile indirectly if the current interpreter is in either400    # optimize mode, or if either optimization level was requested by401    # the caller.402    if direct is None:403        direct = __debug__ and optimize == 0404 405    # "Indirect" byte-compilation: write a temporary script and then406    # run it with the appropriate flags.407    if not direct:408        try:409            from tempfile import mkstemp410 411            (script_fd, script_name) = mkstemp(".py")412        except ImportError:413            from tempfile import mktemp414 415            (script_fd, script_name) = None, mktemp(".py")416        log.info("writing byte-compilation script '%s'", script_name)417        if not dry_run:418            if script_fd is not None:419                script = os.fdopen(script_fd, "w", encoding='utf-8')420            else:  # pragma: no cover421                script = open(script_name, "w", encoding='utf-8')422 423            with script:424                script.write(425                    """\426from distutils.util import byte_compile427files = [428"""429                )430 431                # XXX would be nice to write absolute filenames, just for432                # safety's sake (script should be more robust in the face of433                # chdir'ing before running it).  But this requires abspath'ing434                # 'prefix' as well, and that breaks the hack in build_lib's435                # 'byte_compile()' method that carefully tacks on a trailing436                # slash (os.sep really) to make sure the prefix here is "just437                # right".  This whole prefix business is rather delicate -- the438                # problem is that it's really a directory, but I'm treating it439                # as a dumb string, so trailing slashes and so forth matter.440 441                script.write(",\n".join(map(repr, py_files)) + "]\n")442                script.write(443                    f"""444byte_compile(files, optimize={optimize!r}, force={force!r},445             prefix={prefix!r}, base_dir={base_dir!r},446             verbose={verbose!r}, dry_run=0,447             direct=1)448"""449                )450 451        cmd = [sys.executable]452        cmd.extend(subprocess._optim_args_from_interpreter_flags())453        cmd.append(script_name)454        spawn(cmd, dry_run=dry_run)455        execute(os.remove, (script_name,), "removing %s" % script_name, dry_run=dry_run)456 457    # "Direct" byte-compilation: use the py_compile module to compile458    # right here, right now.  Note that the script generated in indirect459    # mode simply calls 'byte_compile()' in direct mode, a weird sort of460    # cross-process recursion.  Hey, it works!461    else:462        from py_compile import compile463 464        for file in py_files:465            if file[-3:] != ".py":466                # This lets us be lazy and not filter filenames in467                # the "install_lib" command.468                continue469 470            # Terminology from the py_compile module:471            #   cfile - byte-compiled file472            #   dfile - purported source filename (same as 'file' by default)473            if optimize >= 0:474                opt = '' if optimize == 0 else optimize475                cfile = importlib.util.cache_from_source(file, optimization=opt)476            else:477                cfile = importlib.util.cache_from_source(file)478            dfile = file479            if prefix:480                if file[: len(prefix)] != prefix:481                    raise ValueError(482                        f"invalid prefix: filename {file!r} doesn't start with {prefix!r}"483                    )484                dfile = dfile[len(prefix) :]485            if base_dir:486                dfile = os.path.join(base_dir, dfile)487 488            cfile_base = os.path.basename(cfile)489            if direct:490                if force or newer(file, cfile):491                    log.info("byte-compiling %s to %s", file, cfile_base)492                    if not dry_run:493                        compile(file, cfile, dfile)494                else:495                    log.debug("skipping byte-compilation of %s to %s", file, cfile_base)496 497 498def rfc822_escape(header):499    """Return a version of the string escaped for inclusion in an500    RFC-822 header, by ensuring there are 8 spaces space after each newline.501    """502    indent = 8 * " "503    lines = header.splitlines(keepends=True)504 505    # Emulate the behaviour of `str.split`506    # (the terminal line break in `splitlines` does not result in an extra line):507    ends_in_newline = lines and lines[-1].splitlines()[0] != lines[-1]508    suffix = indent if ends_in_newline else ""509 510    return indent.join(lines) + suffix511