codekingpro/portable-devtools

1"""distutils.util2 3Miscellaneous utility functions -- anything that doesn't fit into4one of the other *util.py modules.5"""6 7from __future__ import annotations8 9import functools10import importlib.util11import os12import pathlib13import re14import string15import subprocess16import sys17import sysconfig18import tempfile19from collections.abc import Callable, Iterable, Mapping20from typing import TYPE_CHECKING, AnyStr21 22from jaraco.functools import pass_none23 24from ._log import log25from ._modified import newer26from .errors import DistutilsByteCompileError, DistutilsPlatformError27from .spawn import spawn28 29if TYPE_CHECKING:30    from typing_extensions import TypeVarTuple, Unpack31 32    _Ts = TypeVarTuple("_Ts")33 34 35def get_host_platform() -> str:36    """37    Return a string that identifies the current platform. Use this38    function to distinguish platform-specific build directories and39    platform-specific built distributions.40    """41 42    # This function initially exposed platforms as defined in Python 3.943    # even with older Python versions when distutils was split out.44    # Now it delegates to stdlib sysconfig.45 46    return sysconfig.get_platform()47 48 49def get_platform() -> str:50    if os.name == 'nt':51        TARGET_TO_PLAT = {52            'x86': 'win32',53            'x64': 'win-amd64',54            'arm': 'win-arm32',55            'arm64': 'win-arm64',56        }57        target = os.environ.get('VSCMD_ARG_TGT_ARCH')58        return TARGET_TO_PLAT.get(target) or get_host_platform()59    return get_host_platform()60 61 62if sys.platform == 'darwin':63    _syscfg_macosx_ver = None  # cache the version pulled from sysconfig64MACOSX_VERSION_VAR = 'MACOSX_DEPLOYMENT_TARGET'65 66 67def _clear_cached_macosx_ver():68    """For testing only. Do not call."""69    global _syscfg_macosx_ver70    _syscfg_macosx_ver = None71 72 73def get_macosx_target_ver_from_syscfg():74    """Get the version of macOS latched in the Python interpreter configuration.75    Returns the version as a string or None if can't obtain one. Cached."""76    global _syscfg_macosx_ver77    if _syscfg_macosx_ver is None:78        from distutils import sysconfig79 80        ver = sysconfig.get_config_var(MACOSX_VERSION_VAR) or ''81        if ver:82            _syscfg_macosx_ver = ver83    return _syscfg_macosx_ver84 85 86def get_macosx_target_ver():87    """Return the version of macOS for which we are building.88 89    The target version defaults to the version in sysconfig latched at time90    the Python interpreter was built, unless overridden by an environment91    variable. If neither source has a value, then None is returned"""92 93    syscfg_ver = get_macosx_target_ver_from_syscfg()94    env_ver = os.environ.get(MACOSX_VERSION_VAR)95 96    if env_ver:97        # Validate overridden version against sysconfig version, if have both.98        # Ensure that the deployment target of the build process is not less99        # than 10.3 if the interpreter was built for 10.3 or later.  This100        # ensures extension modules are built with correct compatibility101        # values, specifically LDSHARED which can use102        # '-undefined dynamic_lookup' which only works on >= 10.3.103        if (104            syscfg_ver105            and split_version(syscfg_ver) >= [10, 3]106            and split_version(env_ver) < [10, 3]107        ):108            my_msg = (109                '$' + MACOSX_VERSION_VAR + ' mismatch: '110                f'now "{env_ver}" but "{syscfg_ver}" during configure; '111                'must use 10.3 or later'112            )113            raise DistutilsPlatformError(my_msg)114        return env_ver115    return syscfg_ver116 117 118def split_version(s: str) -> list[int]:119    """Convert a dot-separated string into a list of numbers for comparisons"""120    return [int(n) for n in s.split('.')]121 122 123@pass_none124def convert_path(pathname: str | os.PathLike[str]) -> str:125    r"""126    Allow for pathlib.Path inputs, coax to a native path string.127 128    If None is passed, will just pass it through as129    Setuptools relies on this behavior.130 131    >>> convert_path(None) is None132    True133 134    Removes empty paths.135 136    >>> convert_path('foo/./bar').replace('\\', '/')137    'foo/bar'138    """139    return os.fspath(pathlib.PurePath(pathname))140 141 142def change_root(143    new_root: AnyStr | os.PathLike[AnyStr], pathname: AnyStr | os.PathLike[AnyStr]144) -> AnyStr:145    """Return 'pathname' with 'new_root' prepended.  If 'pathname' is146    relative, this is equivalent to "os.path.join(new_root,pathname)".147    Otherwise, it requires making 'pathname' relative and then joining the148    two, which is tricky on DOS/Windows and Mac OS.149    """150    if os.name == 'posix':151        if not os.path.isabs(pathname):152            return os.path.join(new_root, pathname)153        else:154            return os.path.join(new_root, pathname[1:])155 156    elif os.name == 'nt':157        (drive, path) = os.path.splitdrive(pathname)158        if path[0] == os.sep:159            path = path[1:]160        return os.path.join(new_root, path)161 162    raise DistutilsPlatformError(f"nothing known about platform '{os.name}'")163 164 165@functools.lru_cache166def check_environ() -> None:167    """Ensure that 'os.environ' has all the environment variables we168    guarantee that users can use in config files, command-line options,169    etc.  Currently this includes:170      HOME - user's home directory (Unix only)171      PLAT - description of the current platform, including hardware172             and OS (see 'get_platform()')173    """174    if os.name == 'posix' and 'HOME' not in os.environ:175        try:176            import pwd177 178            os.environ['HOME'] = pwd.getpwuid(os.getuid())[5]179        except (ImportError, KeyError):180            # bpo-10496: if the current user identifier doesn't exist in the181            # password database, do nothing182            pass183 184    if 'PLAT' not in os.environ:185        os.environ['PLAT'] = get_platform()186 187 188def subst_vars(s, local_vars: Mapping[str, object]) -> str:189    """190    Perform variable substitution on 'string'.191    Variables are indicated by format-style braces ("{var}").192    Variable is substituted by the value found in the 'local_vars'193    dictionary or in 'os.environ' if it's not in 'local_vars'.194    'os.environ' is first checked/augmented to guarantee that it contains195    certain values: see 'check_environ()'.  Raise ValueError for any196    variables not found in either 'local_vars' or 'os.environ'.197    """198    check_environ()199    lookup = dict(os.environ)200    lookup.update((name, str(value)) for name, value in local_vars.items())201    try:202        return _subst_compat(s).format_map(lookup)203    except KeyError as var:204        raise ValueError(f"invalid variable {var}")205 206 207def _subst_compat(s):208    """209    Replace shell/Perl-style variable substitution with210    format-style. For compatibility.211    """212 213    def _subst(match):214        return f'{{{match.group(1)}}}'215 216    repl = re.sub(r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, s)217    if repl != s:218        import warnings219 220        warnings.warn(221            "shell/Perl-style substitutions are deprecated",222            DeprecationWarning,223        )224    return repl225 226 227def grok_environment_error(exc: object, prefix: str = "error: ") -> str:228    # Function kept for backward compatibility.229    # Used to try clever things with EnvironmentErrors,230    # but nowadays str(exception) produces good messages.231    return prefix + str(exc)232 233 234# Needed by 'split_quoted()'235_wordchars_re = _squote_re = _dquote_re = None236 237 238def _init_regex():239    global _wordchars_re, _squote_re, _dquote_re240    _wordchars_re = re.compile(rf'[^\\\'\"{string.whitespace} ]*')241    _squote_re = re.compile(r"'(?:[^'\\]|\\.)*'")242    _dquote_re = re.compile(r'"(?:[^"\\]|\\.)*"')243 244 245def split_quoted(s: str) -> list[str]:246    """Split a string up according to Unix shell-like rules for quotes and247    backslashes.  In short: words are delimited by spaces, as long as those248    spaces are not escaped by a backslash, or inside a quoted string.249    Single and double quotes are equivalent, and the quote characters can250    be backslash-escaped.  The backslash is stripped from any two-character251    escape sequence, leaving only the escaped character.  The quote252    characters are stripped from any quoted string.  Returns a list of253    words.254    """255 256    # This is a nice algorithm for splitting up a single string, since it257    # doesn't require character-by-character examination.  It was a little258    # bit of a brain-bender to get it working right, though...259    if _wordchars_re is None:260        _init_regex()261 262    s = s.strip()263    words = []264    pos = 0265 266    while s:267        m = _wordchars_re.match(s, pos)268        end = m.end()269        if end == len(s):270            words.append(s[:end])271            break272 273        if s[end] in string.whitespace:274            # unescaped, unquoted whitespace: now275            # we definitely have a word delimiter276            words.append(s[:end])277            s = s[end:].lstrip()278            pos = 0279 280        elif s[end] == '\\':281            # preserve whatever is being escaped;282            # will become part of the current word283            s = s[:end] + s[end + 1 :]284            pos = end + 1285 286        else:287            if s[end] == "'":  # slurp singly-quoted string288                m = _squote_re.match(s, end)289            elif s[end] == '"':  # slurp doubly-quoted string290                m = _dquote_re.match(s, end)291            else:292                raise RuntimeError(f"this can't happen (bad char '{s[end]}')")293 294            if m is None:295                raise ValueError(f"bad string (mismatched {s[end]} quotes?)")296 297            (beg, end) = m.span()298            s = s[:beg] + s[beg + 1 : end - 1] + s[end:]299            pos = m.end() - 2300 301        if pos >= len(s):302            words.append(s)303            break304 305    return words306 307 308# split_quoted ()309 310 311def execute(312    func: Callable[[Unpack[_Ts]], object],313    args: tuple[Unpack[_Ts]],314    msg: object = None,315    verbose: bool = False,316) -> None:317    """318    Perform some action that affects the outside world (e.g. by319    writing to the filesystem). Was previously used to deal with320    "dry run" operations, but now runs unconditionally.321    """322    if msg is None:323        msg = f"{func.__name__}{args!r}"324        if msg[-2:] == ',)':  # correct for singleton tuple325            msg = msg[0:-2] + ')'326 327    log.info(msg)328    func(*args)329 330 331def strtobool(val: str) -> bool:332    """Convert a string representation of truth to true (1) or false (0).333 334    True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values335    are 'n', 'no', 'f', 'false', 'off', and '0'.  Raises ValueError if336    'val' is anything else.337    """338    val = val.lower()339    if val in ('y', 'yes', 't', 'true', 'on', '1'):340        return True341    elif val in ('n', 'no', 'f', 'false', 'off', '0'):342        return False343    else:344        raise ValueError(f"invalid truth value {val!r}")345 346 347def byte_compile(  # noqa: C901348    py_files: Iterable[str],349    optimize: int = 0,350    force: bool = False,351    prefix: str | None = None,352    base_dir: str | None = None,353    verbose: bool = True,354    direct: bool | None = None,355) -> None:356    """Byte-compile a collection of Python source files to .pyc357    files in a __pycache__ subdirectory.  'py_files' is a list358    of files to compile; any files that don't end in ".py" are silently359    skipped.  'optimize' must be one of the following:360      0 - don't optimize361      1 - normal optimization (like "python -O")362      2 - extra optimization (like "python -OO")363    If 'force' is true, all files are recompiled regardless of364    timestamps.365 366    The source filename encoded in each bytecode file defaults to the367    filenames listed in 'py_files'; you can modify these with 'prefix' and368    'basedir'.  'prefix' is a string that will be stripped off of each369    source filename, and 'base_dir' is a directory name that will be370    prepended (after 'prefix' is stripped).  You can supply either or both371    (or neither) of 'prefix' and 'base_dir', as you wish.372 373    Byte-compilation is either done directly in this interpreter process374    with the standard py_compile module, or indirectly by writing a375    temporary script and executing it.  Normally, you should let376    'byte_compile()' figure out to use direct compilation or not (see377    the source for details).  The 'direct' flag is used by the script378    generated in indirect mode; unless you know what you're doing, leave379    it set to None.380    """381 382    # nothing is done if sys.dont_write_bytecode is True383    if sys.dont_write_bytecode:384        raise DistutilsByteCompileError('byte-compiling is disabled.')385 386    # First, if the caller didn't force us into direct or indirect mode,387    # figure out which mode we should be in.  We take a conservative388    # approach: choose direct mode *only* if the current interpreter is389    # in debug mode and optimize is 0.  If we're not in debug mode (-O390    # or -OO), we don't know which level of optimization this391    # interpreter is running with, so we can't do direct392    # byte-compilation and be certain that it's the right thing.  Thus,393    # always compile indirectly if the current interpreter is in either394    # optimize mode, or if either optimization level was requested by395    # the caller.396    if direct is None:397        direct = __debug__ and optimize == 0398 399    # "Indirect" byte-compilation: write a temporary script and then400    # run it with the appropriate flags.401    if not direct:402        (script_fd, script_name) = tempfile.mkstemp(".py")403        log.info("writing byte-compilation script '%s'", script_name)404        script = os.fdopen(script_fd, "w", encoding='utf-8')405 406        with script:407            script.write(408                """\409from distutils.util import byte_compile410files = [411"""412            )413 414            # XXX would be nice to write absolute filenames, just for415            # safety's sake (script should be more robust in the face of416            # chdir'ing before running it).  But this requires abspath'ing417            # 'prefix' as well, and that breaks the hack in build_lib's418            # 'byte_compile()' method that carefully tacks on a trailing419            # slash (os.sep really) to make sure the prefix here is "just420            # right".  This whole prefix business is rather delicate -- the421            # problem is that it's really a directory, but I'm treating it422            # as a dumb string, so trailing slashes and so forth matter.423 424            script.write(",\n".join(map(repr, py_files)) + "]\n")425            script.write(426                f"""427byte_compile(files, optimize={optimize!r}, force={force!r},428         prefix={prefix!r}, base_dir={base_dir!r},429         verbose={verbose!r},430         direct=True)431"""432            )433 434        cmd = [sys.executable]435        cmd.extend(subprocess._optim_args_from_interpreter_flags())436        cmd.append(script_name)437        spawn(cmd)438        execute(os.remove, (script_name,), f"removing {script_name}")439 440    # "Direct" byte-compilation: use the py_compile module to compile441    # right here, right now.  Note that the script generated in indirect442    # mode simply calls 'byte_compile()' in direct mode, a weird sort of443    # cross-process recursion.  Hey, it works!444    else:445        from py_compile import compile446 447        for file in py_files:448            if file[-3:] != ".py":449                # This lets us be lazy and not filter filenames in450                # the "install_lib" command.451                continue452 453            # Terminology from the py_compile module:454            #   cfile - byte-compiled file455            #   dfile - purported source filename (same as 'file' by default)456            if optimize >= 0:457                opt = '' if optimize == 0 else optimize458                cfile = importlib.util.cache_from_source(file, optimization=opt)459            else:460                cfile = importlib.util.cache_from_source(file)461            dfile = file462            if prefix:463                if file[: len(prefix)] != prefix:464                    raise ValueError(465                        f"invalid prefix: filename {file!r} doesn't start with {prefix!r}"466                    )467                dfile = dfile[len(prefix) :]468            if base_dir:469                dfile = os.path.join(base_dir, dfile)470 471            cfile_base = os.path.basename(cfile)472            if direct:473                if force or newer(file, cfile):474                    log.info("byte-compiling %s to %s", file, cfile_base)475                    compile(file, cfile, dfile)476                else:477                    log.debug("skipping byte-compilation of %s to %s", file, cfile_base)478 479 480def rfc822_escape(header: str) -> str:481    """Return a version of the string escaped for inclusion in an482    RFC-822 header, by ensuring there are 8 spaces space after each newline.483    """484    indent = 8 * " "485    lines = header.splitlines(keepends=True)486 487    # Emulate the behaviour of `str.split`488    # (the terminal line break in `splitlines` does not result in an extra line):489    ends_in_newline = lines and lines[-1].splitlines()[0] != lines[-1]490    suffix = indent if ends_in_newline else ""491 492    return indent.join(lines) + suffix493 494 495def is_mingw() -> bool:496    """Returns True if the current platform is mingw.497 498    Python compiled with Mingw-w64 has sys.platform == 'win32' and499    get_platform() starts with 'mingw'.500    """501    return sys.platform == 'win32' and get_platform().startswith('mingw')502 503 504def is_freethreaded():505    """Return True if the Python interpreter is built with free threading support."""506    return bool(sysconfig.get_config_var('Py_GIL_DISABLED'))507