codekingpro/portable-devtools

1"""setuptools.command.bdist_egg2 3Build .egg distributions"""4 5from __future__ import annotations6 7import marshal8import os9import re10import sys11import textwrap12from collections.abc import Iterator13from sysconfig import get_path, get_platform, get_python_version14from types import CodeType15from typing import TYPE_CHECKING, AnyStr, Literal16 17from setuptools import Command18from setuptools.extension import Library19 20from .._path import StrPath, StrPathT, ensure_directory21 22from distutils import log23from distutils.dir_util import mkpath, remove_tree24 25if TYPE_CHECKING:26    from _typeshed import GenericPath27    from typing_extensions import TypeAlias28 29# Same as zipfile._ZipFileMode from typeshed30_ZipFileMode: TypeAlias = Literal["r", "w", "x", "a"]31 32 33def _get_purelib():34    return get_path("purelib")35 36 37def strip_module(filename):38    if '.' in filename:39        filename = os.path.splitext(filename)[0]40    filename = filename.removesuffix('module')41    return filename42 43 44def sorted_walk(45    dir: GenericPath[AnyStr],46) -> Iterator[tuple[AnyStr, list[AnyStr], list[AnyStr]]]:47    """Do os.walk in a reproducible way,48    independent of indeterministic filesystem readdir order49    """50    for base, dirs, files in os.walk(dir):51        dirs.sort()52        files.sort()53        yield base, dirs, files54 55 56def write_stub(resource, pyfile) -> None:57    _stub_template = textwrap.dedent(58        """59        def __bootstrap__():60            global __bootstrap__, __loader__, __file__61            import sys, importlib.resources as irs, importlib.util62            with irs.as_file(irs.files(__name__).joinpath(%r)) as __file__:63                __loader__ = None; del __bootstrap__, __loader__64                spec = importlib.util.spec_from_file_location(__name__,__file__)65                mod = importlib.util.module_from_spec(spec)66                spec.loader.exec_module(mod)67        __bootstrap__()68        """69    ).lstrip()70    with open(pyfile, 'w', encoding="utf-8") as f:71        f.write(_stub_template % resource)72 73 74class bdist_egg(Command):75    description = 'create an "egg" distribution'76 77    user_options = [78        ('bdist-dir=', 'b', "temporary directory for creating the distribution"),79        (80            'plat-name=',81            'p',82            "platform name to embed in generated filenames "83            "(by default uses `sysconfig.get_platform()`)",84        ),85        ('exclude-source-files', None, "remove all .py files from the generated egg"),86        (87            'keep-temp',88            'k',89            "keep the pseudo-installation tree around after "90            "creating the distribution archive",91        ),92        ('dist-dir=', 'd', "directory to put final built distributions in"),93        ('skip-build', None, "skip rebuilding everything (for testing/debugging)"),94    ]95 96    boolean_options = ['keep-temp', 'skip-build', 'exclude-source-files']97 98    def initialize_options(self):99        self.bdist_dir = None100        self.plat_name = None101        self.keep_temp = False102        self.dist_dir = None103        self.skip_build = False104        self.egg_output = None105        self.exclude_source_files = None106 107    def finalize_options(self) -> None:108        ei_cmd = self.ei_cmd = self.get_finalized_command("egg_info")109        self.egg_info = ei_cmd.egg_info110 111        if self.bdist_dir is None:112            bdist_base = self.get_finalized_command('bdist').bdist_base113            self.bdist_dir = os.path.join(bdist_base, 'egg')114 115        if self.plat_name is None:116            self.plat_name = get_platform()117 118        self.set_undefined_options('bdist', ('dist_dir', 'dist_dir'))119 120        if self.egg_output is None:121            # Compute filename of the output egg122            basename = ei_cmd._get_egg_basename(123                py_version=get_python_version(),124                platform=self.distribution.has_ext_modules() and self.plat_name,125            )126 127            self.egg_output = os.path.join(self.dist_dir, basename + '.egg')128 129    def do_install_data(self) -> None:130        # Hack for packages that install data to install's --install-lib131        self.get_finalized_command('install').install_lib = self.bdist_dir132 133        site_packages = os.path.normcase(os.path.realpath(_get_purelib()))134        old, self.distribution.data_files = self.distribution.data_files, []135 136        for item in old:137            if isinstance(item, tuple) and len(item) == 2:138                if os.path.isabs(item[0]):139                    realpath = os.path.realpath(item[0])140                    normalized = os.path.normcase(realpath)141                    if normalized == site_packages or normalized.startswith(142                        site_packages + os.sep143                    ):144                        item = realpath[len(site_packages) + 1 :], item[1]145                        # XXX else: raise ???146            self.distribution.data_files.append(item)147 148        try:149            log.info("installing package data to %s", self.bdist_dir)150            self.call_command('install_data', force=False, root=None)151        finally:152            self.distribution.data_files = old153 154    def get_outputs(self):155        return [self.egg_output]156 157    def call_command(self, cmdname, **kw):158        """Invoke reinitialized command `cmdname` with keyword args"""159        for dirname in INSTALL_DIRECTORY_ATTRS:160            kw.setdefault(dirname, self.bdist_dir)161        kw.setdefault('skip_build', self.skip_build)162        cmd = self.reinitialize_command(cmdname, **kw)163        self.run_command(cmdname)164        return cmd165 166    def run(self) -> None:  # noqa: C901  # is too complex (14)  # FIXME167        # Generate metadata first168        self.run_command("egg_info")169        # We run install_lib before install_data, because some data hacks170        # pull their data path from the install_lib command.171        log.info("installing library code to %s", self.bdist_dir)172        instcmd = self.get_finalized_command('install')173        old_root = instcmd.root174        instcmd.root = None175        if self.distribution.has_c_libraries() and not self.skip_build:176            self.run_command('build_clib')177        cmd = self.call_command('install_lib', warn_dir=False)178        instcmd.root = old_root179 180        all_outputs, ext_outputs = self.get_ext_outputs()181        self.stubs = []182        to_compile = []183        for p, ext_name in enumerate(ext_outputs):184            filename, _ext = os.path.splitext(ext_name)185            pyfile = os.path.join(self.bdist_dir, strip_module(filename) + '.py')186            self.stubs.append(pyfile)187            log.info("creating stub loader for %s", ext_name)188            write_stub(os.path.basename(ext_name), pyfile)189            to_compile.append(pyfile)190            ext_outputs[p] = ext_name.replace(os.sep, '/')191 192        if to_compile:193            cmd.byte_compile(to_compile)194        if self.distribution.data_files:195            self.do_install_data()196 197        # Make the EGG-INFO directory198        archive_root = self.bdist_dir199        egg_info = os.path.join(archive_root, 'EGG-INFO')200        self.mkpath(egg_info)201        if self.distribution.scripts:202            script_dir = os.path.join(egg_info, 'scripts')203            log.info("installing scripts to %s", script_dir)204            self.call_command('install_scripts', install_dir=script_dir, no_ep=True)205 206        self.copy_metadata_to(egg_info)207        native_libs = os.path.join(egg_info, "native_libs.txt")208        if all_outputs:209            log.info("writing %s", native_libs)210            ensure_directory(native_libs)211            with open(native_libs, 'wt', encoding="utf-8") as libs_file:212                libs_file.write('\n'.join(all_outputs))213                libs_file.write('\n')214        elif os.path.isfile(native_libs):215            log.info("removing %s", native_libs)216            os.unlink(native_libs)217 218        write_safety_flag(os.path.join(archive_root, 'EGG-INFO'), self.zip_safe())219 220        if os.path.exists(os.path.join(self.egg_info, 'depends.txt')):221            log.warn(222                "WARNING: 'depends.txt' will not be used by setuptools 0.6!\n"223                "Use the install_requires/extras_require setup() args instead."224            )225 226        if self.exclude_source_files:227            self.zap_pyfiles()228 229        # Make the archive230        make_zipfile(231            self.egg_output,232            archive_root,233            verbose=self.verbose,234            mode=self.gen_header(),235        )236        if not self.keep_temp:237            remove_tree(self.bdist_dir)238 239        # Add to 'Distribution.dist_files' so that the "upload" command works240        getattr(self.distribution, 'dist_files', []).append((241            'bdist_egg',242            get_python_version(),243            self.egg_output,244        ))245 246    def zap_pyfiles(self) -> None:247        log.info("Removing .py files from temporary directory")248        for base, dirs, files in walk_egg(self.bdist_dir):249            for name in files:250                path = os.path.join(base, name)251 252                if name.endswith('.py'):253                    log.debug("Deleting %s", path)254                    os.unlink(path)255 256                if base.endswith('__pycache__'):257                    path_old = path258 259                    pattern = r'(?P<name>.+)\.(?P<magic>[^.]+)\.pyc'260                    m = re.match(pattern, name)261                    # We shouldn't find any non-pyc files in __pycache__262                    assert m is not None263                    path_new = os.path.join(base, os.pardir, m.group('name') + '.pyc')264                    log.info(f"Renaming file from [{path_old}] to [{path_new}]")265                    try:266                        os.remove(path_new)267                    except OSError:268                        pass269                    os.rename(path_old, path_new)270 271    def zip_safe(self):272        safe = getattr(self.distribution, 'zip_safe', None)273        if safe is not None:274            return safe275        log.warn("zip_safe flag not set; analyzing archive contents...")276        return analyze_egg(self.bdist_dir, self.stubs)277 278    def gen_header(self) -> Literal["w"]:279        return 'w'280 281    def copy_metadata_to(self, target_dir) -> None:282        "Copy metadata (egg info) to the target_dir"283        # normalize the path (so that a forward-slash in egg_info will284        # match using startswith below)285        norm_egg_info = os.path.normpath(self.egg_info)286        prefix = os.path.join(norm_egg_info, '')287        for path in self.ei_cmd.filelist.files:288            if path.startswith(prefix):289                target = os.path.join(target_dir, path[len(prefix) :])290                ensure_directory(target)291                self.copy_file(path, target)292 293    def get_ext_outputs(self):294        """Get a list of relative paths to C extensions in the output distro"""295 296        all_outputs = []297        ext_outputs = []298 299        paths = {self.bdist_dir: ''}300        for base, dirs, files in sorted_walk(self.bdist_dir):301            all_outputs.extend(302                paths[base] + filename303                for filename in files304                if os.path.splitext(filename)[1].lower() in NATIVE_EXTENSIONS305            )306            for filename in dirs:307                paths[os.path.join(base, filename)] = paths[base] + filename + '/'308 309        if self.distribution.has_ext_modules():310            build_cmd = self.get_finalized_command('build_ext')311            for ext in build_cmd.extensions:312                if isinstance(ext, Library):313                    continue314                fullname = build_cmd.get_ext_fullname(ext.name)315                filename = build_cmd.get_ext_filename(fullname)316                if not os.path.basename(filename).startswith('dl-'):317                    if os.path.exists(os.path.join(self.bdist_dir, filename)):318                        ext_outputs.append(filename)319 320        return all_outputs, ext_outputs321 322 323NATIVE_EXTENSIONS: dict[str, None] = dict.fromkeys('.dll .so .dylib .pyd'.split())324 325 326def walk_egg(egg_dir: StrPath) -> Iterator[tuple[str, list[str], list[str]]]:327    """Walk an unpacked egg's contents, skipping the metadata directory"""328    walker = sorted_walk(egg_dir)329    base, dirs, files = next(walker)330    if 'EGG-INFO' in dirs:331        dirs.remove('EGG-INFO')332    yield base, dirs, files333    yield from walker334 335 336def analyze_egg(egg_dir, stubs):337    # check for existing flag in EGG-INFO338    for flag, fn in safety_flags.items():339        if os.path.exists(os.path.join(egg_dir, 'EGG-INFO', fn)):340            return flag341    if not can_scan():342        return False343    safe = True344    for base, dirs, files in walk_egg(egg_dir):345        for name in files:346            if name.endswith(('.py', '.pyw')):347                continue348            elif name.endswith(('.pyc', '.pyo')):349                # always scan, even if we already know we're not safe350                safe = scan_module(egg_dir, base, name, stubs) and safe351    return safe352 353 354def write_safety_flag(egg_dir, safe) -> None:355    # Write or remove zip safety flag file(s)356    for flag, fn in safety_flags.items():357        fn = os.path.join(egg_dir, fn)358        if os.path.exists(fn):359            if safe is None or bool(safe) != flag:360                os.unlink(fn)361        elif safe is not None and bool(safe) == flag:362            with open(fn, 'wt', encoding="utf-8") as f:363                f.write('\n')364 365 366safety_flags = {367    True: 'zip-safe',368    False: 'not-zip-safe',369}370 371 372def scan_module(egg_dir, base, name, stubs):373    """Check whether module possibly uses unsafe-for-zipfile stuff"""374 375    filename = os.path.join(base, name)376    if filename[:-1] in stubs:377        return True  # Extension module378    pkg = base[len(egg_dir) + 1 :].replace(os.sep, '.')379    module = pkg + (pkg and '.' or '') + os.path.splitext(name)[0]380    skip = 16  # skip magic & reserved? & date & file size381    f = open(filename, 'rb')382    f.read(skip)383    code = marshal.load(f)384    f.close()385    safe = True386    symbols = dict.fromkeys(iter_symbols(code))387    for bad in ['__file__', '__path__']:388        if bad in symbols:389            log.warn("%s: module references %s", module, bad)390            safe = False391    if 'inspect' in symbols:392        for bad in [393            'getsource',394            'getabsfile',395            'getfile',396            'getsourcefile',397            'getsourcelines',398            'findsource',399            'getcomments',400            'getframeinfo',401            'getinnerframes',402            'getouterframes',403            'stack',404            'trace',405        ]:406            if bad in symbols:407                log.warn("%s: module MAY be using inspect.%s", module, bad)408                safe = False409    return safe410 411 412def iter_symbols(code: CodeType) -> Iterator[str]:413    """Yield names and strings used by `code` and its nested code objects"""414    yield from code.co_names415    for const in code.co_consts:416        if isinstance(const, str):417            yield const418        elif isinstance(const, CodeType):419            yield from iter_symbols(const)420 421 422def can_scan() -> bool:423    if not sys.platform.startswith('java') and sys.platform != 'cli':424        # CPython, PyPy, etc.425        return True426    log.warn("Unable to analyze compiled code on this platform.")427    log.warn(428        "Please ask the author to include a 'zip_safe'"429        " setting (either True or False) in the package's setup.py"430    )431    return False432 433 434# Attribute names of options for commands that might need to be convinced to435# install to the egg build directory436 437INSTALL_DIRECTORY_ATTRS = ['install_lib', 'install_dir', 'install_data', 'install_base']438 439 440def make_zipfile(441    zip_filename: StrPathT,442    base_dir,443    verbose: bool = False,444    compress=True,445    mode: _ZipFileMode = 'w',446) -> StrPathT:447    """Create a zip file from all the files under 'base_dir'.  The output448    zip file will be named 'base_dir' + ".zip".  Uses either the "zipfile"449    Python module (if available) or the InfoZIP "zip" utility (if installed450    and found on the default search path).  If neither tool is available,451    raises DistutilsExecError.  Returns the name of the output zip file.452    """453    import zipfile454 455    mkpath(os.path.dirname(zip_filename))  # type: ignore[arg-type] # python/mypy#18075456    log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir)457 458    def visit(z, dirname, names):459        for name in names:460            path = os.path.normpath(os.path.join(dirname, name))461            if os.path.isfile(path):462                p = path[len(base_dir) + 1 :]463                z.write(path, p)464                log.debug("adding '%s'", p)465 466    compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED467    z = zipfile.ZipFile(zip_filename, mode, compression=compression)468    for dirname, dirs, files in sorted_walk(base_dir):469        visit(z, dirname, files)470    z.close()471    return zip_filename472