codekingpro/portable-devtools

1"""setuptools.command.bdist_egg2 3Build .egg distributions"""4 5from distutils.dir_util import remove_tree, mkpath6from distutils import log7from types import CodeType8import sys9import os10import re11import textwrap12import marshal13 14from setuptools.extension import Library15from setuptools import Command16from .._path import ensure_directory17 18from sysconfig import get_path, get_python_version19 20 21def _get_purelib():22    return get_path("purelib")23 24 25def strip_module(filename):26    if '.' in filename:27        filename = os.path.splitext(filename)[0]28    if filename.endswith('module'):29        filename = filename[:-6]30    return filename31 32 33def sorted_walk(dir):34    """Do os.walk in a reproducible way,35    independent of indeterministic filesystem readdir order36    """37    for base, dirs, files in os.walk(dir):38        dirs.sort()39        files.sort()40        yield base, dirs, files41 42 43def write_stub(resource, pyfile):44    _stub_template = textwrap.dedent(45        """46        def __bootstrap__():47            global __bootstrap__, __loader__, __file__48            import sys, pkg_resources, importlib.util49            __file__ = pkg_resources.resource_filename(__name__, %r)50            __loader__ = None; del __bootstrap__, __loader__51            spec = importlib.util.spec_from_file_location(__name__,__file__)52            mod = importlib.util.module_from_spec(spec)53            spec.loader.exec_module(mod)54        __bootstrap__()55        """56    ).lstrip()57    with open(pyfile, 'w') as f:58        f.write(_stub_template % resource)59 60 61class bdist_egg(Command):62    description = "create an \"egg\" distribution"63 64    user_options = [65        ('bdist-dir=', 'b', "temporary directory for creating the distribution"),66        (67            'plat-name=',68            'p',69            "platform name to embed in generated filenames "70            "(by default uses `pkg_resources.get_build_platform()`)",71        ),72        ('exclude-source-files', None, "remove all .py files from the generated egg"),73        (74            'keep-temp',75            'k',76            "keep the pseudo-installation tree around after "77            + "creating the distribution archive",78        ),79        ('dist-dir=', 'd', "directory to put final built distributions in"),80        ('skip-build', None, "skip rebuilding everything (for testing/debugging)"),81    ]82 83    boolean_options = ['keep-temp', 'skip-build', 'exclude-source-files']84 85    def initialize_options(self):86        self.bdist_dir = None87        self.plat_name = None88        self.keep_temp = 089        self.dist_dir = None90        self.skip_build = 091        self.egg_output = None92        self.exclude_source_files = None93 94    def finalize_options(self):95        ei_cmd = self.ei_cmd = self.get_finalized_command("egg_info")96        self.egg_info = ei_cmd.egg_info97 98        if self.bdist_dir is None:99            bdist_base = self.get_finalized_command('bdist').bdist_base100            self.bdist_dir = os.path.join(bdist_base, 'egg')101 102        if self.plat_name is None:103            from pkg_resources import get_build_platform104 105            self.plat_name = get_build_platform()106 107        self.set_undefined_options('bdist', ('dist_dir', 'dist_dir'))108 109        if self.egg_output is None:110            # Compute filename of the output egg111            basename = ei_cmd._get_egg_basename(112                py_version=get_python_version(),113                platform=self.distribution.has_ext_modules() and self.plat_name,114            )115 116            self.egg_output = os.path.join(self.dist_dir, basename + '.egg')117 118    def do_install_data(self):119        # Hack for packages that install data to install's --install-lib120        self.get_finalized_command('install').install_lib = self.bdist_dir121 122        site_packages = os.path.normcase(os.path.realpath(_get_purelib()))123        old, self.distribution.data_files = self.distribution.data_files, []124 125        for item in old:126            if isinstance(item, tuple) and len(item) == 2:127                if os.path.isabs(item[0]):128                    realpath = os.path.realpath(item[0])129                    normalized = os.path.normcase(realpath)130                    if normalized == site_packages or normalized.startswith(131                        site_packages + os.sep132                    ):133                        item = realpath[len(site_packages) + 1 :], item[1]134                        # XXX else: raise ???135            self.distribution.data_files.append(item)136 137        try:138            log.info("installing package data to %s", self.bdist_dir)139            self.call_command('install_data', force=0, root=None)140        finally:141            self.distribution.data_files = old142 143    def get_outputs(self):144        return [self.egg_output]145 146    def call_command(self, cmdname, **kw):147        """Invoke reinitialized command `cmdname` with keyword args"""148        for dirname in INSTALL_DIRECTORY_ATTRS:149            kw.setdefault(dirname, self.bdist_dir)150        kw.setdefault('skip_build', self.skip_build)151        kw.setdefault('dry_run', self.dry_run)152        cmd = self.reinitialize_command(cmdname, **kw)153        self.run_command(cmdname)154        return cmd155 156    def run(self):  # noqa: C901  # is too complex (14)  # FIXME157        # Generate metadata first158        self.run_command("egg_info")159        # We run install_lib before install_data, because some data hacks160        # pull their data path from the install_lib command.161        log.info("installing library code to %s", self.bdist_dir)162        instcmd = self.get_finalized_command('install')163        old_root = instcmd.root164        instcmd.root = None165        if self.distribution.has_c_libraries() and not self.skip_build:166            self.run_command('build_clib')167        cmd = self.call_command('install_lib', warn_dir=0)168        instcmd.root = old_root169 170        all_outputs, ext_outputs = self.get_ext_outputs()171        self.stubs = []172        to_compile = []173        for p, ext_name in enumerate(ext_outputs):174            filename, ext = os.path.splitext(ext_name)175            pyfile = os.path.join(self.bdist_dir, strip_module(filename) + '.py')176            self.stubs.append(pyfile)177            log.info("creating stub loader for %s", ext_name)178            if not self.dry_run:179                write_stub(os.path.basename(ext_name), pyfile)180            to_compile.append(pyfile)181            ext_outputs[p] = ext_name.replace(os.sep, '/')182 183        if to_compile:184            cmd.byte_compile(to_compile)185        if self.distribution.data_files:186            self.do_install_data()187 188        # Make the EGG-INFO directory189        archive_root = self.bdist_dir190        egg_info = os.path.join(archive_root, 'EGG-INFO')191        self.mkpath(egg_info)192        if self.distribution.scripts:193            script_dir = os.path.join(egg_info, 'scripts')194            log.info("installing scripts to %s", script_dir)195            self.call_command('install_scripts', install_dir=script_dir, no_ep=1)196 197        self.copy_metadata_to(egg_info)198        native_libs = os.path.join(egg_info, "native_libs.txt")199        if all_outputs:200            log.info("writing %s", native_libs)201            if not self.dry_run:202                ensure_directory(native_libs)203                libs_file = open(native_libs, 'wt')204                libs_file.write('\n'.join(all_outputs))205                libs_file.write('\n')206                libs_file.close()207        elif os.path.isfile(native_libs):208            log.info("removing %s", native_libs)209            if not self.dry_run:210                os.unlink(native_libs)211 212        write_safety_flag(os.path.join(archive_root, 'EGG-INFO'), self.zip_safe())213 214        if os.path.exists(os.path.join(self.egg_info, 'depends.txt')):215            log.warn(216                "WARNING: 'depends.txt' will not be used by setuptools 0.6!\n"217                "Use the install_requires/extras_require setup() args instead."218            )219 220        if self.exclude_source_files:221            self.zap_pyfiles()222 223        # Make the archive224        make_zipfile(225            self.egg_output,226            archive_root,227            verbose=self.verbose,228            dry_run=self.dry_run,229            mode=self.gen_header(),230        )231        if not self.keep_temp:232            remove_tree(self.bdist_dir, dry_run=self.dry_run)233 234        # Add to 'Distribution.dist_files' so that the "upload" command works235        getattr(self.distribution, 'dist_files', []).append((236            'bdist_egg',237            get_python_version(),238            self.egg_output,239        ))240 241    def zap_pyfiles(self):242        log.info("Removing .py files from temporary directory")243        for base, dirs, files in walk_egg(self.bdist_dir):244            for name in files:245                path = os.path.join(base, name)246 247                if name.endswith('.py'):248                    log.debug("Deleting %s", path)249                    os.unlink(path)250 251                if base.endswith('__pycache__'):252                    path_old = path253 254                    pattern = r'(?P<name>.+)\.(?P<magic>[^.]+)\.pyc'255                    m = re.match(pattern, name)256                    path_new = os.path.join(base, os.pardir, m.group('name') + '.pyc')257                    log.info("Renaming file from [%s] to [%s]" % (path_old, path_new))258                    try:259                        os.remove(path_new)260                    except OSError:261                        pass262                    os.rename(path_old, path_new)263 264    def zip_safe(self):265        safe = getattr(self.distribution, 'zip_safe', None)266        if safe is not None:267            return safe268        log.warn("zip_safe flag not set; analyzing archive contents...")269        return analyze_egg(self.bdist_dir, self.stubs)270 271    def gen_header(self):272        return 'w'273 274    def copy_metadata_to(self, target_dir):275        "Copy metadata (egg info) to the target_dir"276        # normalize the path (so that a forward-slash in egg_info will277        # match using startswith below)278        norm_egg_info = os.path.normpath(self.egg_info)279        prefix = os.path.join(norm_egg_info, '')280        for path in self.ei_cmd.filelist.files:281            if path.startswith(prefix):282                target = os.path.join(target_dir, path[len(prefix) :])283                ensure_directory(target)284                self.copy_file(path, target)285 286    def get_ext_outputs(self):287        """Get a list of relative paths to C extensions in the output distro"""288 289        all_outputs = []290        ext_outputs = []291 292        paths = {self.bdist_dir: ''}293        for base, dirs, files in sorted_walk(self.bdist_dir):294            for filename in files:295                if os.path.splitext(filename)[1].lower() in NATIVE_EXTENSIONS:296                    all_outputs.append(paths[base] + filename)297            for filename in dirs:298                paths[os.path.join(base, filename)] = paths[base] + filename + '/'299 300        if self.distribution.has_ext_modules():301            build_cmd = self.get_finalized_command('build_ext')302            for ext in build_cmd.extensions:303                if isinstance(ext, Library):304                    continue305                fullname = build_cmd.get_ext_fullname(ext.name)306                filename = build_cmd.get_ext_filename(fullname)307                if not os.path.basename(filename).startswith('dl-'):308                    if os.path.exists(os.path.join(self.bdist_dir, filename)):309                        ext_outputs.append(filename)310 311        return all_outputs, ext_outputs312 313 314NATIVE_EXTENSIONS = dict.fromkeys('.dll .so .dylib .pyd'.split())315 316 317def walk_egg(egg_dir):318    """Walk an unpacked egg's contents, skipping the metadata directory"""319    walker = sorted_walk(egg_dir)320    base, dirs, files = next(walker)321    if 'EGG-INFO' in dirs:322        dirs.remove('EGG-INFO')323    yield base, dirs, files324    yield from walker325 326 327def analyze_egg(egg_dir, stubs):328    # check for existing flag in EGG-INFO329    for flag, fn in safety_flags.items():330        if os.path.exists(os.path.join(egg_dir, 'EGG-INFO', fn)):331            return flag332    if not can_scan():333        return False334    safe = True335    for base, dirs, files in walk_egg(egg_dir):336        for name in files:337            if name.endswith('.py') or name.endswith('.pyw'):338                continue339            elif name.endswith('.pyc') or name.endswith('.pyo'):340                # always scan, even if we already know we're not safe341                safe = scan_module(egg_dir, base, name, stubs) and safe342    return safe343 344 345def write_safety_flag(egg_dir, safe):346    # Write or remove zip safety flag file(s)347    for flag, fn in safety_flags.items():348        fn = os.path.join(egg_dir, fn)349        if os.path.exists(fn):350            if safe is None or bool(safe) != flag:351                os.unlink(fn)352        elif safe is not None and bool(safe) == flag:353            f = open(fn, 'wt')354            f.write('\n')355            f.close()356 357 358safety_flags = {359    True: 'zip-safe',360    False: 'not-zip-safe',361}362 363 364def scan_module(egg_dir, base, name, stubs):365    """Check whether module possibly uses unsafe-for-zipfile stuff"""366 367    filename = os.path.join(base, name)368    if filename[:-1] in stubs:369        return True  # Extension module370    pkg = base[len(egg_dir) + 1 :].replace(os.sep, '.')371    module = pkg + (pkg and '.' or '') + os.path.splitext(name)[0]372    skip = 16  # skip magic & reserved? & date & file size373    f = open(filename, 'rb')374    f.read(skip)375    code = marshal.load(f)376    f.close()377    safe = True378    symbols = dict.fromkeys(iter_symbols(code))379    for bad in ['__file__', '__path__']:380        if bad in symbols:381            log.warn("%s: module references %s", module, bad)382            safe = False383    if 'inspect' in symbols:384        for bad in [385            'getsource',386            'getabsfile',387            'getsourcefile',388            'getfile' 'getsourcelines',389            'findsource',390            'getcomments',391            'getframeinfo',392            'getinnerframes',393            'getouterframes',394            'stack',395            'trace',396        ]:397            if bad in symbols:398                log.warn("%s: module MAY be using inspect.%s", module, bad)399                safe = False400    return safe401 402 403def iter_symbols(code):404    """Yield names and strings used by `code` and its nested code objects"""405    yield from code.co_names406    for const in code.co_consts:407        if isinstance(const, str):408            yield const409        elif isinstance(const, CodeType):410            yield from iter_symbols(const)411 412 413def can_scan():414    if not sys.platform.startswith('java') and sys.platform != 'cli':415        # CPython, PyPy, etc.416        return True417    log.warn("Unable to analyze compiled code on this platform.")418    log.warn(419        "Please ask the author to include a 'zip_safe'"420        " setting (either True or False) in the package's setup.py"421    )422    return False423 424 425# Attribute names of options for commands that might need to be convinced to426# install to the egg build directory427 428INSTALL_DIRECTORY_ATTRS = ['install_lib', 'install_dir', 'install_data', 'install_base']429 430 431def make_zipfile(zip_filename, base_dir, verbose=0, dry_run=0, compress=True, mode='w'):432    """Create a zip file from all the files under 'base_dir'.  The output433    zip file will be named 'base_dir' + ".zip".  Uses either the "zipfile"434    Python module (if available) or the InfoZIP "zip" utility (if installed435    and found on the default search path).  If neither tool is available,436    raises DistutilsExecError.  Returns the name of the output zip file.437    """438    import zipfile439 440    mkpath(os.path.dirname(zip_filename), dry_run=dry_run)441    log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir)442 443    def visit(z, dirname, names):444        for name in names:445            path = os.path.normpath(os.path.join(dirname, name))446            if os.path.isfile(path):447                p = path[len(base_dir) + 1 :]448                if not dry_run:449                    z.write(path, p)450                log.debug("adding '%s'", p)451 452    compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED453    if not dry_run:454        z = zipfile.ZipFile(zip_filename, mode, compression=compression)455        for dirname, dirs, files in sorted_walk(base_dir):456            visit(z, dirname, files)457        z.close()458    else:459        for dirname, dirs, files in sorted_walk(base_dir):460            visit(None, dirname, files)461    return zip_filename462