codekingpro/portable-devtools

1"""distutils.filelist2 3Provides the FileList class, used for poking about the filesystem4and building lists of files.5"""6 7import fnmatch8import functools9import os10import re11 12from ._log import log13from .errors import DistutilsInternalError, DistutilsTemplateError14from .util import convert_path15 16 17class FileList:18    """A list of files built by on exploring the filesystem and filtered by19    applying various patterns to what we find there.20 21    Instance attributes:22      dir23        directory from which files will be taken -- only used if24        'allfiles' not supplied to constructor25      files26        list of filenames currently being built/filtered/manipulated27      allfiles28        complete list of files under consideration (ie. without any29        filtering applied)30    """31 32    def __init__(self, warn=None, debug_print=None):33        # ignore argument to FileList, but keep them for backwards34        # compatibility35        self.allfiles = None36        self.files = []37 38    def set_allfiles(self, allfiles):39        self.allfiles = allfiles40 41    def findall(self, dir=os.curdir):42        self.allfiles = findall(dir)43 44    def debug_print(self, msg):45        """Print 'msg' to stdout if the global DEBUG (taken from the46        DISTUTILS_DEBUG environment variable) flag is true.47        """48        from distutils.debug import DEBUG49 50        if DEBUG:51            print(msg)52 53    # Collection methods54 55    def append(self, item):56        self.files.append(item)57 58    def extend(self, items):59        self.files.extend(items)60 61    def sort(self):62        # Not a strict lexical sort!63        sortable_files = sorted(map(os.path.split, self.files))64        self.files = []65        for sort_tuple in sortable_files:66            self.files.append(os.path.join(*sort_tuple))67 68    # Other miscellaneous utility methods69 70    def remove_duplicates(self):71        # Assumes list has been sorted!72        for i in range(len(self.files) - 1, 0, -1):73            if self.files[i] == self.files[i - 1]:74                del self.files[i]75 76    # "File template" methods77 78    def _parse_template_line(self, line):79        words = line.split()80        action = words[0]81 82        patterns = dir = dir_pattern = None83 84        if action in ('include', 'exclude', 'global-include', 'global-exclude'):85            if len(words) < 2:86                raise DistutilsTemplateError(87                    "'%s' expects <pattern1> <pattern2> ..." % action88                )89            patterns = [convert_path(w) for w in words[1:]]90        elif action in ('recursive-include', 'recursive-exclude'):91            if len(words) < 3:92                raise DistutilsTemplateError(93                    "'%s' expects <dir> <pattern1> <pattern2> ..." % action94                )95            dir = convert_path(words[1])96            patterns = [convert_path(w) for w in words[2:]]97        elif action in ('graft', 'prune'):98            if len(words) != 2:99                raise DistutilsTemplateError(100                    "'%s' expects a single <dir_pattern>" % action101                )102            dir_pattern = convert_path(words[1])103        else:104            raise DistutilsTemplateError("unknown action '%s'" % action)105 106        return (action, patterns, dir, dir_pattern)107 108    def process_template_line(self, line):  # noqa: C901109        # Parse the line: split it up, make sure the right number of words110        # is there, and return the relevant words.  'action' is always111        # defined: it's the first word of the line.  Which of the other112        # three are defined depends on the action; it'll be either113        # patterns, (dir and patterns), or (dir_pattern).114        (action, patterns, dir, dir_pattern) = self._parse_template_line(line)115 116        # OK, now we know that the action is valid and we have the117        # right number of words on the line for that action -- so we118        # can proceed with minimal error-checking.119        if action == 'include':120            self.debug_print("include " + ' '.join(patterns))121            for pattern in patterns:122                if not self.include_pattern(pattern, anchor=1):123                    log.warning("warning: no files found matching '%s'", pattern)124 125        elif action == 'exclude':126            self.debug_print("exclude " + ' '.join(patterns))127            for pattern in patterns:128                if not self.exclude_pattern(pattern, anchor=1):129                    log.warning(130                        (131                            "warning: no previously-included files "132                            "found matching '%s'"133                        ),134                        pattern,135                    )136 137        elif action == 'global-include':138            self.debug_print("global-include " + ' '.join(patterns))139            for pattern in patterns:140                if not self.include_pattern(pattern, anchor=0):141                    log.warning(142                        (143                            "warning: no files found matching '%s' "144                            "anywhere in distribution"145                        ),146                        pattern,147                    )148 149        elif action == 'global-exclude':150            self.debug_print("global-exclude " + ' '.join(patterns))151            for pattern in patterns:152                if not self.exclude_pattern(pattern, anchor=0):153                    log.warning(154                        (155                            "warning: no previously-included files matching "156                            "'%s' found anywhere in distribution"157                        ),158                        pattern,159                    )160 161        elif action == 'recursive-include':162            self.debug_print("recursive-include {} {}".format(dir, ' '.join(patterns)))163            for pattern in patterns:164                if not self.include_pattern(pattern, prefix=dir):165                    msg = "warning: no files found matching '%s' under directory '%s'"166                    log.warning(msg, pattern, dir)167 168        elif action == 'recursive-exclude':169            self.debug_print("recursive-exclude {} {}".format(dir, ' '.join(patterns)))170            for pattern in patterns:171                if not self.exclude_pattern(pattern, prefix=dir):172                    log.warning(173                        (174                            "warning: no previously-included files matching "175                            "'%s' found under directory '%s'"176                        ),177                        pattern,178                        dir,179                    )180 181        elif action == 'graft':182            self.debug_print("graft " + dir_pattern)183            if not self.include_pattern(None, prefix=dir_pattern):184                log.warning("warning: no directories found matching '%s'", dir_pattern)185 186        elif action == 'prune':187            self.debug_print("prune " + dir_pattern)188            if not self.exclude_pattern(None, prefix=dir_pattern):189                log.warning(190                    ("no previously-included directories found matching '%s'"),191                    dir_pattern,192                )193        else:194            raise DistutilsInternalError(195                "this cannot happen: invalid action '%s'" % action196            )197 198    # Filtering/selection methods199 200    def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0):201        """Select strings (presumably filenames) from 'self.files' that202        match 'pattern', a Unix-style wildcard (glob) pattern.  Patterns203        are not quite the same as implemented by the 'fnmatch' module: '*'204        and '?'  match non-special characters, where "special" is platform-205        dependent: slash on Unix; colon, slash, and backslash on206        DOS/Windows; and colon on Mac OS.207 208        If 'anchor' is true (the default), then the pattern match is more209        stringent: "*.py" will match "foo.py" but not "foo/bar.py".  If210        'anchor' is false, both of these will match.211 212        If 'prefix' is supplied, then only filenames starting with 'prefix'213        (itself a pattern) and ending with 'pattern', with anything in between214        them, will match.  'anchor' is ignored in this case.215 216        If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and217        'pattern' is assumed to be either a string containing a regex or a218        regex object -- no translation is done, the regex is just compiled219        and used as-is.220 221        Selected strings will be added to self.files.222 223        Return True if files are found, False otherwise.224        """225        # XXX docstring lying about what the special chars are?226        files_found = False227        pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)228        self.debug_print("include_pattern: applying regex r'%s'" % pattern_re.pattern)229 230        # delayed loading of allfiles list231        if self.allfiles is None:232            self.findall()233 234        for name in self.allfiles:235            if pattern_re.search(name):236                self.debug_print(" adding " + name)237                self.files.append(name)238                files_found = True239        return files_found240 241    def exclude_pattern(self, pattern, anchor=1, prefix=None, is_regex=0):242        """Remove strings (presumably filenames) from 'files' that match243        'pattern'.  Other parameters are the same as for244        'include_pattern()', above.245        The list 'self.files' is modified in place.246        Return True if files are found, False otherwise.247        """248        files_found = False249        pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)250        self.debug_print("exclude_pattern: applying regex r'%s'" % pattern_re.pattern)251        for i in range(len(self.files) - 1, -1, -1):252            if pattern_re.search(self.files[i]):253                self.debug_print(" removing " + self.files[i])254                del self.files[i]255                files_found = True256        return files_found257 258 259# Utility functions260 261 262def _find_all_simple(path):263    """264    Find all files under 'path'265    """266    all_unique = _UniqueDirs.filter(os.walk(path, followlinks=True))267    results = (268        os.path.join(base, file) for base, dirs, files in all_unique for file in files269    )270    return filter(os.path.isfile, results)271 272 273class _UniqueDirs(set):274    """275    Exclude previously-seen dirs from walk results,276    avoiding infinite recursion.277    Ref https://bugs.python.org/issue44497.278    """279 280    def __call__(self, walk_item):281        """282        Given an item from an os.walk result, determine283        if the item represents a unique dir for this instance284        and if not, prevent further traversal.285        """286        base, dirs, files = walk_item287        stat = os.stat(base)288        candidate = stat.st_dev, stat.st_ino289        found = candidate in self290        if found:291            del dirs[:]292        self.add(candidate)293        return not found294 295    @classmethod296    def filter(cls, items):297        return filter(cls(), items)298 299 300def findall(dir=os.curdir):301    """302    Find all files under 'dir' and return the list of full filenames.303    Unless dir is '.', return full filenames with dir prepended.304    """305    files = _find_all_simple(dir)306    if dir == os.curdir:307        make_rel = functools.partial(os.path.relpath, start=dir)308        files = map(make_rel, files)309    return list(files)310 311 312def glob_to_re(pattern):313    """Translate a shell-like glob pattern to a regular expression; return314    a string containing the regex.  Differs from 'fnmatch.translate()' in315    that '*' does not match "special characters" (which are316    platform-specific).317    """318    pattern_re = fnmatch.translate(pattern)319 320    # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which321    # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,322    # and by extension they shouldn't match such "special characters" under323    # any OS.  So change all non-escaped dots in the RE to match any324    # character except the special characters (currently: just os.sep).325    sep = os.sep326    if os.sep == '\\':327        # we're using a regex to manipulate a regex, so we need328        # to escape the backslash twice329        sep = r'\\\\'330    escaped = r'\1[^%s]' % sep331    pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re)332    return pattern_re333 334 335def translate_pattern(pattern, anchor=1, prefix=None, is_regex=0):336    """Translate a shell-like wildcard pattern to a compiled regular337    expression.  Return the compiled regex.  If 'is_regex' true,338    then 'pattern' is directly compiled to a regex (if it's a string)339    or just returned as-is (assumes it's a regex object).340    """341    if is_regex:342        if isinstance(pattern, str):343            return re.compile(pattern)344        else:345            return pattern346 347    # ditch start and end characters348    start, _, end = glob_to_re('_').partition('_')349 350    if pattern:351        pattern_re = glob_to_re(pattern)352        assert pattern_re.startswith(start) and pattern_re.endswith(end)353    else:354        pattern_re = ''355 356    if prefix is not None:357        prefix_re = glob_to_re(prefix)358        assert prefix_re.startswith(start) and prefix_re.endswith(end)359        prefix_re = prefix_re[len(start) : len(prefix_re) - len(end)]360        sep = os.sep361        if os.sep == '\\':362            sep = r'\\'363        pattern_re = pattern_re[len(start) : len(pattern_re) - len(end)]364        pattern_re = rf'{start}\A{prefix_re}{sep}.*{pattern_re}{end}'365    else:  # no prefix -- respect anchor flag366        if anchor:367            pattern_re = rf'{start}\A{pattern_re[len(start) :]}'368 369    return re.compile(pattern_re)370