codekingpro/portable-devtools

1"""API and implementations for loading templates from different data2sources.3"""4 5import importlib.util6import os7import posixpath8import sys9import typing as t10import weakref11import zipimport12from collections import abc13from hashlib import sha114from importlib import import_module15from types import ModuleType16 17from .exceptions import TemplateNotFound18from .utils import internalcode19 20if t.TYPE_CHECKING:21    from .environment import Environment22    from .environment import Template23 24 25def split_template_path(template: str) -> t.List[str]:26    """Split a path into segments and perform a sanity check.  If it detects27    '..' in the path it will raise a `TemplateNotFound` error.28    """29    pieces = []30    for piece in template.split("/"):31        if (32            os.path.sep in piece33            or (os.path.altsep and os.path.altsep in piece)34            or piece == os.path.pardir35        ):36            raise TemplateNotFound(template)37        elif piece and piece != ".":38            pieces.append(piece)39    return pieces40 41 42class BaseLoader:43    """Baseclass for all loaders.  Subclass this and override `get_source` to44    implement a custom loading mechanism.  The environment provides a45    `get_template` method that calls the loader's `load` method to get the46    :class:`Template` object.47 48    A very basic example for a loader that looks up templates on the file49    system could look like this::50 51        from jinja2 import BaseLoader, TemplateNotFound52        from os.path import join, exists, getmtime53 54        class MyLoader(BaseLoader):55 56            def __init__(self, path):57                self.path = path58 59            def get_source(self, environment, template):60                path = join(self.path, template)61                if not exists(path):62                    raise TemplateNotFound(template)63                mtime = getmtime(path)64                with open(path) as f:65                    source = f.read()66                return source, path, lambda: mtime == getmtime(path)67    """68 69    #: if set to `False` it indicates that the loader cannot provide access70    #: to the source of templates.71    #:72    #: .. versionadded:: 2.473    has_source_access = True74 75    def get_source(76        self, environment: "Environment", template: str77    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:78        """Get the template source, filename and reload helper for a template.79        It's passed the environment and template name and has to return a80        tuple in the form ``(source, filename, uptodate)`` or raise a81        `TemplateNotFound` error if it can't locate the template.82 83        The source part of the returned tuple must be the source of the84        template as a string. The filename should be the name of the85        file on the filesystem if it was loaded from there, otherwise86        ``None``. The filename is used by Python for the tracebacks87        if no loader extension is used.88 89        The last item in the tuple is the `uptodate` function.  If auto90        reloading is enabled it's always called to check if the template91        changed.  No arguments are passed so the function must store the92        old state somewhere (for example in a closure).  If it returns `False`93        the template will be reloaded.94        """95        if not self.has_source_access:96            raise RuntimeError(97                f"{type(self).__name__} cannot provide access to the source"98            )99        raise TemplateNotFound(template)100 101    def list_templates(self) -> t.List[str]:102        """Iterates over all templates.  If the loader does not support that103        it should raise a :exc:`TypeError` which is the default behavior.104        """105        raise TypeError("this loader cannot iterate over all templates")106 107    @internalcode108    def load(109        self,110        environment: "Environment",111        name: str,112        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,113    ) -> "Template":114        """Loads a template.  This method looks up the template in the cache115        or loads one by calling :meth:`get_source`.  Subclasses should not116        override this method as loaders working on collections of other117        loaders (such as :class:`PrefixLoader` or :class:`ChoiceLoader`)118        will not call this method but `get_source` directly.119        """120        code = None121        if globals is None:122            globals = {}123 124        # first we try to get the source for this template together125        # with the filename and the uptodate function.126        source, filename, uptodate = self.get_source(environment, name)127 128        # try to load the code from the bytecode cache if there is a129        # bytecode cache configured.130        bcc = environment.bytecode_cache131        if bcc is not None:132            bucket = bcc.get_bucket(environment, name, filename, source)133            code = bucket.code134 135        # if we don't have code so far (not cached, no longer up to136        # date) etc. we compile the template137        if code is None:138            code = environment.compile(source, name, filename)139 140        # if the bytecode cache is available and the bucket doesn't141        # have a code so far, we give the bucket the new code and put142        # it back to the bytecode cache.143        if bcc is not None and bucket.code is None:144            bucket.code = code145            bcc.set_bucket(bucket)146 147        return environment.template_class.from_code(148            environment, code, globals, uptodate149        )150 151 152class FileSystemLoader(BaseLoader):153    """Load templates from a directory in the file system.154 155    The path can be relative or absolute. Relative paths are relative to156    the current working directory.157 158    .. code-block:: python159 160        loader = FileSystemLoader("templates")161 162    A list of paths can be given. The directories will be searched in163    order, stopping at the first matching template.164 165    .. code-block:: python166 167        loader = FileSystemLoader(["/override/templates", "/default/templates"])168 169    :param searchpath: A path, or list of paths, to the directory that170        contains the templates.171    :param encoding: Use this encoding to read the text from template172        files.173    :param followlinks: Follow symbolic links in the path.174 175    .. versionchanged:: 2.8176        Added the ``followlinks`` parameter.177    """178 179    def __init__(180        self,181        searchpath: t.Union[182            str, "os.PathLike[str]", t.Sequence[t.Union[str, "os.PathLike[str]"]]183        ],184        encoding: str = "utf-8",185        followlinks: bool = False,186    ) -> None:187        if not isinstance(searchpath, abc.Iterable) or isinstance(searchpath, str):188            searchpath = [searchpath]189 190        self.searchpath = [os.fspath(p) for p in searchpath]191        self.encoding = encoding192        self.followlinks = followlinks193 194    def get_source(195        self, environment: "Environment", template: str196    ) -> t.Tuple[str, str, t.Callable[[], bool]]:197        pieces = split_template_path(template)198 199        for searchpath in self.searchpath:200            # Use posixpath even on Windows to avoid "drive:" or UNC201            # segments breaking out of the search directory.202            filename = posixpath.join(searchpath, *pieces)203 204            if os.path.isfile(filename):205                break206        else:207            raise TemplateNotFound(template)208 209        with open(filename, encoding=self.encoding) as f:210            contents = f.read()211 212        mtime = os.path.getmtime(filename)213 214        def uptodate() -> bool:215            try:216                return os.path.getmtime(filename) == mtime217            except OSError:218                return False219 220        # Use normpath to convert Windows altsep to sep.221        return contents, os.path.normpath(filename), uptodate222 223    def list_templates(self) -> t.List[str]:224        found = set()225        for searchpath in self.searchpath:226            walk_dir = os.walk(searchpath, followlinks=self.followlinks)227            for dirpath, _, filenames in walk_dir:228                for filename in filenames:229                    template = (230                        os.path.join(dirpath, filename)[len(searchpath) :]231                        .strip(os.path.sep)232                        .replace(os.path.sep, "/")233                    )234                    if template[:2] == "./":235                        template = template[2:]236                    if template not in found:237                        found.add(template)238        return sorted(found)239 240 241class PackageLoader(BaseLoader):242    """Load templates from a directory in a Python package.243 244    :param package_name: Import name of the package that contains the245        template directory.246    :param package_path: Directory within the imported package that247        contains the templates.248    :param encoding: Encoding of template files.249 250    The following example looks up templates in the ``pages`` directory251    within the ``project.ui`` package.252 253    .. code-block:: python254 255        loader = PackageLoader("project.ui", "pages")256 257    Only packages installed as directories (standard pip behavior) or258    zip/egg files (less common) are supported. The Python API for259    introspecting data in packages is too limited to support other260    installation methods the way this loader requires.261 262    There is limited support for :pep:`420` namespace packages. The263    template directory is assumed to only be in one namespace264    contributor. Zip files contributing to a namespace are not265    supported.266 267    .. versionchanged:: 3.0268        No longer uses ``setuptools`` as a dependency.269 270    .. versionchanged:: 3.0271        Limited PEP 420 namespace package support.272    """273 274    def __init__(275        self,276        package_name: str,277        package_path: "str" = "templates",278        encoding: str = "utf-8",279    ) -> None:280        package_path = os.path.normpath(package_path).rstrip(os.path.sep)281 282        # normpath preserves ".", which isn't valid in zip paths.283        if package_path == os.path.curdir:284            package_path = ""285        elif package_path[:2] == os.path.curdir + os.path.sep:286            package_path = package_path[2:]287 288        self.package_path = package_path289        self.package_name = package_name290        self.encoding = encoding291 292        # Make sure the package exists. This also makes namespace293        # packages work, otherwise get_loader returns None.294        import_module(package_name)295        spec = importlib.util.find_spec(package_name)296        assert spec is not None, "An import spec was not found for the package."297        loader = spec.loader298        assert loader is not None, "A loader was not found for the package."299        self._loader = loader300        self._archive = None301        template_root = None302 303        if isinstance(loader, zipimport.zipimporter):304            self._archive = loader.archive305            pkgdir = next(iter(spec.submodule_search_locations))  # type: ignore306            template_root = os.path.join(pkgdir, package_path).rstrip(os.path.sep)307        else:308            roots: t.List[str] = []309 310            # One element for regular packages, multiple for namespace311            # packages, or None for single module file.312            if spec.submodule_search_locations:313                roots.extend(spec.submodule_search_locations)314            # A single module file, use the parent directory instead.315            elif spec.origin is not None:316                roots.append(os.path.dirname(spec.origin))317 318            for root in roots:319                root = os.path.join(root, package_path)320 321                if os.path.isdir(root):322                    template_root = root323                    break324 325        if template_root is None:326            raise ValueError(327                f"The {package_name!r} package was not installed in a"328                " way that PackageLoader understands."329            )330 331        self._template_root = template_root332 333    def get_source(334        self, environment: "Environment", template: str335    ) -> t.Tuple[str, str, t.Optional[t.Callable[[], bool]]]:336        # Use posixpath even on Windows to avoid "drive:" or UNC337        # segments breaking out of the search directory. Use normpath to338        # convert Windows altsep to sep.339        p = os.path.normpath(340            posixpath.join(self._template_root, *split_template_path(template))341        )342        up_to_date: t.Optional[t.Callable[[], bool]]343 344        if self._archive is None:345            # Package is a directory.346            if not os.path.isfile(p):347                raise TemplateNotFound(template)348 349            with open(p, "rb") as f:350                source = f.read()351 352            mtime = os.path.getmtime(p)353 354            def up_to_date() -> bool:355                return os.path.isfile(p) and os.path.getmtime(p) == mtime356 357        else:358            # Package is a zip file.359            try:360                source = self._loader.get_data(p)  # type: ignore361            except OSError as e:362                raise TemplateNotFound(template) from e363 364            # Could use the zip's mtime for all template mtimes, but365            # would need to safely reload the module if it's out of366            # date, so just report it as always current.367            up_to_date = None368 369        return source.decode(self.encoding), p, up_to_date370 371    def list_templates(self) -> t.List[str]:372        results: t.List[str] = []373 374        if self._archive is None:375            # Package is a directory.376            offset = len(self._template_root)377 378            for dirpath, _, filenames in os.walk(self._template_root):379                dirpath = dirpath[offset:].lstrip(os.path.sep)380                results.extend(381                    os.path.join(dirpath, name).replace(os.path.sep, "/")382                    for name in filenames383                )384        else:385            if not hasattr(self._loader, "_files"):386                raise TypeError(387                    "This zip import does not have the required"388                    " metadata to list templates."389                )390 391            # Package is a zip file.392            prefix = (393                self._template_root[len(self._archive) :].lstrip(os.path.sep)394                + os.path.sep395            )396            offset = len(prefix)397 398            for name in self._loader._files.keys():399                # Find names under the templates directory that aren't directories.400                if name.startswith(prefix) and name[-1] != os.path.sep:401                    results.append(name[offset:].replace(os.path.sep, "/"))402 403        results.sort()404        return results405 406 407class DictLoader(BaseLoader):408    """Loads a template from a Python dict mapping template names to409    template source.  This loader is useful for unittesting:410 411    >>> loader = DictLoader({'index.html': 'source here'})412 413    Because auto reloading is rarely useful this is disabled per default.414    """415 416    def __init__(self, mapping: t.Mapping[str, str]) -> None:417        self.mapping = mapping418 419    def get_source(420        self, environment: "Environment", template: str421    ) -> t.Tuple[str, None, t.Callable[[], bool]]:422        if template in self.mapping:423            source = self.mapping[template]424            return source, None, lambda: source == self.mapping.get(template)425        raise TemplateNotFound(template)426 427    def list_templates(self) -> t.List[str]:428        return sorted(self.mapping)429 430 431class FunctionLoader(BaseLoader):432    """A loader that is passed a function which does the loading.  The433    function receives the name of the template and has to return either434    a string with the template source, a tuple in the form ``(source,435    filename, uptodatefunc)`` or `None` if the template does not exist.436 437    >>> def load_template(name):438    ...     if name == 'index.html':439    ...         return '...'440    ...441    >>> loader = FunctionLoader(load_template)442 443    The `uptodatefunc` is a function that is called if autoreload is enabled444    and has to return `True` if the template is still up to date.  For more445    details have a look at :meth:`BaseLoader.get_source` which has the same446    return value.447    """448 449    def __init__(450        self,451        load_func: t.Callable[452            [str],453            t.Optional[454                t.Union[455                    str, t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]456                ]457            ],458        ],459    ) -> None:460        self.load_func = load_func461 462    def get_source(463        self, environment: "Environment", template: str464    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:465        rv = self.load_func(template)466 467        if rv is None:468            raise TemplateNotFound(template)469 470        if isinstance(rv, str):471            return rv, None, None472 473        return rv474 475 476class PrefixLoader(BaseLoader):477    """A loader that is passed a dict of loaders where each loader is bound478    to a prefix.  The prefix is delimited from the template by a slash per479    default, which can be changed by setting the `delimiter` argument to480    something else::481 482        loader = PrefixLoader({483            'app1':     PackageLoader('mypackage.app1'),484            'app2':     PackageLoader('mypackage.app2')485        })486 487    By loading ``'app1/index.html'`` the file from the app1 package is loaded,488    by loading ``'app2/index.html'`` the file from the second.489    """490 491    def __init__(492        self, mapping: t.Mapping[str, BaseLoader], delimiter: str = "/"493    ) -> None:494        self.mapping = mapping495        self.delimiter = delimiter496 497    def get_loader(self, template: str) -> t.Tuple[BaseLoader, str]:498        try:499            prefix, name = template.split(self.delimiter, 1)500            loader = self.mapping[prefix]501        except (ValueError, KeyError) as e:502            raise TemplateNotFound(template) from e503        return loader, name504 505    def get_source(506        self, environment: "Environment", template: str507    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:508        loader, name = self.get_loader(template)509        try:510            return loader.get_source(environment, name)511        except TemplateNotFound as e:512            # re-raise the exception with the correct filename here.513            # (the one that includes the prefix)514            raise TemplateNotFound(template) from e515 516    @internalcode517    def load(518        self,519        environment: "Environment",520        name: str,521        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,522    ) -> "Template":523        loader, local_name = self.get_loader(name)524        try:525            return loader.load(environment, local_name, globals)526        except TemplateNotFound as e:527            # re-raise the exception with the correct filename here.528            # (the one that includes the prefix)529            raise TemplateNotFound(name) from e530 531    def list_templates(self) -> t.List[str]:532        result = []533        for prefix, loader in self.mapping.items():534            for template in loader.list_templates():535                result.append(prefix + self.delimiter + template)536        return result537 538 539class ChoiceLoader(BaseLoader):540    """This loader works like the `PrefixLoader` just that no prefix is541    specified.  If a template could not be found by one loader the next one542    is tried.543 544    >>> loader = ChoiceLoader([545    ...     FileSystemLoader('/path/to/user/templates'),546    ...     FileSystemLoader('/path/to/system/templates')547    ... ])548 549    This is useful if you want to allow users to override builtin templates550    from a different location.551    """552 553    def __init__(self, loaders: t.Sequence[BaseLoader]) -> None:554        self.loaders = loaders555 556    def get_source(557        self, environment: "Environment", template: str558    ) -> t.Tuple[str, t.Optional[str], t.Optional[t.Callable[[], bool]]]:559        for loader in self.loaders:560            try:561                return loader.get_source(environment, template)562            except TemplateNotFound:563                pass564        raise TemplateNotFound(template)565 566    @internalcode567    def load(568        self,569        environment: "Environment",570        name: str,571        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,572    ) -> "Template":573        for loader in self.loaders:574            try:575                return loader.load(environment, name, globals)576            except TemplateNotFound:577                pass578        raise TemplateNotFound(name)579 580    def list_templates(self) -> t.List[str]:581        found = set()582        for loader in self.loaders:583            found.update(loader.list_templates())584        return sorted(found)585 586 587class _TemplateModule(ModuleType):588    """Like a normal module but with support for weak references"""589 590 591class ModuleLoader(BaseLoader):592    """This loader loads templates from precompiled templates.593 594    Example usage:595 596    >>> loader = ChoiceLoader([597    ...     ModuleLoader('/path/to/compiled/templates'),598    ...     FileSystemLoader('/path/to/templates')599    ... ])600 601    Templates can be precompiled with :meth:`Environment.compile_templates`.602    """603 604    has_source_access = False605 606    def __init__(607        self,608        path: t.Union[609            str, "os.PathLike[str]", t.Sequence[t.Union[str, "os.PathLike[str]"]]610        ],611    ) -> None:612        package_name = f"_jinja2_module_templates_{id(self):x}"613 614        # create a fake module that looks for the templates in the615        # path given.616        mod = _TemplateModule(package_name)617 618        if not isinstance(path, abc.Iterable) or isinstance(path, str):619            path = [path]620 621        mod.__path__ = [os.fspath(p) for p in path]622 623        sys.modules[package_name] = weakref.proxy(624            mod, lambda x: sys.modules.pop(package_name, None)625        )626 627        # the only strong reference, the sys.modules entry is weak628        # so that the garbage collector can remove it once the629        # loader that created it goes out of business.630        self.module = mod631        self.package_name = package_name632 633    @staticmethod634    def get_template_key(name: str) -> str:635        return "tmpl_" + sha1(name.encode("utf-8")).hexdigest()636 637    @staticmethod638    def get_module_filename(name: str) -> str:639        return ModuleLoader.get_template_key(name) + ".py"640 641    @internalcode642    def load(643        self,644        environment: "Environment",645        name: str,646        globals: t.Optional[t.MutableMapping[str, t.Any]] = None,647    ) -> "Template":648        key = self.get_template_key(name)649        module = f"{self.package_name}.{key}"650        mod = getattr(self.module, module, None)651 652        if mod is None:653            try:654                mod = __import__(module, None, None, ["root"])655            except ImportError as e:656                raise TemplateNotFound(name) from e657 658            # remove the entry from sys.modules, we only want the attribute659            # on the module object we have stored on the loader.660            sys.modules.pop(module, None)661 662        if globals is None:663            globals = {}664 665        return environment.template_class.from_module_dict(666            environment, mod.__dict__, globals667        )668