Team Ai
Datasetpublic

codekingpro/portable-devtools

sourceHugging Faceupdated 5mo agoView on Hugging Face
1likes14kdownloads
token.py179 linesDownload Raw Back to markdown_it
1from __future__ import annotations2 3from collections.abc import Callable, MutableMapping4import dataclasses as dc5from typing import Any, Literal6import warnings7 8 9def convert_attrs(value: Any) -> Any:10    """Convert Token.attrs set as ``None`` or ``[[key, value], ...]`` to a dict.11 12    This improves compatibility with upstream markdown-it.13    """14    if not value:15        return {}16    if isinstance(value, list):17        return dict(value)18    return value19 20 21@dc.dataclass(slots=True)22class Token:23    type: str24    """Type of the token (string, e.g. "paragraph_open")"""25 26    tag: str27    """HTML tag name, e.g. 'p'"""28 29    nesting: Literal[-1, 0, 1]30    """Level change (number in {-1, 0, 1} set), where:31    -  `1` means the tag is opening32    -  `0` means the tag is self-closing33    - `-1` means the tag is closing34    """35 36    attrs: dict[str, str | int | float] = dc.field(default_factory=dict)37    """HTML attributes.38    Note this differs from the upstream "list of lists" format,39    although than an instance can still be initialised with this format.40    """41 42    map: list[int] | None = None43    """Source map info. Format: `[ line_begin, line_end ]`"""44 45    level: int = 046    """Nesting level, the same as `state.level`"""47 48    children: list[Token] | None = None49    """Array of child nodes (inline and img tokens)."""50 51    content: str = ""52    """Inner content, in the case of a self-closing tag (code, html, fence, etc.),"""53 54    markup: str = ""55    """'*' or '_' for emphasis, fence string for fence, etc."""56 57    info: str = ""58    """Additional information:59    - Info string for "fence" tokens60    - The value "auto" for autolink "link_open" and "link_close" tokens61    - The string value of the item marker for ordered-list "list_item_open" tokens62    """63 64    meta: dict[Any, Any] = dc.field(default_factory=dict)65    """A place for plugins to store any arbitrary data"""66 67    block: bool = False68    """True for block-level tokens, false for inline tokens.69    Used in renderer to calculate line breaks70    """71 72    hidden: bool = False73    """If true, ignore this element when rendering.74    Used for tight lists to hide paragraphs.75    """76 77    def __post_init__(self) -> None:78        self.attrs = convert_attrs(self.attrs)79 80    def attrIndex(self, name: str) -> int:81        warnings.warn(  # noqa: B02882            "Token.attrIndex should not be used, since Token.attrs is a dictionary",83            UserWarning,84        )85        if name not in self.attrs:86            return -187        return list(self.attrs.keys()).index(name)88 89    def attrItems(self) -> list[tuple[str, str | int | float]]:90        """Get (key, value) list of attrs."""91        return list(self.attrs.items())92 93    def attrPush(self, attrData: tuple[str, str | int | float]) -> None:94        """Add `[ name, value ]` attribute to list. Init attrs if necessary."""95        name, value = attrData96        self.attrSet(name, value)97 98    def attrSet(self, name: str, value: str | int | float) -> None:99        """Set `name` attribute to `value`. Override old value if exists."""100        self.attrs[name] = value101 102    def attrGet(self, name: str) -> None | str | int | float:103        """Get the value of attribute `name`, or null if it does not exist."""104        return self.attrs.get(name, None)105 106    def attrJoin(self, name: str, value: str) -> None:107        """Join value to existing attribute via space.108        Or create new attribute if not exists.109        Useful to operate with token classes.110        """111        if name in self.attrs:112            current = self.attrs[name]113            if not isinstance(current, str):114                raise TypeError(115                    f"existing attr 'name' is not a str: {self.attrs[name]}"116                )117            self.attrs[name] = f"{current} {value}"118        else:119            self.attrs[name] = value120 121    def copy(self, **changes: Any) -> Token:122        """Return a shallow copy of the instance."""123        return dc.replace(self, **changes)124 125    def as_dict(126        self,127        *,128        children: bool = True,129        as_upstream: bool = True,130        meta_serializer: Callable[[dict[Any, Any]], Any] | None = None,131        filter: Callable[[str, Any], bool] | None = None,132        dict_factory: Callable[..., MutableMapping[str, Any]] = dict,133    ) -> MutableMapping[str, Any]:134        """Return the token as a dictionary.135 136        :param children: Also convert children to dicts137        :param as_upstream: Ensure the output dictionary is equal to that created by markdown-it138            For example, attrs are converted to null or lists139        :param meta_serializer: hook for serializing ``Token.meta``140        :param filter: A callable whose return code determines whether an141            attribute or element is included (``True``) or dropped (``False``).142            Is called with the (key, value) pair.143        :param dict_factory: A callable to produce dictionaries from.144            For example, to produce ordered dictionaries instead of normal Python145            dictionaries, pass in ``collections.OrderedDict``.146 147        """148        mapping = dict_factory((f.name, getattr(self, f.name)) for f in dc.fields(self))149        if filter:150            mapping = dict_factory((k, v) for k, v in mapping.items() if filter(k, v))151        if as_upstream and "attrs" in mapping:152            mapping["attrs"] = (153                None154                if not mapping["attrs"]155                else [[k, v] for k, v in mapping["attrs"].items()]156            )157        if meta_serializer and "meta" in mapping:158            mapping["meta"] = meta_serializer(mapping["meta"])159        if children and mapping.get("children", None):160            mapping["children"] = [161                child.as_dict(162                    children=children,163                    filter=filter,164                    dict_factory=dict_factory,165                    as_upstream=as_upstream,166                    meta_serializer=meta_serializer,167                )168                for child in mapping["children"]169            ]170        return mapping171 172    @classmethod173    def from_dict(cls, dct: MutableMapping[str, Any]) -> Token:174        """Convert a dict to a Token."""175        token = cls(**dct)176        if token.children:177            token.children = [cls.from_dict(c) for c in token.children]  # type: ignore[arg-type]178        return token179 
codekingpro/portable-devtools · Team Ai