codekingpro/portable-devtools

1"""2    pygments.lexers.mime3    ~~~~~~~~~~~~~~~~~~~~4 5    Lexer for Multipurpose Internet Mail Extensions (MIME) data.6 7    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.8    :license: BSD, see LICENSE for details.9"""10 11import re12 13from pygments.lexer import RegexLexer, include14from pygments.lexers import get_lexer_for_mimetype15from pygments.token import Text, Name, String, Operator, Comment, Other16from pygments.util import get_int_opt, ClassNotFound17 18__all__ = ["MIMELexer"]19 20 21class MIMELexer(RegexLexer):22    """23    Lexer for Multipurpose Internet Mail Extensions (MIME) data. This lexer is24    designed to process nested multipart data.25 26    It assumes that the given data contains both header and body (and is27    split at an empty line). If no valid header is found, then the entire data28    will be treated as body.29 30    Additional options accepted:31 32    `MIME-max-level`33        Max recursion level for nested MIME structure. Any negative number34        would treated as unlimited. (default: -1)35 36    `Content-Type`37        Treat the data as a specific content type. Useful when header is38        missing, or this lexer would try to parse from header. (default:39        `text/plain`)40 41    `Multipart-Boundary`42        Set the default multipart boundary delimiter. This option is only used43        when `Content-Type` is `multipart` and header is missing. This lexer44        would try to parse from header by default. (default: None)45 46    `Content-Transfer-Encoding`47        Treat the data as a specific encoding. Or this lexer would try to parse48        from header by default. (default: None)49    """50 51    name = "MIME"52    aliases = ["mime"]53    mimetypes = ["multipart/mixed",54                 "multipart/related",55                 "multipart/alternative"]56    url = 'https://en.wikipedia.org/wiki/MIME'57    version_added = '2.5'58 59    def __init__(self, **options):60        super().__init__(**options)61        self.boundary = options.get("Multipart-Boundary")62        self.content_transfer_encoding = options.get("Content_Transfer_Encoding")63        self.content_type = options.get("Content_Type", "text/plain")64        self.max_nested_level = get_int_opt(options, "MIME-max-level", -1)65 66    def get_header_tokens(self, match):67        field = match.group(1)68 69        if field.lower() in self.attention_headers:70            yield match.start(1), Name.Tag, field + ":"71            yield match.start(2), Text.Whitespace, match.group(2)72 73            pos = match.end(2)74            body = match.group(3)75            for i, t, v in self.get_tokens_unprocessed(body, ("root", field.lower())):76                yield pos + i, t, v77 78        else:79            yield match.start(), Comment, match.group()80 81    def get_body_tokens(self, match):82        pos_body_start = match.start()83        entire_body = match.group()84 85        # skip first newline86        if entire_body[0] == '\n':87            yield pos_body_start, Text.Whitespace, '\n'88            pos_body_start = pos_body_start + 189            entire_body = entire_body[1:]90 91        # if it is not a multipart92        if not self.content_type.startswith("multipart") or not self.boundary:93            for i, t, v in self.get_bodypart_tokens(entire_body):94                yield pos_body_start + i, t, v95            return96 97        # find boundary98        bdry_pattern = rf"^--{re.escape(self.boundary)}(--)?\n"99        bdry_matcher = re.compile(bdry_pattern, re.MULTILINE)100 101        # some data has prefix text before first boundary102        m = bdry_matcher.search(entire_body)103        if m:104            pos_part_start = pos_body_start + m.end()105            pos_iter_start = lpos_end = m.end()106            yield pos_body_start, Text, entire_body[:m.start()]107            yield pos_body_start + lpos_end, String.Delimiter, m.group()108        else:109            pos_part_start = pos_body_start110            pos_iter_start = 0111 112        # process tokens of each body part113        for m in bdry_matcher.finditer(entire_body, pos_iter_start):114            # bodypart115            lpos_start = pos_part_start - pos_body_start116            lpos_end = m.start()117            part = entire_body[lpos_start:lpos_end]118            for i, t, v in self.get_bodypart_tokens(part):119                yield pos_part_start + i, t, v120 121            # boundary122            yield pos_body_start + lpos_end, String.Delimiter, m.group()123            pos_part_start = pos_body_start + m.end()124 125        # some data has suffix text after last boundary126        lpos_start = pos_part_start - pos_body_start127        if lpos_start != len(entire_body):128            yield pos_part_start, Text, entire_body[lpos_start:]129 130    def get_bodypart_tokens(self, text):131        # return if:132        #  * no content133        #  * no content type specific134        #  * content encoding is not readable135        #  * max recurrsion exceed136        if not text.strip() or not self.content_type:137            return [(0, Other, text)]138 139        cte = self.content_transfer_encoding140        if cte and cte not in {"8bit", "7bit", "quoted-printable"}:141            return [(0, Other, text)]142 143        if self.max_nested_level == 0:144            return [(0, Other, text)]145 146        # get lexer147        try:148            lexer = get_lexer_for_mimetype(self.content_type)149        except ClassNotFound:150            return [(0, Other, text)]151 152        if isinstance(lexer, type(self)):153            lexer.max_nested_level = self.max_nested_level - 1154 155        return lexer.get_tokens_unprocessed(text)156 157    def store_content_type(self, match):158        self.content_type = match.group(1)159 160        prefix_len = match.start(1) - match.start(0)161        yield match.start(0), Text.Whitespace, match.group(0)[:prefix_len]162        yield match.start(1), Name.Label, match.group(2)163        yield match.end(2), String.Delimiter, '/'164        yield match.start(3), Name.Label, match.group(3)165 166    def get_content_type_subtokens(self, match):167        yield match.start(1), Text, match.group(1)168        yield match.start(2), Text.Whitespace, match.group(2)169        yield match.start(3), Name.Attribute, match.group(3)170        yield match.start(4), Operator, match.group(4)171        yield match.start(5), String, match.group(5)172 173        if match.group(3).lower() == "boundary":174            boundary = match.group(5).strip()175            if boundary[0] == '"' and boundary[-1] == '"':176                boundary = boundary[1:-1]177            self.boundary = boundary178 179    def store_content_transfer_encoding(self, match):180        self.content_transfer_encoding = match.group(0).lower()181        yield match.start(0), Name.Constant, match.group(0)182 183    attention_headers = {"content-type", "content-transfer-encoding"}184 185    tokens = {186        "root": [187            (r"^([\w-]+):( *)([\s\S]*?\n)(?![ \t])", get_header_tokens),188            (r"^$[\s\S]+", get_body_tokens),189        ],190        "header": [191            # folding192            (r"\n[ \t]", Text.Whitespace),193            (r"\n(?![ \t])", Text.Whitespace, "#pop"),194        ],195        "content-type": [196            include("header"),197            (198                r"^\s*((multipart|application|audio|font|image|model|text|video"199                r"|message)/([\w-]+))",200                store_content_type,201            ),202            (r'(;)((?:[ \t]|\n[ \t])*)([\w:-]+)(=)([\s\S]*?)(?=;|\n(?![ \t]))',203             get_content_type_subtokens),204            (r';[ \t]*\n(?![ \t])', Text, '#pop'),205        ],206        "content-transfer-encoding": [207            include("header"),208            (r"([\w-]+)", store_content_transfer_encoding),209        ],210    }211