codekingpro/portable-devtools

1"""Tokenizers — fast, batteries-included tokenization library.
2
3Free-threaded Python (3.14t) note:
4    Wheels built against free-threaded CPython declare ``Py_MOD_GIL_NOT_USED``
5    and use ``RwLock``-guarded interior mutability so component setters are
6    safe to call from multiple threads. Compound mutations
7    (``tokenizer.post_processor.special_tokens = …``) are still not atomic —
8    use a Python lock if you need the read-then-write to be serialized.
9    See ``docs/free-threading-audit.md`` for the full analysis.
10"""
11
12from enum import Enum
13from typing import List, Tuple, Union
14
15
16Offsets = Tuple[int, int]
17
18TextInputSequence = str
19"""A :obj:`str` that represents an input sequence """
20
21PreTokenizedInputSequence = Union[List[str], Tuple[str]]
22"""A pre-tokenized input sequence. Can be one of:
23
24    - A :obj:`List` of :obj:`str`
25    - A :obj:`Tuple` of :obj:`str`
26"""
27
28TextEncodeInput = Union[
29    TextInputSequence,
30    Tuple[TextInputSequence, TextInputSequence],
31    List[TextInputSequence],
32]
33"""Represents a textual input for encoding. Can be either:
34
35    - A single sequence: :data:`~tokenizers.TextInputSequence`
36    - A pair of sequences:
37
38      - A :obj:`Tuple` of :data:`~tokenizers.TextInputSequence`
39      - Or a :obj:`List` of :data:`~tokenizers.TextInputSequence` of size 2
40"""
41
42PreTokenizedEncodeInput = Union[
43    PreTokenizedInputSequence,
44    Tuple[PreTokenizedInputSequence, PreTokenizedInputSequence],
45    List[PreTokenizedInputSequence],
46]
47"""Represents a pre-tokenized input for encoding. Can be either:
48
49    - A single sequence: :data:`~tokenizers.PreTokenizedInputSequence`
50    - A pair of sequences:
51
52      - A :obj:`Tuple` of :data:`~tokenizers.PreTokenizedInputSequence`
53      - Or a :obj:`List` of :data:`~tokenizers.PreTokenizedInputSequence` of size 2
54"""
55
56InputSequence = Union[TextInputSequence, PreTokenizedInputSequence]
57"""Represents all the possible types of input sequences for encoding. Can be:
58
59    - When ``is_pretokenized=False``: :data:`~TextInputSequence`
60    - When ``is_pretokenized=True``: :data:`~PreTokenizedInputSequence`
61"""
62
63EncodeInput = Union[TextEncodeInput, PreTokenizedEncodeInput]
64"""Represents all the possible types of input for encoding. Can be:
65
66    - When ``is_pretokenized=False``: :data:`~TextEncodeInput`
67    - When ``is_pretokenized=True``: :data:`~PreTokenizedEncodeInput`
68"""
69
70
71class OffsetReferential(Enum):
72    ORIGINAL = "original"
73    NORMALIZED = "normalized"
74
75
76class OffsetType(Enum):
77    BYTE = "byte"
78    CHAR = "char"
79
80
81class SplitDelimiterBehavior(Enum):
82    REMOVED = "removed"
83    ISOLATED = "isolated"
84    MERGED_WITH_PREVIOUS = "merged_with_previous"
85    MERGED_WITH_NEXT = "merged_with_next"
86    CONTIGUOUS = "contiguous"
87
88
89from .tokenizers import (
90    AddedToken,
91    Encoding,
92    NormalizedString,
93    PreTokenizedString,
94    Regex,
95    Token,
96    Tokenizer,
97    decoders,
98    models,
99    normalizers,
100    pre_tokenizers,
101    processors,
102    trainers,
103    __version__,
104)
105from .implementations import (
106    BertWordPieceTokenizer,
107    ByteLevelBPETokenizer,
108    CharBPETokenizer,
109    SentencePieceBPETokenizer,
110    SentencePieceUnigramTokenizer,
111)
112