Datasetpublic

codekingpro/portable-devtools

sourceHugging Faceupdated 5mo agoView on Hugging Face

1likes14kdownloads

rootpythonuser_packagesPython313site-packagestokenizerspre_tokenizers__init__.py

__init__.py18 linesDownload Raw Back to pre_tokenizers

1# Generated content DO NOT EDIT
2
3from .. import pre_tokenizers
4
5BertPreTokenizer = pre_tokenizers.BertPreTokenizer
6ByteLevel = pre_tokenizers.ByteLevel
7CharDelimiterSplit = pre_tokenizers.CharDelimiterSplit
8Digits = pre_tokenizers.Digits
9FixedLength = pre_tokenizers.FixedLength
10Metaspace = pre_tokenizers.Metaspace
11PreTokenizer = pre_tokenizers.PreTokenizer
12Punctuation = pre_tokenizers.Punctuation
13Sequence = pre_tokenizers.Sequence
14Split = pre_tokenizers.Split
15UnicodeScripts = pre_tokenizers.UnicodeScripts
16Whitespace = pre_tokenizers.Whitespace
17WhitespaceSplit = pre_tokenizers.WhitespaceSplit
18