Datasetpublic

codekingpro/portable-devtools

sourceHugging Faceupdated 5mo agoView on Hugging Face

1likes14kdownloads

rootpythonuser_packagesPython313site-packagestokenizersnormalizers__init__.py

__init__.py30 linesDownload Raw Back to normalizers

1from .. import normalizers
2
3
4Normalizer = normalizers.Normalizer
5BertNormalizer = normalizers.BertNormalizer
6NFD = normalizers.NFD
7NFKD = normalizers.NFKD
8NFC = normalizers.NFC
9NFKC = normalizers.NFKC
10Sequence = normalizers.Sequence
11Lowercase = normalizers.Lowercase
12Prepend = normalizers.Prepend
13Strip = normalizers.Strip
14StripAccents = normalizers.StripAccents
15Nmt = normalizers.Nmt
16Precompiled = normalizers.Precompiled
17Replace = normalizers.Replace
18ByteLevel = normalizers.ByteLevel
19
20NORMALIZERS = {"nfc": NFC, "nfd": NFD, "nfkc": NFKC, "nfkd": NFKD}
21
22
23def unicode_normalizer_from_str(normalizer: str) -> Normalizer:
24    if normalizer not in NORMALIZERS:
25        raise ValueError(
26            "{} is not a known unicode normalizer. Available are {}".format(normalizer, NORMALIZERS.keys())
27        )
28
29    return NORMALIZERS[normalizer]()
30