huggingface/text-data-filtering

1import re2from typing import Dict3 4 5non_printing_characters_re = re.compile(6    f"[{''.join(map(chr, list(range(0,32)) + list(range(127,160))))}]"7)8 9digits_re: re.Pattern = re.compile(r"\d")10 11unicode_punctuation: Dict[str, str] = {12    "，": ",",13    "。": ".",14    "、": ",",15    "„": '"',16    "”": '"',17    "“": '"',18    "«": '"',19    "»": '"',20    "１": '"',21    "」": '"',22    "「": '"',23    "《": '"',24    "》": '"',25    "´": "'",26    "∶": ":",27    "：": ":",28    "？": "?",29    "！": "!",30    "（": "(",31    "）": ")",32    "；": ";",33    "–": "-",34    "—": " - ",35    "．": ". ",36    "～": "~",37    "’": "'",38    "…": "...",39    "━": "-",40    "〈": "<",41    "〉": ">",42    "【": "[",43    "】": "]",44    "％": "%",45    "►": "-",46}47 48normalization = {49    "non_printing_characters_re": non_printing_characters_re,50    "digits_re": digits_re,51    "unicode_punctuation": unicode_punctuation,52}53