binuser007/Toxic_comment_classification_using_Bert

1import re
2import nltk
3from nltk.tokenize import word_tokenize
4from nltk.corpus import stopwords
5from nltk.stem import WordNetLemmatizer
6from typing import List, Optional
7
8class TextPreprocessor:
9    def __init__(self):
10        nltk.download('punkt')
11        nltk.download('stopwords')
12        nltk.download('wordnet')
13        self.stop_words = set(stopwords.words('english'))
14        self.lemmatizer = WordNetLemmatizer()
15
16    def clean_text(self, text: str) -> str:
17        """Clean and normalize text"""
18        # Convert to lowercase
19        text = text.lower()
20        
21        # Remove special characters and numbers
22        text = re.sub(r'[^a-zA-Z\s]', '', text)
23        
24        # Remove extra whitespace
25        text = re.sub(r'\s+', ' ', text).strip()
26        
27        return text
28
29    def tokenize(self, text: str) -> List[str]:
30        """Tokenize text into words"""
31        return word_tokenize(text)
32
33    def remove_stopwords(self, tokens: List[str]) -> List[str]:
34        """Remove stop words from token list"""
35        return [token for token in tokens if token not in self.stop_words]
36
37    def lemmatize(self, tokens: List[str]) -> List[str]:
38        """Lemmatize tokens"""
39        return [self.lemmatizer.lemmatize(token) for token in tokens]
40
41    def process(self, text: str) -> List[str]:
42        """Complete preprocessing pipeline"""
43        cleaned_text = self.clean_text(text)
44        tokens = self.tokenize(cleaned_text)
45        tokens = self.remove_stopwords(tokens)
46        tokens = self.lemmatize(tokens)
47        return tokens