AntNikYab/NaturalLanguageProcessing

1import re2import string3import numpy as np4import torch5import torch.nn as nn6from transformers import BertTokenizer, BertModel7from sklearn.linear_model import LogisticRegression8from nltk.stem import SnowballStemmer9 10from nltk.corpus import stopwords11import nltk12nltk.download('stopwords')13stop_words = set(stopwords.words('russian'))14stemmer = SnowballStemmer('russian') 15sw = stopwords.words('russian')   16 17tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')18 19class LSTMClassifier(nn.Module):20    def __init__(self, embedding_dim: int, hidden_size:int, embedding: torch.nn.modules.sparse.Embedding) -> None:21        super().__init__()22 23        self.embedding_dim = embedding_dim24        self.hidden_size = hidden_size25        self.embedding = embedding26 27        self.lstm = nn.LSTM(28            input_size=self.embedding_dim,29            hidden_size=self.hidden_size,30            batch_first=True31        )32        self.clf = nn.Linear(self.hidden_size, 1)33 34    def forward(self, x):35        embeddings = self.embedding(x)36        _, (h_n, _) = self.lstm(embeddings)37        out = self.clf(h_n.squeeze())38        return out39 40 41def data_preprocessing(text: str) -> str:42    """preprocessing string: lowercase, removing html-tags, punctuation, 43                            stopwords, digits44 45    Args:46        text (str): input string for preprocessing47 48    Returns:49        str: preprocessed string50    """    51 52    text = text.lower()53    text = re.sub('<.*?>', '', text) # html tags54    text = ''.join([c for c in text if c not in string.punctuation])# Remove punctuation55    text = ' '.join([word for word in text.split() if word not in stop_words])56    text = [word for word in text.split() if not word.isdigit()]57    text = ' '.join(text)58    return text59 60def get_words_by_freq(sorted_words: list, n: int = 10) -> list:61    return list(filter(lambda x: x[1] > n, sorted_words))62 63def padding(review_int: list, seq_len: int) -> np.array: # type: ignore64    """Make left-sided padding for input list of tokens65 66    Args:67        review_int (list): input list of tokens68        seq_len (int): max length of sequence, it len(review_int[i]) > seq_len it will be trimmed, else it will be padded by zeros69 70    Returns:71        np.array: padded sequences72    """    73    features = np.zeros((len(review_int), seq_len), dtype = int)74    for i, review in enumerate(review_int):75        if len(review) <= seq_len:76            zeros = list(np.zeros(seq_len - len(review)))77            new = zeros + review78        else:79            new = review[: seq_len]80        features[i, :] = np.array(new)81            82    return features83 84def preprocess_single_string(85    input_string: str, 86    seq_len: int, 87    vocab_to_int: dict,88    ) -> torch.tensor:89    """Function for all preprocessing steps on a single string90 91    Args:92        input_string (str): input single string for preprocessing93        seq_len (int): max length of sequence, it len(review_int[i]) > seq_len it will be trimmed, else it will be padded by zeros94        vocab_to_int (dict, optional): word corpus {'word' : int index}. Defaults to vocab_to_int.95 96    Returns:97        list: preprocessed string98    """    99 100    preprocessed_string = data_preprocessing(input_string)101    result_list = []102    for word in preprocessed_string.split():103        try: 104            result_list.append(vocab_to_int[word])105        except KeyError as e:106            print(f'{e}: not in dictionary!')107    result_padded = padding([result_list], seq_len)[0]108 109    return torch.tensor(result_padded)110 111def predict_sentence(text: str, model: nn.Module, seq_len: int, vocab_to_int: dict) -> str: 112    p_str = preprocess_single_string(text, seq_len, vocab_to_int).unsqueeze(0)113    model.eval()114    pred = model(p_str)115    output = pred.sigmoid().round().item() 116    if output == 0: 117        return 'Негативный отзыв'118    else: 119        return 'Позитивный отзыв'120    121def predict_single_string(text: str,122                          model:  BertModel,123                          loaded_model: LogisticRegression124) -> str:125 126    with torch.no_grad():127        encoded_input = tokenizer(text, return_tensors='pt')128        output = model(**encoded_input)129        vector = output[0][:,0,:]130        pred0 = loaded_model.predict_proba(vector)[0][0]131        pred1 = loaded_model.predict_proba(vector)[0][1]132    if pred0 > pred1:133        return 'Негативный отзыв'134    else:135        return 'Позитивный отзыв'136    137def clean(text):138 139    text = text.lower()140    text = re.sub(r'\s+', ' ', text)  # заменить два и более пробела на один пробел141    text = re.sub(r'\d+', ' ', text) # удаляем числа142    text = text.translate(str.maketrans('', '', string.punctuation)) # удаляем знаки пунктуации 143    text = re.sub(r'\n+', ' ', text) # удаляем символ перевод строки 144    145    return text146 147def tokin(text):148    text = clean(text)149    text = ' '.join([stemmer.stem(word) for word in text.split()])150    text = ' '.join([word for word in text.split() if word not in sw])151    return text152 153 154def predict_ml_class(text, loaded_vectorizer, loaded_classifier):155 156    t = tokin(text).split('    ')157    new_text_bow = loaded_vectorizer.transform(t)158    predicted_label = loaded_classifier.predict(new_text_bow)159    if predicted_label == 0: 160        return 'Негативный отзыв'161    else: 162        return 'Позитивный отзыв'