Jayesh13/Toxic-Comment-Classification

1# Importing necessary libraries2import streamlit as st3import os4import numpy as np5import pandas as pd6import matplotlib.pyplot as plt7import re8 9 10st.title('Toxic Comment Classification')11comment = st.text_area("Enter Your Text", "Type Here")12 13comment_input = []14comment_input.append(comment)15test_df = pd.DataFrame()16test_df['comment_text'] = comment_input17cols = {'toxic':[0], 'severe_toxic':[0], 'obscene':[0], 'threat':[0], 'insult':[0], 'identity_hate':[0], 'non_toxic': [0]}18for key in cols.keys():19    test_df[key] = cols[key]20test_df = test_df.reset_index()21test_df.drop(columns=["index"], inplace=True)22 23# Data Cleaning and Preprocessing24# creating copy of data for data cleaning and preprocessing25cleaned_data = test_df.copy()26 27# Removing Hyperlinks from text28cleaned_data["comment_text"] = cleaned_data["comment_text"].map(lambda x: re.sub(r"https?://\S+|www\.\S+","",x) )29 30# Removing emojis from text31cleaned_data["comment_text"] = cleaned_data["comment_text"].map(lambda x: re.sub("["32                                                                                   u"\U0001F600-\U0001F64F"33                                                                                   u"\U0001F300-\U0001F5FF"  34                                                                                   u"\U0001F680-\U0001F6FF" 35                                                                                   u"\U0001F1E0-\U0001F1FF"  36                                                                                   u"\U00002702-\U000027B0"37                                                                                   u"\U000024C2-\U0001F251"38                                                                                   "]+","", x, flags=re.UNICODE))39 40# Removing IP addresses from text 41cleaned_data["comment_text"] = cleaned_data["comment_text"].map(lambda x: re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}","",x))42 43# Removing html tags from text 44cleaned_data["comment_text"] = cleaned_data["comment_text"].map(lambda x: re.sub(r"<.*?>","",x))45 46# There are some comments which contain double quoted words like --> ""words""  we will convert these to --> "words" 47cleaned_data["comment_text"] = cleaned_data["comment_text"].map(lambda x: re.sub(r"\"\"", "\"",x))   # replacing "" with "48cleaned_data["comment_text"] = cleaned_data["comment_text"].map(lambda x: re.sub(r"^\"", "",x))      # removing quotation from start and the end of the string49cleaned_data["comment_text"] = cleaned_data["comment_text"].map(lambda x: re.sub(r"\"$", "",x))50 51# Removing Punctuation / Special characters (;:'".?@!%&*+) which appears more than twice in the text 52cleaned_data["comment_text"] = cleaned_data["comment_text"].map(lambda x: re.sub(r"[^a-zA-Z0-9\s][^a-zA-Z0-9\s]+", " ",x))53 54# Removing Special characters 55cleaned_data["comment_text"] = cleaned_data["comment_text"].map(lambda x: re.sub(r"[^a-zA-Z0-9\s\"\',:;?!.()]", " ",x))56 57# Removing extra spaces in text58cleaned_data["comment_text"] = cleaned_data["comment_text"].map(lambda x: re.sub(r"\s\s+", " ",x))59 60Final_data = cleaned_data.copy()61 62# Model Building63from transformers import DistilBertTokenizer64import torch65import torch.nn as nn66from torch.utils.data import DataLoader, Dataset67 68# Using Pretrained DistilBertTokenizer69tokenizer = DistilBertTokenizer.from_pretrained("distilbert-base-uncased")70 71# Creating Dataset class for Toxic comments and Labels 72class Toxic_Dataset(Dataset):73    def __init__(self, Comments_, Labels_):74        self.comments = Comments_.copy()75        self.labels = Labels_.copy()76        77        self.comments["comment_text"] = self.comments["comment_text"].map(lambda x: tokenizer(x, padding="max_length", truncation=True, return_tensors="pt"))78    79    def __len__(self):80        return len(self.labels)81    82    def __getitem__(self, idx):83        comment = self.comments.loc[idx,"comment_text"]84        label = np.array(self.labels.loc[idx,:])85        86        return comment, label87 88X_test = pd.DataFrame(test_df.iloc[:, 0])89Y_test = test_df.iloc[:, 1:]90Test_data = Toxic_Dataset(X_test, Y_test)91Test_Loader = DataLoader(Test_data, shuffle=False)92 93# Loading pre-trained weights of DistilBert model for sequence classification94# and changing classifiers output to 7 because we have 7 labels to classify.95# DistilBERT96 97from transformers import DistilBertForSequenceClassification98 99Distil_bert = DistilBertForSequenceClassification.from_pretrained("distilbert-base-uncased")100 101Distil_bert.classifier = nn.Sequential(102                    nn.Linear(768,7),103                    nn.Sigmoid()104                  )105# print(Distil_bert)106 107# Instantiating the model and loading the weights108model = Distil_bert109model.to('cpu')110model = torch.load('dsbert_toxic_balanced.pt', map_location=torch.device('cpu'))111 112# Making Predictions113for comments, labels in Test_Loader:114    labels = labels.to('cpu')115    labels = labels.float()116    masks = comments['attention_mask'].squeeze(1).to('cpu')117    input_ids = comments['input_ids'].squeeze(1).to('cpu')118    119    output = model(input_ids, masks)120    op = output.logits121    122    res = []123    for i in range(7):124        res.append(op[0, i])125    # print(res)126 127preds = []128 129for i in range(len(res)):130    preds.append(res[i].tolist())131 132classes = ['Toxic', 'Severe Toxic', 'Obscene', 'Threat', 'Insult', 'Identity Hate', 'Non Toxic']133 134if st.button('Classify'):135    for i in range(len(res)):136        st.write(f"{classes[i]} : {round(preds[i], 2)}\n")137    st.success('These are the outputs')138    139