Jayesh13/Toxic_comment_classification

1import streamlit as st 2import pickle 3import numpy as np4import pandas as pd5import re6import tensorflow 7from tensorflow import keras8from keras.preprocessing import text,sequence,utils9import html10import string11import nltk12from nltk.stem.porter import PorterStemmer13from nltk.stem import WordNetLemmatizer14from nltk.tokenize import word_tokenize 15from nltk.corpus import stopwords16stop_words = stopwords.words('english')17from tensorflow.keras.preprocessing.text import text_to_word_sequence18from tensorflow.keras.preprocessing.text import Tokenizer  19from tensorflow.keras.preprocessing.sequence import pad_sequences20from tensorflow.keras import models21from tensorflow.keras import layers22from tensorflow.keras import losses23from tensorflow.keras import metrics24from tensorflow.keras import optimizers25from tensorflow.keras.utils import plot_model26 27def remove_special_chars(text):28    re1 = re.compile(r'  +')29    x1 = text.lower().replace('#39;', "'").replace('amp;', '&').replace('#146;', "'").replace(30        'nbsp;', ' ').replace('#36;', '$').replace('\\n', "\n").replace('quot;', "'").replace(31        '<br />', "\n").replace('\\"', '"').replace('<unk>', 'u_n').replace(' @.@ ', '.').replace(32        ' @-@ ', '-').replace('\\', ' \\ ')33    return re1.sub(' ', html.unescape(x1))34 35def to_lowercase(text):36    return text.lower()37 38def remove_punctuation(text):39    """Remove punctuation from list of tokenized words"""40    translator = str.maketrans('', '', string.punctuation)41    return text.translate(translator)42 43def replace_numbers(text):44    """Replace all interger occurrences in list of tokenized words with textual representation"""45    return re.sub(r'\d+', '', text)46 47def remove_whitespaces(text):48    return text.strip()49 50def remove_stopwords(words, stop_words):51    return [word for word in words if word not in stop_words]52 53def stem_words(words):54    """Stem words in text"""55    stemmer = PorterStemmer()56    return [stemmer.stem(word) for word in words]57 58def lemmatize_words(words):59    """Lemmatize words in text"""60 61    lemmatizer = WordNetLemmatizer()62    return [lemmatizer.lemmatize(word) for word in words]63 64def lemmatize_verbs(words):65    """Lemmatize verbs in text"""66 67    lemmatizer = WordNetLemmatizer()68    return ' '.join([lemmatizer.lemmatize(word, pos='v') for word in words])69 70def text2words(text):71    return word_tokenize(text)72 73def clean_text( text):74    text = remove_special_chars(text)75    text = remove_punctuation(text)76    text = to_lowercase(text)77    text = replace_numbers(text)78    words = text2words(text)79    words = remove_stopwords(words, stop_words)80    #words = stem_words(words)# Either stem ovocar lemmatize81    words = lemmatize_words(words)82    words = lemmatize_verbs(words)83 84    return ''.join(words)85 86#df = pd.read_csv('train.csv.zip')87#df['comment_text'] = df['comment_text'].apply(lambda x: clean_text(x))88 89model = pickle.load(open('tox_model.h5','rb'))90 91st.title('Toxic comment classification')92input = st.text_area('Enter your comment')93 94input = input.apply(lambda x: clean_text(x))95tok = Tokenizer(num_words=1000, oov_token='UNK')96#tok.fit_on_texts(df['comment_text'] )97 98x_test = tok.texts_to_sequence(input)99input_text = pad_sequences(x_test,100                            maxlen=50, 101                            truncating='post', 102                            padding='post'103                               )104if input:105    out = model.predict(input_text)106    st.json(out)107 108 109