TFI/K-Means_Clustering_Algorithm
0
1import gradio as gr2import pandas as pd3from sklearn.feature_extraction.text import TfidfVectorizer4from sklearn.cluster import KMeans5import re6from io import BytesIO7import tempfile8from wordcloud import WordCloud, STOPWORDS9import matplotlib.pyplot as plt10import plotly.express as px11from PIL import Image12 13categories_keywords = {14 "Application Status": ["application status", "application", "status", "submitted", "processing", "pending", "approval", "rejected", "accepted", "apply", "how to apply", "can I apply"],15 "Follow-Ups": ['update', 'updates', 'any updates', 'any news', 'response from you', 'any reply'],16 "Firki": ['firki'],17 "Interviews": ['interview', 'set up interview', 'phone interview'],18 "Volunteering": ["volunteer", "volunteering", "help out", "assist", "volunteer work", "volunteer opportunities"],19 "Certificates": ["certificate", "certificates", "completion", "certification", "accreditation", "proof", "document", "certified"],20 "Job Opportunities": ["job", "opportunity", "career", "vacancy", "position", "employment", "hiring", "recruitment", "internship", "post", "posts", "available", "teacher", "teaching", "opportunities", "looking for"],21 "Surveys and Forms": ["survey", "form", "forms", "questionnaire", "feedback form", "response", "fill out", "submission"],22 "Spam": ["spam", "unsubscribe", "remove", "stop", "junk", "block", "opt-out"],23 "Rescheduling and Postponing": ["reschedule", "postpone", "delay", "change date", "new time", "rearrange", "shift", "adjust timing"],24 "Contact and Communication Issues": ["contact", "communicate", "communication", "reach out", "phone", "email", "address", "details"],25 "Email and Credentials Issues": ["email", "credentials", "login", "password", "gmail", "username", "verification", "reset"],26 "Timing and Scheduling": ["timing", "schedule", "scheduling", "time", "appointment", "availability", "calendar", "book", "slot"],27 "Salary and Benefits": ["salary", "benefits", "pay", "compensation", "wages", "earnings", "package", "remuneration", "incentives"],28 "Technical Issues": ["technical", "issue", "problem", "error", "bug", "glitch", "fix", "troubleshoot", "support"],29 "End of Conversation": ["bye", "thank you", "thanks", "goodbye", "end conversation", "ok", "ok thanks"],30 "Feedback": ["feedback", "comments", "review", "opinion", "suggestion", "critique", "rating"],31 "Event Inquiries": ["event", "webinar", "meeting", "conference", "session", "seminar", "workshop", "invitation"],32 "Payment Issues": ["payment", "billing", "transaction", "charge", "fee", "invoice", "refund", "receipt"],33 "Registration Issues": ["registration", "register", "sign up", "enroll", "join", "signup", "enrollment"],34 "Service Requests": ["service", "support", "request", "assistance", "help", "aid", "maintenance"],35 "Account Issues": ["account", "profile", "update", "activation", "deactivation", "reset", "account password"],36 "Product Information": ["product", "service", "details", "info", "information", "specifications", "features"],37 "Order Status": ["order", "status", "tracking", "shipment", "delivery", "purchase", "dispatch"],38 "Miscellaneous": []39}40 41 42def categorize_question(question):43 words = question.split()44 45 # words to exclude from End Conversation46 exclusion_words = {'is', 'please', 'not resolved', 'unresolved', 'problem', 'help', 'issue', 'webinar', 'office', 'leave', 'approved', 'notice', 'period', 'good morning', 'when', 'where', 'why', 'how', 'which', 'and when'}47 48 49 # Categorization50 for category, keywords in categories_keywords.items():51 if any(keyword.lower() in question.lower() for keyword in keywords):52 return category53 54 # Secondary check for 'End of Conversation' category55 if "end of conversation" in question.lower() and not any(exclusion_word in question.lower() for exclusion_word in exclusion_words):56 return "End of Conversation"57 58 return "Miscellaneous"59 60 61def preprocess_data(df):62 df.rename(columns={'Question Asked': 'texts'}, inplace=True)63 df['texts'] = df['texts'].astype(str).str.lower()64 df['texts'] = df['texts'].apply(lambda text: re.sub(r'https?://\S+|www\.\S+', '', text))65 66 def remove_emoji(string):67 emoji_pattern = re.compile("["68 u"\U0001F600-\U0001F64F"69 u"\U0001F300-\U0001F5FF"70 u"\U0001F680-\U0001F6FF"71 u"\U0001F1E0-\U0001F1FF"72 u"\U00002702-\U000027B0"73 u"\U000024C2-\U0001F251"74 "]+", flags=re.UNICODE)75 return emoji_pattern.sub(r'', string)76 77 df['texts'] = df['texts'].apply(remove_emoji)78 79 custom_synonyms = {80 'application': ['form'],81 'apply': ['fill', 'applied'],82 'work': ['job'],83 'salary': ['stipend', 'pay', 'payment', 'paid'],84 'test': ['online test', 'amcat test', 'exam', 'assessment'],85 'pass': ['clear', 'selected', 'pass or not'],86 'result': ['outcome', 'mark', 'marks'],87 'thanks': ["thanks a lot to you", "thankyou so much", "thank you so much", "tysm", "thank you",88 "okaythank", "thx", "ty", "thankyou", "thank", "thank u"],89 'interview': ["pi"]90 }91 92 for original_word, synonym_list in custom_synonyms.items():93 for synonym in synonym_list:94 pattern = r"\b" + synonym + r"\b"95 df['texts'] = df['texts'].str.replace(pattern, original_word, regex=True)96 97 spam_list = ["click here", "free", "recharge", "limited", "discount", "money back guarantee", "aaj", "kal", "mein",98 "how can i help you", "how can we help you", "how we can help you", "follow", "king", "contacting", "gar",99 "kirke", "subscribe", "youtube", "jio", "insta", "make money", "b2b", "sent using truecaller"]100 101 for spam_phrase in spam_list:102 pattern = r"\b" + re.escape(spam_phrase) + r"\b"103 df = df[~df['texts'].str.contains(pattern)]104 105 def remove_punctuations(text):106 return re.sub(r'[^\w\s]', '', text)107 108 df['texts'] = df['texts'].apply(remove_punctuations)109 df['texts'] = df['texts'].str.strip()110 df = df[df['texts'] != '']111 112 # Categorize113 df['Category'] = df['texts'].apply(categorize_question)114 115 return df116 117def cluster_data(df, num_clusters):118 vectorizer = TfidfVectorizer(stop_words='english')119 X = vectorizer.fit_transform(df['texts'])120 121 kmeans = KMeans(n_clusters=num_clusters, random_state=0)122 kmeans.fit(X)123 df['Cluster'] = kmeans.labels_124 125 return df, kmeans126 127def generate_wordcloud(df):128 text = " ".join(df['texts'].tolist())129 stopwords = set(STOPWORDS)130 wordcloud = WordCloud(131 width=800,132 height=400,133 background_color='white',134 max_words=300,135 collocations=False,136 min_font_size=10,137 max_font_size=200,138 stopwords=stopwords,139 prefer_horizontal=1.0,140 scale=2,141 relative_scaling=0.5,142 random_state=42143 ).generate(text)144 145 plt.figure(figsize=(15, 7))146 plt.imshow(wordcloud, interpolation='bilinear')147 plt.axis('off')148 buf = BytesIO()149 plt.savefig(buf, format='png')150 buf.seek(0)151 img = Image.open(buf)152 return img153 154def generate_bar_chart(df, num_clusters_to_display):155 # Exclude common words156 common_words = {'i', 'you', 'thanks', 'thank', 'ok', 'okay', 'sure', 'done', 'to', 'for', 'and', 'but', 'so'}157 158 top_categories = df['Category'].value_counts().index[:num_clusters_to_display]159 df_top_categories = df[df['Category'].isin(top_categories)]160 161 category_top_words = df_top_categories.groupby('Category', observed=False)['texts'].apply(lambda x: ' '.join(x)).reset_index()162 category_top_words['top_word'] = category_top_words['texts'].apply(lambda x: ' '.join([word for word in pd.Series(x.split()).value_counts().index if word not in common_words][:3]))163 category_sizes = df_top_categories['Category'].value_counts().reset_index()164 category_sizes.columns = ['Category', 'Count']165 category_sizes = category_sizes.merge(category_top_words[['Category', 'top_word']], on='Category')166 167 fig = px.bar(category_sizes, x='Category', y='Count', text='top_word', title='Category Frequency with Top Words')168 fig.update_traces(textposition='outside')169 fig.update_layout(xaxis_title='Category', yaxis_title='Frequency', showlegend=False)170 171 buf = BytesIO()172 fig.write_image(buf, format='png')173 buf.seek(0)174 img = Image.open(buf)175 return img176 177def main(file, num_clusters_to_display):178 try:179 df = pd.read_csv(file)180 181 # Filter by 'Fallback Message shown'182 df = df[df['Answer'] == 'Fallback Message shown']183 184 df = preprocess_data(df)185 186 # Clustering187 num_clusters = 12 188 df, kmeans = cluster_data(df, num_clusters)189 190 # Categorization191 df['Category'] = df['texts'].apply(categorize_question)192 193 df = df[df['Category'] != 'Miscellaneous']194 195 # Sorting (ascending order)196 category_sizes = df['Category'].value_counts().reset_index()197 category_sizes.columns = ['Category', 'Count']198 sorted_categories = category_sizes.sort_values(by='Count', ascending=False)['Category'].tolist()199 sorted_categories_sm = category_sizes.sort_values(by='Count', ascending=True)['Category'].tolist()200 201 # Display (according to input slider)202 largest_categories = sorted_categories[:num_clusters_to_display]203 smallest_categories = sorted_categories_sm[:num_clusters_to_display]204 205 # Filtering (according to input slider)206 filtered_df = df[df['Category'].isin(largest_categories)]207 filtered_cloud_df = df[df['Category'].isin(smallest_categories)]208 209 # Sort the output file by Category and Cluster210 filtered_df = filtered_df.sort_values(by=['Category', 'Cluster'])211 filtered_cloud_df = filtered_cloud_df.sort_values(by='Category')212 213 wordcloud_img = generate_wordcloud(filtered_cloud_df)214 bar_chart_img = generate_bar_chart(df, num_clusters_to_display)215 216 with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as tmpfile:217 filtered_df.to_csv(tmpfile.name, index=False)218 csv_file_path = tmpfile.name219 220 return csv_file_path, wordcloud_img, bar_chart_img221 except Exception as e:222 print(f"Error: {e}")223 return str(e), None, None224 225interface = gr.Interface(226 fn=main,227 inputs=[228 gr.File(label="Upload CSV File (.csv)"),229 gr.Slider(label="Number of Categories to Display", minimum=1, maximum=15, step=1, value=5)230 ],231 outputs=[232 gr.File(label="Categorized Data CSV"),233 gr.Image(label="Word Cloud"),234 gr.Image(label="Bar Chart")235 ],236 title="Unanswered User Queries Categorization",237)238 239interface.launch(share=True)240