Decoder2704/python-chatbot-jay
0
1{2 "cells": [3 {4 "cell_type": "code",5 "execution_count": 1,6 "id": "cc66e702",7 "metadata": {8 "id": "cc66e702"9 },10 "outputs": [],11 "source": [12 "import pandas as pd\n",13 "import numpy as np\n",14 "\n",15 "import matplotlib.pyplot as plt\n",16 "import matplotlib.lines as mlines\n",17 "\n",18 "import nltk\n",19 "from nltk.tokenize import ToktokTokenizer\n",20 "from nltk.stem.wordnet import WordNetLemmatizer\n",21 "from nltk.corpus import stopwords\n",22 "from string import punctuation\n",23 "\n",24 "import re\n",25 "from bs4 import BeautifulSoup\n",26 "\n",27 "from sklearn.feature_extraction.text import TfidfVectorizer\n",28 "from sklearn.model_selection import train_test_split\n",29 "from sklearn.model_selection import learning_curve\n",30 "from sklearn.model_selection import ShuffleSplit\n",31 "from sklearn.preprocessing import MultiLabelBinarizer\n",32 "from sklearn.linear_model import SGDClassifier\n",33 "from sklearn.multiclass import OneVsRestClassifier\n",34 "from sklearn.metrics import hamming_loss\n",35 "from sklearn.metrics import confusion_matrix\n",36 "\n",37 "from scipy.sparse import hstack"38 ]39 },40 {41 "cell_type": "code",42 "execution_count": 2,43 "id": "ed1d7799",44 "metadata": {45 "id": "ed1d7799",46 "outputId": "81334488-9472-4dc5-fe18-eea42ac2f16d"47 },48 "outputs": [49 {50 "data": {51 "text/html": [52 "<div>\n",53 "<style scoped>\n",54 " .dataframe tbody tr th:only-of-type {\n",55 " vertical-align: middle;\n",56 " }\n",57 "\n",58 " .dataframe tbody tr th {\n",59 " vertical-align: top;\n",60 " }\n",61 "\n",62 " .dataframe thead th {\n",63 " text-align: right;\n",64 " }\n",65 "</style>\n",66 "<table border=\"1\" class=\"dataframe\">\n",67 " <thead>\n",68 " <tr style=\"text-align: right;\">\n",69 " <th></th>\n",70 " <th>Id</th>\n",71 " <th>OwnerUserId</th>\n",72 " <th>CreationDate</th>\n",73 " <th>Score</th>\n",74 " <th>Title</th>\n",75 " <th>Body</th>\n",76 " </tr>\n",77 " </thead>\n",78 " <tbody>\n",79 " <tr>\n",80 " <th>0</th>\n",81 " <td>469</td>\n",82 " <td>147.0</td>\n",83 " <td>2008-08-02T15:11:16Z</td>\n",84 " <td>21</td>\n",85 " <td>How can I find the full path to a font from it...</td>\n",86 " <td><p>I am using the Photoshop's javascript API t...</td>\n",87 " </tr>\n",88 " <tr>\n",89 " <th>1</th>\n",90 " <td>502</td>\n",91 " <td>147.0</td>\n",92 " <td>2008-08-02T17:01:58Z</td>\n",93 " <td>27</td>\n",94 " <td>Get a preview JPEG of a PDF on Windows?</td>\n",95 " <td><p>I have a cross-platform (Python) applicatio...</td>\n",96 " </tr>\n",97 " <tr>\n",98 " <th>2</th>\n",99 " <td>535</td>\n",100 " <td>154.0</td>\n",101 " <td>2008-08-02T18:43:54Z</td>\n",102 " <td>40</td>\n",103 " <td>Continuous Integration System for a Python Cod...</td>\n",104 " <td><p>I'm starting work on a hobby project with a...</td>\n",105 " </tr>\n",106 " <tr>\n",107 " <th>3</th>\n",108 " <td>594</td>\n",109 " <td>116.0</td>\n",110 " <td>2008-08-03T01:15:08Z</td>\n",111 " <td>25</td>\n",112 " <td>cx_Oracle: How do I iterate over a result set?</td>\n",113 " <td><p>There are several ways to iterate over a re...</td>\n",114 " </tr>\n",115 " <tr>\n",116 " <th>4</th>\n",117 " <td>683</td>\n",118 " <td>199.0</td>\n",119 " <td>2008-08-03T13:19:16Z</td>\n",120 " <td>28</td>\n",121 " <td>Using 'in' to match an attribute of Python obj...</td>\n",122 " <td><p>I don't remember whether I was dreaming or ...</td>\n",123 " </tr>\n",124 " </tbody>\n",125 "</table>\n",126 "</div>"127 ],128 "text/plain": [129 " Id OwnerUserId CreationDate Score \\\n",130 "0 469 147.0 2008-08-02T15:11:16Z 21 \n",131 "1 502 147.0 2008-08-02T17:01:58Z 27 \n",132 "2 535 154.0 2008-08-02T18:43:54Z 40 \n",133 "3 594 116.0 2008-08-03T01:15:08Z 25 \n",134 "4 683 199.0 2008-08-03T13:19:16Z 28 \n",135 "\n",136 " Title \\\n",137 "0 How can I find the full path to a font from it... \n",138 "1 Get a preview JPEG of a PDF on Windows? \n",139 "2 Continuous Integration System for a Python Cod... \n",140 "3 cx_Oracle: How do I iterate over a result set? \n",141 "4 Using 'in' to match an attribute of Python obj... \n",142 "\n",143 " Body \n",144 "0 <p>I am using the Photoshop's javascript API t... \n",145 "1 <p>I have a cross-platform (Python) applicatio... \n",146 "2 <p>I'm starting work on a hobby project with a... \n",147 "3 <p>There are several ways to iterate over a re... \n",148 "4 <p>I don't remember whether I was dreaming or ... "149 ]150 },151 "execution_count": 2,152 "metadata": {},153 "output_type": "execute_result"154 }155 ],156 "source": [157 "questions = pd.read_csv(\"Dataset/Questions.csv\",encoding='latin-1')\n",158 "questions.head()"159 ]160 },161 {162 "cell_type": "code",163 "execution_count": 3,164 "id": "56d3b4b7",165 "metadata": {166 "id": "56d3b4b7",167 "outputId": "16898e46-8a10-4740-c26d-8379a21235b6"168 },169 "outputs": [170 {171 "name": "stdout",172 "output_type": "stream",173 "text": [174 "<class 'pandas.core.frame.DataFrame'>\n",175 "RangeIndex: 607282 entries, 0 to 607281\n",176 "Data columns (total 6 columns):\n",177 " # Column Non-Null Count Dtype \n",178 "--- ------ -------------- ----- \n",179 " 0 Id 607282 non-null int64 \n",180 " 1 OwnerUserId 601070 non-null float64\n",181 " 2 CreationDate 607282 non-null object \n",182 " 3 Score 607282 non-null int64 \n",183 " 4 Title 607282 non-null object \n",184 " 5 Body 607282 non-null object \n",185 "dtypes: float64(1), int64(2), object(3)\n",186 "memory usage: 27.8+ MB\n"187 ]188 }189 ],190 "source": [191 "questions.info()"192 ]193 },194 {195 "cell_type": "code",196 "execution_count": 4,197 "id": "1e5fe457",198 "metadata": {199 "id": "1e5fe457",200 "outputId": "ab28ae61-d924-49d7-daa8-0e708059b6aa"201 },202 "outputs": [203 {204 "data": {205 "text/html": [206 "<div>\n",207 "<style scoped>\n",208 " .dataframe tbody tr th:only-of-type {\n",209 " vertical-align: middle;\n",210 " }\n",211 "\n",212 " .dataframe tbody tr th {\n",213 " vertical-align: top;\n",214 " }\n",215 "\n",216 " .dataframe thead th {\n",217 " text-align: right;\n",218 " }\n",219 "</style>\n",220 "<table border=\"1\" class=\"dataframe\">\n",221 " <thead>\n",222 " <tr style=\"text-align: right;\">\n",223 " <th></th>\n",224 " <th>Id</th>\n",225 " <th>Tag</th>\n",226 " </tr>\n",227 " </thead>\n",228 " <tbody>\n",229 " <tr>\n",230 " <th>0</th>\n",231 " <td>469</td>\n",232 " <td>python</td>\n",233 " </tr>\n",234 " <tr>\n",235 " <th>1</th>\n",236 " <td>469</td>\n",237 " <td>osx</td>\n",238 " </tr>\n",239 " <tr>\n",240 " <th>2</th>\n",241 " <td>469</td>\n",242 " <td>fonts</td>\n",243 " </tr>\n",244 " <tr>\n",245 " <th>3</th>\n",246 " <td>469</td>\n",247 " <td>photoshop</td>\n",248 " </tr>\n",249 " <tr>\n",250 " <th>4</th>\n",251 " <td>502</td>\n",252 " <td>python</td>\n",253 " </tr>\n",254 " </tbody>\n",255 "</table>\n",256 "</div>"257 ],258 "text/plain": [259 " Id Tag\n",260 "0 469 python\n",261 "1 469 osx\n",262 "2 469 fonts\n",263 "3 469 photoshop\n",264 "4 502 python"265 ]266 },267 "execution_count": 4,268 "metadata": {},269 "output_type": "execute_result"270 }271 ],272 "source": [273 "tags = pd.read_csv(\"Dataset/Tags.csv\", dtype={'Tag':str})\n",274 "tags.head()"275 ]276 },277 {278 "cell_type": "code",279 "execution_count": 5,280 "id": "323604e7",281 "metadata": {282 "id": "323604e7",283 "outputId": "eb612a70-558f-4cdf-ef09-01be4d703c59"284 },285 "outputs": [286 {287 "name": "stdout",288 "output_type": "stream",289 "text": [290 "<class 'pandas.core.frame.DataFrame'>\n",291 "RangeIndex: 1885078 entries, 0 to 1885077\n",292 "Data columns (total 2 columns):\n",293 " # Column Dtype \n",294 "--- ------ ----- \n",295 " 0 Id int64 \n",296 " 1 Tag object\n",297 "dtypes: int64(1), object(1)\n",298 "memory usage: 28.8+ MB\n"299 ]300 }301 ],302 "source": [303 "tags.info()"304 ]305 },306 {307 "cell_type": "code",308 "execution_count": 6,309 "id": "ad32fc75",310 "metadata": {311 "id": "ad32fc75"312 },313 "outputs": [],314 "source": [315 "tags['Tag'] = tags['Tag'].astype(str)"316 ]317 },318 {319 "cell_type": "code",320 "execution_count": 7,321 "id": "feb025cc",322 "metadata": {323 "id": "feb025cc"324 },325 "outputs": [],326 "source": [327 "grouped_tags = tags.groupby(\"Id\")['Tag'].apply(lambda tags: ' '.join(tags))"328 ]329 },330 {331 "cell_type": "code",332 "execution_count": 8,333 "id": "b0b0d092",334 "metadata": {335 "id": "b0b0d092",336 "outputId": "d18ca49e-f61c-4503-df7f-6caa18831f7f"337 },338 "outputs": [339 {340 "data": {341 "text/plain": [342 "Id\n",343 "469 python osx fonts photoshop\n",344 "502 python windows image pdf\n",345 "535 python continuous-integration extreme-programming\n",346 "594 python sql database oracle cx-oracle\n",347 "683 python arrays iteration\n",348 "Name: Tag, dtype: object"349 ]350 },351 "execution_count": 8,352 "metadata": {},353 "output_type": "execute_result"354 }355 ],356 "source": [357 "grouped_tags.head()"358 ]359 },360 {361 "cell_type": "code",362 "execution_count": 9,363 "id": "f0362a5b",364 "metadata": {365 "id": "f0362a5b",366 "outputId": "e7d6c82e-0e3e-4b3f-f333-c8f47c307a7e"367 },368 "outputs": [369 {370 "data": {371 "text/html": [372 "<div>\n",373 "<style scoped>\n",374 " .dataframe tbody tr th:only-of-type {\n",375 " vertical-align: middle;\n",376 " }\n",377 "\n",378 " .dataframe tbody tr th {\n",379 " vertical-align: top;\n",380 " }\n",381 "\n",382 " .dataframe thead th {\n",383 " text-align: right;\n",384 " }\n",385 "</style>\n",386 "<table border=\"1\" class=\"dataframe\">\n",387 " <thead>\n",388 " <tr style=\"text-align: right;\">\n",389 " <th></th>\n",390 " <th>Id</th>\n",391 " <th>Tag</th>\n",392 " </tr>\n",393 " </thead>\n",394 " <tbody>\n",395 " <tr>\n",396 " <th>0</th>\n",397 " <td>469</td>\n",398 " <td>python osx fonts photoshop</td>\n",399 " </tr>\n",400 " <tr>\n",401 " <th>1</th>\n",402 " <td>502</td>\n",403 " <td>python windows image pdf</td>\n",404 " </tr>\n",405 " <tr>\n",406 " <th>2</th>\n",407 " <td>535</td>\n",408 " <td>python continuous-integration extreme-programming</td>\n",409 " </tr>\n",410 " <tr>\n",411 " <th>3</th>\n",412 " <td>594</td>\n",413 " <td>python sql database oracle cx-oracle</td>\n",414 " </tr>\n",415 " <tr>\n",416 " <th>4</th>\n",417 " <td>683</td>\n",418 " <td>python arrays iteration</td>\n",419 " </tr>\n",420 " <tr>\n",421 " <th>...</th>\n",422 " <td>...</td>\n",423 " <td>...</td>\n",424 " </tr>\n",425 " <tr>\n",426 " <th>607278</th>\n",427 " <td>40143190</td>\n",428 " <td>python bash multiline</td>\n",429 " </tr>\n",430 " <tr>\n",431 " <th>607279</th>\n",432 " <td>40143228</td>\n",433 " <td>python selenium-webdriver</td>\n",434 " </tr>\n",435 " <tr>\n",436 " <th>607280</th>\n",437 " <td>40143267</td>\n",438 " <td>python django django-rest-framework</td>\n",439 " </tr>\n",440 " <tr>\n",441 " <th>607281</th>\n",442 " <td>40143338</td>\n",443 " <td>python</td>\n",444 " </tr>\n",445 " <tr>\n",446 " <th>607282</th>\n",447 " <td>40143365</td>\n",448 " <td>python numpy theano nnet</td>\n",449 " </tr>\n",450 " </tbody>\n",451 "</table>\n",452 "<p>607283 rows × 2 columns</p>\n",453 "</div>"454 ],455 "text/plain": [456 " Id Tag\n",457 "0 469 python osx fonts photoshop\n",458 "1 502 python windows image pdf\n",459 "2 535 python continuous-integration extreme-programming\n",460 "3 594 python sql database oracle cx-oracle\n",461 "4 683 python arrays iteration\n",462 "... ... ...\n",463 "607278 40143190 python bash multiline\n",464 "607279 40143228 python selenium-webdriver\n",465 "607280 40143267 python django django-rest-framework\n",466 "607281 40143338 python\n",467 "607282 40143365 python numpy theano nnet\n",468 "\n",469 "[607283 rows x 2 columns]"470 ]471 },472 "execution_count": 9,473 "metadata": {},474 "output_type": "execute_result"475 }476 ],477 "source": [478 "grouped_tags.reset_index()"479 ]480 },481 {482 "cell_type": "code",483 "execution_count": 10,484 "id": "81906d51",485 "metadata": {486 "id": "81906d51"487 },488 "outputs": [],489 "source": [490 "grouped_tags_final = pd.DataFrame({'Id':grouped_tags.index, 'Tags':grouped_tags.values})"491 ]492 },493 {494 "cell_type": "code",495 "execution_count": 11,496 "id": "e4675f81",497 "metadata": {498 "id": "e4675f81",499 "outputId": "5c8db22c-1b76-483c-df7c-ae17e8d02703"500 },501 "outputs": [502 {503 "data": {504 "text/html": [505 "<div>\n",506 "<style scoped>\n",507 " .dataframe tbody tr th:only-of-type {\n",508 " vertical-align: middle;\n",509 " }\n",510 "\n",511 " .dataframe tbody tr th {\n",512 " vertical-align: top;\n",513 " }\n",514 "\n",515 " .dataframe thead th {\n",516 " text-align: right;\n",517 " }\n",518 "</style>\n",519 "<table border=\"1\" class=\"dataframe\">\n",520 " <thead>\n",521 " <tr style=\"text-align: right;\">\n",522 " <th></th>\n",523 " <th>Id</th>\n",524 " <th>Tags</th>\n",525 " </tr>\n",526 " </thead>\n",527 " <tbody>\n",528 " <tr>\n",529 " <th>0</th>\n",530 " <td>469</td>\n",531 " <td>python osx fonts photoshop</td>\n",532 " </tr>\n",533 " <tr>\n",534 " <th>1</th>\n",535 " <td>502</td>\n",536 " <td>python windows image pdf</td>\n",537 " </tr>\n",538 " <tr>\n",539 " <th>2</th>\n",540 " <td>535</td>\n",541 " <td>python continuous-integration extreme-programming</td>\n",542 " </tr>\n",543 " <tr>\n",544 " <th>3</th>\n",545 " <td>594</td>\n",546 " <td>python sql database oracle cx-oracle</td>\n",547 " </tr>\n",548 " <tr>\n",549 " <th>4</th>\n",550 " <td>683</td>\n",551 " <td>python arrays iteration</td>\n",552 " </tr>\n",553 " </tbody>\n",554 "</table>\n",555 "</div>"556 ],557 "text/plain": [558 " Id Tags\n",559 "0 469 python osx fonts photoshop\n",560 "1 502 python windows image pdf\n",561 "2 535 python continuous-integration extreme-programming\n",562 "3 594 python sql database oracle cx-oracle\n",563 "4 683 python arrays iteration"564 ]565 },566 "execution_count": 11,567 "metadata": {},568 "output_type": "execute_result"569 }570 ],571 "source": [572 "grouped_tags_final.head(5)"573 ]574 },575 {576 "cell_type": "code",577 "execution_count": 12,578 "id": "6e9338e8",579 "metadata": {580 "id": "6e9338e8"581 },582 "outputs": [],583 "source": [584 "questions.drop(columns=['OwnerUserId', 'CreationDate'], inplace=True)"585 ]586 },587 {588 "cell_type": "code",589 "execution_count": 13,590 "id": "1bb6ed96",591 "metadata": {592 "id": "1bb6ed96",593 "outputId": "bc607267-5c44-4993-a71a-c9ed3e305f7b"594 },595 "outputs": [596 {597 "data": {598 "text/html": [599 "<div>\n",600 "<style scoped>\n",601 " .dataframe tbody tr th:only-of-type {\n",602 " vertical-align: middle;\n",603 " }\n",604 "\n",605 " .dataframe tbody tr th {\n",606 " vertical-align: top;\n",607 " }\n",608 "\n",609 " .dataframe thead th {\n",610 " text-align: right;\n",611 " }\n",612 "</style>\n",613 "<table border=\"1\" class=\"dataframe\">\n",614 " <thead>\n",615 " <tr style=\"text-align: right;\">\n",616 " <th></th>\n",617 " <th>Id</th>\n",618 " <th>Score</th>\n",619 " <th>Title</th>\n",620 " <th>Body</th>\n",621 " <th>Tags</th>\n",622 " </tr>\n",623 " </thead>\n",624 " <tbody>\n",625 " <tr>\n",626 " <th>0</th>\n",627 " <td>469</td>\n",628 " <td>21</td>\n",629 " <td>How can I find the full path to a font from it...</td>\n",630 " <td><p>I am using the Photoshop's javascript API t...</td>\n",631 " <td>python osx fonts photoshop</td>\n",632 " </tr>\n",633 " <tr>\n",634 " <th>1</th>\n",635 " <td>502</td>\n",636 " <td>27</td>\n",637 " <td>Get a preview JPEG of a PDF on Windows?</td>\n",638 " <td><p>I have a cross-platform (Python) applicatio...</td>\n",639 " <td>python windows image pdf</td>\n",640 " </tr>\n",641 " <tr>\n",642 " <th>2</th>\n",643 " <td>535</td>\n",644 " <td>40</td>\n",645 " <td>Continuous Integration System for a Python Cod...</td>\n",646 " <td><p>I'm starting work on a hobby project with a...</td>\n",647 " <td>python continuous-integration extreme-programming</td>\n",648 " </tr>\n",649 " <tr>\n",650 " <th>3</th>\n",651 " <td>594</td>\n",652 " <td>25</td>\n",653 " <td>cx_Oracle: How do I iterate over a result set?</td>\n",654 " <td><p>There are several ways to iterate over a re...</td>\n",655 " <td>python sql database oracle cx-oracle</td>\n",656 " </tr>\n",657 " <tr>\n",658 " <th>4</th>\n",659 " <td>683</td>\n",660 " <td>28</td>\n",661 " <td>Using 'in' to match an attribute of Python obj...</td>\n",662 " <td><p>I don't remember whether I was dreaming or ...</td>\n",663 " <td>python arrays iteration</td>\n",664 " </tr>\n",665 " </tbody>\n",666 "</table>\n",667 "</div>"668 ],669 "text/plain": [670 " Id Score Title \\\n",671 "0 469 21 How can I find the full path to a font from it... \n",672 "1 502 27 Get a preview JPEG of a PDF on Windows? \n",673 "2 535 40 Continuous Integration System for a Python Cod... \n",674 "3 594 25 cx_Oracle: How do I iterate over a result set? \n",675 "4 683 28 Using 'in' to match an attribute of Python obj... \n",676 "\n",677 " Body \\\n",678 "0 <p>I am using the Photoshop's javascript API t... \n",679 "1 <p>I have a cross-platform (Python) applicatio... \n",680 "2 <p>I'm starting work on a hobby project with a... \n",681 "3 <p>There are several ways to iterate over a re... \n",682 "4 <p>I don't remember whether I was dreaming or ... \n",683 "\n",684 " Tags \n",685 "0 python osx fonts photoshop \n",686 "1 python windows image pdf \n",687 "2 python continuous-integration extreme-programming \n",688 "3 python sql database oracle cx-oracle \n",689 "4 python arrays iteration "690 ]691 },692 "execution_count": 13,693 "metadata": {},694 "output_type": "execute_result"695 }696 ],697 "source": [698 "questions = questions.merge(grouped_tags_final, on='Id')\n",699 "questions.head()"700 ]701 },702 {703 "cell_type": "code",704 "execution_count": 14,705 "id": "85013340",706 "metadata": {707 "id": "85013340"708 },709 "outputs": [],710 "source": [711 "filtered_questions = questions[questions['Score']>5]"712 ]713 },714 {715 "cell_type": "code",716 "execution_count": 15,717 "id": "b198e5ec",718 "metadata": {719 "id": "b198e5ec",720 "outputId": "d226609b-cdf6-4195-e483-619229de5781"721 },722 "outputs": [723 {724 "name": "stderr",725 "output_type": "stream",726 "text": [727 "C:\\Users\\win 10\\AppData\\Local\\Temp\\ipykernel_16056\\3611255864.py:1: SettingWithCopyWarning: \n",728 "A value is trying to be set on a copy of a slice from a DataFrame\n",729 "\n",730 "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",731 " filtered_questions.drop(columns = ['Id', 'Score'], inplace=True)\n"732 ]733 }734 ],735 "source": [736 "filtered_questions.drop(columns = ['Id', 'Score'], inplace=True)"737 ]738 },739 {740 "cell_type": "code",741 "execution_count": 16,742 "id": "8b6903da",743 "metadata": {744 "id": "8b6903da",745 "outputId": "34f06ed9-b46f-4de5-a145-c6613ef38c19"746 },747 "outputs": [748 {749 "data": {750 "text/html": [751 "<div>\n",752 "<style scoped>\n",753 " .dataframe tbody tr th:only-of-type {\n",754 " vertical-align: middle;\n",755 " }\n",756 "\n",757 " .dataframe tbody tr th {\n",758 " vertical-align: top;\n",759 " }\n",760 "\n",761 " .dataframe thead th {\n",762 " text-align: right;\n",763 " }\n",764 "</style>\n",765 "<table border=\"1\" class=\"dataframe\">\n",766 " <thead>\n",767 " <tr style=\"text-align: right;\">\n",768 " <th></th>\n",769 " <th>Title</th>\n",770 " <th>Body</th>\n",771 " <th>Tags</th>\n",772 " </tr>\n",773 " </thead>\n",774 " <tbody>\n",775 " <tr>\n",776 " <th>0</th>\n",777 " <td>How can I find the full path to a font from it...</td>\n",778 " <td><p>I am using the Photoshop's javascript API t...</td>\n",779 " <td>python osx fonts photoshop</td>\n",780 " </tr>\n",781 " <tr>\n",782 " <th>1</th>\n",783 " <td>Get a preview JPEG of a PDF on Windows?</td>\n",784 " <td><p>I have a cross-platform (Python) applicatio...</td>\n",785 " <td>python windows image pdf</td>\n",786 " </tr>\n",787 " <tr>\n",788 " <th>2</th>\n",789 " <td>Continuous Integration System for a Python Cod...</td>\n",790 " <td><p>I'm starting work on a hobby project with a...</td>\n",791 " <td>python continuous-integration extreme-programming</td>\n",792 " </tr>\n",793 " <tr>\n",794 " <th>3</th>\n",795 " <td>cx_Oracle: How do I iterate over a result set?</td>\n",796 " <td><p>There are several ways to iterate over a re...</td>\n",797 " <td>python sql database oracle cx-oracle</td>\n",798 " </tr>\n",799 " <tr>\n",800 " <th>4</th>\n",801 " <td>Using 'in' to match an attribute of Python obj...</td>\n",802 " <td><p>I don't remember whether I was dreaming or ...</td>\n",803 " <td>python arrays iteration</td>\n",804 " </tr>\n",805 " </tbody>\n",806 "</table>\n",807 "</div>"808 ],809 "text/plain": [810 " Title \\\n",811 "0 How can I find the full path to a font from it... \n",812 "1 Get a preview JPEG of a PDF on Windows? \n",813 "2 Continuous Integration System for a Python Cod... \n",814 "3 cx_Oracle: How do I iterate over a result set? \n",815 "4 Using 'in' to match an attribute of Python obj... \n",816 "\n",817 " Body \\\n",818 "0 <p>I am using the Photoshop's javascript API t... \n",819 "1 <p>I have a cross-platform (Python) applicatio... \n",820 "2 <p>I'm starting work on a hobby project with a... \n",821 "3 <p>There are several ways to iterate over a re... \n",822 "4 <p>I don't remember whether I was dreaming or ... \n",823 "\n",824 " Tags \n",825 "0 python osx fonts photoshop \n",826 "1 python windows image pdf \n",827 "2 python continuous-integration extreme-programming \n",828 "3 python sql database oracle cx-oracle \n",829 "4 python arrays iteration "830 ]831 },832 "execution_count": 16,833 "metadata": {},834 "output_type": "execute_result"835 }836 ],837 "source": [838 "filtered_questions.head()"839 ]840 },841 {842 "cell_type": "code",843 "execution_count": 17,844 "id": "eaf22ab1",845 "metadata": {846 "id": "eaf22ab1",847 "outputId": "f58ed740-9b41-4cca-ea18-9ecf3e69bad1"848 },849 "outputs": [850 {851 "name": "stdout",852 "output_type": "stream",853 "text": [854 "134101\n",855 "6384\n"856 ]857 },858 {859 "name": "stderr",860 "output_type": "stream",861 "text": [862 "C:\\Users\\win 10\\AppData\\Local\\Temp\\ipykernel_16056\\2466576371.py:1: SettingWithCopyWarning: \n",863 "A value is trying to be set on a copy of a slice from a DataFrame.\n",864 "Try using .loc[row_indexer,col_indexer] = value instead\n",865 "\n",866 "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",867 " filtered_questions['Tags'] = filtered_questions['Tags'].apply(lambda x: x.split())\n"868 ]869 }870 ],871 "source": [872 "filtered_questions['Tags'] = filtered_questions['Tags'].apply(lambda x: x.split())\n",873 "\n",874 "all_tags = [item for sublist in filtered_questions['Tags'].values for item in sublist]\n",875 "\n",876 "print(len(all_tags))\n",877 "\n",878 "my_set = set(all_tags)\n",879 "unique_tags = list(my_set)\n",880 "print(len(unique_tags))"881 ]882 },883 {884 "cell_type": "code",885 "execution_count": 18,886 "id": "1cca4d0e",887 "metadata": {888 "id": "1cca4d0e"889 },890 "outputs": [],891 "source": [892 "flat_list = [item for sublist in filtered_questions['Tags'].values for item in sublist]\n",893 "\n",894 "keywords = nltk.FreqDist(flat_list)\n",895 "\n",896 "keywords = nltk.FreqDist(keywords)\n",897 "\n",898 "frequencies_words = keywords.most_common(100)\n",899 "tags_features = [word[0] for word in frequencies_words]"900 ]901 },902 {903 "cell_type": "code",904 "execution_count": 19,905 "id": "949ae3b9",906 "metadata": {907 "id": "949ae3b9",908 "outputId": "599254f3-ae0a-46db-9bf3-426de74525a5"909 },910 "outputs": [911 {912 "data": {913 "text/plain": [914 "['python',\n",915 " 'django',\n",916 " 'numpy',\n",917 " 'matplotlib',\n",918 " 'pandas',\n",919 " 'python-3.x',\n",920 " 'python-2.7',\n",921 " 'list',\n",922 " 'string',\n",923 " 'flask',\n",924 " 'dictionary',\n",925 " 'scipy',\n",926 " 'regex',\n",927 " 'performance',\n",928 " 'google-app-engine',\n",929 " 'sqlalchemy',\n",930 " 'arrays',\n",931 " 'pip',\n",932 " 'windows',\n",933 " 'algorithm',\n",934 " 'unit-testing',\n",935 " 'linux',\n",936 " 'unicode',\n",937 " 'multithreading',\n",938 " 'django-models',\n",939 " 'osx',\n",940 " 'json',\n",941 " 'datetime',\n",942 " 'c++',\n",943 " 'mysql',\n",944 " 'virtualenv',\n",945 " 'multiprocessing',\n",946 " 'subprocess',\n",947 " 'class',\n",948 " 'java',\n",949 " 'c',\n",950 " 'ipython',\n",951 " 'file',\n",952 " 'csv',\n",953 " 'logging',\n",954 " 'exception',\n",955 " 'opencv',\n",956 " 'sorting',\n",957 " 'selenium',\n",958 " 'tkinter',\n",959 " 'python-imaging-library',\n",960 " 'javascript',\n",961 " 'module',\n",962 " 'celery',\n",963 " 'function',\n",964 " 'import',\n",965 " 'parsing',\n",966 " 'cython',\n",967 " 'scikit-learn',\n",968 " 'math',\n",969 " 'xml',\n",970 " 'dataframe',\n",971 " 'html',\n",972 " 'pycharm',\n",973 " 'machine-learning',\n",974 " 'beautifulsoup',\n",975 " 'plot',\n",976 " 'list-comprehension',\n",977 " 'generator',\n",978 " 'django-admin',\n",979 " 'pyqt',\n",980 " 'postgresql',\n",981 " 'debugging',\n",982 " 'nltk',\n",983 " 'tuples',\n",984 " 'file-io',\n",985 " 'urllib2',\n",986 " 'oop',\n",987 " 'setuptools',\n",988 " 'sockets',\n",989 " 'decorator',\n",990 " 'image-processing',\n",991 " 'image',\n",992 " 'ruby',\n",993 " 'scrapy',\n",994 " 'sql',\n",995 " 'ubuntu',\n",996 " 'database',\n",997 " 'argparse',\n",998 " 'http',\n",999 " 'random',\n",1000 " 'optimization',\n",1001 " 'python-2.x',\n",1002 " 'pickle',\n",1003 " 'python-import',\n",1004 " 'python-requests',\n",1005 " 'inheritance',\n",1006 " 'nlp',\n",1007 " 'lxml',\n",1008 " 'twisted',\n",1009 " 'jinja2',\n",1010 " 'php',\n",1011 " 'django-templates',\n",1012 " 'mongodb',\n",1013 " 'sqlite']"1014 ]1015 },1016 "execution_count": 19,1017 "metadata": {},1018 "output_type": "execute_result"1019 }1020 ],1021 "source": [1022 "tags_features"1023 ]1024 },1025 {1026 "cell_type": "code",1027 "execution_count": 20,1028 "id": "fce64d64",1029 "metadata": {1030 "id": "fce64d64"1031 },1032 "outputs": [],1033 "source": [1034 "def most_common(tags):\n",1035 " tags_filtered = []\n",1036 " for i in range(0, len(tags)):\n",1037 " if tags[i] in tags_features:\n",1038 " tags_filtered.append(tags[i])\n",1039 " return tags_filtered"1040 ]1041 },1042 {1043 "cell_type": "code",1044 "execution_count": 21,1045 "id": "5d487c7c",1046 "metadata": {1047 "id": "5d487c7c",1048 "outputId": "bc916bbb-01e2-4ffc-88e3-6f6c2847404c"1049 },1050 "outputs": [1051 {1052 "name": "stderr",1053 "output_type": "stream",1054 "text": [1055 "C:\\Users\\win 10\\AppData\\Local\\Temp\\ipykernel_16056\\2458735045.py:1: SettingWithCopyWarning: \n",1056 "A value is trying to be set on a copy of a slice from a DataFrame.\n",1057 "Try using .loc[row_indexer,col_indexer] = value instead\n",1058 "\n",1059 "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",1060 " filtered_questions['Tags'] = filtered_questions['Tags'].apply(lambda x: most_common(x))\n",1061 "C:\\Users\\win 10\\AppData\\Local\\Temp\\ipykernel_16056\\2458735045.py:2: SettingWithCopyWarning: \n",1062 "A value is trying to be set on a copy of a slice from a DataFrame.\n",1063 "Try using .loc[row_indexer,col_indexer] = value instead\n",1064 "\n",1065 "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",1066 " filtered_questions['Tags'] = filtered_questions['Tags'].apply(lambda x: x if len(x)>0 else None)\n"1067 ]1068 }1069 ],1070 "source": [1071 "filtered_questions['Tags'] = filtered_questions['Tags'].apply(lambda x: most_common(x))\n",1072 "filtered_questions['Tags'] = filtered_questions['Tags'].apply(lambda x: x if len(x)>0 else None)"1073 ]1074 },1075 {1076 "cell_type": "code",1077 "execution_count": 22,1078 "id": "2f042823",1079 "metadata": {1080 "id": "2f042823",1081 "outputId": "c50d57e8-a9a5-4541-e874-475439ba22ad"1082 },1083 "outputs": [1084 {1085 "data": {1086 "text/html": [1087 "<div>\n",1088 "<style scoped>\n",1089 " .dataframe tbody tr th:only-of-type {\n",1090 " vertical-align: middle;\n",1091 " }\n",1092 "\n",1093 " .dataframe tbody tr th {\n",1094 " vertical-align: top;\n",1095 " }\n",1096 "\n",1097 " .dataframe thead th {\n",1098 " text-align: right;\n",1099 " }\n",1100 "</style>\n",1101 "<table border=\"1\" class=\"dataframe\">\n",1102 " <thead>\n",1103 " <tr style=\"text-align: right;\">\n",1104 " <th></th>\n",1105 " <th>Title</th>\n",1106 " <th>Body</th>\n",1107 " <th>Tags</th>\n",1108 " </tr>\n",1109 " </thead>\n",1110 " <tbody>\n",1111 " <tr>\n",1112 " <th>0</th>\n",1113 " <td>How can I find the full path to a font from it...</td>\n",1114 " <td><p>I am using the Photoshop's javascript API t...</td>\n",1115 " <td>[python, osx]</td>\n",1116 " </tr>\n",1117 " <tr>\n",1118 " <th>1</th>\n",1119 " <td>Get a preview JPEG of a PDF on Windows?</td>\n",1120 " <td><p>I have a cross-platform (Python) applicatio...</td>\n",1121 " <td>[python, windows, image]</td>\n",1122 " </tr>\n",1123 " <tr>\n",1124 " <th>2</th>\n",1125 " <td>Continuous Integration System for a Python Cod...</td>\n",1126 " <td><p>I'm starting work on a hobby project with a...</td>\n",1127 " <td>[python]</td>\n",1128 " </tr>\n",1129 " <tr>\n",1130 " <th>3</th>\n",1131 " <td>cx_Oracle: How do I iterate over a result set?</td>\n",1132 " <td><p>There are several ways to iterate over a re...</td>\n",1133 " <td>[python, sql, database]</td>\n",1134 " </tr>\n",1135 " <tr>\n",1136 " <th>4</th>\n",1137 " <td>Using 'in' to match an attribute of Python obj...</td>\n",1138 " <td><p>I don't remember whether I was dreaming or ...</td>\n",1139 " <td>[python, arrays]</td>\n",1140 " </tr>\n",1141 " </tbody>\n",1142 "</table>\n",1143 "</div>"1144 ],1145 "text/plain": [1146 " Title \\\n",1147 "0 How can I find the full path to a font from it... \n",1148 "1 Get a preview JPEG of a PDF on Windows? \n",1149 "2 Continuous Integration System for a Python Cod... \n",1150 "3 cx_Oracle: How do I iterate over a result set? \n",1151 "4 Using 'in' to match an attribute of Python obj... \n",1152 "\n",1153 " Body Tags \n",1154 "0 <p>I am using the Photoshop's javascript API t... [python, osx] \n",1155 "1 <p>I have a cross-platform (Python) applicatio... [python, windows, image] \n",1156 "2 <p>I'm starting work on a hobby project with a... [python] \n",1157 "3 <p>There are several ways to iterate over a re... [python, sql, database] \n",1158 "4 <p>I don't remember whether I was dreaming or ... [python, arrays] "1159 ]1160 },1161 "execution_count": 22,1162 "metadata": {},1163 "output_type": "execute_result"1164 }1165 ],1166 "source": [1167 "filtered_questions.head()"1168 ]1169 },1170 {1171 "cell_type": "code",1172 "execution_count": 23,1173 "id": "55f91d6c",1174 "metadata": {1175 "id": "55f91d6c",1176 "outputId": "b3947c73-1859-4780-a5d8-7913f83ce5fa"1177 },1178 "outputs": [1179 {1180 "data": {1181 "text/plain": [1182 "(42420, 3)"1183 ]1184 },1185 "execution_count": 23,1186 "metadata": {},1187 "output_type": "execute_result"1188 }1189 ],1190 "source": [1191 "filtered_questions.shape"1192 ]1193 },1194 {1195 "cell_type": "code",1196 "execution_count": 24,1197 "id": "8b6729c9",1198 "metadata": {1199 "id": "8b6729c9",1200 "outputId": "779ea659-5200-4e09-9855-f9ef5875a191"