arminch83/Predicting-programmers-salaries
0
1{2 "cells": [3 {4 "cell_type": "markdown",5 "id": "bd3b8298-0dcf-49a4-91e9-2f23968436a8",6 "metadata": {},7 "source": [8 "# Import"9 ]10 },11 {12 "cell_type": "code",13 "execution_count": 89,14 "id": "b24a2d26-cac1-4219-bd8e-82e1209fc652",15 "metadata": {},16 "outputs": [],17 "source": [18 "import numpy as np\n",19 "import pandas as pd\n",20 "\n",21 "import matplotlib.pyplot as plt\n",22 "import plotly.express as px\n",23 "\n",24 "import plotly.io as pio\n",25 "pio.renderers.default = \"notebook\"\n",26 "\n",27 "from sklearn.preprocessing import OneHotEncoder\n",28 "from sklearn.pipeline import Pipeline\n",29 "from sklearn.compose import ColumnTransformer\n",30 "from xgboost import XGBRegressor\n",31 "from sklearn.model_selection import train_test_split\n",32 "from sklearn.metrics import r2_score,mean_squared_error, mean_absolute_error\n",33 "from sklearn.model_selection import GridSearchCV\n",34 "from sklearn.model_selection import cross_val_score\n",35 "from sklearn.preprocessing import StandardScaler, PolynomialFeatures\n",36 "from sklearn.feature_selection import SelectKBest, f_regression\n",37 "\n",38 "from sklearn.ensemble import RandomForestRegressor, StackingRegressor\n",39 "from sklearn.neighbors import KNeighborsRegressor\n",40 "from sklearn.linear_model import LinearRegression\n",41 "from sklearn.linear_model import Ridge\n",42 "\n",43 "from sklearn.neural_network import MLPRegressor\n",44 "\n",45 "from catboost import CatBoostRegressor\n",46 "from catboost import Pool\n",47 "from catboost.utils import get_roc_curve, create_cd\n",48 "from catboost import cv\n",49 "\n",50 "import joblib\n",51 "\n",52 "import warnings\n",53 "warnings. filterwarnings('ignore')"54 ]55 },56 {57 "cell_type": "markdown",58 "id": "67e2af54-5350-4cc1-a0cf-f34e4d803479",59 "metadata": {},60 "source": [61 "# Import data"62 ]63 },64 {65 "cell_type": "code",66 "execution_count": 3,67 "id": "8ff5aa5b-ef56-4213-80da-19771abcff00",68 "metadata": {},69 "outputs": [],70 "source": [71 "data = pd.read_csv('F:/survey_results_public.csv')"72 ]73 },74 {75 "cell_type": "code",76 "execution_count": 4,77 "id": "409cb9e3-9d63-48b1-9a46-6b03a7eadacc",78 "metadata": {79 "scrolled": true80 },81 "outputs": [82 {83 "data": {84 "text/html": [85 "<div>\n",86 "<style scoped>\n",87 " .dataframe tbody tr th:only-of-type {\n",88 " vertical-align: middle;\n",89 " }\n",90 "\n",91 " .dataframe tbody tr th {\n",92 " vertical-align: top;\n",93 " }\n",94 "\n",95 " .dataframe thead th {\n",96 " text-align: right;\n",97 " }\n",98 "</style>\n",99 "<table border=\"1\" class=\"dataframe\">\n",100 " <thead>\n",101 " <tr style=\"text-align: right;\">\n",102 " <th></th>\n",103 " <th>ResponseId</th>\n",104 " <th>MainBranch</th>\n",105 " <th>Employment</th>\n",106 " <th>RemoteWork</th>\n",107 " <th>CodingActivities</th>\n",108 " <th>EdLevel</th>\n",109 " <th>LearnCode</th>\n",110 " <th>LearnCodeOnline</th>\n",111 " <th>LearnCodeCoursesCert</th>\n",112 " <th>YearsCode</th>\n",113 " <th>...</th>\n",114 " <th>TimeSearching</th>\n",115 " <th>TimeAnswering</th>\n",116 " <th>Onboarding</th>\n",117 " <th>ProfessionalTech</th>\n",118 " <th>TrueFalse_1</th>\n",119 " <th>TrueFalse_2</th>\n",120 " <th>TrueFalse_3</th>\n",121 " <th>SurveyLength</th>\n",122 " <th>SurveyEase</th>\n",123 " <th>ConvertedCompYearly</th>\n",124 " </tr>\n",125 " </thead>\n",126 " <tbody>\n",127 " <tr>\n",128 " <th>0</th>\n",129 " <td>1</td>\n",130 " <td>None of these</td>\n",131 " <td>NaN</td>\n",132 " <td>NaN</td>\n",133 " <td>NaN</td>\n",134 " <td>NaN</td>\n",135 " <td>NaN</td>\n",136 " <td>NaN</td>\n",137 " <td>NaN</td>\n",138 " <td>NaN</td>\n",139 " <td>...</td>\n",140 " <td>NaN</td>\n",141 " <td>NaN</td>\n",142 " <td>NaN</td>\n",143 " <td>NaN</td>\n",144 " <td>NaN</td>\n",145 " <td>NaN</td>\n",146 " <td>NaN</td>\n",147 " <td>NaN</td>\n",148 " <td>NaN</td>\n",149 " <td>NaN</td>\n",150 " </tr>\n",151 " <tr>\n",152 " <th>1</th>\n",153 " <td>2</td>\n",154 " <td>I am a developer by profession</td>\n",155 " <td>Employed, full-time</td>\n",156 " <td>Fully remote</td>\n",157 " <td>Hobby;Contribute to open-source projects</td>\n",158 " <td>NaN</td>\n",159 " <td>NaN</td>\n",160 " <td>NaN</td>\n",161 " <td>NaN</td>\n",162 " <td>NaN</td>\n",163 " <td>...</td>\n",164 " <td>NaN</td>\n",165 " <td>NaN</td>\n",166 " <td>NaN</td>\n",167 " <td>NaN</td>\n",168 " <td>NaN</td>\n",169 " <td>NaN</td>\n",170 " <td>NaN</td>\n",171 " <td>Too long</td>\n",172 " <td>Difficult</td>\n",173 " <td>NaN</td>\n",174 " </tr>\n",175 " <tr>\n",176 " <th>2</th>\n",177 " <td>3</td>\n",178 " <td>I am not primarily a developer, but I write co...</td>\n",179 " <td>Employed, full-time</td>\n",180 " <td>Hybrid (some remote, some in-person)</td>\n",181 " <td>Hobby</td>\n",182 " <td>Master’s degree (M.A., M.S., M.Eng., MBA, etc.)</td>\n",183 " <td>Books / Physical media;Friend or family member...</td>\n",184 " <td>Technical documentation;Blogs;Programming Game...</td>\n",185 " <td>NaN</td>\n",186 " <td>14</td>\n",187 " <td>...</td>\n",188 " <td>NaN</td>\n",189 " <td>NaN</td>\n",190 " <td>NaN</td>\n",191 " <td>NaN</td>\n",192 " <td>NaN</td>\n",193 " <td>NaN</td>\n",194 " <td>NaN</td>\n",195 " <td>Appropriate in length</td>\n",196 " <td>Neither easy nor difficult</td>\n",197 " <td>40205.0</td>\n",198 " </tr>\n",199 " <tr>\n",200 " <th>3</th>\n",201 " <td>4</td>\n",202 " <td>I am a developer by profession</td>\n",203 " <td>Employed, full-time</td>\n",204 " <td>Fully remote</td>\n",205 " <td>I don’t code outside of work</td>\n",206 " <td>Bachelor’s degree (B.A., B.S., B.Eng., etc.)</td>\n",207 " <td>Books / Physical media;School (i.e., Universit...</td>\n",208 " <td>NaN</td>\n",209 " <td>NaN</td>\n",210 " <td>20</td>\n",211 " <td>...</td>\n",212 " <td>NaN</td>\n",213 " <td>NaN</td>\n",214 " <td>NaN</td>\n",215 " <td>NaN</td>\n",216 " <td>NaN</td>\n",217 " <td>NaN</td>\n",218 " <td>NaN</td>\n",219 " <td>Appropriate in length</td>\n",220 " <td>Easy</td>\n",221 " <td>215232.0</td>\n",222 " </tr>\n",223 " <tr>\n",224 " <th>4</th>\n",225 " <td>5</td>\n",226 " <td>I am a developer by profession</td>\n",227 " <td>Employed, full-time</td>\n",228 " <td>Hybrid (some remote, some in-person)</td>\n",229 " <td>Hobby</td>\n",230 " <td>Bachelor’s degree (B.A., B.S., B.Eng., etc.)</td>\n",231 " <td>Other online resources (e.g., videos, blogs, f...</td>\n",232 " <td>Technical documentation;Blogs;Stack Overflow;O...</td>\n",233 " <td>NaN</td>\n",234 " <td>8</td>\n",235 " <td>...</td>\n",236 " <td>NaN</td>\n",237 " <td>NaN</td>\n",238 " <td>NaN</td>\n",239 " <td>NaN</td>\n",240 " <td>NaN</td>\n",241 " <td>NaN</td>\n",242 " <td>NaN</td>\n",243 " <td>Too long</td>\n",244 " <td>Easy</td>\n",245 " <td>NaN</td>\n",246 " </tr>\n",247 " </tbody>\n",248 "</table>\n",249 "<p>5 rows × 79 columns</p>\n",250 "</div>"251 ],252 "text/plain": [253 " ResponseId MainBranch \\\n",254 "0 1 None of these \n",255 "1 2 I am a developer by profession \n",256 "2 3 I am not primarily a developer, but I write co... \n",257 "3 4 I am a developer by profession \n",258 "4 5 I am a developer by profession \n",259 "\n",260 " Employment RemoteWork \\\n",261 "0 NaN NaN \n",262 "1 Employed, full-time Fully remote \n",263 "2 Employed, full-time Hybrid (some remote, some in-person) \n",264 "3 Employed, full-time Fully remote \n",265 "4 Employed, full-time Hybrid (some remote, some in-person) \n",266 "\n",267 " CodingActivities \\\n",268 "0 NaN \n",269 "1 Hobby;Contribute to open-source projects \n",270 "2 Hobby \n",271 "3 I don’t code outside of work \n",272 "4 Hobby \n",273 "\n",274 " EdLevel \\\n",275 "0 NaN \n",276 "1 NaN \n",277 "2 Master’s degree (M.A., M.S., M.Eng., MBA, etc.) \n",278 "3 Bachelor’s degree (B.A., B.S., B.Eng., etc.) \n",279 "4 Bachelor’s degree (B.A., B.S., B.Eng., etc.) \n",280 "\n",281 " LearnCode \\\n",282 "0 NaN \n",283 "1 NaN \n",284 "2 Books / Physical media;Friend or family member... \n",285 "3 Books / Physical media;School (i.e., Universit... \n",286 "4 Other online resources (e.g., videos, blogs, f... \n",287 "\n",288 " LearnCodeOnline LearnCodeCoursesCert \\\n",289 "0 NaN NaN \n",290 "1 NaN NaN \n",291 "2 Technical documentation;Blogs;Programming Game... NaN \n",292 "3 NaN NaN \n",293 "4 Technical documentation;Blogs;Stack Overflow;O... NaN \n",294 "\n",295 " YearsCode ... TimeSearching TimeAnswering Onboarding ProfessionalTech \\\n",296 "0 NaN ... NaN NaN NaN NaN \n",297 "1 NaN ... NaN NaN NaN NaN \n",298 "2 14 ... NaN NaN NaN NaN \n",299 "3 20 ... NaN NaN NaN NaN \n",300 "4 8 ... NaN NaN NaN NaN \n",301 "\n",302 " TrueFalse_1 TrueFalse_2 TrueFalse_3 SurveyLength \\\n",303 "0 NaN NaN NaN NaN \n",304 "1 NaN NaN NaN Too long \n",305 "2 NaN NaN NaN Appropriate in length \n",306 "3 NaN NaN NaN Appropriate in length \n",307 "4 NaN NaN NaN Too long \n",308 "\n",309 " SurveyEase ConvertedCompYearly \n",310 "0 NaN NaN \n",311 "1 Difficult NaN \n",312 "2 Neither easy nor difficult 40205.0 \n",313 "3 Easy 215232.0 \n",314 "4 Easy NaN \n",315 "\n",316 "[5 rows x 79 columns]"317 ]318 },319 "execution_count": 4,320 "metadata": {},321 "output_type": "execute_result"322 }323 ],324 "source": [325 "data.head()"326 ]327 },328 {329 "cell_type": "code",330 "execution_count": 5,331 "id": "a7e25d22-d4dc-40a0-a168-943b51762c13",332 "metadata": {},333 "outputs": [334 {335 "data": {336 "text/plain": [337 "(73268, 79)"338 ]339 },340 "execution_count": 5,341 "metadata": {},342 "output_type": "execute_result"343 }344 ],345 "source": [346 "data.shape"347 ]348 },349 {350 "cell_type": "code",351 "execution_count": 6,352 "id": "70738b2b-056c-4fa5-8fc0-21d6aafd7668",353 "metadata": {354 "scrolled": true355 },356 "outputs": [357 {358 "name": "stdout",359 "output_type": "stream",360 "text": [361 "<class 'pandas.core.frame.DataFrame'>\n",362 "RangeIndex: 73268 entries, 0 to 73267\n",363 "Data columns (total 79 columns):\n",364 " # Column Non-Null Count Dtype \n",365 "--- ------ -------------- ----- \n",366 " 0 ResponseId 73268 non-null int64 \n",367 " 1 MainBranch 73268 non-null object \n",368 " 2 Employment 71709 non-null object \n",369 " 3 RemoteWork 58958 non-null object \n",370 " 4 CodingActivities 58899 non-null object \n",371 " 5 EdLevel 71571 non-null object \n",372 " 6 LearnCode 71580 non-null object \n",373 " 7 LearnCodeOnline 50685 non-null object \n",374 " 8 LearnCodeCoursesCert 29389 non-null object \n",375 " 9 YearsCode 71331 non-null object \n",376 " 10 YearsCodePro 51833 non-null object \n",377 " 11 DevType 61302 non-null object \n",378 " 12 OrgSize 51039 non-null object \n",379 " 13 PurchaseInfluence 50969 non-null object \n",380 " 14 BuyNewTool 67963 non-null object \n",381 " 15 Country 71771 non-null object \n",382 " 16 Currency 51264 non-null object \n",383 " 17 CompTotal 38422 non-null float64\n",384 " 18 CompFreq 44425 non-null object \n",385 " 19 LanguageHaveWorkedWith 70975 non-null object \n",386 " 20 LanguageWantToWorkWith 67027 non-null object \n",387 " 21 DatabaseHaveWorkedWith 60121 non-null object \n",388 " 22 DatabaseWantToWorkWith 51014 non-null object \n",389 " 23 PlatformHaveWorkedWith 49924 non-null object \n",390 " 24 PlatformWantToWorkWith 40415 non-null object \n",391 " 25 WebframeHaveWorkedWith 53544 non-null object \n",392 " 26 WebframeWantToWorkWith 46122 non-null object \n",393 " 27 MiscTechHaveWorkedWith 44992 non-null object \n",394 " 28 MiscTechWantToWorkWith 36810 non-null object \n",395 " 29 ToolsTechHaveWorkedWith 54171 non-null object \n",396 " 30 ToolsTechWantToWorkWith 46566 non-null object \n",397 " 31 NEWCollabToolsHaveWorkedWith 70347 non-null object \n",398 " 32 NEWCollabToolsWantToWorkWith 64108 non-null object \n",399 " 33 OpSysProfessional use 65503 non-null object \n",400 " 34 OpSysPersonal use 70963 non-null object \n",401 " 35 VersionControlSystem 71379 non-null object \n",402 " 36 VCInteraction 68156 non-null object \n",403 " 37 VCHostingPersonal use 0 non-null float64\n",404 " 38 VCHostingProfessional use 0 non-null float64\n",405 " 39 OfficeStackAsyncHaveWorkedWith 46223 non-null object \n",406 " 40 OfficeStackAsyncWantToWorkWith 32072 non-null object \n",407 " 41 OfficeStackSyncHaveWorkedWith 62128 non-null object \n",408 " 42 OfficeStackSyncWantToWorkWith 47688 non-null object \n",409 " 43 Blockchain 71071 non-null object \n",410 " 44 NEWSOSites 71365 non-null object \n",411 " 45 SOVisitFreq 70961 non-null object \n",412 " 46 SOAccount 71572 non-null object \n",413 " 47 SOPartFreq 58229 non-null object \n",414 " 48 SOComm 71408 non-null object \n",415 " 49 Age 70946 non-null object \n",416 " 50 Gender 70853 non-null object \n",417 " 51 Trans 70315 non-null object \n",418 " 52 Sexuality 66565 non-null object \n",419 " 53 Ethnicity 69474 non-null object \n",420 " 54 Accessibility 67244 non-null object \n",421 " 55 MentalHealth 66447 non-null object \n",422 " 56 TBranch 52670 non-null object \n",423 " 57 ICorPM 36283 non-null object \n",424 " 58 WorkExp 36769 non-null float64\n",425 " 59 Knowledge_1 35804 non-null object \n",426 " 60 Knowledge_2 34973 non-null object \n",427 " 61 Knowledge_3 35133 non-null object \n",428 " 62 Knowledge_4 35097 non-null object \n",429 " 63 Knowledge_5 35014 non-null object \n",430 " 64 Knowledge_6 34991 non-null object \n",431 " 65 Knowledge_7 34977 non-null object \n",432 " 66 Frequency_1 35371 non-null object \n",433 " 67 Frequency_2 35344 non-null object \n",434 " 68 Frequency_3 34515 non-null object \n",435 " 69 TimeSearching 36198 non-null object \n",436 " 70 TimeAnswering 36022 non-null object \n",437 " 71 Onboarding 35679 non-null object \n",438 " 72 ProfessionalTech 34906 non-null object \n",439 " 73 TrueFalse_1 35819 non-null object \n",440 " 74 TrueFalse_2 35715 non-null object \n",441 " 75 TrueFalse_3 35749 non-null object \n",442 " 76 SurveyLength 70444 non-null object \n",443 " 77 SurveyEase 70508 non-null object \n",444 " 78 ConvertedCompYearly 38071 non-null float64\n",445 "dtypes: float64(5), int64(1), object(73)\n",446 "memory usage: 44.2+ MB\n"447 ]448 }449 ],450 "source": [451 "data.info()"452 ]453 },454 {455 "cell_type": "code",456 "execution_count": 7,457 "id": "ea43972a-a4ce-467e-9dc4-0b85e1f74fae",458 "metadata": {459 "scrolled": true460 },461 "outputs": [462 {463 "data": {464 "text/plain": [465 "Index(['ResponseId', 'MainBranch', 'Employment', 'RemoteWork',\n",466 " 'CodingActivities', 'EdLevel', 'LearnCode', 'LearnCodeOnline',\n",467 " 'LearnCodeCoursesCert', 'YearsCode', 'YearsCodePro', 'DevType',\n",468 " 'OrgSize', 'PurchaseInfluence', 'BuyNewTool', 'Country', 'Currency',\n",469 " 'CompTotal', 'CompFreq', 'LanguageHaveWorkedWith',\n",470 " 'LanguageWantToWorkWith', 'DatabaseHaveWorkedWith',\n",471 " 'DatabaseWantToWorkWith', 'PlatformHaveWorkedWith',\n",472 " 'PlatformWantToWorkWith', 'WebframeHaveWorkedWith',\n",473 " 'WebframeWantToWorkWith', 'MiscTechHaveWorkedWith',\n",474 " 'MiscTechWantToWorkWith', 'ToolsTechHaveWorkedWith',\n",475 " 'ToolsTechWantToWorkWith', 'NEWCollabToolsHaveWorkedWith',\n",476 " 'NEWCollabToolsWantToWorkWith', 'OpSysProfessional use',\n",477 " 'OpSysPersonal use', 'VersionControlSystem', 'VCInteraction',\n",478 " 'VCHostingPersonal use', 'VCHostingProfessional use',\n",479 " 'OfficeStackAsyncHaveWorkedWith', 'OfficeStackAsyncWantToWorkWith',\n",480 " 'OfficeStackSyncHaveWorkedWith', 'OfficeStackSyncWantToWorkWith',\n",481 " 'Blockchain', 'NEWSOSites', 'SOVisitFreq', 'SOAccount', 'SOPartFreq',\n",482 " 'SOComm', 'Age', 'Gender', 'Trans', 'Sexuality', 'Ethnicity',\n",483 " 'Accessibility', 'MentalHealth', 'TBranch', 'ICorPM', 'WorkExp',\n",484 " 'Knowledge_1', 'Knowledge_2', 'Knowledge_3', 'Knowledge_4',\n",485 " 'Knowledge_5', 'Knowledge_6', 'Knowledge_7', 'Frequency_1',\n",486 " 'Frequency_2', 'Frequency_3', 'TimeSearching', 'TimeAnswering',\n",487 " 'Onboarding', 'ProfessionalTech', 'TrueFalse_1', 'TrueFalse_2',\n",488 " 'TrueFalse_3', 'SurveyLength', 'SurveyEase', 'ConvertedCompYearly'],\n",489 " dtype='object')"490 ]491 },492 "execution_count": 7,493 "metadata": {},494 "output_type": "execute_result"495 }496 ],497 "source": [498 "data.columns"499 ]500 },501 {502 "cell_type": "markdown",503 "id": "2de7daac-a1d9-4734-a585-29a0ba131cee",504 "metadata": {},505 "source": [506 "# Visualization"507 ]508 },509 {510 "cell_type": "code",511 "execution_count": 8,512 "id": "0f5e0c49-0987-4f71-9920-6bf18dbfed82",513 "metadata": {},514 "outputs": [],515 "source": [516 "def plot_line_chart(df, column, line=\"\"):\n",517 " if line == '':\n",518 " line = df[column].value_counts().keys()[0:20]\n",519 " data = df[column].value_counts()[0:20]\n",520 " fig=px.line(x=line,y=data)\n",521 " fig.show()\n",522 " \n",523 "def plot_bar_chart(df, column, line=\"\"):\n",524 " if line == '':\n",525 " line = df[column].value_counts().keys()[0:20]\n",526 " data = df[column].value_counts()[0:20]\n",527 " fig=px.bar(x=line,y=data)\n",528 " fig.show()\n",529 " \n",530 "def plot_pie_chart(df, column, line = ''):\n",531 " if line == '':\n",532 " line = df[column].value_counts().keys()[0:20]\n",533 " data = df[column].value_counts()[0:20]\n",534 " fig=px.pie(names=line,values=data)\n",535 " fig.show()"536 ]537 },538 {539 "cell_type": "code",540 "execution_count": 9,541 "id": "6394ccf1-9336-423c-a7b6-4b2d5ed7da5f",542 "metadata": {543 "scrolled": true544 },545 "outputs": [],546 "source": [547 "#plot_bar_chart(data,\"Country\")\n",548 "#plot_line_chart(data,\"Country\")"549 ]550 },551 {552 "cell_type": "code",553 "execution_count": 10,554 "id": "b0e41253-7245-45f3-a690-7cb5dec44dc7",555 "metadata": {556 "scrolled": true557 },558 "outputs": [],559 "source": [560 "#line = ['developer', 'student', 'half-developer', 'hobby', 'used-developer', 'neither']\n",561 "#plot_line_chart(data, 'MainBranch', line)\n",562 "#plot_bar_chart(data, 'MainBranch', line)\n",563 "#plot_pie_chart(data, 'MainBranch', line)"564 ]565 },566 {567 "cell_type": "code",568 "execution_count": 11,569 "id": "d12f307b-2c9b-4131-895f-3577d8eaf726",570 "metadata": {571 "scrolled": true572 },573 "outputs": [],574 "source": [575 "#plot_bar_chart(data,\"Employment\")\n",576 "#plot_line_chart(data,\"Employment\")\n",577 "#plot_pie_chart(data,\"Employment\")"578 ]579 },580 {581 "cell_type": "code",582 "execution_count": 12,583 "id": "10a57132-0d6d-4a29-aacd-7ace798b8acb",584 "metadata": {585 "scrolled": true586 },587 "outputs": [],588 "source": [589 "#plot_bar_chart(data,\"EdLevel\")\n",590 "#plot_pie_chart(data,\"EdLevel\")"591 ]592 },593 {594 "cell_type": "code",595 "execution_count": 13,596 "id": "c709bff9-45c6-4ac2-8361-035560d32a81",597 "metadata": {598 "scrolled": true599 },600 "outputs": [],601 "source": [602 "#plot_bar_chart(data,\"YearsCodePro\")\n",603 "#plot_pie_chart(data,\"YearsCodePro\")"604 ]605 },606 {607 "cell_type": "markdown",608 "id": "f1077547-695f-4426-a1ce-7e311836c3d4",609 "metadata": {},610 "source": [611 "# EDA"612 ]613 },614 {615 "cell_type": "code",616 "execution_count": 14,617 "id": "bce82caa-381b-429d-a489-8f47c824dfe5",618 "metadata": {},619 "outputs": [],620 "source": [621 "data = data[['Country', 'EdLevel', 'YearsCodePro','DevType', 'LanguageHaveWorkedWith', 'ConvertedCompYearly']]"622 ]623 },624 {625 "cell_type": "code",626 "execution_count": 15,627 "id": "71e3fabb-24de-4df1-9f8d-58dd28fc8a4a",628 "metadata": {},629 "outputs": [630 {631 "data": {632 "text/plain": [633 "Index(['Country', 'EdLevel', 'YearsCodePro', 'DevType',\n",634 " 'LanguageHaveWorkedWith', 'ConvertedCompYearly'],\n",635 " dtype='object')"636 ]637 },638 "execution_count": 15,639 "metadata": {},640 "output_type": "execute_result"641 }642 ],643 "source": [644 "data.columns"645 ]646 },647 {648 "cell_type": "code",649 "execution_count": 16,650 "id": "bf3a4e1d-42bc-49f9-92e6-577dd4bde1cf",651 "metadata": {},652 "outputs": [653 {654 "data": {655 "text/html": [656 "<div>\n",657 "<style scoped>\n",658 " .dataframe tbody tr th:only-of-type {\n",659 " vertical-align: middle;\n",660 " }\n",661 "\n",662 " .dataframe tbody tr th {\n",663 " vertical-align: top;\n",664 " }\n",665 "\n",666 " .dataframe thead th {\n",667 " text-align: right;\n",668 " }\n",669 "</style>\n",670 "<table border=\"1\" class=\"dataframe\">\n",671 " <thead>\n",672 " <tr style=\"text-align: right;\">\n",673 " <th></th>\n",674 " <th>Country</th>\n",675 " <th>EdLevel</th>\n",676 " <th>YearsCodePro</th>\n",677 " <th>DevType</th>\n",678 " <th>Language</th>\n",679 " <th>Salary</th>\n",680 " </tr>\n",681 " </thead>\n",682 " <tbody>\n",683 " <tr>\n",684 " <th>0</th>\n",685 " <td>NaN</td>\n",686 " <td>NaN</td>\n",687 " <td>NaN</td>\n",688 " <td>NaN</td>\n",689 " <td>NaN</td>\n",690 " <td>NaN</td>\n",691 " </tr>\n",692 " <tr>\n",693 " <th>1</th>\n",694 " <td>Canada</td>\n",695 " <td>NaN</td>\n",696 " <td>NaN</td>\n",697 " <td>NaN</td>\n",698 " <td>JavaScript;TypeScript</td>\n",699 " <td>NaN</td>\n",700 " </tr>\n",701 " <tr>\n",702 " <th>2</th>\n",703 " <td>United Kingdom of Great Britain and Northern I...</td>\n",704 " <td>Master’s degree (M.A., M.S., M.Eng., MBA, etc.)</td>\n",705 " <td>5</td>\n",706 " <td>Data scientist or machine learning specialist;...</td>\n",707 " <td>C#;C++;HTML/CSS;JavaScript;Python</td>\n",708 " <td>40205.0</td>\n",709 " </tr>\n",710 " <tr>\n",711 " <th>3</th>\n",712 " <td>Israel</td>\n",713 " <td>Bachelor’s degree (B.A., B.S., B.Eng., etc.)</td>\n",714 " <td>17</td>\n",715 " <td>Developer, full-stack</td>\n",716 " <td>C#;JavaScript;SQL;TypeScript</td>\n",717 " <td>215232.0</td>\n",718 " </tr>\n",719 " <tr>\n",720 " <th>4</th>\n",721 " <td>United States of America</td>\n",722 " <td>Bachelor’s degree (B.A., B.S., B.Eng., etc.)</td>\n",723 " <td>3</td>\n",724 " <td>Developer, front-end;Developer, full-stack;Dev...</td>\n",725 " <td>C#;HTML/CSS;JavaScript;SQL;Swift;TypeScript</td>\n",726 " <td>NaN</td>\n",727 " </tr>\n",728 " </tbody>\n",729 "</table>\n",730 "</div>"731 ],732 "text/plain": [733 " Country \\\n",734 "0 NaN \n",735 "1 Canada \n",736 "2 United Kingdom of Great Britain and Northern I... \n",737 "3 Israel \n",738 "4 United States of America \n",739 "\n",740 " EdLevel YearsCodePro \\\n",741 "0 NaN NaN \n",742 "1 NaN NaN \n",743 "2 Master’s degree (M.A., M.S., M.Eng., MBA, etc.) 5 \n",744 "3 Bachelor’s degree (B.A., B.S., B.Eng., etc.) 17 \n",745 "4 Bachelor’s degree (B.A., B.S., B.Eng., etc.) 3 \n",746 "\n",747 " DevType \\\n",748 "0 NaN \n",749 "1 NaN \n",750 "2 Data scientist or machine learning specialist;... \n",751 "3 Developer, full-stack \n",752 "4 Developer, front-end;Developer, full-stack;Dev... \n",753 "\n",754 " Language Salary \n",755 "0 NaN NaN \n",756 "1 JavaScript;TypeScript NaN \n",757 "2 C#;C++;HTML/CSS;JavaScript;Python 40205.0 \n",758 "3 C#;JavaScript;SQL;TypeScript 215232.0 \n",759 "4 C#;HTML/CSS;JavaScript;SQL;Swift;TypeScript NaN "760 ]761 },762 "execution_count": 16,763 "metadata": {},764 "output_type": "execute_result"765 }766 ],767 "source": [768 "data = data.rename({'ConvertedCompYearly' : 'Salary', 'LanguageHaveWorkedWith' : 'Language'}, axis=1)\n",769 "data.head()"770 ]771 },772 {773 "cell_type": "code",774 "execution_count": 17,775 "id": "921e1dde-2f09-46fb-b5f4-c346464e0886",776 "metadata": {},777 "outputs": [],778 "source": [779 "data.dropna(inplace=True)"780 ]781 },782 {783 "cell_type": "code",784 "execution_count": 18,785 "id": "74f80d39-cf7b-4f6f-a6ea-466d3f412b17",786 "metadata": {},787 "outputs": [788 {789 "data": {790 "text/plain": [791 "(37740, 6)"792 ]793 },794 "execution_count": 18,795 "metadata": {},796 "output_type": "execute_result"797 }798 ],799 "source": [800 "data.shape"801 ]802 },803 {804 "cell_type": "code",805 "execution_count": 19,806 "id": "49e56f86-8d97-4c54-81d1-b25d7b82ac56",807 "metadata": {},808 "outputs": [],809 "source": [810 "data = data.reset_index()"811 ]812 },813 {814 "cell_type": "code",815 "execution_count": 20,816 "id": "b20b3a6a-22f8-4669-abf8-69603021763b",817 "metadata": {},818 "outputs": [],819 "source": [820 "def one_hot_encoding(column) : \n",821 " columns_lst=[]\n",822 " for i in data[column] : \n",823 " for j in i.split(';') :\n",824 " if j not in columns_lst:\n",825 " columns_lst.append(j)\n",826 " return columns_lst"827 ]828 },829 {830 "cell_type": "code",831 "execution_count": 21,832 "id": "ac095c00-cab7-4c60-b60b-daaa77dab15b",833 "metadata": {},834 "outputs": [],835 "source": [836 "language_lst = one_hot_encoding('Language')\n",837 "\n",838 "for i in language_lst :\n",839 " data[i] = 0\n",840 "\n",841 "for i in range(len(data)) :\n",842 " for j in (data.iloc[i, :]['Language']).split(';') :\n",843 " data.loc[i, j] = 1"844 ]845 },846 {847 "cell_type": "code",848 "execution_count": 22,849 "id": "edcb5721-2975-43de-914f-c2f95dfff631",850 "metadata": {},851 "outputs": [],852 "source": [853 "dev_type_lst = one_hot_encoding('DevType')\n",854 "\n",855 "for i in dev_type_lst :\n",856 " data[i] = 0\n",857 "\n",858 "for i in range(len(data)) :\n",859 " for j in (data.iloc[i, :]['DevType']).split(';') :\n",860 " data.loc[i, j] = 1"861 ]862 },863 {864 "cell_type": "code",865 "execution_count": 23,866 "id": "0b67fc38-40cb-49ec-9d80-053c5ca2dc89",867 "metadata": {},868 "outputs": [869 {870 "data": {871 "text/plain": [872 "0 36293\n",873 "1 1447\n",874 "Name: Product manager, dtype: int64"875 ]876 },877 "execution_count": 23,878 "metadata": {},879 "output_type": "execute_result"880 }881 ],882 "source": [883 "data['Product manager'].value_counts()"884 ]885 },886 {887 "cell_type": "code",888 "execution_count": 24,889 "id": "6f359587-56a5-41d1-95f6-9ce7a2e8927f",890 "metadata": {},891 "outputs": [892 {893 "data": {894 "text/plain": [895 "0 31442\n",896 "1 6298\n",897 "Name: C++, dtype: int64"898 ]899 },900 "execution_count": 24,901 "metadata": {},902 "output_type": "execute_result"903 }904 ],905 "source": [906 "data['C++'].value_counts()"907 ]908 },909 {910 "cell_type": "code",911 "execution_count": 25,912 "id": "a153b7a2-a047-468a-aa0a-8c335493428a",913 "metadata": {},914 "outputs": [],915 "source": [916 "data.drop(['Language', 'DevType'], axis=1, inplace=True)"917 ]918 },919 {920 "cell_type": "code",921 "execution_count": 26,922 "id": "c71dd0ab-8861-4596-af8e-c0f233fc29e2",923 "metadata": {},924 "outputs": [925 {926 "data": {927 "text/plain": [928 "United States of America 8642\n",929 "Germany 2887\n",930 "United Kingdom of Great Britain and Northern Ireland 2637\n",931 "India 2140\n",932 "Canada 1470\n",933 " ... \n",934 "Suriname 1\n",935 "Haiti 1\n",936 "Somalia 1\n",937 "Cape Verde 1\n",938 "Seychelles 1\n",939 "Name: Country, Length: 159, dtype: int64"940 ]941 },942 "execution_count": 26,943 "metadata": {},944 "output_type": "execute_result"945 }946 ],947 "source": [948 "data.Country.value_counts()"949 ]950 },951 {952 "cell_type": "code",953 "execution_count": 27,954 "id": "75d10571-d02d-400c-9ddf-d267076bc20d",955 "metadata": {956 "scrolled": true957 },958 "outputs": [],959 "source": [960 "d = {}\n",961 "\n",962 "for i in data.Country.unique():\n",963 " if i not in d.keys() :\n",964 " d[i] = (data['Country'] == i).sum()\n",965 "\n",966 " \n",967 "for i in range(len(data)) :\n",968 " country = (data.iloc[i]['Country'])\n",969 " if d[country] <= 200 :\n",970 " data.loc[i, 'Country'] = 'Other'"971 ]972 },973 {974 "cell_type": "code",975 "execution_count": 28,976 "id": "3963ae7b-916f-4ad7-8c5f-5ff3d41c4bcd",977 "metadata": {978 "scrolled": true979 },980 "outputs": [981 {982 "data": {983 "text/html": [984 "<div>\n",985 "<style scoped>\n",986 " .dataframe tbody tr th:only-of-type {\n",987 " vertical-align: middle;\n",988 " }\n",989 "\n",990 " .dataframe tbody tr th {\n",991 " vertical-align: top;\n",992 " }\n",993 "\n",994 " .dataframe thead th {\n",995 " text-align: right;\n",996 " }\n",997 "</style>\n",998 "<table border=\"1\" class=\"dataframe\">\n",999 " <thead>\n",1000 " <tr style=\"text-align: right;\">\n",1001 " <th></th>\n",1002 " <th>index</th>\n",1003 " <th>Country</th>\n",1004 " <th>EdLevel</th>\n",1005 " <th>YearsCodePro</th>\n",1006 " <th>Salary</th>\n",1007 " <th>C#</th>\n",1008 " <th>C++</th>\n",1009 " <th>HTML/CSS</th>\n",1010 " <th>JavaScript</th>\n",1011 " <th>Python</th>\n",1012 " <th>...</th>\n",1013 " <th>Project manager</th>\n",1014 " <th>Product manager</th>\n",1015 " <th>Senior Executive (C-Suite, VP, etc.)</th>\n",1016 " <th>System administrator</th>\n",1017 " <th>Marketing or sales professional</th>\n",1018 " <th>Scientist</th>\n",1019 " <th>Security professional</th>\n",1020 " <th>Other (please specify):</th>\n",1021 " <th>Educator</th>\n",1022 " <th>Blockchain</th>\n",1023 " </tr>\n",1024 " </thead>\n",1025 " <tbody>\n",1026 " <tr>\n",1027 " <th>0</th>\n",1028 " <td>2</td>\n",1029 " <td>United Kingdom of Great Britain and Northern I...</td>\n",1030 " <td>Master’s degree (M.A., M.S., M.Eng., MBA, etc.)</td>\n",1031 " <td>5</td>\n",1032 " <td>40205.0</td>\n",1033 " <td>1</td>\n",1034 " <td>1</td>\n",1035 " <td>1</td>\n",1036 " <td>1</td>\n",1037 " <td>1</td>\n",1038 " <td>...</td>\n",1039 " <td>0</td>\n",1040 " <td>0</td>\n",1041 " <td>0</td>\n",1042 " <td>0</td>\n",1043 " <td>0</td>\n",1044 " <td>0</td>\n",1045 " <td>0</td>\n",1046 " <td>0</td>\n",1047 " <td>0</td>\n",1048 " <td>0</td>\n",1049 " </tr>\n",1050 " <tr>\n",1051 " <th>1</th>\n",1052 " <td>3</td>\n",1053 " <td>Israel</td>\n",1054 " <td>Bachelor’s degree (B.A., B.S., B.Eng., etc.)</td>\n",1055 " <td>17</td>\n",1056 " <td>215232.0</td>\n",1057 " <td>1</td>\n",1058 " <td>0</td>\n",1059 " <td>0</td>\n",1060 " <td>1</td>\n",1061 " <td>0</td>\n",1062 " <td>...</td>\n",1063 " <td>0</td>\n",1064 " <td>0</td>\n",1065 " <td>0</td>\n",1066 " <td>0</td>\n",1067 " <td>0</td>\n",1068 " <td>0</td>\n",1069 " <td>0</td>\n",1070 " <td>0</td>\n",1071 " <td>0</td>\n",1072 " <td>0</td>\n",1073 " </tr>\n",1074 " <tr>\n",1075 " <th>2</th>\n",1076 " <td>10</td>\n",1077 " <td>United Kingdom of Great Britain and Northern I...</td>\n",1078 " <td>Bachelor’s degree (B.A., B.S., B.Eng., etc.)</td>\n",1079 " <td>2</td>\n",1080 " <td>60307.0</td>\n",1081 " <td>1</td>\n",1082 " <td>0</td>\n",1083 " <td>1</td>\n",1084 " <td>1</td>\n",1085 " <td>0</td>\n",1086 " <td>...</td>\n",1087 " <td>0</td>\n",1088 " <td>0</td>\n",1089 " <td>0</td>\n",1090 " <td>0</td>\n",1091 " <td>0</td>\n",1092 " <td>0</td>\n",1093 " <td>0</td>\n",1094 " <td>0</td>\n",1095 " <td>0</td>\n",1096 " <td>0</td>\n",1097 " </tr>\n",1098 " <tr>\n",1099 " <th>3</th>\n",1100 " <td>11</td>\n",1101 " <td>United States of America</td>\n",1102 " <td>Bachelor’s degree (B.A., B.S., B.Eng., etc.)</td>\n",1103 " <td>10</td>\n",1104 " <td>194400.0</td>\n",1105 " <td>1</td>\n",1106 " <td>0</td>\n",1107 " <td>1</td>\n",1108 " <td>1</td>\n",1109 " <td>1</td>\n",1110 " <td>...</td>\n",1111 " <td>0</td>\n",1112 " <td>0</td>\n",1113 " <td>0</td>\n",1114 " <td>0</td>\n",1115 " <td>0</td>\n",1116 " <td>0</td>\n",1117 " <td>0</td>\n",1118 " <td>0</td>\n",1119 " <td>0</td>\n",1120 " <td>0</td>\n",1121 " </tr>\n",1122 " <tr>\n",1123 " <th>4</th>\n",1124 " <td>12</td>\n",1125 " <td>United States of America</td>\n",1126 " <td>Bachelor’s degree (B.A., B.S., B.Eng., etc.)</td>\n",1127 " <td>5</td>\n",1128 " <td>65000.0</td>\n",1129 " <td>0</td>\n",1130 " <td>0</td>\n",1131 " <td>1</td>\n",1132 " <td>0</td>\n",1133 " <td>0</td>\n",1134 " <td>...</td>\n",1135 " <td>0</td>\n",1136 " <td>0</td>\n",1137 " <td>0</td>\n",1138 " <td>0</td>\n",1139 " <td>0</td>\n",1140 " <td>0</td>\n",1141 " <td>0</td>\n",1142 " <td>0</td>\n",1143 " <td>0</td>\n",1144 " <td>0</td>\n",1145 " </tr>\n",1146 " <tr>\n",1147 " <th>...</th>\n",1148 " <td>...</td>\n",1149 " <td>...</td>\n",1150 " <td>...</td>\n",1151 " <td>...</td>\n",1152 " <td>...</td>\n",1153 " <td>...</td>\n",1154 " <td>...</td>\n",1155 " <td>...</td>\n",1156 " <td>...</td>\n",1157 " <td>...</td>\n",1158 " <td>...</td>\n",1159 " <td>...</td>\n",1160 " <td>...</td>\n",1161 " <td>...</td>\n",1162 " <td>...</td>\n",1163 " <td>...</td>\n",1164 " <td>...</td>\n",1165 " <td>...</td>\n",1166 " <td>...</td>\n",1167 " <td>...</td>\n",1168 " <td>...</td>\n",1169 " </tr>\n",1170 " <tr>\n",1171 " <th>37735</th>\n",1172 " <td>73116</td>\n",1173 " <td>United States of America</td>\n",1174 " <td>Bachelor’s degree (B.A., B.S., B.Eng., etc.)</td>\n",1175 " <td>16</td>\n",1176 " <td>115000.0</td>\n",1177 " <td>1</td>\n",1178 " <td>0</td>\n",1179 " <td>1</td>\n",1180 " <td>1</td>\n",1181 " <td>0</td>\n",1182 " <td>...</td>\n",1183 " <td>0</td>\n",1184 " <td>1</td>\n",1185 " <td>0</td>\n",1186 " <td>0</td>\n",1187 " <td>0</td>\n",1188 " <td>0</td>\n",1189 " <td>0</td>\n",1190 " <td>0</td>\n",1191 " <td>0</td>\n",1192 " <td>0</td>\n",1193 " </tr>\n",1194 " <tr>\n",1195 " <th>37736</th>\n",1196 " <td>73118</td>\n",1197 " <td>Other</td>\n",1198 " <td>Bachelor’s degree (B.A., B.S., B.Eng., etc.)</td>\n",1199 " <td>3</td>\n",1200 " <td>57720.0</td>\n",