main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cb1a7ad7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datasist.structdata import detect_outliers\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, train_test_split, cross_val_score\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "from sklearn.ensemble import (\n",
    "    BaggingClassifier,\n",
    "    ExtraTreesClassifier,\n",
    "    RandomForestClassifier,\n",
    "    StackingClassifier,\n",
    "    HistGradientBoostingClassifier\n",
    ")\n",
    "from xgboost import XGBClassifier\n",
    "\n",
    "import joblib ## for model exportation\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "07da0f7f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Annual_Income</th>\n",
       "      <th>Monthly_Inhand_Salary</th>\n",
       "      <th>Num_Bank_Accounts</th>\n",
       "      <th>Num_Credit_Card</th>\n",
       "      <th>Interest_Rate</th>\n",
       "      <th>Num_of_Loan</th>\n",
       "      <th>Delay_from_due_date</th>\n",
       "      <th>Num_of_Delayed_Payment</th>\n",
       "      <th>Changed_Credit_Limit</th>\n",
       "      <th>Num_Credit_Inquiries</th>\n",
       "      <th>Credit_Mix</th>\n",
       "      <th>Outstanding_Debt</th>\n",
       "      <th>Credit_Utilization_Ratio</th>\n",
       "      <th>Credit_History_Age</th>\n",
       "      <th>Total_EMI_per_month</th>\n",
       "      <th>Amount_invested_monthly</th>\n",
       "      <th>Monthly_Balance</th>\n",
       "      <th>Credit_Score</th>\n",
       "      <th>Credit-Builder Loan</th>\n",
       "      <th>Personal Loan</th>\n",
       "      <th>Debt Consolidation Loan</th>\n",
       "      <th>Student Loan</th>\n",
       "      <th>Payday Loan</th>\n",
       "      <th>Mortgage Loan</th>\n",
       "      <th>Auto Loan</th>\n",
       "      <th>Home Equity Loan</th>\n",
       "      <th>Occupation_Accountant</th>\n",
       "      <th>Occupation_Architect</th>\n",
       "      <th>Occupation_Developer</th>\n",
       "      <th>Occupation_Doctor</th>\n",
       "      <th>Occupation_Engineer</th>\n",
       "      <th>Occupation_Entrepreneur</th>\n",
       "      <th>Occupation_Journalist</th>\n",
       "      <th>Occupation_Lawyer</th>\n",
       "      <th>Occupation_Manager</th>\n",
       "      <th>Occupation_Mechanic</th>\n",
       "      <th>Occupation_Media_Manager</th>\n",
       "      <th>Occupation_Musician</th>\n",
       "      <th>Occupation_Scientist</th>\n",
       "      <th>Occupation_Teacher</th>\n",
       "      <th>Occupation_Writer</th>\n",
       "      <th>Payment_of_Min_Amount_No</th>\n",
       "      <th>Payment_of_Min_Amount_Yes</th>\n",
       "      <th>Payment_Behaviour_High_spent_Large_value_payments</th>\n",
       "      <th>Payment_Behaviour_High_spent_Medium_value_payments</th>\n",
       "      <th>Payment_Behaviour_High_spent_Small_value_payments</th>\n",
       "      <th>Payment_Behaviour_Low_spent_Large_value_payments</th>\n",
       "      <th>Payment_Behaviour_Low_spent_Medium_value_payments</th>\n",
       "      <th>Payment_Behaviour_Low_spent_Small_value_payments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23.0</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>1824.843333</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>809.98</td>\n",
       "      <td>26.822620</td>\n",
       "      <td>265.0</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>80.415295</td>\n",
       "      <td>312.494089</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>23.0</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3095.978333</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.944960</td>\n",
       "      <td>219.0</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>118.280222</td>\n",
       "      <td>284.629162</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23.0</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3095.978333</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>809.98</td>\n",
       "      <td>31.377862</td>\n",
       "      <td>268.0</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>199.458074</td>\n",
       "      <td>223.451310</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23.0</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>1824.843333</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>11.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>809.98</td>\n",
       "      <td>24.797347</td>\n",
       "      <td>269.0</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>41.420153</td>\n",
       "      <td>341.489231</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>23.0</td>\n",
       "      <td>19114.12</td>\n",
       "      <td>3095.978333</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.27</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>809.98</td>\n",
       "      <td>27.262259</td>\n",
       "      <td>270.0</td>\n",
       "      <td>49.574949</td>\n",
       "      <td>62.430172</td>\n",
       "      <td>340.479212</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97219</th>\n",
       "      <td>25.0</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>3359.415833</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.663572</td>\n",
       "      <td>378.0</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>60.971333</td>\n",
       "      <td>479.866228</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97220</th>\n",
       "      <td>25.0</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>3359.415833</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>502.38</td>\n",
       "      <td>40.565631</td>\n",
       "      <td>379.0</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>54.185950</td>\n",
       "      <td>496.651610</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97221</th>\n",
       "      <td>25.0</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>3359.415833</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>502.38</td>\n",
       "      <td>41.255522</td>\n",
       "      <td>380.0</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>24.028477</td>\n",
       "      <td>516.809083</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97222</th>\n",
       "      <td>25.0</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>3359.415833</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>502.38</td>\n",
       "      <td>33.638208</td>\n",
       "      <td>381.0</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>251.672582</td>\n",
       "      <td>319.164979</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97223</th>\n",
       "      <td>25.0</td>\n",
       "      <td>39628.99</td>\n",
       "      <td>3359.415833</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11.50</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>502.38</td>\n",
       "      <td>34.192463</td>\n",
       "      <td>382.0</td>\n",
       "      <td>35.104023</td>\n",
       "      <td>167.163865</td>\n",
       "      <td>393.673696</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>97224 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Age  Annual_Income  Monthly_Inhand_Salary  Num_Bank_Accounts  \\\n",
       "0      23.0       19114.12            1824.843333                3.0   \n",
       "1      23.0       19114.12            3095.978333                3.0   \n",
       "2      23.0       19114.12            3095.978333                3.0   \n",
       "3      23.0       19114.12            1824.843333                3.0   \n",
       "4      23.0       19114.12            3095.978333                3.0   \n",
       "...     ...            ...                    ...                ...   \n",
       "97219  25.0       39628.99            3359.415833                4.0   \n",
       "97220  25.0       39628.99            3359.415833                4.0   \n",
       "97221  25.0       39628.99            3359.415833                4.0   \n",
       "97222  25.0       39628.99            3359.415833                4.0   \n",
       "97223  25.0       39628.99            3359.415833                4.0   \n",
       "\n",
       "       Num_Credit_Card  Interest_Rate  Num_of_Loan  Delay_from_due_date  \\\n",
       "0                  4.0            3.0          4.0                  3.0   \n",
       "1                  4.0            3.0          4.0                 -1.0   \n",
       "2                  4.0            3.0          4.0                  5.0   \n",
       "3                  4.0            3.0          4.0                  6.0   \n",
       "4                  4.0            3.0          4.0                  8.0   \n",
       "...                ...            ...          ...                  ...   \n",
       "97219              6.0            7.0          2.0                 23.0   \n",
       "97220              6.0            7.0          2.0                 18.0   \n",
       "97221              6.0           13.0          2.0                 27.0   \n",
       "97222              6.0            7.0          2.0                 20.0   \n",
       "97223              6.0            7.0          2.0                 18.0   \n",
       "\n",
       "       Num_of_Delayed_Payment  Changed_Credit_Limit  Num_Credit_Inquiries  \\\n",
       "0                         7.0                 11.27                   4.0   \n",
       "1                        14.0                 11.27                   4.0   \n",
       "2                         4.0                  6.27                   4.0   \n",
       "3                        14.0                 11.27                   4.0   \n",
       "4                         4.0                  9.27                   4.0   \n",
       "...                       ...                   ...                   ...   \n",
       "97219                     7.0                 11.50                   3.0   \n",
       "97220                     7.0                 11.50                   3.0   \n",
       "97221                     6.0                 11.50                   3.0   \n",
       "97222                    14.0                 11.50                   3.0   \n",
       "97223                     6.0                 11.50                   3.0   \n",
       "\n",
       "       Credit_Mix  Outstanding_Debt  Credit_Utilization_Ratio  \\\n",
       "0             1.0            809.98                 26.822620   \n",
       "1             2.0            809.98                 31.944960   \n",
       "2             2.0            809.98                 31.377862   \n",
       "3             2.0            809.98                 24.797347   \n",
       "4             2.0            809.98                 27.262259   \n",
       "...           ...               ...                       ...   \n",
       "97219         1.0            502.38                 34.663572   \n",
       "97220         1.0            502.38                 40.565631   \n",
       "97221         2.0            502.38                 41.255522   \n",
       "97222         2.0            502.38                 33.638208   \n",
       "97223         2.0            502.38                 34.192463   \n",
       "\n",
       "       Credit_History_Age  Total_EMI_per_month  Amount_invested_monthly  \\\n",
       "0                   265.0            49.574949                80.415295   \n",
       "1                   219.0            49.574949               118.280222   \n",
       "2                   268.0            49.574949               199.458074   \n",
       "3                   269.0            49.574949                41.420153   \n",
       "4                   270.0            49.574949                62.430172   \n",
       "...                   ...                  ...                      ...   \n",
       "97219               378.0            35.104023                60.971333   \n",
       "97220               379.0            35.104023                54.185950   \n",
       "97221               380.0            35.104023                24.028477   \n",
       "97222               381.0            35.104023               251.672582   \n",
       "97223               382.0            35.104023               167.163865   \n",
       "\n",
       "       Monthly_Balance  Credit_Score  Credit-Builder Loan  Personal Loan  \\\n",
       "0           312.494089             2                  1.0            1.0   \n",
       "1           284.629162             2                  1.0            1.0   \n",
       "2           223.451310             2                  1.0            1.0   \n",
       "3           341.489231             2                  1.0            1.0   \n",
       "4           340.479212             2                  1.0            1.0   \n",
       "...                ...           ...                  ...            ...   \n",
       "97219       479.866228             0                  0.0            0.0   \n",
       "97220       496.651610             0                  0.0            0.0   \n",
       "97221       516.809083             0                  0.0            0.0   \n",
       "97222       319.164979             1                  0.0            0.0   \n",
       "97223       393.673696             0                  0.0            0.0   \n",
       "\n",
       "       Debt Consolidation Loan  Student Loan  Payday Loan  Mortgage Loan  \\\n",
       "0                          0.0           0.0          0.0            0.0   \n",
       "1                          0.0           0.0          0.0            0.0   \n",
       "2                          0.0           0.0          0.0            0.0   \n",
       "3                          0.0           0.0          0.0            0.0   \n",
       "4                          0.0           0.0          0.0            0.0   \n",
       "...                        ...           ...          ...            ...   \n",
       "97219                      0.0           1.0          0.0            0.0   \n",
       "97220                      0.0           1.0          0.0            0.0   \n",
       "97221                      0.0           1.0          0.0            0.0   \n",
       "97222                      0.0           1.0          0.0            0.0   \n",
       "97223                      0.0           1.0          0.0            0.0   \n",
       "\n",
       "       Auto Loan  Home Equity Loan  Occupation_Accountant  \\\n",
       "0            1.0               1.0                      0   \n",
       "1            1.0               1.0                      0   \n",
       "2            1.0               1.0                      0   \n",
       "3            1.0               1.0                      0   \n",
       "4            1.0               1.0                      0   \n",
       "...          ...               ...                    ...   \n",
       "97219        1.0               0.0                      0   \n",
       "97220        1.0               0.0                      0   \n",
       "97221        1.0               0.0                      0   \n",
       "97222        1.0               0.0                      0   \n",
       "97223        1.0               0.0                      0   \n",
       "\n",
       "       Occupation_Architect  Occupation_Developer  Occupation_Doctor  \\\n",
       "0                         0                     0                  0   \n",
       "1                         0                     0                  0   \n",
       "2                         0                     0                  0   \n",
       "3                         0                     0                  0   \n",
       "4                         0                     0                  0   \n",
       "...                     ...                   ...                ...   \n",
       "97219                     0                     0                  0   \n",
       "97220                     0                     0                  0   \n",
       "97221                     0                     0                  0   \n",
       "97222                     0                     0                  0   \n",
       "97223                     0                     0                  0   \n",
       "\n",
       "       Occupation_Engineer  Occupation_Entrepreneur  Occupation_Journalist  \\\n",
       "0                        0                        0                      0   \n",
       "1                        0                        0                      0   \n",
       "2                        0                        0                      0   \n",
       "3                        0                        0                      0   \n",
       "4                        0                        0                      0   \n",
       "...                    ...                      ...                    ...   \n",
       "97219                    0                        0                      0   \n",
       "97220                    0                        0                      0   \n",
       "97221                    0                        0                      0   \n",
       "97222                    0                        0                      0   \n",
       "97223                    0                        0                      0   \n",
       "\n",
       "       Occupation_Lawyer  Occupation_Manager  Occupation_Mechanic  \\\n",
       "0                      0                   0                    0   \n",
       "1                      0                   0                    0   \n",
       "2                      0                   0                    0   \n",
       "3                      0                   0                    0   \n",
       "4                      0                   0                    0   \n",
       "...                  ...                 ...                  ...   \n",
       "97219                  0                   0                    1   \n",
       "97220                  0                   0                    1   \n",
       "97221                  0                   0                    1   \n",
       "97222                  0                   0                    1   \n",
       "97223                  0                   0                    1   \n",
       "\n",
       "       Occupation_Media_Manager  Occupation_Musician  Occupation_Scientist  \\\n",
       "0                             0                    0                     1   \n",
       "1                             0                    0                     1   \n",
       "2                             0                    0                     1   \n",
       "3                             0                    0                     1   \n",
       "4                             0                    0                     1   \n",
       "...                         ...                  ...                   ...   \n",
       "97219                         0                    0                     0   \n",
       "97220                         0                    0                     0   \n",
       "97221                         0                    0                     0   \n",
       "97222                         0                    0                     0   \n",
       "97223                         0                    0                     0   \n",
       "\n",
       "       Occupation_Teacher  Occupation_Writer  Payment_of_Min_Amount_No  \\\n",
       "0                       0                  0                         1   \n",
       "1                       0                  0                         1   \n",
       "2                       0                  0                         1   \n",
       "3                       0                  0                         1   \n",
       "4                       0                  0                         1   \n",
       "...                   ...                ...                       ...   \n",
       "97219                   0                  0                         1   \n",
       "97220                   0                  0                         1   \n",
       "97221                   0                  0                         1   \n",
       "97222                   0                  0                         1   \n",
       "97223                   0                  0                         1   \n",
       "\n",
       "       Payment_of_Min_Amount_Yes  \\\n",
       "0                              0   \n",
       "1                              0   \n",
       "2                              0   \n",
       "3                              0   \n",
       "4                              0   \n",
       "...                          ...   \n",
       "97219                          0   \n",
       "97220                          0   \n",
       "97221                          0   \n",
       "97222                          0   \n",
       "97223                          0   \n",
       "\n",
       "       Payment_Behaviour_High_spent_Large_value_payments  \\\n",
       "0                                                      0   \n",
       "1                                                      0   \n",
       "2                                                      0   \n",
       "3                                                      0   \n",
       "4                                                      0   \n",
       "...                                                  ...   \n",
       "97219                                                  1   \n",
       "97220                                                  0   \n",
       "97221                                                  1   \n",
       "97222                                                  0   \n",
       "97223                                                  0   \n",
       "\n",
       "       Payment_Behaviour_High_spent_Medium_value_payments  \\\n",
       "0                                                      0    \n",
       "1                                                      0    \n",
       "2                                                      0    \n",
       "3                                                      1    \n",
       "4                                                      0    \n",
       "...                                                  ...    \n",
       "97219                                                  0    \n",
       "97220                                                  1    \n",
       "97221                                                  0    \n",
       "97222                                                  0    \n",
       "97223                                                  0    \n",
       "\n",
       "       Payment_Behaviour_High_spent_Small_value_payments  \\\n",
       "0                                                      1   \n",
       "1                                                      0   \n",
       "2                                                      0   \n",
       "3                                                      0   \n",
       "4                                                      0   \n",
       "...                                                  ...   \n",
       "97219                                                  0   \n",
       "97220                                                  0   \n",
       "97221                                                  0   \n",
       "97222                                                  0   \n",
       "97223                                                  0   \n",
       "\n",
       "       Payment_Behaviour_Low_spent_Large_value_payments  \\\n",
       "0                                                     0   \n",
       "1                                                     1   \n",
       "2                                                     0   \n",
       "3                                                     0   \n",
       "4                                                     0   \n",
       "...                                                 ...   \n",
       "97219                                                 0   \n",
       "97220                                                 0   \n",
       "97221                                                 0   \n",
       "97222                                                 1   \n",
       "97223                                                 0   \n",
       "\n",
       "       Payment_Behaviour_Low_spent_Medium_value_payments  \\\n",
       "0                                                      0   \n",
       "1                                                      0   \n",
       "2                                                      0   \n",
       "3                                                      0   \n",
       "4                                                      0   \n",
       "...                                                  ...   \n",
       "97219                                                  0   \n",
       "97220                                                  0   \n",
       "97221                                                  0   \n",
       "97222                                                  0   \n",
       "97223                                                  0   \n",
       "\n",
       "       Payment_Behaviour_Low_spent_Small_value_payments  \n",
       "0                                                     0  \n",
       "1                                                     0  \n",
       "2                                                     1  \n",
       "3                                                     0  \n",
       "4                                                     1  \n",
       "...                                                 ...  \n",
       "97219                                                 0  \n",
       "97220                                                 0  \n",
       "97221                                                 0  \n",
       "97222                                                 0  \n",
       "97223                                                 1  \n",
       "\n",
       "[97224 rows x 50 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', 500)\n",
    "\n",
    "df = pd.read_csv(\"cleaned_data.csv\", low_memory=False)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e16dbea0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 97224 entries, 0 to 97223\n",
      "Data columns (total 50 columns):\n",
      " #   Column                                              Non-Null Count  Dtype  \n",
      "---  ------                                              --------------  -----  \n",
      " 0   Age                                                 97224 non-null  float64\n",
      " 1   Annual_Income                                       97224 non-null  float64\n",
      " 2   Monthly_Inhand_Salary                               97224 non-null  float64\n",
      " 3   Num_Bank_Accounts                                   97224 non-null  float64\n",
      " 4   Num_Credit_Card                                     97224 non-null  float64\n",
      " 5   Interest_Rate                                       97224 non-null  float64\n",
      " 6   Num_of_Loan                                         97224 non-null  float64\n",
      " 7   Delay_from_due_date                                 97224 non-null  float64\n",
      " 8   Num_of_Delayed_Payment                              97224 non-null  float64\n",
      " 9   Changed_Credit_Limit                                97224 non-null  float64\n",
      " 10  Num_Credit_Inquiries                                97224 non-null  float64\n",
      " 11  Credit_Mix                                          97224 non-null  float64\n",
      " 12  Outstanding_Debt                                    97224 non-null  float64\n",
      " 13  Credit_Utilization_Ratio                            97224 non-null  float64\n",
      " 14  Credit_History_Age                                  97224 non-null  float64\n",
      " 15  Total_EMI_per_month                                 97224 non-null  float64\n",
      " 16  Amount_invested_monthly                             97224 non-null  float64\n",
      " 17  Monthly_Balance                                     97224 non-null  float64\n",
      " 18  Credit_Score                                        97224 non-null  int64  \n",
      " 19  Credit-Builder Loan                                 97224 non-null  float64\n",
      " 20  Personal Loan                                       97224 non-null  float64\n",
      " 21  Debt Consolidation Loan                             97224 non-null  float64\n",
      " 22  Student Loan                                        97224 non-null  float64\n",
      " 23  Payday Loan                                         97224 non-null  float64\n",
      " 24  Mortgage Loan                                       97224 non-null  float64\n",
      " 25  Auto Loan                                           97224 non-null  float64\n",
      " 26  Home Equity Loan                                    97224 non-null  float64\n",
      " 27  Occupation_Accountant                               97224 non-null  int64  \n",
      " 28  Occupation_Architect                                97224 non-null  int64  \n",
      " 29  Occupation_Developer                                97224 non-null  int64  \n",
      " 30  Occupation_Doctor                                   97224 non-null  int64  \n",
      " 31  Occupation_Engineer                                 97224 non-null  int64  \n",
      " 32  Occupation_Entrepreneur                             97224 non-null  int64  \n",
      " 33  Occupation_Journalist                               97224 non-null  int64  \n",
      " 34  Occupation_Lawyer                                   97224 non-null  int64  \n",
      " 35  Occupation_Manager                                  97224 non-null  int64  \n",
      " 36  Occupation_Mechanic                                 97224 non-null  int64  \n",
      " 37  Occupation_Media_Manager                            97224 non-null  int64  \n",
      " 38  Occupation_Musician                                 97224 non-null  int64  \n",
      " 39  Occupation_Scientist                                97224 non-null  int64  \n",
      " 40  Occupation_Teacher                                  97224 non-null  int64  \n",
      " 41  Occupation_Writer                                   97224 non-null  int64  \n",
      " 42  Payment_of_Min_Amount_No                            97224 non-null  int64  \n",
      " 43  Payment_of_Min_Amount_Yes                           97224 non-null  int64  \n",
      " 44  Payment_Behaviour_High_spent_Large_value_payments   97224 non-null  int64  \n",
      " 45  Payment_Behaviour_High_spent_Medium_value_payments  97224 non-null  int64  \n",
      " 46  Payment_Behaviour_High_spent_Small_value_payments   97224 non-null  int64  \n",
      " 47  Payment_Behaviour_Low_spent_Large_value_payments    97224 non-null  int64  \n",
      " 48  Payment_Behaviour_Low_spent_Medium_value_payments   97224 non-null  int64  \n",
      " 49  Payment_Behaviour_Low_spent_Small_value_payments    97224 non-null  int64  \n",
      "dtypes: float64(26), int64(24)\n",
      "memory usage: 37.1 MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "122c8230",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    0.532009\n",
       "0    0.290103\n",
       "2    0.177888\n",
       "Name: Credit_Score, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"Credit_Score\"].value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dcf34a5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    0.333333\n",
       "1    0.333333\n",
       "0    0.333333\n",
       "Name: Credit_Score, dtype: float64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X, y = df.drop(\"Credit_Score\",axis=1).values , df[\"Credit_Score\"] \n",
    "rus = SMOTE(sampling_strategy='auto')\n",
    "X_data_rus, y_data_rus = rus.fit_resample(X, y)\n",
    "y_data_rus.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "52d8780a",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_data_rus, y_data_rus, test_size=0.2, \n",
    "                                                    random_state=42, stratify=y_data_rus)\n",
    "\n",
    "from sklearn.preprocessing import PowerTransformer\n",
    "scalar = PowerTransformer(method=\"yeo-johnson\", standardize=True).fit(X_train)\n",
    "X_train = scalar.transform(X_train)\n",
    "X_test = scalar.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4eda4f3d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Hyperparameter tuning for SGD Classifier:\n",
      "Best hyperparameters: {'alpha': 0.0001, 'eta0': 0.1, 'learning_rate': 'invscaling', 'max_iter': 1000}\n",
      "Training Score: 0.7220731933267277\n",
      "Test Accuracy: 0.723505719349122\n",
      "Mean Cross-Validation Score: 0.20038354451224105\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['sgd_classifier_model.pkl']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# SGD Classifier\n",
    "sgd_classifier = SGDClassifier(random_state=42)\n",
    "\n",
    "# Hyperparameter tuning\n",
    "# sgd_param_grid = {\n",
    "#     'alpha': [0.0001, 0.001, 0.01, 0.1],\n",
    "#     'learning_rate': ['constant', 'optimal', 'invscaling'],\n",
    "#     'max_iter': [1000, 2000, 3000, 4000],  # Increase the max_iter values\n",
    "#     'eta0': [0.01, 0.1, 0.5]\n",
    "# }\n",
    "\n",
    "sgd_param_grid = {\n",
    "    'alpha': [0.0001],\n",
    "    'learning_rate': ['invscaling'],\n",
    "    'max_iter': [1000],  \n",
    "    'eta0': [0.1]\n",
    "}\n",
    "\n",
    "\n",
    "grid_search_sgd = GridSearchCV(sgd_classifier, sgd_param_grid, cv=5, scoring='accuracy', n_jobs=-1)\n",
    "grid_search_sgd.fit(X_train, y_train)\n",
    "\n",
    "# Print results\n",
    "print(\"\\nHyperparameter tuning for SGD Classifier:\")\n",
    "print(f\"Best hyperparameters: {grid_search_sgd.best_params_}\")\n",
    "best_model_sgd = grid_search_sgd.best_estimator_\n",
    "\n",
    "# Training score\n",
    "training_score_sgd = best_model_sgd.score(X_train, y_train)\n",
    "print(f\"Training Score: {training_score_sgd}\")\n",
    "\n",
    "# Test score\n",
    "y_pred_sgd = best_model_sgd.predict(X_test)\n",
    "test_score_sgd = accuracy_score(y_test, y_pred_sgd)\n",
    "print(f\"Test Accuracy: {test_score_sgd}\")\n",
    "\n",
    "# Mean cross-validation score\n",
    "cross_val_scores_sgd = cross_val_score(best_model_sgd, X, y, cv=5)\n",
    "mean_cv_score_sgd = cross_val_scores_sgd.mean()\n",
    "print(f\"Mean Cross-Validation Score: {mean_cv_score_sgd}\")\n",
    "\n",
    "joblib.dump(best_model_sgd, 'sgd_classifier_model.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "839a24f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# kneighbors_classifier = KNeighborsClassifier()\n",
    "\n",
    "# # Hyperparameter tuning\n",
    "# # kneighbors_param_grid = {\n",
    "# #     'n_neighbors': [3, 5, 7, 9]\n",
    "# # }\n",
    "\n",
    "# kneighbors_param_grid = {\n",
    "#     'n_neighbors': [7, 9, 11, 13, 15],  # Fine-tune 'n_neighbors'\n",
    "#     'weights': ['uniform', 'distance'],\n",
    "#     'algorithm': ['auto'],\n",
    "#     'p': [1, 2],\n",
    "#     'leaf_size': [10, 20, 30]\n",
    "# }\n",
    "\n",
    "\n",
    "# grid_search_kneighbors = GridSearchCV(kneighbors_classifier, kneighbors_param_grid, cv=5, scoring='accuracy', n_jobs=-1)\n",
    "# grid_search_kneighbors.fit(X_train, y_train)\n",
    "\n",
    "# # Print results\n",
    "# print(\"\\nHyperparameter tuning for K-Nearest Neighbors:\")\n",
    "# print(f\"Best hyperparameters: {grid_search_kneighbors.best_params_}\")\n",
    "# best_model_kneighbors = grid_search_kneighbors.best_estimator_\n",
    "\n",
    "# # Training score\n",
    "# training_score_kneighbors = best_model_kneighbors.score(X_train, y_train)\n",
    "# print(f\"Training Score: {training_score_kneighbors}\")\n",
    "\n",
    "# # Test score\n",
    "# y_pred_kneighbors = best_model_kneighbors.predict(X_test)\n",
    "# test_score_kneighbors = accuracy_score(y_test, y_pred_kneighbors)\n",
    "# print(f\"Test Accuracy: {test_score_kneighbors}\")\n",
    "\n",
    "# # Mean cross-validation score\n",
    "# cross_val_scores_kneighbors = cross_val_score(best_model_kneighbors, X, y, cv=5)\n",
    "# mean_cv_score_kneighbors = cross_val_scores_kneighbors.mean()\n",
    "# print(f\"Mean Cross-Validation Score: {mean_cv_score_kneighbors}\")\n",
    "\n",
    "# # joblib.dump(best_model_kneighbors, 'kneighbors_classifier_model.pkl')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9ec78ebd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # linear_svc_classifier = LinearSVC(random_state=42)\n",
    "# ##############################################################################################################\n",
    "# # Hyperparameter tuning\n",
    "# linear_svc_param_grid = {\n",
    "#     'C': [0.1, 1, 10, 100],\n",
    "#     'max_iter': [1000, 2000, 3000, 4000],\n",
    "#     'penalty': ['l1', 'l2'],\n",
    "#     'dual': [True, False]\n",
    "# }\n",
    "\n",
    "\n",
    "# grid_search_linear_svc = GridSearchCV(linear_svc_classifier, linear_svc_param_grid, cv=5, scoring='accuracy', n_jobs=-1)\n",
    "# grid_search_linear_svc.fit(X_train, y_train)\n",
    "\n",
    "# # Print results\n",
    "# print(\"\\nHyperparameter tuning for Linear SVC:\")\n",
    "# print(f\"Best hyperparameters: {grid_search_linear_svc.best_params_}\")\n",
    "# best_model_linear_svc = grid_search_linear_svc.best_estimator_\n",
    "\n",
    "# # Training score\n",
    "# training_score_linear_svc = best_model_linear_svc.score(X_train, y_train)\n",
    "# print(f\"Training Score: {training_score_linear_svc}\")\n",
    "\n",
    "# # Test score\n",
    "# y_pred_linear_svc = best_model_linear_svc.predict(X_test)\n",
    "# test_score_linear_svc = accuracy_score(y_test, y_pred_linear_svc)\n",
    "# print(f\"Test Accuracy: {test_score_linear_svc}\")\n",
    "\n",
    "# # Mean cross-validation score\n",
    "# cross_val_scores_linear_svc = cross_val_score(best_model_linear_svc, X, y, cv=5)\n",
    "# mean_cv_score_linear_svc = cross_val_scores_linear_svc.mean()\n",
    "# print(f\"Mean Cross-Validation Score: {mean_cv_score_linear_svc}\")\n",
    "\n",
    "# # joblib.dump(best_model_linear_svc, 'linear_svc_model.pkl')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "aa3e2c30",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\bossr\\anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:1244: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.7254565520352514\n",
      "Test Accuracy: 0.7252456903496053\n",
      "Mean Cross-validation Score: 0.6338456152259346\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['linear_svc_model.pkl']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Best hyperparameters: {'C': 0.1, 'dual': False, 'max_iter': 1000, 'penalty': 'l1'}\n",
    "\n",
    "LinearSVC_Model = LinearSVC(\n",
    "    C=0.1, dual=False, max_iter = 10000, penalty = 'l1')\n",
    "\n",
    "\n",
    "\n",
    "LinearSVC_Model.fit(X_train, y_train)\n",
    "y_train_pred = LinearSVC_Model.predict(X_train)\n",
    "\n",
    "train_accuracy = accuracy_score(y_train, y_train_pred)\n",
    "print(\"Training Accuracy:\", train_accuracy)\n",
    "\n",
    "y_test_pred = LinearSVC_Model.predict(X_test)\n",
    "test_accuracy = accuracy_score(y_test, y_test_pred)\n",
    "print(\"Test Accuracy:\", test_accuracy)\n",
    "\n",
    "cv_scores = cross_val_score(LinearSVC_Model, X, y, cv=5, n_jobs=-1)\n",
    "# print(\"Cross-validation Scores:\", cv_scores)\n",
    "print(\"Mean Cross-validation Score:\", cv_scores.mean())\n",
    "\n",
    "joblib.dump(LinearSVC_Model, 'linear_svc_model.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0e213793",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.9999919443840273\n",
      "Test Accuracy: 0.8504913806992106\n",
      "Mean Cross-validation Score: 0.6876594929156548\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['RandomForest.pkl']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RFmodel = RandomForestClassifier()\n",
    "RFmodel.fit(X_train, y_train)\n",
    "\n",
    "y_train_pred = RFmodel.predict(X_train)\n",
    "\n",
    "train_accuracy = accuracy_score(y_train, y_train_pred)\n",
    "print(\"Training Accuracy:\", train_accuracy)\n",
    "\n",
    "y_test_pred = RFmodel.predict(X_test)\n",
    "test_accuracy = accuracy_score(y_test, y_test_pred)\n",
    "print(\"Test Accuracy:\", test_accuracy)\n",
    "\n",
    "cv_scores = cross_val_score(RFmodel, X, y, cv=5, n_jobs=-1)\n",
    "# print(\"Cross-validation Scores:\", cv_scores)\n",
    "print(\"Mean Cross-validation Score:\", cv_scores.mean())\n",
    "\n",
    "joblib.dump(RFmodel, 'RandomForest.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "02073b85",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.9898660351063744\n",
      "Test Accuracy: 0.8258417915256968\n",
      "Mean Cross-validation Score: 0.6608451590894389\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['BaggingModel.pkl']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BaggingModel = BaggingClassifier(n_jobs=-1)\n",
    "BaggingModel.fit(X_train, y_train)\n",
    "\n",
    "y_train_pred = BaggingModel.predict(X_train)\n",
    "\n",
    "train_accuracy = accuracy_score(y_train, y_train_pred)\n",
    "print(\"Training Accuracy:\", train_accuracy)\n",
    "\n",
    "y_test_pred = BaggingModel.predict(X_test)\n",
    "test_accuracy = accuracy_score(y_test, y_test_pred)\n",
    "print(\"Test Accuracy:\", test_accuracy)\n",
    "\n",
    "cv_scores = cross_val_score(BaggingModel, X, y, cv=5, n_jobs=-1)\n",
    "# print(\"Cross-validation Scores:\", cv_scores)\n",
    "print(\"Mean Cross-validation Score:\", cv_scores.mean())\n",
    "\n",
    "joblib.dump(BaggingModel, 'BaggingModel.pkl')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7f0802f8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.735244125442052\n",
      "Test Accuracy: 0.7300789431287257\n",
      "Mean Cross-validation Score: 0.6557126962152625\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['ExtraTrees.pkl']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ETmodel = ExtraTreesClassifier(max_depth=10, n_jobs=-1)\n",
    "ETmodel.fit(X_train, y_train)\n",
    "\n",
    "y_train_pred = ETmodel.predict(X_train)\n",
    "\n",
    "train_accuracy = accuracy_score(y_train, y_train_pred)\n",
    "print(\"Training Accuracy:\", train_accuracy)\n",
    "\n",
    "y_test_pred = ETmodel.predict(X_test)\n",
    "test_accuracy = accuracy_score(y_test, y_test_pred)\n",
    "print(\"Test Accuracy:\", test_accuracy)\n",
    "\n",
    "cv_scores = cross_val_score(ETmodel, X, y, cv=5, n_jobs=-1)\n",
    "# print(\"Cross-validation Scores:\", cv_scores)\n",
    "print(\"Mean Cross-validation Score:\", cv_scores.mean())\n",
    "\n",
    "joblib.dump(ETmodel, 'ExtraTrees.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8f97367e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.8260228618381305\n",
      "Test Accuracy: 0.8063476719832448\n",
      "Mean Cross-validation Score: 0.6934502745361947\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['HistGradientBoosting.pkl']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "HistGradientBoosting = HistGradientBoostingClassifier()\n",
    "HistGradientBoosting.fit(X_train, y_train)\n",
    "\n",
    "y_train_pred = HistGradientBoosting.predict(X_train)\n",
    "\n",
    "train_accuracy = accuracy_score(y_train, y_train_pred)\n",
    "print(\"Training Accuracy:\", train_accuracy)\n",
    "\n",
    "y_test_pred = HistGradientBoosting.predict(X_test)\n",
    "test_accuracy = accuracy_score(y_test, y_test_pred)\n",
    "print(\"Test Accuracy:\", test_accuracy)\n",
    "\n",
    "cv_scores = cross_val_score(HistGradientBoosting, X, y, cv=5, n_jobs=-1)\n",
    "# print(\"Cross-validation Scores:\", cv_scores)\n",
    "print(\"Mean Cross-validation Score:\", cv_scores.mean())\n",
    "\n",
    "joblib.dump(HistGradientBoosting, 'HistGradientBoosting.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ec0ece7b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.8590589429420721\n",
      "Test Accuracy: 0.8210085387465764\n",
      "Mean Cross-validation Score: 0.6856332857236789\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['XGB.pkl']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "XGB = XGBClassifier(n_jobs=-1)\n",
    "XGB.fit(X_train, y_train)\n",
    "\n",
    "y_train_pred = XGB.predict(X_train)\n",
    "\n",
    "train_accuracy = accuracy_score(y_train, y_train_pred)\n",
    "print(\"Training Accuracy:\", train_accuracy)\n",
    "\n",
    "y_test_pred = XGB.predict(X_test)\n",
    "test_accuracy = accuracy_score(y_test, y_test_pred)\n",
    "print(\"Test Accuracy:\", test_accuracy)\n",
    "\n",
    "cv_scores = cross_val_score(XGB, X, y, cv=5, n_jobs=-1)\n",
    "# print(\"Cross-validation Scores:\", cv_scores)\n",
    "print(\"Mean Cross-validation Score:\", cv_scores.mean())\n",
    "\n",
    "joblib.dump(XGB, 'XGB.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d9c62a7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy: 1.0\n",
      "Test Accuracy: 0.8797164491702916\n"
     ]
    }
   ],
   "source": [
    "knn_model = KNeighborsClassifier(algorithm='auto', \n",
    "                                 leaf_size=10, \n",
    "                                 n_neighbors=7, \n",
    "                                 p=1, \n",
    "                                 weights='distance')\n",
    "\n",
    "# Fit the model to your training data\n",
    "knn_model.fit(X_train, y_train)\n",
    "train_predictions = knn_model.predict(X_train)\n",
    "test_predictions = knn_model.predict(X_test)\n",
    "\n",
    "# Calculate accuracy for training and testing data\n",
    "train_accuracy = accuracy_score(y_train, train_predictions)\n",
    "test_accuracy = accuracy_score(y_test, test_predictions)\n",
    "\n",
    "print(\"Train Accuracy:\", train_accuracy)\n",
    "print(\"Test Accuracy:\", test_accuracy)\n",
    "\n",
    "\n",
    "# Export the trained model to a file\n",
    "# joblib.dump(knn_model, 'knn_model.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "529ed81c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['knn_model.pkl']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# joblib.dump(knn_model, 'knn_model.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92b01141",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}