{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Sahel-Eskandar/Project-Notebooks/blob/main/1_LogReg_XGB_MLP_CLF_Classifier.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RfFtonm8VyQl"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.model_selection import KFold\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.metrics import roc_auc_score\n",
        "from sklearn.metrics import roc_curve\n",
        "from sklearn.metrics import accuracy_score, confusion_matrix, precision_score, recall_score, classification_report\n",
        "from sklearn.dummy import DummyClassifier\n",
        "from sklearn import svm\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from xgboost import XGBClassifier\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from sklearn.metrics import roc_curve\n",
        "from sklearn.metrics import auc\n",
        "from sklearn.ensemble import IsolationForest\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "%matplotlib inline "
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Data Prepration"
      ],
      "metadata": {
        "id": "iVacmLdYkESs"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9CUsv11XVyQm"
      },
      "outputs": [],
      "source": [
        "path_y = Output + 'df_y.csv'\n",
        "path_X = Output + 'df_X.csv'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "a1wVoG-HVyQn"
      },
      "outputs": [],
      "source": [
        "X = pd.read_csv(path_X)\n",
        "y = pd.read_csv(path_y)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bMBYXfN_VyQn",
        "outputId": "3c81c124-45e6-4db4-c4b1-9b93f2870e69"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(69790, 1)\n",
            "(69790, 10)\n"
          ]
        }
      ],
      "source": [
        "print(y.shape)\n",
        "print(X.shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Lupo4AcVyQo"
      },
      "source": [
        "##### KFold cross validation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Mps4gU1MVyQp"
      },
      "outputs": [],
      "source": [
        "kf = KFold(n_splits = 10, shuffle = True, random_state = 4)\n",
        "skf = StratifiedKFold(n_splits = 10, shuffle = True, random_state = 4)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E3BNo8FLVyQp"
      },
      "outputs": [],
      "source": [
        "for train_index,test_index in kf.split(X,y):\n",
        "    X_train,X_test = X.loc[train_index],X.loc[test_index]\n",
        "    y_train,y_test = y.loc[train_index],y.loc[test_index]"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Model Training & Evaluation"
      ],
      "metadata": {
        "id": "Ye6-TuLuj9mk"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a06YxtiuVyQq"
      },
      "source": [
        "###### Logistic regression model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QDBUd2U7VyQq"
      },
      "outputs": [],
      "source": [
        "logreg = LogisticRegression()\n",
        "grid={\"C\":np.logspace(-3,3,7), \"penalty\":[\"l1\",\"l2\"]}\n",
        "logreg_cv=GridSearchCV(logreg,grid,cv=10)\n",
        "model = logreg_cv.fit(X_train, y_train)\n",
        "y_pred = logreg_cv.predict(X_test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SvoeuL_UVyQq"
      },
      "outputs": [],
      "source": [
        "scores = logreg_cv.score(X_test,y_test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2s6HQFNrVyQr",
        "outputId": "80448f12-2dfd-49bb-cd37-62085a951f33"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy of logistic regression classifier on test set: 0.85\n"
          ]
        }
      ],
      "source": [
        "print('Accuracy of logistic regression classifier on test set: {:.2f}'.format(scores))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BuGxikfWVyQr",
        "outputId": "954bd403-5058-4233-c20c-6593f840eb6b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[2876,  566],\n",
              "       [ 469, 3068]])"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "confusion_matrix = confusion_matrix(y_test, y_pred)\n",
        "confusion_matrix"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IYNJYQMXVyQr",
        "outputId": "f6a9e872-a9e5-40eb-ddf2-1b34250b705f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.86      0.84      0.85      3442\n",
            "           1       0.84      0.87      0.86      3537\n",
            "\n",
            "    accuracy                           0.85      6979\n",
            "   macro avg       0.85      0.85      0.85      6979\n",
            "weighted avg       0.85      0.85      0.85      6979\n",
            "\n"
          ]
        }
      ],
      "source": [
        "print(classification_report(y_test, y_pred))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "glvGgYxYVyQr",
        "outputId": "4698d4f7-adba-4d26-ca43-55ea50056345"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "[encoded data, 23988 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "logit_roc_auc = roc_auc_score(y_test, y_pred)\n",
        "\n",
        "fpr, tpr, thresholds = roc_curve(y_test, logreg_cv.predict_proba(X_test)[:,1])\n",
        "\n",
        "\n",
        "plt.figure()\n",
        "plt.plot(fpr, tpr, label='Logistic Regression (area = %0.2f)' % logit_roc_auc)\n",
        "plt.plot([0, 1], [0, 1],'r--')\n",
        "plt.xlim([0.0, 1.0])\n",
        "plt.ylim([0.0, 1.05])\n",
        "plt.xlabel('False Positive Rate')\n",
        "plt.ylabel('True Positive Rate')\n",
        "plt.title('Receiver operating characteristic')\n",
        "plt.legend(loc=\"lower right\")\n",
        "plt.savefig('Log_ROC')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "afbLAD91VyQr"
      },
      "outputs": [],
      "source": [
        "def logreg(X, y, cv):\n",
        "    \"\"\"\n",
        "    Creates folds manually, perform \n",
        "    Returns an array of validation (recall) scores\n",
        "    \"\"\"\n",
        "    if cv == 'kf':\n",
        "        cv = KFold(n_splits = 10, shuffle = True, random_state = 4)\n",
        "    elif cv == 'skf':\n",
        "        cv = StratifiedKFold(n_splits = 10, shuffle = True, random_state = 4)\n",
        "    else:\n",
        "        cv == None\n",
        "    \n",
        "    scores = []\n",
        "    logreg = LogisticRegression()\n",
        "    grid={\"C\":np.logspace(-3,3,7), \"penalty\":[\"l1\",\"l2\"]}\n",
        "    logreg_cv=GridSearchCV(logreg,grid,cv=10)\n",
        "    for train_index,test_index in cv.split(X,y):\n",
        "        X_train,X_test = X.loc[train_index],X.loc[test_index]\n",
        "        y_train,y_test = y.loc[train_index],y.loc[test_index]\n",
        "\n",
        "        # Fit the model on the training data\n",
        "        model_obj = logreg_cv.fit(X_train, y_train)\n",
        "        # Score the model on the validation data\n",
        "        score = recall_score(y_test, model_obj.predict(X_test))\n",
        "        scores.append(score)\n",
        "    return np.array(scores).mean()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-LMeCxGVVyQs",
        "outputId": "798411df-3eff-49e5-ce85-746282de9803"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8569456268661846"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ],
      "source": [
        "logreg(X, y, kf)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u4x6sWVGVyQs",
        "outputId": "58bdfc28-e3c4-49bb-ce16-90734742f689"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8572003702179835"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "logreg(X, y, skf)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bfJH9QOXVyQs"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import confusion_matrix\n",
        "def model_classifier(model, X, y, cv):\n",
        "    \"\"\"\n",
        "    Creates folds manually, perform \n",
        "    Returns an array of validation (recall) scores\n",
        "    \"\"\"\n",
        "    if cv == 'kf':\n",
        "        cv = KFold(n_splits = 10, shuffle = True, random_state = 4)\n",
        "    elif cv == 'skf':\n",
        "        cv = StratifiedKFold(n_splits = 10, shuffle = True, random_state = 4)\n",
        "    else:\n",
        "        cv == None\n",
        "    \n",
        "    scores = []\n",
        "    \n",
        "    \n",
        "    for train_index,test_index in cv.split(X,y):\n",
        "        X_train,X_test = X.loc[train_index],X.loc[test_index]\n",
        "        y_train,y_test = y.loc[train_index],y.loc[test_index]\n",
        "\n",
        "        # Fit the model on the training data\n",
        "        model_obj = model.fit(X_train, y_train)\n",
        "        y_pred = model_obj.predict(X_test)\n",
        "        # Score the model on the validation data\n",
        "        score = accuracy_score(y_test, y_pred)\n",
        "        report = classification_report(y_test, y_pred)\n",
        "        conf_matrix = confusion_matrix(y_test, y_pred) \n",
        "        \n",
        "        scores.append(score)\n",
        "        mean_score = np.array(scores).mean()\n",
        "    import seaborn as sns\n",
        "    array = conf_matrix\n",
        "    #df_cm = pd.DataFrame(array, index = ['No','Yes'], columns = ['No','Yes'])\n",
        "    plt.figure(figsize = (6,4))\n",
        "    #sns.heatmap(df_cm, annot=True,fmt='d')\n",
        "    group_names = ['True Neg','False Pos','False Neg','True Pos']\n",
        "    group_counts = ['{0:0.0f}'.format(value) for value in array.flatten()]\n",
        "    group_percentages = ['{0:.2%}'.format(value) for value in array.flatten()/np.sum(array)]\n",
        "    labels = [f\"{v1}\\n{v2}\\n{v3}\" for v1, v2, v3 in zip(group_names,group_counts,group_percentages)]\n",
        "    labels = np.asarray(labels).reshape(2,2)\n",
        "    sns.heatmap(array, annot=labels, fmt='', cmap='Blues')\n",
        "    print('\\n Accuracy scores of the model: {:.2f}'.format(mean_score))\n",
        "    print('\\n Classification report of the model')\n",
        "    print('--------------------------------------')\n",
        "    print(report)\n",
        "    \n",
        "    print('\\n Confusion Matrix of the model')\n",
        "    print('--------------------------------------')\n",
        "    #print(conf_matrix)\n",
        "    \n",
        "    \n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Bo15L8mGVyQs"
      },
      "outputs": [],
      "source": [
        "logreg = LogisticRegression()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 604
        },
        "id": "QxX7MUOlVyQt",
        "outputId": "e0767738-bf3f-4d43-be5d-e05320b800cb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " Accuracy scores of the model: 0.84\n",
            "\n",
            " Classification report of the model\n",
            "--------------------------------------\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.86      0.84      0.85      3442\n",
            "           1       0.84      0.87      0.86      3537\n",
            "\n",
            "    accuracy                           0.85      6979\n",
            "   macro avg       0.85      0.85      0.85      6979\n",
            "weighted avg       0.85      0.85      0.85      6979\n",
            "\n",
            "\n",
            " Confusion Matrix of the model\n",
            "--------------------------------------\n",
            "CPU times: user 2.34 s, sys: 1.52 s, total: 3.86 s\n",
            "Wall time: 3.03 s\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ],
            "image/png": "[encoded data, 16424 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "%%time\n",
        "model_classifier(logreg, X, y, kf)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 572
        },
        "id": "Lf8NrW56VyQt",
        "outputId": "e43a4b15-39d7-4b04-9fae-59b3fb2a48c6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " Accuracy scores of the model: 0.84\n",
            "\n",
            " Classification report of the model\n",
            "--------------------------------------\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.86      0.84      0.85      3489\n",
            "           1       0.84      0.87      0.85      3490\n",
            "\n",
            "    accuracy                           0.85      6979\n",
            "   macro avg       0.85      0.85      0.85      6979\n",
            "weighted avg       0.85      0.85      0.85      6979\n",
            "\n",
            "\n",
            " Confusion Matrix of the model\n",
            "--------------------------------------\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ],
            "image/png": "[encoded data, 16248 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "model_classifier(logreg, X, y, skf)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AxWQay-EVyQt"
      },
      "outputs": [],
      "source": [
        "xgb = XGBClassifier()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 604
        },
        "id": "wIg9sO8iVyQt",
        "outputId": "7338835a-e85d-4d91-f55c-898f89810289"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " Accuracy scores of the model: 0.87\n",
            "\n",
            " Classification report of the model\n",
            "--------------------------------------\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.91      0.84      0.87      3442\n",
            "           1       0.86      0.92      0.89      3537\n",
            "\n",
            "    accuracy                           0.88      6979\n",
            "   macro avg       0.88      0.88      0.88      6979\n",
            "weighted avg       0.88      0.88      0.88      6979\n",
            "\n",
            "\n",
            " Confusion Matrix of the model\n",
            "--------------------------------------\n",
            "CPU times: user 46.6 s, sys: 132 ms, total: 46.7 s\n",
            "Wall time: 46.6 s\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ],
            "image/png": "[encoded data, 16396 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "%%time\n",
        "model_classifier(xgb, X, y, kf)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 569
        },
        "id": "jHUxkjlCVyQt",
        "outputId": "dbed55f7-c5b1-4091-a8a9-78ec0e861917"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " Accuracy scores of the model: 0.87\n",
            "\n",
            " Classification report of the model\n",
            "--------------------------------------\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.91      0.83      0.87      3489\n",
            "           1       0.85      0.91      0.88      3490\n",
            "\n",
            "    accuracy                           0.87      6979\n",
            "   macro avg       0.88      0.87      0.87      6979\n",
            "weighted avg       0.88      0.87      0.87      6979\n",
            "\n",
            "\n",
            " Confusion Matrix of the model\n",
            "--------------------------------------\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ],
            "image/png": "[encoded data, 16184 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "model_classifier(xgb, X, y, skf)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gBhWnJuzVyQt"
      },
      "outputs": [],
      "source": [
        "mlp = MLPClassifier()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 604
        },
        "id": "3LXDx13PVyQt",
        "outputId": "bf63db42-4cc7-46bb-c15f-a29c85423bd2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " Accuracy scores of the model: 0.90\n",
            "\n",
            " Classification report of the model\n",
            "--------------------------------------\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.95      0.86      0.90      3442\n",
            "           1       0.87      0.95      0.91      3537\n",
            "\n",
            "    accuracy                           0.91      6979\n",
            "   macro avg       0.91      0.91      0.91      6979\n",
            "weighted avg       0.91      0.91      0.91      6979\n",
            "\n",
            "\n",
            " Confusion Matrix of the model\n",
            "--------------------------------------\n",
            "CPU times: user 14min 13s, sys: 11min 23s, total: 25min 37s\n",
            "Wall time: 13min 12s\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ],
            "image/png": "[encoded data, 16404 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "%%time\n",
        "model_classifier(mlp, X, y, kf)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 569
        },
        "id": "Lhe69_SdVyQt",
        "outputId": "2a343791-b8a4-427b-cde6-191865673bc8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " Accuracy scores of the model: 0.89\n",
            "\n",
            " Classification report of the model\n",
            "--------------------------------------\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.86      0.90      3489\n",
            "           1       0.87      0.95      0.91      3490\n",
            "\n",
            "    accuracy                           0.91      6979\n",
            "   macro avg       0.91      0.91      0.91      6979\n",
            "weighted avg       0.91      0.91      0.91      6979\n",
            "\n",
            "\n",
            " Confusion Matrix of the model\n",
            "--------------------------------------\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ],
            "image/png": "[encoded data, 16276 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "model_classifier(mlp, X, y, skf)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CqC2zGmwVyQt"
      },
      "outputs": [],
      "source": [
        "clf = svm.SVC(kernel='linear')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 604
        },
        "id": "Bddc2gJEVyQu",
        "outputId": "0fa32245-8017-4599-d39e-6fec1dd9da3c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            " Accuracy scores of the model: 0.85\n",
            "\n",
            " Classification report of the model\n",
            "--------------------------------------\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.88      0.82      0.85      3442\n",
            "           1       0.84      0.89      0.86      3537\n",
            "\n",
            "    accuracy                           0.86      6979\n",
            "   macro avg       0.86      0.86      0.86      6979\n",
            "weighted avg       0.86      0.86      0.86      6979\n",
            "\n",
            "\n",
            " Confusion Matrix of the model\n",
            "--------------------------------------\n",
            "CPU times: user 18min, sys: 2.66 s, total: 18min 3s\n",
            "Wall time: 18min\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ],
            "image/png": "[encoded data, 16476 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "%%time\n",
        "model_classifier(clf, X, y, kf)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QOFiCTy7VyQu"
      },
      "source": [
        "###### ROC plots "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Lpxgw08CVyQu"
      },
      "outputs": [],
      "source": [
        "def roc_plot(model, X, y, cv):\n",
        "    for train_index,test_index in cv.split(X,y):\n",
        "        X_train,X_test = X.loc[train_index],X.loc[test_index]\n",
        "        y_train,y_test = y.loc[train_index],y.loc[test_index]\n",
        "        model_obj = model.fit(X_train, y_train)\n",
        "        y_pred = model_obj.predict(X_test)\n",
        "        y_pred_prob = model_obj.predict_proba(X_test)[:,1]\n",
        "        \n",
        "        logit_roc_auc = roc_auc_score(y_test, y_pred)\n",
        "        fpr, tpr, thresholds = roc_curve(y_test, y_pred_prob)\n",
        "    plt.figure()\n",
        "    val_model = input(\"Enter your model name: \")\n",
        "    plt.plot(fpr, tpr, label= val_model + ' (area = %0.2f)' % logit_roc_auc)\n",
        "    plt.plot([0, 1], [0, 1],'r--')\n",
        "    plt.xlim([0.0, 1.0])\n",
        "    plt.ylim([0.0, 1.05])\n",
        "    plt.xlabel('False Positive Rate')\n",
        "    plt.ylabel('True Positive Rate')\n",
        "    plt.title('Receiver operating characteristic')\n",
        "    plt.legend(loc=\"lower right\")\n",
        "    my_fig = val_model + '.png'\n",
        "    #plt.savefig(os.path.join(my_path, my_fig))\n",
        "    plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 313
        },
        "id": "PG_evzCQVyQu",
        "outputId": "b2925a48-712e-4cdc-98d4-ed34d0867780"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter your model name: Logistic Regression \n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "[encoded data, 23796 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "roc_plot(logreg, X, y, kf)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 313
        },
        "id": "tUB9YXe8VyQu",
        "outputId": "d2460657-1f5a-4c42-bd5e-56708e4a84c0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter your model name: Logistic Regression skf\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "[encoded data, 24280 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "roc_plot(logreg, X, y, skf)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 313
        },
        "id": "6RW8N6DJVyQu",
        "outputId": "340d70b2-ff47-48c7-83e1-f11170afc1be"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter your model name: XGBoost\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "[encoded data, 22460 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "roc_plot(xgb, X, y, kf)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 313
        },
        "id": "iBzabbz7VyQu",
        "outputId": "f8b509e2-75a7-48fa-c9da-9c3cac3d4f1a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter your model name: XGBoost skf\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "[encoded data, 22888 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "roc_plot(xgb, X, y, skf)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 313
        },
        "id": "v-32WMskVyQu",
        "outputId": "8d64fc86-1714-4efe-f04f-d09d1bdbed99"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Enter your model name: MLP\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "[encoded data, 21248 characters]\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "roc_plot(mlp, X, y, kf)"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "0n3jEfmRarvr"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    },
    "toc": {
      "base_numbering": 1,
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": false,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": false
    },
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}