{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W&B has already been installed\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "### Go to Quick start https://docs.wandb.com/quickstart and perform the steps before going further\n",
    "found=$(pip show wandb || true)\n",
    "if [[ -z \"${found}\" ]]; then\n",
    "    pip install --upgrade wandb\n",
    "    echo \"W&B installed successfully\"\n",
    "else\n",
    "    echo \"W&B has already been installed\"\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Falling back to default / hard-coded value set in the program\n",
      "SOURCE_ROOT_DATA_FOLDER = ../data/\n",
      "Falling back to default / hard-coded value set in the program\n",
      "TARGET_ROOT_DATA_FOLDER = ../data/\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# supress warnings\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "DEFAULT_MAX_ROWS = 1000\n",
    "\n",
    "### Kaggle Kernel settings - uncomment on when running on Kaggle\n",
    "# SOURCE_ROOT_DATA_FOLDER = '../input/sbfxcompetitionprocessdatasets/'\n",
    "# TARGET_ROOT_DATA_FOLDER = '../working'\n",
    "\n",
    "### Comment below two lines when running on Kaggle\n",
    "SOURCE_ROOT_DATA_FOLDER = '../data'\n",
    "TARGET_ROOT_DATA_FOLDER = '../data'\n",
    "\n",
    "import sys\n",
    "sys.path.insert(0, '..')\n",
    "sys.path.insert(0, '../library')\n",
    "\n",
    "import importlib\n",
    "visualisation_library = __import__(\"visualisation_library\")\n",
    "utility_library = __import__(\"utility_library\")\n",
    "process_data = __import__(\"process-data-lib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import metrics\n",
    "from scipy.stats import spearmanr\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# visualization tools\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# show plots without need of calling `.show()`\n",
    "%matplotlib inline\n",
    "\n",
    "# prettify plots\n",
    "plt.rcParams['figure.figsize'] = [20.0, 5.0]\n",
    "sns.set_palette(sns.color_palette(\"muted\"))\n",
    "sns.set_style(\"ticks\")\n",
    "\n",
    "# supress warnings\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "wandb: WARNING If you're specifying your api key in code, ensure this code is not shared publically.\n",
      "wandb: WARNING Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n",
      "wandb: WARNING Calling wandb.login() without arguments from jupyter should prompt you for an api key.\n",
      "wandb: Appending key for api.wandb.ai to your netrc file: /Users/swami/.netrc\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
       "                Project page: <a href=\"https://app.wandb.ai/sadhak001/sb-fx-comp\" target=\"_blank\">https://app.wandb.ai/sadhak001/sb-fx-comp</a><br/>\n",
       "                Run page: <a href=\"https://app.wandb.ai/sadhak001/sb-fx-comp/runs/yyuwp9wa\" target=\"_blank\">https://app.wandb.ai/sadhak001/sb-fx-comp/runs/yyuwp9wa</a><br/>\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logged in successfully on W&B\n"
     ]
    }
   ],
   "source": [
    "# When using Kaggle kernels, uncomment the below comments\n",
    "# from kaggle_secrets import UserSecretsClient\n",
    "# user_secrets = UserSecretsClient()\n",
    "# api_token = user_secrets.get_secret(\"WANDB_TOKEN\")\n",
    "\n",
    "# Ensure you have started the Jupyter instance after setting the environment variable WANDB_TOKEN, \n",
    "# with your W&B Access token that can be found in your project page on https://www.wandb.com/\n",
    "api_token = os.environ[\"WANDB_TOKEN\"]\n",
    "\n",
    "# You can permanently save the token in your .bashrc, .bash_profile, or .zshrc\n",
    "\n",
    "# W&B Imports\n",
    "import wandb\n",
    "wandb.login(key=api_token)\n",
    "wandb.init(reinit=True, project=\"sb-fx-comp\")\n",
    "print(\"Logged in successfully on W&B\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "wandb.config.stage=\"04\"\n",
    "wandb.config.method=\"02\"\n",
    "wandb.config.model=\"Linear Regression model\"\n",
    "wandb.config.RANDOM_SEED = 42\n",
    "wandb.config.FEATURE_IMPORTANCE_CUTOFF = 0.00099 ### A rule-off thumb cutoff point, to filter out features of less or no importance\n",
    "wandb.config.nRowsRead = 4500\n",
    "wandb.config.test_size=0.20\n",
    "wandb.config.input_training_dataset_filename = 'train.csv'\n",
    "wandb.config.input_test_dataset_filename = 'test.csv'\n",
    "wandb.config.feature_importance_filename = f'method-{wandb.config.method}-linear-model-feature-importance.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 4177 rows and 229 columns in ../data/04-train-228-features-removed-redundant-columns-after-02.csv\n",
      "There are 1000 rows and 228 columns in ../data/04-test-228-features-removed-redundant-columns-after-02.csv\n"
     ]
    }
   ],
   "source": [
    "final_training_dataset = process_data.load_dataset(SOURCE_ROOT_DATA_FOLDER, wandb.config.input_training_dataset_filename, nRowsRead=wandb.config.nRowsRead)\n",
    "final_test_dataset = process_data.load_dataset(SOURCE_ROOT_DATA_FOLDER, wandb.config.input_test_dataset_filename, nRowsRead=wandb.config.nRowsRead)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_training_dataset.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_column_names = [col for col in final_training_dataset.columns]\n",
    "feature_column_names.remove('id')\n",
    "feature_column_names.remove('target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = final_training_dataset[feature_column_names]\n",
    "y = final_training_dataset['target']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=wandb.config.test_size, random_state=wandb.config.RANDOM_SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://earlacademy.com/linear-regression-and-elasticnet-with-sklearn/\n",
    "# We will use Linear model instead\n",
    "\n",
    "linearModel = LinearRegression()\n",
    "linearModel.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = linearModel.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ratios and metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spearman Correlation: -0.013464546759619446\n"
     ]
    }
   ],
   "source": [
    "print(f'Spearman Correlation: {spearmanr(y_test, predictions)[0]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=-0.013464546759619446, pvalue=0.6974651538686936)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spearmanr(y_test, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Intercept:  1.0001440337328698\n"
     ]
    }
   ],
   "source": [
    "print(\"Intercept: \", linearModel.intercept_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Actual v/s predicted target variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 66124 characters]\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "visualisation_library.plot_between(y_test, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49 features, reduction of 78.41% (feature importance cutoff score: 0.00099 and higher)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a26ecdb38>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 65604 characters]\n",
      "text/plain": [
       "<Figure size 1296x1800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# https://www.kaggle.com/den3b81/elasticnet-lb-0-547-and-feature-importance#Feature-importance\n",
    "\n",
    "feature_importance = pd.Series(index = X_train.columns, data = np.abs(linearModel.coef_))\n",
    "\n",
    "n_selected_features = (feature_importance > wandb.config.FEATURE_IMPORTANCE_CUTOFF).sum()\n",
    "reduction_in_features = (1 - n_selected_features / len(feature_importance)) * 100\n",
    "print('{0:d} features, reduction of {1:2.2f}% (feature importance cutoff score: {2:0.5f} and higher)'.format(\n",
    "    n_selected_features, reduction_in_features, wandb.config.FEATURE_IMPORTANCE_CUTOFF)\n",
    ")\n",
    "\n",
    "wandb.log({\"features with importance\": n_selected_features, \n",
    "           \"reduction in features\": reduction_in_features, \n",
    "           \"total features\": len(feature_column_names)})\n",
    "\n",
    "plt.rcParams.update({'font.size': 22})\n",
    "feature_importance.sort_values().tail(30).plot(kind = 'barh', figsize = (18,25))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved the feature importance columns in descending order of their importance to method-02-linear-model-feature-importance.csv\n"
     ]
    }
   ],
   "source": [
    "feature_importance.sort_values(ascending=False).to_csv(os.path.join(TARGET_ROOT_DATA_FOLDER, wandb.config.feature_importance_filename)) ### in this case we DO NOT use index=False, because we want to save the index\n",
    "feature_importance.sort_values(ascending=False).to_csv(os.path.join(wandb.run.dir, wandb.config.feature_importance_filename)) ### in this case we DO NOT use index=False, because we want to save the index\n",
    "print(f'Saved the feature importance columns in descending order of their importance to {wandb.config.feature_importance_filename}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualisation of residual (difference between predicted and actual)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a26f93860>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 27224 characters]\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot((y_test-predictions), bins=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training & testing metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Explained Variance Score (EVS)</th>\n",
       "      <th>Mean Absolute Error (MAE)</th>\n",
       "      <th>Mean Squared Error (MSE)</th>\n",
       "      <th>Root Mean Square Error (RMSE)</th>\n",
       "      <th>R2</th>\n",
       "      <th>Spearman Correlation</th>\n",
       "      <th>Spearman p-value</th>\n",
       "      <th>Intercept</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.0701693766</td>\n",
       "      <td>0.0158473539</td>\n",
       "      <td>0.0004184923</td>\n",
       "      <td>0.0204570837</td>\n",
       "      <td>-0.0745973389</td>\n",
       "      <td>-0.0134645468</td>\n",
       "      <td>0.6974651539</td>\n",
       "      <td>1.0001440337</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Explained Variance Score (EVS)  Mean Absolute Error (MAE)  \\\n",
       "0                   -0.0701693766               0.0158473539   \n",
       "\n",
       "   Mean Squared Error (MSE)  Root Mean Square Error (RMSE)            R2  \\\n",
       "0              0.0004184923                   0.0204570837 -0.0745973389   \n",
       "\n",
       "   Spearman Correlation  Spearman p-value    Intercept  \n",
       "0         -0.0134645468      0.6974651539 1.0001440337  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_scores = {'Explained Variance Score (EVS)': metrics.explained_variance_score(y_test, predictions),\n",
    "'Mean Absolute Error (MAE)': metrics.mean_absolute_error(y_test, predictions),\n",
    "'Mean Squared Error (MSE)': metrics.mean_squared_error(y_test, predictions),\n",
    "'Root Mean Square Error (RMSE)': np.sqrt(metrics.mean_squared_error(y_test, predictions)),\n",
    "'R2': metrics.r2_score(y_test, predictions),\n",
    "'Spearman Correlation': spearmanr(y_test, predictions)[0],\n",
    "'Spearman p-value': spearmanr(y_test, predictions)[1],\n",
    "'Intercept': linearModel.intercept_\n",
    "}\n",
    "\n",
    "wandb.log(metrics_scores)\n",
    "metrics_scores_df = pd.DataFrame(metrics_scores, index=[0])\n",
    "metrics_scores_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a29a7ca20>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 23084 characters]\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "metrics_scores_df.plot.barh()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Validation using the separate test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_test = final_test_dataset[feature_column_names]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_predictions = linearModel.predict(validation_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submission and model creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_id = pd.DataFrame(final_test_dataset['id'])\n",
    "df_target = pd.DataFrame(validation_predictions, columns = ['target'])\n",
    "df_submission = pd.concat([df_id, df_target], axis=1, sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# method-02 == using Linear model\n",
    "# https://earlacademy.com/linear-regression-and-elasticnet-with-sklearn/\n",
    "\n",
    "def save_model_and_submission(model, dataframe, folder, method):\n",
    "    saved_filename = utility_library.save_submission(dataframe, folder, method)\n",
    "    \n",
    "    model_filename, ext = os.path.splitext(saved_filename)\n",
    "    model_filename = os.path.basename(model_filename) + \".pkl\"\n",
    "\n",
    "    print(\"~~~ Saving artifacts to disk \")\n",
    "    utility_library.save_model(model, folder, model_filename)\n",
    "\n",
    ""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved submission file to ../data/submission-method-02-96e757e-6ff2ae4.csv\n",
      "~~~ Saving artifacts to disk \n",
      "Saved model to file ../data/submission-method-02-96e757e-6ff2ae4.pkl\n",
      "Saved submission file to /home/user/workspace/wandb/run-20200107_164322-yyuwp9wa/submission-method-02-96e757e-6ff2ae4.csv\n",
      "~~~ Saving artifacts to disk \n",
      "Saved model to file /Users/home/user/workspace/wandb/run-20200107_164322-yyuwp9wa/submission-method-02-96e757e-6ff2ae4.pkl\n",
      "~~~ Saving artifacts into W&B folder for further upload\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['/home/user/workspace/wandb/run-20200107_164322-yyuwp9wa/submission-method-02-96e757e-6ff2ae4.pkl',\n",
       " '/home/user/workspace/wandb/run-20200107_164322-yyuwp9wa/submission-method-02.pkl']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "method=f'method-{wandb.config.method}'\n",
    "save_model_and_submission(linearModel, df_submission, TARGET_ROOT_DATA_FOLDER, method)\n",
    "save_model_and_submission(linearModel, df_submission, wandb.run.dir, method)\n",
    "\n",
    "print(\"~~~ Saving artifacts into W&B folder for further upload\")\n",
    "wandb.save(os.path.join(TARGET_ROOT_DATA_FOLDER, f'*{method}*.csv'))\n",
    "wandb.save(os.path.join(TARGET_ROOT_DATA_FOLDER, f'*{method}*.pkl'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 158224 characters]\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "visualisation_library.plot_comparsions_sns(final_training_dataset['target'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
