{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import warnings\n",
    "from math import pi\n",
    "import seaborn as sns\n",
    "import plotly.express as px\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import HTML,display\n",
    "\n",
    "sns.set(style=\"whitegrid\", font_scale=1.75)\n",
    "\n",
    "\n",
    "# prettify plots\n",
    "plt.rcParams['figure.figsize'] = [20.0, 5.0]\n",
    "    \n",
    "%matplotlib inline\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deepnetts-linear-regression-validation.csv\n",
      "deepnetts-tribuo-4-lr-ensembles-average.csv\n",
      "deepnetts-tribuo-4-lr-ensembles-geomean.csv\n",
      "deepnetts-tribuo-4-lr-ensembles-harmonic-mean.csv\n",
      "deepnetts-tribuo-4-lr-ensembles-median.csv\n",
      "linear-for-deepnetts.csv\n",
      "linear-for-tribuo.csv\n",
      "tribuo-3-lr-ensembles-average.csv\n",
      "tribuo-3-lr-ensembles-geomean.csv\n",
      "tribuo-3-lr-ensembles-harmonic-mean.csv\n",
      "tribuo-3-lr-ensembles-median.csv\n",
      "tribuo-linear-regression-ada-validation.csv\n",
      "tribuo-linear-regression-cart-validation.csv\n",
      "tribuo-linear-regression-sgd-validation.csv\n",
      "tribuo-linear-regression-xgb-validation.csv\n"
     ]
    }
   ],
   "source": [
    "DATASETS_FOLDER=\"../datasets/\"\n",
    "!ls ../datasets/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_names = ['X_input', 'y_prediction']\n",
    "\n",
    "deepnetts_lr_validation = pd.read_csv(f'{DATASETS_FOLDER}/deepnetts-linear-regression-validation.csv')\n",
    "deepnetts_lr_validation.columns = column_names\n",
    "\n",
    "tribuo_lr_sgd_validation = pd.read_csv(f'{DATASETS_FOLDER}/tribuo-linear-regression-sgd-validation.csv')\n",
    "tribuo_lr_sgd_validation.columns = column_names\n",
    "\n",
    "tribuo_lr_ada_validation = pd.read_csv(f'{DATASETS_FOLDER}/tribuo-linear-regression-ada-validation.csv')\n",
    "tribuo_lr_ada_validation.columns = column_names\n",
    "tribuo_lr_cart_validation = pd.read_csv(f'{DATASETS_FOLDER}/tribuo-linear-regression-cart-validation.csv')\n",
    "tribuo_lr_cart_validation.columns = column_names\n",
    "tribuo_lr_xgb_validation = pd.read_csv(f'{DATASETS_FOLDER}/tribuo-linear-regression-xgb-validation.csv')\n",
    "tribuo_lr_xgb_validation.columns = column_names\n",
    "\n",
    "tribuo_3_lr_ensembles_average = pd.read_csv(f'{DATASETS_FOLDER}/tribuo-3-lr-ensembles-average.csv')\n",
    "tribuo_3_lr_ensembles_average.columns = column_names\n",
    "tribuo_3_lr_ensembles_median = pd.read_csv(f'{DATASETS_FOLDER}/tribuo-3-lr-ensembles-median.csv')\n",
    "tribuo_3_lr_ensembles_median.columns = column_names\n",
    "tribuo_3_lr_ensembles_geomean = pd.read_csv(f'{DATASETS_FOLDER}/tribuo-3-lr-ensembles-geomean.csv')\n",
    "tribuo_3_lr_ensembles_geomean.columns = column_names\n",
    "tribuo_3_lr_ensembles_harmonic_mean = pd.read_csv(f'{DATASETS_FOLDER}/tribuo-3-lr-ensembles-harmonic-mean.csv')\n",
    "tribuo_3_lr_ensembles_harmonic_mean.columns = column_names                                            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def geometric_mean_of(distribution):\n",
    "    index = 0\n",
    "    N = len(distribution)\n",
    "    result = 1\n",
    "    for index in range(0, N):\n",
    "        result = result * distribution[index]\n",
    "    if N != 0:\n",
    "        return abs(result) ** (1 / N)\n",
    "    else:\n",
    "        raise ValueError('No distribution has been passed in, cannot compute Geometric Mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def harmonic_mean_of(distribution):\n",
    "    index = 0\n",
    "    N = len(distribution)\n",
    "    result = 0\n",
    "    for index in range(0, N):\n",
    "        if distribution[index] != 0:\n",
    "            result = result + (1 / distribution[index])\n",
    "\n",
    "    if result == 0:\n",
    "        raise ValueError('Distribution contains one or more zeros, cannot compute Harmonic Mean')\n",
    "    else:\n",
    "        return 1 / result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Legend\n",
    "\n",
    "- **NN**: Neural Networks\n",
    "- **SGD**: Stochastic Gradient Descent\n",
    "- **LR**: Linear Regression\n",
    "- **ADA**: Adam Optimiser\n",
    "- **CART**: Classification And Regression Trees (Ensemble)\n",
    "- **XGB**: XGBoost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plotting all the input data (they should be the same)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x11b6437b8>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 191516 characters]\n",
      "text/plain": [
       "<Figure size 1440x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 8.0]\n",
    "deepnetts_lr_validation.X_input.plot()\n",
    "tribuo_lr_sgd_validation.X_input.plot()\n",
    "tribuo_lr_ada_validation.X_input.plot()\n",
    "tribuo_lr_cart_validation.X_input.plot()\n",
    "tribuo_lr_xgb_validation.X_input.plot()\n",
    "plt.legend(['deepnetts: Linear Regression/NN', \n",
    "            'tribuo: Linear Regression/SGD',\n",
    "            'tribuo: Linear Regression/ADA',\n",
    "            'tribuo: Linear Regression/CART',\n",
    "            'tribuo: Linear Regression/XGB'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Taking a closer look at the prediction results of 'tribuo: LR/SGD' (Tribuo's Linear Regression using the SGD algorithm)\n",
    "\n",
    "We won't be using this one for ensembling as it seems to be on a different scale."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x11b7725c0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 154544 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 10.0]\n",
    "tribuo_lr_sgd_validation.y_prediction.plot()\n",
    "plt.legend(['tribuo: Linear Regression/SGD'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparing results from DeepNetts: NN and Tribuo Linear Regression: ADA, CART and XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x122a4ba58>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 326644 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 10.0]\n",
    "deepnetts_lr_validation.y_prediction.plot()\n",
    "tribuo_lr_ada_validation.y_prediction.plot()\n",
    "tribuo_lr_cart_validation.y_prediction.plot()\n",
    "tribuo_lr_xgb_validation.y_prediction.plot()\n",
    "plt.legend(['deepnetts: Linear Regression/NN', \n",
    "            'tribuo: Linear Regression/ADA', \n",
    "            'tribuo: Linear Regression/CART', \n",
    "            'tribuo: Linear Regression/XGBoost'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparing all of the above methods (DeepNetts: NN, and Tribuo Linear Regression: ADA, CART and XGBoost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x123188f60>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 436140 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 10.0]\n",
    "deepnetts_lr_validation.y_prediction.plot(color=\"darkgreen\")\n",
    "tribuo_3_lr_ensembles_average.y_prediction.plot(linestyle=\"dotted\")\n",
    "tribuo_3_lr_ensembles_median.y_prediction.plot(linestyle=\"dashed\")\n",
    "tribuo_3_lr_ensembles_geomean.y_prediction.plot(linestyle=\"dashdot\")\n",
    "tribuo_3_lr_ensembles_harmonic_mean.y_prediction.plot(color=\"blue\")\n",
    "plt.legend(['deepnetts: Linear Regression/NN', \n",
    "            'tribuo: Linear Regression/Ensemble (averaging)', \n",
    "            'tribuo: Linear Regression/Ensemble (median)', \n",
    "            'tribuo: Linear Regression/Ensemble (geomean)',\n",
    "            'tribuo: Linear Regression/Ensemble (harmonic mean)'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Taking a closer look at the ensembled results using the harmonic mean\n",
    "\n",
    "We won't be using this one for further ensembling as it seems to be on a different scale as compared to the others."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1231889e8>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 117816 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "tribuo_3_lr_ensembles_harmonic_mean.y_prediction.plot(color=\"blue\")\n",
    "plt.legend(['tribuo: Linear Regression/Ensemble (harmonic mean)'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ensembling by averaging method (DeepNetts: NN & Tribuo Linear Regression: ADA, CART and XGBoost (all 3 ensembled))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "deepnetts_tribuo_4_lr_ensembles_average = tribuo_lr_ada_validation.copy()\n",
    "deepnetts_tribuo_4_lr_ensembles_average.y_prediction = (deepnetts_lr_validation.y_prediction \\\n",
    "            + tribuo_lr_ada_validation.y_prediction + tribuo_lr_cart_validation.y_prediction + \\\n",
    "            tribuo_lr_xgb_validation.y_prediction) / 4\n",
    "deepnetts_tribuo_4_lr_ensembles_average.to_csv(f'{DATASETS_FOLDER}/deepnetts-tribuo-4-lr-ensembles-average.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x131a40208>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 266876 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 10.0]\n",
    "deepnetts_lr_validation.y_prediction.plot(color=\"darkgreen\")\n",
    "tribuo_3_lr_ensembles_average.y_prediction.plot(color=\"pink\")\n",
    "deepnetts_tribuo_4_lr_ensembles_average.y_prediction.plot(color=\"blue\")\n",
    "plt.legend(['deepnetts: Linear Regression/NN', \n",
    "            'tribuo: Linear Regression/Ensemble (average)',\n",
    "            'deepnetts: NN, tribuo: Linear Regression/Ensemble (averaging)'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ensembling by median method (DeepNetts: NN & Tribuo Linear Regression: ADA, CART and XGBoost (all 3 ensembled))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "deepnetts_tribuo_4_lr_ensembles_median = tribuo_lr_ada_validation.copy()\n",
    "concatenated_df = pd.concat([deepnetts_lr_validation.y_prediction, \\\n",
    "                             tribuo_lr_ada_validation.y_prediction, \\\n",
    "                             tribuo_lr_cart_validation.y_prediction, \\\n",
    "                             tribuo_lr_xgb_validation.y_prediction], axis=1)\n",
    "deepnetts_tribuo_4_lr_ensembles_median.y_prediction = np.median(concatenated_df, axis=1)\n",
    "deepnetts_tribuo_4_lr_ensembles_median.to_csv(f'{DATASETS_FOLDER}/deepnetts-tribuo-4-lr-ensembles-median.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x132a05630>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 296224 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 10.0]\n",
    "deepnetts_lr_validation.y_prediction.plot(color=\"pink\")\n",
    "tribuo_3_lr_ensembles_median.y_prediction.plot(linestyle=\"dashed\")\n",
    "deepnetts_tribuo_4_lr_ensembles_median.y_prediction.plot(color=\"blue\")\n",
    "plt.legend(['deepnetts: Linear Regression/NN', \n",
    "            'tribuo: Linear Regression/Ensemble (median)',\n",
    "            'deepnetts: NN, tribuo: Linear Regression/Ensemble (median)'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ensembling by geometric mean method (DeepNetts: NN & Tribuo Linear Regression: ADA, CART and XGBoost (all 3 ensembled))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "deepnetts_tribuo_4_lr_ensembles_geomean = tribuo_lr_ada_validation.copy()\n",
    "deepnetts_tribuo_4_lr_ensembles_geomean.y_prediction = concatenated_df.y_prediction.apply(geometric_mean_of, axis=1)\n",
    "deepnetts_tribuo_4_lr_ensembles_geomean.to_csv(f'{DATASETS_FOLDER}/deepnetts-tribuo-4-lr-ensembles-geomean.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x133a23a90>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 279000 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 10.0]\n",
    "deepnetts_lr_validation.y_prediction.plot(color=\"pink\")\n",
    "tribuo_3_lr_ensembles_geomean.y_prediction.plot(linestyle=\"dashed\")\n",
    "deepnetts_tribuo_4_lr_ensembles_geomean.y_prediction.plot(color=\"blue\")\n",
    "plt.legend(['deepnetts: Linear Regression/NN', \n",
    "            'tribuo: Linear Regression/Ensemble (geomean)',\n",
    "            'deepnetts: NN, tribuo: Linear Regression/Ensemble (geomean)'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ensembling by harmonic mean method (DeepNetts: NN & Tribuo Linear Regression: ADA, CART and XGBoost (all 3 ensembled))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "deepnetts_tribuo_4_lr_ensembles_harmonic_mean = tribuo_lr_ada_validation.copy()\n",
    "deepnetts_tribuo_4_lr_ensembles_harmonic_mean.y_prediction = concatenated_df.y_prediction.apply(harmonic_mean_of, axis=1)\n",
    "deepnetts_tribuo_4_lr_ensembles_harmonic_mean.to_csv(f'{DATASETS_FOLDER}/deepnetts-tribuo-4-lr-ensembles-harmonic-mean.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x13410e0b8>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 220520 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 10.0]\n",
    "deepnetts_lr_validation.y_prediction.plot(color=\"pink\")\n",
    "tribuo_3_lr_ensembles_harmonic_mean.y_prediction.plot(linestyle=\"dashed\")\n",
    "deepnetts_tribuo_4_lr_ensembles_harmonic_mean.y_prediction.plot(color=\"blue\")\n",
    "plt.legend(['deepnetts: Linear Regression/NN', \n",
    "            'tribuo: Linear Regression/Ensemble (harmonic mean)',\n",
    "            'deepnetts: NN, tribuo: Linear Regression/Ensemble (harmonic mean)'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparing all of the above methods (DeepNetts: NN and Tribuo Linear Regression: ADA, CART and XGBoost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x133446438>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 372308 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 10.0]\n",
    "deepnetts_tribuo_4_lr_ensembles_average.y_prediction.plot(linestyle=\"dotted\")\n",
    "deepnetts_tribuo_4_lr_ensembles_median.y_prediction.plot(linestyle=\"dashed\")\n",
    "deepnetts_tribuo_4_lr_ensembles_geomean.y_prediction.plot(linestyle=\"dashdot\")\n",
    "deepnetts_tribuo_4_lr_ensembles_harmonic_mean.y_prediction.plot(color=\"blue\")\n",
    "plt.legend(['deepnetts: NN, tribuo: Linear Regression/Ensemble (averaging)', \n",
    "            'deepnetts: NN, tribuo: Linear Regression/Ensemble (median)', \n",
    "            'deepnetts: NN, tribuo: Linear Regression/Ensemble (geomean)',\n",
    "            'deepnetts: NN, tribuo: Linear Regression/Ensemble (harmonic mean)'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We could stop here or continue creating more levels of ensembles."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
