{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import warnings\n",
    "from math import pi\n",
    "import seaborn as sns\n",
    "import plotly.express as px\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import HTML,display\n",
    "\n",
    "sns.set(style=\"whitegrid\", font_scale=1.75)\n",
    "\n",
    "\n",
    "# prettify plots\n",
    "plt.rcParams['figure.figsize'] = [20.0, 5.0]\n",
    "    \n",
    "%matplotlib inline\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deepnetts-linear-regression-validation.csv\n",
      "deepnetts-tribuo-4-lr-ensembles-average.csv\n",
      "deepnetts-tribuo-4-lr-ensembles-geomean.csv\n",
      "deepnetts-tribuo-4-lr-ensembles-harmonic-mean.csv\n",
      "deepnetts-tribuo-4-lr-ensembles-median.csv\n",
      "linear-for-deepnetts.csv\n",
      "linear-for-tribuo.csv\n",
      "tribuo-3-lr-ensembles-average.csv\n",
      "tribuo-3-lr-ensembles-geomean.csv\n",
      "tribuo-3-lr-ensembles-harmonic-mean.csv\n",
      "tribuo-3-lr-ensembles-median.csv\n",
      "tribuo-linear-regression-ada-validation.csv\n",
      "tribuo-linear-regression-cart-validation.csv\n",
      "tribuo-linear-regression-sgd-validation.csv\n",
      "tribuo-linear-regression-xgb-validation.csv\n"
     ]
    }
   ],
   "source": [
    "DATASETS_FOLDER=\"../datasets/\"\n",
    "!ls ../datasets/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_names = ['X_input', 'y_prediction']\n",
    "\n",
    "tribuo_lr_sgd_validation = pd.read_csv(f'{DATASETS_FOLDER}/tribuo-linear-regression-sgd-validation.csv')\n",
    "tribuo_lr_sgd_validation.columns = column_names\n",
    "\n",
    "tribuo_lr_ada_validation = pd.read_csv(f'{DATASETS_FOLDER}/tribuo-linear-regression-ada-validation.csv')\n",
    "tribuo_lr_ada_validation.columns = column_names\n",
    "tribuo_lr_cart_validation = pd.read_csv(f'{DATASETS_FOLDER}/tribuo-linear-regression-cart-validation.csv')\n",
    "tribuo_lr_cart_validation.columns = column_names\n",
    "tribuo_lr_xgb_validation = pd.read_csv(f'{DATASETS_FOLDER}/tribuo-linear-regression-xgb-validation.csv')\n",
    "tribuo_lr_xgb_validation.columns = column_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def geometric_mean_of(distribution):\n",
    "    index = 0\n",
    "    N = len(distribution)\n",
    "    result = 1\n",
    "    for index in range(0, N):\n",
    "        result = result * distribution[index]\n",
    "    if N != 0:\n",
    "        return abs(result) ** (1 / N)\n",
    "    else:\n",
    "        raise ValueError('No distribution has been passed in, cannot compute Geometric Mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def harmonic_mean_of(distribution):\n",
    "    index = 0\n",
    "    N = len(distribution)\n",
    "    result = 0\n",
    "    for index in range(0, N):\n",
    "        if distribution[index] != 0:\n",
    "            result = result + (1 / distribution[index])\n",
    "\n",
    "    if result == 0:\n",
    "        raise ValueError('Distribution contains one or more zeros, cannot compute Harmonic Mean')\n",
    "    else:\n",
    "        return 1 / result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Legend\n",
    "\n",
    "- **NN**: Neural Networks\n",
    "- **SGD**: Stochastic Gradient Descent\n",
    "- **LR**: Linear Regression\n",
    "- **ADA**: Adam Optimiser\n",
    "- **CART**: Classification And Regression Trees (Ensemble)\n",
    "- **XGB**: XGBoost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plotting all the input data (they should be the same)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x12205dc50>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 184072 characters]\n",
      "text/plain": [
       "<Figure size 1440x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 8.0]\n",
    "tribuo_lr_sgd_validation.X_input.plot()\n",
    "tribuo_lr_ada_validation.X_input.plot()\n",
    "tribuo_lr_cart_validation.X_input.plot()\n",
    "tribuo_lr_xgb_validation.X_input.plot()\n",
    "plt.legend(['tribuo: Linear Regression/SGD',\n",
    "            'tribuo: Linear Regression/ADA',\n",
    "            'tribuo: Linear Regression/CART',\n",
    "            'tribuo: Linear Regression/XGB'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Taking a closer look at the prediction results of 'tribuo: LR/SGD' (Tribuo's Linear Regression using the SGD algorithm)\n",
    "\n",
    "We won't be using this one for ensembling as it seems to be on a different scale."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 10.0]\n",
    "tribuo_lr_sgd_validation.y_prediction.plot()\n",
    "plt.legend(['tribuo: Linear Regression/SGD'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x13d39ae80>"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 271332 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 10.0]\n",
    "tribuo_lr_ada_validation.y_prediction.plot()\n",
    "tribuo_lr_cart_validation.y_prediction.plot()\n",
    "tribuo_lr_xgb_validation.y_prediction.plot()\n",
    "plt.legend(['tribuo: Linear Regression/ADA', \n",
    "            'tribuo: Linear Regression/CART', \n",
    "            'tribuo: Linear Regression/XGBoost'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ensembling by averaging method (Tribuo Linear Regression: ADA, CART and XGBoost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "tribuo_3_lr_ensembles_average = tribuo_lr_ada_validation.copy()\n",
    "tribuo_3_lr_ensembles_average.y_prediction = \\\n",
    "           (tribuo_lr_ada_validation.y_prediction + tribuo_lr_cart_validation.y_prediction + \\\n",
    "            tribuo_lr_xgb_validation.y_prediction) / 3\n",
    "tribuo_3_lr_ensembles_average.to_csv(f'{DATASETS_FOLDER}/tribuo-3-lr-ensembles-average.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x13d709cf8>"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 379912 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 10.0]\n",
    "tribuo_lr_ada_validation.y_prediction.plot(linestyle=\"dotted\")\n",
    "tribuo_lr_cart_validation.y_prediction.plot(linestyle=\"dashed\")\n",
    "tribuo_lr_xgb_validation.y_prediction.plot(linestyle=\"dashdot\")\n",
    "tribuo_3_lr_ensembles_average.y_prediction.plot(color=\"blue\")\n",
    "plt.legend(['tribuo: Linear Regression/ADA', \n",
    "            'tribuo: Linear Regression/CART', \n",
    "            'tribuo: Linear Regression/XGBoost',\n",
    "            'tribuo: Linear Regression/Ensemble (average)'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ensembling by median method (Tribuo Linear Regression: ADA, CART and XGBoost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "tribuo_3_lr_ensembles_median = tribuo_lr_ada_validation.copy()\n",
    "concatenated_df = pd.concat([tribuo_lr_ada_validation.y_prediction, \n",
    "                     tribuo_lr_cart_validation.y_prediction, \n",
    "                     tribuo_lr_xgb_validation.y_prediction], axis=1)\n",
    "tribuo_3_lr_ensembles_median.y_prediction = np.median(concatenated_df, axis=1)\n",
    "tribuo_3_lr_ensembles_median.to_csv(f'{DATASETS_FOLDER}/tribuo-3-lr-ensembles-median.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x13987e6a0>"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 348928 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 10.0]\n",
    "tribuo_lr_ada_validation.y_prediction.plot(linestyle=\"dotted\")\n",
    "tribuo_lr_cart_validation.y_prediction.plot(linestyle=\"dashed\")\n",
    "tribuo_lr_xgb_validation.y_prediction.plot(linestyle=\"dashdot\")\n",
    "tribuo_3_lr_ensembles_median.y_prediction.plot(color=\"blue\")\n",
    "plt.legend(['tribuo: Linear Regression/ADA', \n",
    "            'tribuo: Linear Regression/CART', \n",
    "            'tribuo: Linear Regression/XGBoost',\n",
    "            'tribuo: Linear Regression/Ensemble (median)'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ensembling by geometric mean method (Tribuo Linear Regression: ADA, CART and XGBoost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "tribuo_3_lr_ensembles_geomean = tribuo_lr_ada_validation.copy()\n",
    "tribuo_3_lr_ensembles_geomean.y_prediction = concatenated_df.y_prediction.apply(geometric_mean_of, axis=1)\n",
    "tribuo_3_lr_ensembles_geomean.to_csv(f'{DATASETS_FOLDER}/tribuo-3-lr-ensembles-geomean.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x13d9fda20>"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 384800 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 10.0]\n",
    "tribuo_lr_ada_validation.y_prediction.plot(linestyle=\"dotted\")\n",
    "tribuo_lr_cart_validation.y_prediction.plot(linestyle=\"dashed\")\n",
    "tribuo_lr_xgb_validation.y_prediction.plot(linestyle=\"dashdot\")\n",
    "tribuo_3_lr_ensembles_geomean.y_prediction.plot(color=\"blue\")\n",
    "plt.legend(['tribuo: Linear Regression/ADA', \n",
    "            'tribuo: Linear Regression/CART', \n",
    "            'tribuo: Linear Regression/XGBoost',\n",
    "            'tribuo: Linear Regression/Ensemble (geomean)'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ensembling by harmonic mean method (Tribuo Linear Regression: ADA, CART and XGBoost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "tribuo_3_lr_ensembles_harmonic_mean = tribuo_lr_ada_validation.copy()\n",
    "tribuo_3_lr_ensembles_harmonic_mean.y_prediction = concatenated_df.y_prediction.apply(harmonic_mean_of, axis=1)\n",
    "tribuo_3_lr_ensembles_harmonic_mean.to_csv(f'{DATASETS_FOLDER}/tribuo-3-lr-ensembles-harmonic-mean.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x13e256d30>"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 392164 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 10.0]\n",
    "tribuo_lr_ada_validation.y_prediction.plot(linestyle=\"dotted\")\n",
    "tribuo_lr_cart_validation.y_prediction.plot(linestyle=\"dashed\")\n",
    "tribuo_lr_xgb_validation.y_prediction.plot(linestyle=\"dashdot\")\n",
    "tribuo_3_lr_ensembles_harmonic_mean.y_prediction.plot(color=\"blue\")\n",
    "plt.legend(['tribuo: Linear Regression/ADA', \n",
    "            'tribuo: Linear Regression/CART', \n",
    "            'tribuo: Linear Regression/XGBoost',\n",
    "            'tribuo: Linear Regression/Ensemble (harmonic mean)'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparing all of the above methods (Tribuo Linear Regression: ADA, CART and XGBoost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x12c53b4a8>"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 393840 characters]\n",
      "text/plain": [
       "<Figure size 1440x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20.0, 10.0]\n",
    "tribuo_3_lr_ensembles_average.y_prediction.plot(linestyle=\"dotted\")\n",
    "tribuo_3_lr_ensembles_median.y_prediction.plot(linestyle=\"dashed\")\n",
    "tribuo_3_lr_ensembles_geomean.y_prediction.plot(linestyle=\"dashdot\")\n",
    "tribuo_3_lr_ensembles_harmonic_mean.y_prediction.plot(color=\"blue\")\n",
    "plt.legend(['tribuo: Linear Regression/Ensemble (averaging)', \n",
    "            'tribuo: Linear Regression/Ensemble (median)', \n",
    "            'tribuo: Linear Regression/Ensemble (geomean)',\n",
    "            'tribuo: Linear Regression/Ensemble (harmonic mean)'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
