{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Turns timeseries samples into summary statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20000, 5, 560) (20230, 5, 564) (6705, 5, 564)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from helpers.utils import smart_to_numpy, smart_load\n",
    "\n",
    "def transform_to_features(samples):\n",
    "    assert isinstance(samples, np.ndarray), \"samples must be a numpy array\"\n",
    "    corrected_samples = np.zeros((samples.shape[0], 5, samples.shape[2]))       # NOTE: the following code is specific to eICU dataset and features trained on 7.24\n",
    "    for i in range(0, 7, 2):\n",
    "        corrected_samples[:, int(i/2), :] = np.where(np.round(samples[:, i+1, :]) == 1, np.nan, samples[:, i, :])\n",
    "    corrected_samples[:, 4, :] = np.round(samples[:, -1, :])\n",
    "    return corrected_samples\n",
    "\n",
    "sync_path=\"results/samples/ddpm_eicu_all_48hrs_1690178780_samples.npy\"\n",
    "train_path=\"data/eicu-extract/TRAIN-eicu_multiple_60_2880_564.pt\"\n",
    "test_path=\"data/eicu-extract/TEST-eicu_multiple_60_2880_564.pt\"\n",
    "\n",
    "sync_data, train_data, test_data = smart_to_numpy(smart_load(sync_path)), smart_to_numpy(smart_load(train_path)), smart_to_numpy(smart_load(test_path))\n",
    "sync_data, train_data, test_data = sync_data, transform_to_features(train_data), transform_to_features(test_data) \n",
    "print(sync_data.shape, train_data.shape, test_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from scipy.stats import kurtosis, skew, mode\n",
    "from tqdm import tqdm\n",
    "\n",
    "def turn_timeseries_to_summary_stats_df(timeseries_sample, channels_to_features):\n",
    "    \"\"\"\n",
    "    Turn timeseries dataframe to summary statistics dataframe. Probably there is a better way to write this but I don't bother anymore :(\n",
    "    \"\"\"\n",
    "    result_df = {}\n",
    "    for patient_idx in tqdm(range(timeseries_sample.shape[0]), desc=\"Turning timeseries to summary stats...\"):\n",
    "        for channel_idx in range(timeseries_sample.shape[1]):\n",
    "            if channel_idx == 4:            # for label column\n",
    "                try:\n",
    "                    result_df['hospital_expire_flag'].append(np.unique(timeseries_sample[patient_idx, channel_idx, :])[0])\n",
    "                except:\n",
    "                    result_df['hospital_expire_flag'] = [np.unique(timeseries_sample[patient_idx, channel_idx, :])[0]]\n",
    "                continue\n",
    "            feature_name = channels_to_features[channel_idx]\n",
    "            arr = timeseries_sample[patient_idx, channel_idx, :]\n",
    "            arr = np.squeeze(arr[~np.isnan(arr)])\n",
    "            if arr.size <= 1:\n",
    "                first, min, max, range_, mean, std, median, mode_, kurtosis_, lower_quartile, upper_quartile, iqr, skewness = np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan\n",
    "            else:\n",
    "                if arr.size == 1:\n",
    "                    first = arr\n",
    "                else:\n",
    "                    first = arr[0]\n",
    "                min = arr.min()\n",
    "                max = arr.max()\n",
    "                range_ = max - min\n",
    "                mean = arr.mean()\n",
    "                std = arr.std()\n",
    "                mode_ = mode(arr, keepdims=True)[0][0]\n",
    "                skewness = skew(arr)\n",
    "            \n",
    "            try:\n",
    "                result_df[f\"{feature_name}_first\"].append(first)\n",
    "                result_df[f\"{feature_name}_min\"].append(min)\n",
    "                result_df[f\"{feature_name}_max\"].append(max)\n",
    "                result_df[f\"{feature_name}_range\"].append(range_)\n",
    "                result_df[f\"{feature_name}_mean\"].append(mean)\n",
    "                result_df[f\"{feature_name}_std\"].append(std)\n",
    "                result_df[f\"{feature_name}_mode\"].append(mode_)\n",
    "                result_df[f\"{feature_name}_skewness\"].append(skewness)\n",
    "            except:\n",
    "                result_df[f\"{feature_name}_first\"] = [first]\n",
    "                result_df[f\"{feature_name}_min\"] = [min]\n",
    "                result_df[f\"{feature_name}_max\"] = [max]\n",
    "                result_df[f\"{feature_name}_range\"] = [range_]\n",
    "                result_df[f\"{feature_name}_mean\"] = [mean]\n",
    "                result_df[f\"{feature_name}_std\"] = [std]\n",
    "                result_df[f\"{feature_name}_mode\"] = [mode_]\n",
    "                result_df[f\"{feature_name}_skewness\"] = [skewness]\n",
    "            \n",
    "    result_df = pd.DataFrame(result_df)\n",
    "    return result_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "channels_to_features = {0: \"heartrate\", 1: 'resprate', 2: \"spo2\", 3: 'meanbp', 4: 'hospital_expire_flag'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Turning timeseries to summary stats...: 100%|██████████| 20000/20000 [00:40<00:00, 496.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20000, 53)\n",
      "(5550, 53)\n"
     ]
    }
   ],
   "source": [
    "sync_df = turn_timeseries_to_summary_stats_df(timeseries_sample=sync_data, channels_to_features=channels_to_features)\n",
    "print(sync_df.shape)\n",
    "sync_df.to_csv(\"results/processed/ddpm_eicu_all_48hrs_1690178780_samples.csv\", index=False)\n",
    "\n",
    "sync_df_nonan = sync_df.dropna()\n",
    "print(sync_df_nonan.shape)\n",
    "sync_df_nonan.dropna().to_csv(\"results/processed/ddpm_eicu_all_48hrs_1690178780_samples_dropna.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Turning timeseries to summary stats...:   1%|          | 199/20230 [00:00<00:41, 486.95it/s]/var/folders/t0/wpl_qfjd5s3cczhlskc092xm0000gp/T/ipykernel_91978/2504547031.py:35: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  kurtosis_ = kurtosis(arr)\n",
      "/var/folders/t0/wpl_qfjd5s3cczhlskc092xm0000gp/T/ipykernel_91978/2504547031.py:39: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  skewness = skew(arr)\n",
      "Turning timeseries to summary stats...: 100%|██████████| 20230/20230 [00:40<00:00, 497.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20230, 53)\n",
      "(6109, 53)\n"
     ]
    }
   ],
   "source": [
    "train_df = turn_timeseries_to_summary_stats_df(timeseries_sample=train_data, channels_to_features=channels_to_features)\n",
    "print(train_df.shape)\n",
    "train_df.to_csv(\"results/processed/TRAIN-eicu_multiple_60_2880_564.csv\", index=False)\n",
    "\n",
    "train_df_nonan = train_df.dropna()\n",
    "print(train_df_nonan.shape)\n",
    "train_df_nonan.dropna().to_csv(\"results/processed/TRAIN-eicu_multiple_60_2880_564_dropna.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Turning timeseries to summary stats...:   0%|          | 0/6705 [00:00<?, ?it/s]/var/folders/t0/wpl_qfjd5s3cczhlskc092xm0000gp/T/ipykernel_91978/2504547031.py:35: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  kurtosis_ = kurtosis(arr)\n",
      "/var/folders/t0/wpl_qfjd5s3cczhlskc092xm0000gp/T/ipykernel_91978/2504547031.py:39: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  skewness = skew(arr)\n",
      "Turning timeseries to summary stats...: 100%|██████████| 6705/6705 [00:13<00:00, 499.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6705, 53)\n",
      "(2024, 53)\n"
     ]
    }
   ],
   "source": [
    "test_df = turn_timeseries_to_summary_stats_df(timeseries_sample=test_data, channels_to_features=channels_to_features)\n",
    "print(test_df.shape)\n",
    "test_df.to_csv(\"results/processed/TEST-eicu_multiple_60_2880_564.csv\", index=False)\n",
    "\n",
    "test_df_nonan = test_df.dropna()\n",
    "print(test_df_nonan.shape)\n",
    "test_df_nonan.dropna().to_csv(\"results/processed/TEST-eicu_multiple_60_2880_564_dropna.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Class Imbalance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mortality Rate\n",
      "Train 24 hrs: 0.10703749625892514\n",
      "Test 24 hrs: 0.10628389154704944\n",
      "Train 48 hrs: 0.14320316361838853\n",
      "Test 48 hrs: 0.14571215510812827\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "train_24hrs = pd.read_csv(\"results/processed/TRAIN-eicu_multiple_60_1440_276.csv\")\n",
    "test_24hrs = pd.read_csv(\"results/processed/TEST-eicu_multiple_60_1440_276.csv\")\n",
    "train_48hrs = pd.read_csv(\"results/processed/TRAIN-eicu_multiple_60_2880_564.csv\")\n",
    "test_48hrs = pd.read_csv('results/processed/TEST-eicu_multiple_60_2880_564.csv')\n",
    "print(f\"Mortality Rate\\nTrain 24 hrs: {train_24hrs['hospital_expire_flag'].mean()}\\nTest 24 hrs: {test_24hrs['hospital_expire_flag'].mean()}\\nTrain 48 hrs: {train_48hrs['hospital_expire_flag'].mean()}\\nTest 48 hrs: {test_48hrs['hospital_expire_flag'].mean()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.10805"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(\"results/processed/ddpm_eicu_all_24hrs_1690178742_samples.csv\")['hospital_expire_flag'].mean()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gen",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
