{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract eICU Time Series Data\n",
    "Those features are considered:\n",
    "* Heart Rate\n",
    "* Respiratory Rate\n",
    "* SpO2 (oxygen saturation, it's actually SpO2 although it's called SaO2, see official paper for details)\n",
    "* Mean Blood Pressure\n",
    "\n",
    "Didn't consider systolic pressure and diastolic pressure since they are used to calculate mean blood pressure, and they follow the same trend for most of the time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pre-req: build eICU database and run `sql/patient_id.sql`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from helpers import DataBaseLoader\n",
    "load = DataBaseLoader(user='mt361', password='tian01050417', dbname='eicu_gen', schema='eicu')\n",
    "\n",
    "SQL_SCRIPT = \"\"\"\n",
    "SELECT\n",
    "\tvitalPeriodic.patientunitstayid,\n",
    "\tobservationoffset,\n",
    "\theartrate,\n",
    "\tCASE\n",
    "\t\tWHEN heartrate IS NULL THEN 1\n",
    "\t\tELSE 0\n",
    "\tEND AS heartrate_null,\n",
    "\trespiration,\n",
    "\tCASE\n",
    "\t\tWHEN respiration IS NULL THEN 1\n",
    "\t\tELSE 0\n",
    "\tEND AS respiration_null,\n",
    "\tsao2,\n",
    "\tCASE\n",
    "\t\tWHEN sao2 IS NULL THEN 1\n",
    "\t\tELSE 0\n",
    "\tEND AS sao2_null,\n",
    "\tsystemicMean,\n",
    "\tCASE\n",
    "\t\tWHEN systemicMean IS NULL THEN 1\n",
    "\t\tELSE 0\n",
    "\tEND AS systemicmean_null\n",
    " \n",
    "FROM vitalPeriodic\n",
    "\n",
    "WHERE vitalPeriodic.patientunitstayid = {id}\n",
    "AND\n",
    "(SELECT MIN(observationoffset) FROM vitalPeriodic WHERE patientunitstayid = {id} GROUP BY patientunitstayid) <= {start_time} AND (SELECT MAX(observationoffset) FROM vitalPeriodic WHERE patientunitstayid = {id} GROUP BY patientunitstayid) >= {end_time}\n",
    "AND\n",
    "observationoffset <= {end_time} AND observationoffset >= {start_time}\n",
    "\n",
    "ORDER BY observationoffset\n",
    "\"\"\"\n",
    "\n",
    "def extract(id, start_time, end_time):\n",
    "    df = load.query(SQL_SCRIPT.format(id=id, end_time=end_time, start_time=start_time))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get patients and do train test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 170586 patients in the database.\n"
     ]
    }
   ],
   "source": [
    "unitstay_id = load['patient_id']\n",
    "unitstay_id = unitstay_id.dropna()          # drop ones without labels, this is caused by the join operation\n",
    "unitstay_id[['sepsis', 'ami', 'akf', 'heart_failure']] = unitstay_id[['sepsis', 'ami', 'akf', 'heart_failure']].astype(int)\n",
    "unitstay_id.to_csv('data/eicu-extract/unitstay_id.csv', index=False)\n",
    "print(f\"There are {len(unitstay_id)} patients in the database.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seed set to 2023\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from helpers import seed_everything\n",
    "from sklearn.model_selection import train_test_split\n",
    "seed_everything(2023)\n",
    "X, y = unitstay_id.drop('hospital_expire_flag', axis=1), unitstay_id['hospital_expire_flag']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=2023, stratify=y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "check label and proportion of each group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mortality rate in training set: 9.27%\n",
      "Mortality rate in testing set: 9.27%\n"
     ]
    }
   ],
   "source": [
    "print(f\"Mortality rate in training set: {100*y_train.sum() / len(y_train):.2f}%\\nMortality rate in testing set: {100*y_test.sum() / len(y_test):.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set size: 127939\n",
      "Testing set size: 42647\n"
     ]
    }
   ],
   "source": [
    "print(f\"Training set size: {len(X_train)}\\nTesting set size: {len(X_test)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAINING\n",
      "Sepsis: 14.40%\n",
      "AMI: 5.90%\n",
      "Heart Failure: 9.75%\n",
      "AKF: 11.30%\n",
      "\n",
      "TESTING\n",
      "Sepsis: 14.33%\n",
      "AMI: 5.96%\n",
      "Heart Failure: 9.71%\n",
      "AKF: 11.07%\n"
     ]
    }
   ],
   "source": [
    "print(f\"TRAINING\\nSepsis: {100*len(X_train[X_train['sepsis'] == 1])/len(X_train):.2f}%\\nAMI: {100*len(X_train[X_train['ami'] == 1])/len(X_train):.2f}%\\nHeart Failure: {100*len(X_train[X_train['heart_failure'] == 1])/len(X_train):.2f}%\\nAKF: {100*len(X_train[X_train['akf'] == 1])/len(X_train):.2f}%\")\n",
    "print(f\"\\nTESTING\\nSepsis: {100*len(X_test[X_test['sepsis'] == 1])/len(X_test):.2f}%\\nAMI: {100*len(X_test[X_test['ami'] == 1])/len(X_test):.2f}%\\nHeart Failure: {100*len(X_test[X_test['heart_failure'] == 1])/len(X_test):.2f}%\\nAKF: {100*len(X_test[X_test['akf'] == 1])/len(X_test):.2f}%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "train = pd.concat([X_train, y_train], axis=1)\n",
    "test = pd.concat([X_test, y_test], axis=1)\n",
    "train.to_csv('data/eicu-extract/TRAIN-unitstay_id.csv', index=False)\n",
    "test.to_csv('data/eicu-extract/TEST-unitstay_id.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract Time Series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patientunitstayid</th>\n",
       "      <th>sepsis</th>\n",
       "      <th>ami</th>\n",
       "      <th>heart_failure</th>\n",
       "      <th>akf</th>\n",
       "      <th>hospital_expire_flag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1086277</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1747167</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3140062</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3349404</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1792064</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   patientunitstayid  sepsis  ami  heart_failure  akf  hospital_expire_flag\n",
       "0            1086277       0    0              0    0                     0\n",
       "1            1747167       0    0              0    0                     0\n",
       "2            3140062       0    0              0    0                     0\n",
       "3            3349404       0    0              0    0                     0\n",
       "4            1792064       0    0              0    0                     0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "train = pd.read_csv('data/eicu-extract/TRAIN-unitstay_id.csv')\n",
    "test = pd.read_csv('data/eicu-extract/TEST-unitstay_id.csv')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['ami'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['heart_failure'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['akf'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import seaborn as sns\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "def extract_and_add_labels(id, start_time, end_time, mode='train', data_frame=None):\n",
    "    df = extract(id, start_time=start_time, end_time=end_time)\n",
    "    \n",
    "    if data_frame is not None:\n",
    "        patient_df = data_frame[data_frame['patientunitstayid'] == id][['hospital_expire_flag']]    \n",
    "    else:\n",
    "        if mode == 'train':\n",
    "            patient_df = train[train['patientunitstayid'] == id][['hospital_expire_flag']]\n",
    "        elif mode == 'test':\n",
    "            patient_df = test[test['patientunitstayid'] == id][['hospital_expire_flag']]\n",
    "        else:\n",
    "            raise ValueError('mode must be either train or test')\n",
    "        \n",
    "    for col in patient_df.columns:\n",
    "        dummy = np.full((df.shape[0], ), patient_df[col].values)\n",
    "        df[col] = dummy.astype(int)\n",
    "    \n",
    "    return df\n",
    "\n",
    "def lets_extract(ids, start_time, end_time, timesteps, mode='train', data_frame=None):\n",
    "    result = []\n",
    "    for id in tqdm(ids, desc='Extracting...'):\n",
    "        sample = extract_and_add_labels(id, start_time=start_time, end_time=end_time, mode=mode, data_frame=data_frame)\n",
    "        diff = sample['observationoffset'].diff()\n",
    "        if len(sample) > 0 and (diff.min() == diff.max() == 5):     # check time interval is consistently 5 minutes, if not, move on\n",
    "            sample = sample.drop(['patientunitstayid', 'observationoffset'], axis=1).to_numpy().transpose()\n",
    "            if sample.shape[1] < timesteps:\n",
    "                continue\n",
    "            else:\n",
    "                sample = sample[:, :timesteps]\n",
    "        else:\n",
    "            continue\n",
    "        result.append(sample)\n",
    "        \n",
    "    return np.asarray(result)\n",
    "\n",
    "def lets_extract_no_restrict(ids, start_time, end_time, timesteps):\n",
    "    result = []\n",
    "    for id in tqdm(ids, desc='Extracting...'):\n",
    "        sample = extract_and_add_labels(id, start_time=start_time, end_time=end_time)\n",
    "        if len(sample) > 0:\n",
    "            sample = sample.drop(['patientunitstayid', 'observationoffset'], axis=1).to_numpy().transpose()\n",
    "            if sample.shape[1] < timesteps:\n",
    "                continue\n",
    "            else:\n",
    "                sample = sample[:, :timesteps]\n",
    "        else:\n",
    "            continue\n",
    "        result.append(sample)\n",
    "        \n",
    "    return np.asarray(result)\n",
    "\n",
    "def visualize(ids, start_time, end_time, cumulative = True, xlim = None):\n",
    "    result = []\n",
    "    for id in tqdm(ids, desc='Collecting...'):\n",
    "    # for id in ids:\n",
    "        sample = extract(id, start_time=start_time, end_time=end_time)\n",
    "        if len(sample) == 0:\n",
    "            continue\n",
    "        result.append(len(sample))\n",
    "    sns.histplot(-np.asarray(result), cumulative = cumulative)\n",
    "    plt.xlim(xlim)\n",
    "    plt.xlabel('Length of time series')\n",
    "    plt.ylabel('Number of patients')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Collecting...: 100%|██████████| 127939/127939 [10:43<00:00, 198.89it/s]\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 23516 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "visualize(train['patientunitstayid'].values, xlim=(-800, 0), start_time = 60, end_time = 2880)            # use negatives in order to view >= as cumulative distribution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "extract with time interval of 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 127939/127939 [08:53<00:00, 239.77it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(20230, 9, 564)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = lets_extract(train['patientunitstayid'].values, start_time=60, end_time=2880, timesteps=564)\n",
    "result.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 127939/127939 [07:22<00:00, 288.82it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(46778, 9, 276)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result3 = lets_extract(train['patientunitstayid'].values, start_time=60, end_time=1440, timesteps=276)\n",
    "result3.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "torch.save(torch.tensor(result.astype(np.float64)), 'data/eicu-extract/TRAIN-eicu_multiple_60_2880_564.pt')\n",
    "torch.save(torch.tensor(result3.astype(np.float64)), 'data/eicu-extract/TRAIN-eicu_multiple_60_1440_276.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "use same procedure as training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 42647/42647 [04:55<00:00, 144.54it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(6705, 9, 564)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_48hrs = lets_extract(test['patientunitstayid'].values, start_time=60, end_time=2880, timesteps=564, mode='test')\n",
    "test_48hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 42647/42647 [04:46<00:00, 148.70it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(15675, 9, 276)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_24hrs = lets_extract(test['patientunitstayid'].values, start_time=60, end_time=1440, timesteps=276, mode='test')\n",
    "test_24hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "torch.save(torch.tensor(test_48hrs.astype(np.float64)), 'data/eicu-extract/TEST-eicu_multiple_60_2880_564.pt')\n",
    "torch.save(torch.tensor(test_24hrs.astype(np.float64)), 'data/eicu-extract/TEST-eicu_multiple_60_1440_276.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sub-populations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** TRAINING ***\n",
      "Number of sepsis patients: 18419\n",
      "Number of ami patients: 7549\n",
      "Number of heart failure patients: 12478\n",
      "Number of akf patients: 14455\n",
      "\n",
      "*** TESTING ***\n",
      "Number of sepsis patients: 6111\n",
      "Number of ami patients: 2541\n",
      "Number of heart failure patients: 4143\n",
      "Number of akf patients: 4719\n"
     ]
    }
   ],
   "source": [
    "train_sepsis_id = train[train['sepsis'] ==1][['patientunitstayid', 'hospital_expire_flag']]\n",
    "train_ami_id = train[train['ami'] ==1][['patientunitstayid', 'hospital_expire_flag']]\n",
    "train_heart_failure_id = train[train['heart_failure'] ==1][['patientunitstayid', 'hospital_expire_flag']]\n",
    "train_akf_id = train[train['akf'] ==1][['patientunitstayid', 'hospital_expire_flag']]\n",
    "\n",
    "test_sepsis_id = test[test['sepsis'] ==1][['patientunitstayid', 'hospital_expire_flag']]\n",
    "test_ami_id = test[test['ami'] ==1][['patientunitstayid', 'hospital_expire_flag']]\n",
    "test_heart_failure_id = test[test['heart_failure'] ==1][['patientunitstayid', 'hospital_expire_flag']]\n",
    "test_akf_id = test[test['akf'] ==1][['patientunitstayid', 'hospital_expire_flag']]\n",
    "\n",
    "print(f\"*** TRAINING ***\\nNumber of sepsis patients: {len(train_sepsis_id)}\\nNumber of ami patients: {len(train_ami_id)}\\nNumber of heart failure patients: {len(train_heart_failure_id)}\\nNumber of akf patients: {len(train_akf_id)}\\n\")\n",
    "print(f\"*** TESTING ***\\nNumber of sepsis patients: {len(test_sepsis_id)}\\nNumber of ami patients: {len(test_ami_id)}\\nNumber of heart failure patients: {len(test_heart_failure_id)}\\nNumber of akf patients: {len(test_akf_id)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "save those IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_sepsis_id.to_csv(\"data/eicu-extract/TRAIN-sepsis_id.csv\", index=False)\n",
    "train_ami_id.to_csv(\"data/eicu-extract/TRAIN-ami_id.csv\", index=False)\n",
    "train_heart_failure_id.to_csv(\"data/eicu-extract/TRAIN-heart_failure_id.csv\", index=False)\n",
    "train_akf_id.to_csv(\"data/eicu-extract/TRAIN-akf_id.csv\", index=False)\n",
    "\n",
    "test_sepsis_id.to_csv(\"data/eicu-extract/TEST-sepsis_id.csv\", index=False)\n",
    "test_ami_id.to_csv(\"data/eicu-extract/TEST-ami_id.csv\", index=False)\n",
    "test_heart_failure_id.to_csv(\"data/eicu-extract/TEST-heart_failure_id.csv\", index=False)\n",
    "test_akf_id.to_csv(\"data/eicu-extract/TEST-akf_id.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sepsis/Septicemia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...:   0%|          | 0/18419 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 18419/18419 [02:17<00:00, 133.54it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(4839, 9, 564)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "train_sepsis_id = pd.read_csv(\"data/eicu-extract/TRAIN-sepsis_id.csv\")\n",
    "train_ami_id = pd.read_csv(\"data/eicu-extract/TRAIN-ami_id.csv\")\n",
    "train_hear_failure_id = pd.read_csv(\"data/eicu-extract/TRAIN-heart_failure_id.csv\")\n",
    "train_akf_id = pd.read_csv(\"data/eicu-extract/TRAIN-akf_id.csv\")\n",
    "\n",
    "train_sepsis_48hrs = lets_extract(train_sepsis_id['patientunitstayid'].values, start_time=60, end_time=2880, timesteps=564, data_frame=train_sepsis_id)\n",
    "train_sepsis_48hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 18419/18419 [02:21<00:00, 130.09it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(8813, 9, 276)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sepsis_24hrs = lets_extract(train_sepsis_id['patientunitstayid'].values, start_time=60, end_time=1440, timesteps=276, data_frame=train_sepsis_id)\n",
    "train_sepsis_24hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "torch.save(torch.tensor(train_sepsis_48hrs.astype(np.float64)), 'data/eicu-extract/TRAIN-SEPSIS-eicu_multiple_60_2880_564.pt')\n",
    "torch.save(torch.tensor(train_sepsis_24hrs.astype(np.float64)), 'data/eicu-extract/TRAIN-SEPSIS-eicu_multiple_60_1440_276.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Acite Myocardial Infarction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 7549/7549 [00:49<00:00, 153.75it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1183, 9, 564)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ami_48hrs = lets_extract(train_ami_id['patientunitstayid'].values, start_time=60, end_time=2880, timesteps=564, data_frame=train_ami_id)\n",
    "train_ami_48hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 7549/7549 [00:47<00:00, 158.22it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2927, 9, 276)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ami_24hrs = lets_extract(train_ami_id['patientunitstayid'].values, start_time=60, end_time=1440, timesteps=276, data_frame=train_ami_id)\n",
    "train_ami_24hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "torch.save(torch.tensor(train_ami_48hrs.astype(np.float64)), 'data/eicu-extract/TRAIN-AMI-eicu_multiple_60_2880_564.pt')\n",
    "torch.save(torch.tensor(train_ami_24hrs.astype(np.float64)), 'data/eicu-extract/TRAIN-AMI-eicu_multiple_60_1440_276.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Heart Failure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 12478/12478 [01:34<00:00, 131.70it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(3156, 9, 564)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_heart_failure_48hrs = lets_extract(train_heart_failure_id['patientunitstayid'].values, start_time=60, end_time=2880, timesteps=564, data_frame=train_heart_failure_id)\n",
    "train_heart_failure_48hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 12478/12478 [01:34<00:00, 132.59it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(6102, 9, 276)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_heart_failure_24hrs = lets_extract(train_heart_failure_id['patientunitstayid'].values, start_time=60, end_time=1440, timesteps=276, data_frame=train_heart_failure_id)\n",
    "train_heart_failure_24hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "torch.save(torch.tensor(train_heart_failure_48hrs.astype(np.float64)), 'data/eicu-extract/TRAIN-HF-eicu_multiple_60_2880_564.pt')\n",
    "torch.save(torch.tensor(train_heart_failure_24hrs.astype(np.float64)), 'data/eicu-extract/TRAIN-HF-eicu_multiple_60_1440_276.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Acute Kidney Failure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 14455/14455 [01:56<00:00, 124.37it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(3856, 9, 564)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_akf_48hrs = lets_extract(train_akf_id['patientunitstayid'].values, start_time=60, end_time=2880, timesteps=564, data_frame=train_akf_id)\n",
    "train_akf_48hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 14455/14455 [01:53<00:00, 127.23it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(6981, 9, 276)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_akf_24hrs = lets_extract(train_akf_id['patientunitstayid'].values, start_time=60, end_time=1440, timesteps=276, data_frame=train_akf_id)\n",
    "train_akf_24hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "torch.save(torch.tensor(train_akf_48hrs.astype(np.float64)), 'data/eicu-extract/TRAIN-AKF-eicu_multiple_60_2880_564.pt')\n",
    "torch.save(torch.tensor(train_akf_24hrs.astype(np.float64)), 'data/eicu-extract/TRAIN-AKF-eicu_multiple_60_1440_276.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sepsis/Septicemia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 6111/6111 [00:45<00:00, 133.55it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1568, 9, 564)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "test_sepsis_id = pd.read_csv(\"data/eicu-extract/TEST-sepsis_id.csv\")\n",
    "test_ami_id = pd.read_csv(\"data/eicu-extract/TEST-ami_id.csv\")\n",
    "test_hear_failure_id = pd.read_csv(\"data/eicu-extract/TEST-heart_failure_id.csv\")\n",
    "test_akf_id = pd.read_csv(\"data/eicu-extract/TEST-akf_id.csv\")\n",
    "\n",
    "test_sepsis_48hrs = lets_extract(test_sepsis_id['patientunitstayid'].values, start_time=60, end_time=2880, timesteps=564, data_frame=test_sepsis_id)\n",
    "test_sepsis_48hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 6111/6111 [00:41<00:00, 146.09it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2858, 9, 276)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_sepsis_24hrs = lets_extract(test_sepsis_id['patientunitstayid'].values, start_time=60, end_time=1440, timesteps=276, data_frame=test_sepsis_id)\n",
    "test_sepsis_24hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "torch.save(torch.tensor(test_sepsis_48hrs.astype(np.float64)), 'data/eicu-extract/TEST-SEPSIS-eicu_multiple_60_2880_564.pt')\n",
    "torch.save(torch.tensor(test_sepsis_24hrs.astype(np.float64)), 'data/eicu-extract/TEST-SEPSIS-eicu_multiple_60_1440_276.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Acute Myocardial Infarction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 2541/2541 [00:17<00:00, 146.12it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(382, 9, 564)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_ami_48hrs = lets_extract(test_ami_id['patientunitstayid'].values, start_time=60, end_time=2880, timesteps=564, data_frame=test_ami_id)\n",
    "test_ami_48hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 2541/2541 [00:16<00:00, 151.23it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(953, 9, 276)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_ami_24hrs = lets_extract(test_ami_id['patientunitstayid'].values, start_time=60, end_time=1440, timesteps=276, data_frame=test_ami_id)\n",
    "test_ami_24hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "torch.save(torch.tensor(test_ami_48hrs.astype(np.float64)), 'data/eicu-extract/TEST-AMI-eicu_multiple_60_2880_564.pt')\n",
    "torch.save(torch.tensor(test_ami_24hrs.astype(np.float64)), 'data/eicu-extract/TEST-AMI-eicu_multiple_60_1440_276.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Heart Failure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 4143/4143 [00:29<00:00, 138.76it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1064, 9, 564)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_heart_failure_48hrs = lets_extract(test_heart_failure_id['patientunitstayid'].values, start_time=60, end_time=2880, timesteps=564, data_frame=test_heart_failure_id)\n",
    "test_heart_failure_48hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 4143/4143 [00:28<00:00, 147.22it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2090, 9, 276)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_heart_failure_24hrs = lets_extract(test_heart_failure_id['patientunitstayid'].values, start_time=60, end_time=1440, timesteps=276, data_frame=test_heart_failure_id)\n",
    "test_heart_failure_24hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "torch.save(torch.tensor(test_heart_failure_48hrs.astype(np.float64)), 'data/eicu-extract/TEST-HF-eicu_multiple_60_2880_564.pt')\n",
    "torch.save(torch.tensor(test_heart_failure_24hrs.astype(np.float64)), 'data/eicu-extract/TEST-HF-eicu_multiple_60_1440_276.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Acute Kidney Failure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 4719/4719 [00:35<00:00, 131.14it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(1276, 9, 564)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_akf_48hrs = lets_extract(test_akf_id['patientunitstayid'].values, start_time=60, end_time=2880, timesteps=564, data_frame=test_akf_id)\n",
    "test_akf_48hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Extracting...: 100%|██████████| 4719/4719 [00:35<00:00, 134.42it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2270, 9, 276)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_akf_24hrs = lets_extract(test_akf_id['patientunitstayid'].values, start_time=60, end_time=1440, timesteps=276, data_frame=test_akf_id)\n",
    "test_akf_24hrs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "torch.save(torch.tensor(test_akf_48hrs.astype(np.float64)), 'data/eicu-extract/TEST-AKF-eicu_multiple_60_2880_564.pt')\n",
    "torch.save(torch.tensor(test_akf_24hrs.astype(np.float64)), 'data/eicu-extract/TEST-AKF-eicu_multiple_60_1440_276.pt')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gen",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
