{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_data = \"/path/to/RESPIN_ASRU_Challenge_2023\"\n",
    "corpus = \"bn\"\n",
    "origin_file_path = [f\"{dir_data}/corpus/{corpus}/train/text\",\n",
    "                    f\"{dir_data}/corpus/{corpus}/dev/text\"]\n",
    "\n",
    "config = \"../config.yaml\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from text.symbols import UNK_ID\n",
    "from text import tokenizer, detokenizer\n",
    "from utils.hparams import get_hparams_from_file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get hyperparameters from config file\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['phonemize_text', 'tokenize_text', 'add_bos_eos']\n"
     ]
    }
   ],
   "source": [
    "hps = get_hparams_from_file(config)\n",
    "text_cleaners = hps.data.text_cleaners\n",
    "print(text_cleaners)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Separate text cleaners by `phonemize_text` flag\n",
    "\n",
    "Used for faster text processing.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['phonemize_text'], ['tokenize_text', 'add_bos_eos']]\n"
     ]
    }
   ],
   "source": [
    "def separate_text_cleaners(text_cleaners):\n",
    "    final_list = []\n",
    "    temp_list = []\n",
    "\n",
    "    for cleaner in text_cleaners:\n",
    "        if cleaner == \"phonemize_text\":\n",
    "            if temp_list:\n",
    "                final_list.append(temp_list)\n",
    "            final_list.append([cleaner])\n",
    "            temp_list = []\n",
    "        else:\n",
    "            temp_list.append(cleaner)\n",
    "\n",
    "    if temp_list:\n",
    "        final_list.append(temp_list)\n",
    "\n",
    "    return final_list\n",
    "\n",
    "\n",
    "text_cleaners = separate_text_cleaners(text_cleaners)\n",
    "print(text_cleaners)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read and combine data from all files\n",
    "\n",
    "The data looks like this:\n",
    "\n",
    "`sid` _ `txtid` _ `uttid` `text`\n",
    "\n",
    "`16777288` _ `629046` _ `281474981563595` `\"বাঁশের প্রায়\"`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of lines: 581236\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16777288_629046_281474981563595 বাঁশের প্রায় দ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16777288_629072_281474981405386 এক ধরনের পদ্ধত...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16777288_629108_281474981563619 গুগলি বা ঝিনুক...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16777288_629112_281474981581650 বাগ আরশোলা ইত্...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16777288_629121_281474981582236 প্রাকৃতিক সব জ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   0\n",
       "0  16777288_629046_281474981563595 বাঁশের প্রায় দ...\n",
       "1  16777288_629072_281474981405386 এক ধরনের পদ্ধত...\n",
       "2  16777288_629108_281474981563619 গুগলি বা ঝিনুক...\n",
       "3  16777288_629112_281474981581650 বাগ আরশোলা ইত্...\n",
       "4  16777288_629121_281474981582236 প্রাকৃতিক সব জ..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.concat([pd.read_csv(path, sep=\"\\t\", header=None)\n",
    "                 for path in origin_file_path], ignore_index=True)\n",
    "print(\"Number of lines:\", len(data))\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert the data to the format of sid txtid uttid text, where each value in own column\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spkid</th>\n",
       "      <th>txtid</th>\n",
       "      <th>uttid</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16777288</td>\n",
       "      <td>629046</td>\n",
       "      <td>281474981563595</td>\n",
       "      <td>বাঁশের প্রায় দশ হাজার প্রজাতি হয় যেমন বাম্বুসা...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16777288</td>\n",
       "      <td>629072</td>\n",
       "      <td>281474981405386</td>\n",
       "      <td>এক ধরনের পদ্ধতি যেটা দিয়ে শস্য থেকে খোসা ছাড়ান...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16777288</td>\n",
       "      <td>629108</td>\n",
       "      <td>281474981563619</td>\n",
       "      <td>গুগলি বা ঝিনুকের মানে হল ওয়েস্টার আমরা খাই</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16777288</td>\n",
       "      <td>629112</td>\n",
       "      <td>281474981581650</td>\n",
       "      <td>বাগ আরশোলা ইত্যাদি সব পোকা গুলোর চাষ হয়</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16777288</td>\n",
       "      <td>629121</td>\n",
       "      <td>281474981582236</td>\n",
       "      <td>প্রাকৃতিক সব জিনিস গুলো দিয়া যখন চাষ করা হয়</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      spkid   txtid            uttid  \\\n",
       "0  16777288  629046  281474981563595   \n",
       "1  16777288  629072  281474981405386   \n",
       "2  16777288  629108  281474981563619   \n",
       "3  16777288  629112  281474981581650   \n",
       "4  16777288  629121  281474981582236   \n",
       "\n",
       "                                                text  \n",
       "0  বাঁশের প্রায় দশ হাজার প্রজাতি হয় যেমন বাম্বুসা...  \n",
       "1  এক ধরনের পদ্ধতি যেটা দিয়ে শস্য থেকে খোসা ছাড়ান...  \n",
       "2         গুগলি বা ঝিনুকের মানে হল ওয়েস্টার আমরা খাই  \n",
       "3            বাগ আরশোলা ইত্যাদি সব পোকা গুলোর চাষ হয়  \n",
       "4        প্রাকৃতিক সব জিনিস গুলো দিয়া যখন চাষ করা হয়  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data[0].str.split(\"_\", n=2, expand=True)\n",
    "\n",
    "data = pd.concat(\n",
    "    [data[0], data[1], data[2].str.split(\" \", n=1, expand=True)], axis=1)\n",
    "data.columns = [\"spkid\", \"txtid\", \"uttid\", \"text\"]\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Map speaker ids to indices with sorted order\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spkidx</th>\n",
       "      <th>spkid</th>\n",
       "      <th>txtid</th>\n",
       "      <th>uttid</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>16777288</td>\n",
       "      <td>629046</td>\n",
       "      <td>281474981563595</td>\n",
       "      <td>বাঁশের প্রায় দশ হাজার প্রজাতি হয় যেমন বাম্বুসা...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>16777288</td>\n",
       "      <td>629072</td>\n",
       "      <td>281474981405386</td>\n",
       "      <td>এক ধরনের পদ্ধতি যেটা দিয়ে শস্য থেকে খোসা ছাড়ান...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>16777288</td>\n",
       "      <td>629108</td>\n",
       "      <td>281474981563619</td>\n",
       "      <td>গুগলি বা ঝিনুকের মানে হল ওয়েস্টার আমরা খাই</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>16777288</td>\n",
       "      <td>629112</td>\n",
       "      <td>281474981581650</td>\n",
       "      <td>বাগ আরশোলা ইত্যাদি সব পোকা গুলোর চাষ হয়</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>16777288</td>\n",
       "      <td>629121</td>\n",
       "      <td>281474981582236</td>\n",
       "      <td>প্রাকৃতিক সব জিনিস গুলো দিয়া যখন চাষ করা হয়</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   spkidx     spkid   txtid            uttid  \\\n",
       "0       0  16777288  629046  281474981563595   \n",
       "1       0  16777288  629072  281474981405386   \n",
       "2       0  16777288  629108  281474981563619   \n",
       "3       0  16777288  629112  281474981581650   \n",
       "4       0  16777288  629121  281474981582236   \n",
       "\n",
       "                                                text  \n",
       "0  বাঁশের প্রায় দশ হাজার প্রজাতি হয় যেমন বাম্বুসা...  \n",
       "1  এক ধরনের পদ্ধতি যেটা দিয়ে শস্য থেকে খোসা ছাড়ান...  \n",
       "2         গুগলি বা ঝিনুকের মানে হল ওয়েস্টার আমরা খাই  \n",
       "3            বাগ আরশোলা ইত্যাদি সব পোকা গুলোর চাষ হয়  \n",
       "4        প্রাকৃতিক সব জিনিস গুলো দিয়া যখন চাষ করা হয়  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spkid_to_idx = {spkid: idx for idx,\n",
    "                spkid in enumerate(sorted(data.spkid.unique()))}\n",
    "spkidx = data[\"spkid\"].map(spkid_to_idx)\n",
    "\n",
    "spkidx = pd.DataFrame(spkidx)\n",
    "spkidx.columns = [\"spkidx\"]\n",
    "data = pd.concat([spkidx, data], axis=1)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of speakers: 2011\n",
      "Number of lines: 581236\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of speakers:\", len(data.spkidx.unique()))\n",
    "print(\"Number of lines:\", len(data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert the text to tokens\n",
    "\n",
    "It may take a while, so better to preprocess the text and save it to a file in advance.\n",
    "\n",
    "**Note** `phonemize_text` takes the longest time.`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_norm = data[\"text\"].tolist()\n",
    "for cleaners in text_cleaners:\n",
    "    if \"phonemize_text\" in cleaners:\n",
    "        text_norm = tokenizer(text_norm, cleaners, hps.data.language)\n",
    "    else:\n",
    "        for idx, text in enumerate(text_norm):\n",
    "            temp = tokenizer(text, cleaners, hps.data.language)\n",
    "            # print(f\"{text}\\n{detokenizer(temp)}\")\n",
    "            assert UNK_ID not in temp, f\"Found unknown symbol:\\n{text}\\n{detokenizer(temp)}\"\n",
    "            text_norm[idx] = temp\n",
    "    print(f\"Finished tokenizing with {cleaners}\")\n",
    "\n",
    "text_norm = [\"\\t\".join(map(str, text)) for text in text_norm]\n",
    "data = data.assign(cleaned_text=text_norm)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save the data to .csv file\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data = data.iloc[:100]\n",
    "data.to_csv(f\"../filelists/metadata.csv\", sep=\"|\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py10",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
