{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TF_emotion_detection.ipynb",
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyNmGyEUG5lc7EJ+9YF/hh07",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mrm8488/shared_colab_notebooks/blob/master/TF_emotion_detection.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Iyp18lEI_CBb",
        "colab_type": "text"
      },
      "source": [
        "# Text Emotion classifier with Tensorflow 2\n",
        "\n",
        "> Author: [Manuel Romero](https://twitter.com/mrm8488)\n",
        "\n",
        "> Dataset and idea from: [Elvis](https://twitter.com/omarsar0)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sbi_vGGzO4PY",
        "colab_type": "code",
        "outputId": "6e6f894b-0759-4f9e-bf2a-2b903b0237ba",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 319
        }
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mon May  4 16:51:06 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 440.64.00    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   32C    P0    27W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4UzDWp3jmOjg",
        "colab_type": "text"
      },
      "source": [
        "## Download GloVe word embeddings\n",
        "We will use it to create the embedding layer so we will not have to train it in the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vurgcIbSmSaK",
        "colab_type": "code",
        "outputId": "67026eeb-3047-4fde-d1d4-640bce81e90e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 372
        }
      },
      "source": [
        "!wget http://nlp.stanford.edu/data/glove.6B.zip"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-05-04 16:51:07--  http://nlp.stanford.edu/data/glove.6B.zip\n",
            "Resolving nlp.stanford.edu (nlp.stanford.edu)... 171.64.67.140\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:80... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://nlp.stanford.edu/data/glove.6B.zip [following]\n",
            "--2020-05-04 16:51:08--  https://nlp.stanford.edu/data/glove.6B.zip\n",
            "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:443... connected.\n",
            "HTTP request sent, awaiting response... 301 Moved Permanently\n",
            "Location: http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip [following]\n",
            "--2020-05-04 16:51:08--  http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip\n",
            "Resolving downloads.cs.stanford.edu (downloads.cs.stanford.edu)... 171.64.64.22\n",
            "Connecting to downloads.cs.stanford.edu (downloads.cs.stanford.edu)|171.64.64.22|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 862182613 (822M) [application/zip]\n",
            "Saving to: ‘glove.6B.zip’\n",
            "\n",
            "glove.6B.zip        100%[===================>] 822.24M  2.10MB/s    in 6m 27s  \n",
            "\n",
            "2020-05-04 16:57:35 (2.13 MB/s) - ‘glove.6B.zip’ saved [862182613/862182613]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GylfwnLJmXua",
        "colab_type": "code",
        "outputId": "1817a061-036c-43eb-f463-733a35fea460",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 106
        }
      },
      "source": [
        "!unzip glove*.zip"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  glove.6B.zip\n",
            "  inflating: glove.6B.50d.txt        \n",
            "  inflating: glove.6B.100d.txt       \n",
            "  inflating: glove.6B.200d.txt       \n",
            "  inflating: glove.6B.300d.txt       \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "feKOoXEln9P9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!rm -rf glove.6B.50d.txt\n",
        "!rm -rf glove.6B.200d.txt\n",
        "!rm -rf glove.6B.300d.txt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VrMkgEBSPMW_",
        "colab_type": "text"
      },
      "source": [
        "## Download the dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JtpqJ8gIPCTO",
        "colab_type": "code",
        "outputId": "0df1eadb-743f-40ff-cbd8-bd1c92efd439",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 464
        }
      },
      "source": [
        "!wget https://www.dropbox.com/s/607ptdakxuh5i4s/merged_training.pkl"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-05-04 16:58:03--  https://www.dropbox.com/s/607ptdakxuh5i4s/merged_training.pkl\n",
            "Resolving www.dropbox.com (www.dropbox.com)... 162.125.8.1, 2620:100:6016:1::a27d:101\n",
            "Connecting to www.dropbox.com (www.dropbox.com)|162.125.8.1|:443... connected.\n",
            "HTTP request sent, awaiting response... 301 Moved Permanently\n",
            "Location: /s/raw/607ptdakxuh5i4s/merged_training.pkl [following]\n",
            "--2020-05-04 16:58:08--  https://www.dropbox.com/s/raw/607ptdakxuh5i4s/merged_training.pkl\n",
            "Reusing existing connection to www.dropbox.com:443.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://uc0cc5a32002f45f5701faaddaa6.dl.dropboxusercontent.com/cd/0/inline/A3GjNNXOhAy8HUwPlw90F3CvhhF9apvzFDQ2Jga08qiMFekb9escyFtvI6OpmF9w5VU97uwTDFge7Jkl6EdF9bunULxbl5bG3dhEYZauHJz7tFmL_6q6UKP5W-5xqZZjCvQ/file# [following]\n",
            "--2020-05-04 16:58:08--  https://uc0cc5a32002f45f5701faaddaa6.dl.dropboxusercontent.com/cd/0/inline/A3GjNNXOhAy8HUwPlw90F3CvhhF9apvzFDQ2Jga08qiMFekb9escyFtvI6OpmF9w5VU97uwTDFge7Jkl6EdF9bunULxbl5bG3dhEYZauHJz7tFmL_6q6UKP5W-5xqZZjCvQ/file\n",
            "Resolving uc0cc5a32002f45f5701faaddaa6.dl.dropboxusercontent.com (uc0cc5a32002f45f5701faaddaa6.dl.dropboxusercontent.com)... 162.125.8.6, 2620:100:601b:6::a27d:806\n",
            "Connecting to uc0cc5a32002f45f5701faaddaa6.dl.dropboxusercontent.com (uc0cc5a32002f45f5701faaddaa6.dl.dropboxusercontent.com)|162.125.8.6|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 FOUND\n",
            "Location: /cd/0/inline2/A3FTOnMEx_gJazjN-FO94SucU1VOKDnMK2RaV0EgGPydH7BY1Zss48PaGu50bWt1ofyBhzBZVwM6ZjzOfo8kCECRM5qghFTc_wBGbUf97WJLLEQ5C7yjIFLgyOLJmbASMqW_b51vA4mlh928V9mY1WW-X74YFgJRxSiqaK2PgPIcQrtSYgg8oK3UZ0NSUoLR7LmW5-r6lqr3LMiOHsZarJEulfUrqMriMioXaB-F0rCZxnjP41MbQf_syRqnJgx7HroGVa56psCspw9Yyt_UPBIFpcqRTY3zcKCuW-HWbxUGc83JpwIkZRK5Aw-w4RnGy9_UQ_fJjUN0CjZBrfoBbrQX7B4TiMrIY1LX9epq1nTZTA/file [following]\n",
            "--2020-05-04 16:58:09--  https://uc0cc5a32002f45f5701faaddaa6.dl.dropboxusercontent.com/cd/0/inline2/A3FTOnMEx_gJazjN-FO94SucU1VOKDnMK2RaV0EgGPydH7BY1Zss48PaGu50bWt1ofyBhzBZVwM6ZjzOfo8kCECRM5qghFTc_wBGbUf97WJLLEQ5C7yjIFLgyOLJmbASMqW_b51vA4mlh928V9mY1WW-X74YFgJRxSiqaK2PgPIcQrtSYgg8oK3UZ0NSUoLR7LmW5-r6lqr3LMiOHsZarJEulfUrqMriMioXaB-F0rCZxnjP41MbQf_syRqnJgx7HroGVa56psCspw9Yyt_UPBIFpcqRTY3zcKCuW-HWbxUGc83JpwIkZRK5Aw-w4RnGy9_UQ_fJjUN0CjZBrfoBbrQX7B4TiMrIY1LX9epq1nTZTA/file\n",
            "Reusing existing connection to uc0cc5a32002f45f5701faaddaa6.dl.dropboxusercontent.com:443.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 49991846 (48M) [application/octet-stream]\n",
            "Saving to: ‘merged_training.pkl’\n",
            "\n",
            "merged_training.pkl 100%[===================>]  47.68M  52.5MB/s    in 0.9s    \n",
            "\n",
            "2020-05-04 16:58:10 (52.5 MB/s) - ‘merged_training.pkl’ saved [49991846/49991846]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0rYszFzWPqK6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pickle\n",
        "\n",
        "## helper function\n",
        "def load_from_pickle(directory):\n",
        "    return pickle.load(open(directory,\"rb\"))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_tk_4iQ-PsvY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data = load_from_pickle(directory=\"merged_training.pkl\")\n",
        "\n",
        "## using a sample\n",
        "emotions = [ \"sadness\", \"joy\", \"love\", \"anger\", \"fear\", \"surprise\"]\n",
        "data= data[data[\"emotions\"].isin(emotions)]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dO1xEkykQY8D",
        "colab_type": "code",
        "outputId": "b0583150-ae17-4bf6-c4fa-185586a1ea66",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177
        }
      },
      "source": [
        "data.info()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 416809 entries, 27383 to 64703\n",
            "Data columns (total 2 columns):\n",
            " #   Column    Non-Null Count   Dtype \n",
            "---  ------    --------------   ----- \n",
            " 0   text      416809 non-null  object\n",
            " 1   emotions  416809 non-null  object\n",
            "dtypes: object(2)\n",
            "memory usage: 9.5+ MB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u5ipy2VBQdz-",
        "colab_type": "code",
        "outputId": "ed5ed46e-d562-48a6-ed7b-71d859c93d5c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164
        }
      },
      "source": [
        "data.describe()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>emotions</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>416809</td>\n",
              "      <td>416809</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>unique</th>\n",
              "      <td>393822</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>top</th>\n",
              "      <td>i feel more adventurous willing to take risks</td>\n",
              "      <td>joy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>freq</th>\n",
              "      <td>16</td>\n",
              "      <td>141067</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                 text emotions\n",
              "count                                          416809   416809\n",
              "unique                                         393822        6\n",
              "top     i feel more adventurous willing to take risks      joy\n",
              "freq                                               16   141067"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "laP9BKZsQiIO",
        "colab_type": "code",
        "outputId": "9db24e87-87e5-46ec-c553-e01ed5050a60",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "data.isna().sum()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "text        0\n",
              "emotions    0\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_i-irlEkQoje",
        "colab_type": "code",
        "outputId": "00471253-0889-4143-f685-65c1c21cbc9d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "data.isnull().sum()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "text        0\n",
              "emotions    0\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NgTNDtvnQskR",
        "colab_type": "code",
        "outputId": "97014244-180a-4167-dce5-e08263cbcdca",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 634
        }
      },
      "source": [
        "data.head(20)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>emotions</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>27383</th>\n",
              "      <td>i feel awful about it too because it s my job ...</td>\n",
              "      <td>sadness</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>110083</th>\n",
              "      <td>im alone i feel awful</td>\n",
              "      <td>sadness</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>140764</th>\n",
              "      <td>ive probably mentioned this before but i reall...</td>\n",
              "      <td>joy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>100071</th>\n",
              "      <td>i was feeling a little low few days back</td>\n",
              "      <td>sadness</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2837</th>\n",
              "      <td>i beleive that i am much more sensitive to oth...</td>\n",
              "      <td>love</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18231</th>\n",
              "      <td>i find myself frustrated with christians becau...</td>\n",
              "      <td>love</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10714</th>\n",
              "      <td>i am one of those people who feels like going ...</td>\n",
              "      <td>joy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35177</th>\n",
              "      <td>i feel especially pleased about this as this h...</td>\n",
              "      <td>joy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>122177</th>\n",
              "      <td>i was struggling with these awful feelings and...</td>\n",
              "      <td>joy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26723</th>\n",
              "      <td>i feel so enraged but helpless at the same time</td>\n",
              "      <td>anger</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41979</th>\n",
              "      <td>i said feeling a bit rebellious</td>\n",
              "      <td>anger</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2046</th>\n",
              "      <td>i also feel disillusioned that someone who cla...</td>\n",
              "      <td>sadness</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98659</th>\n",
              "      <td>i mean is on this stupid trip of making the gr...</td>\n",
              "      <td>joy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50434</th>\n",
              "      <td>i woke up feeling particularly vile tried to i...</td>\n",
              "      <td>anger</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9280</th>\n",
              "      <td>i could feel the vile moth burrowing its way i...</td>\n",
              "      <td>anger</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92846</th>\n",
              "      <td>i know its just doing its job and doesnt actua...</td>\n",
              "      <td>joy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>106363</th>\n",
              "      <td>i wish you knew every word i write i write for...</td>\n",
              "      <td>sadness</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23395</th>\n",
              "      <td>i feel weird knowing mine died when i wasn t a...</td>\n",
              "      <td>fear</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31583</th>\n",
              "      <td>i feel assured that there is no such thing as ...</td>\n",
              "      <td>joy</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8271</th>\n",
              "      <td>i feel blessed everyday for our little man and...</td>\n",
              "      <td>love</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                     text emotions\n",
              "27383   i feel awful about it too because it s my job ...  sadness\n",
              "110083                              im alone i feel awful  sadness\n",
              "140764  ive probably mentioned this before but i reall...      joy\n",
              "100071           i was feeling a little low few days back  sadness\n",
              "2837    i beleive that i am much more sensitive to oth...     love\n",
              "18231   i find myself frustrated with christians becau...     love\n",
              "10714   i am one of those people who feels like going ...      joy\n",
              "35177   i feel especially pleased about this as this h...      joy\n",
              "122177  i was struggling with these awful feelings and...      joy\n",
              "26723     i feel so enraged but helpless at the same time    anger\n",
              "41979                     i said feeling a bit rebellious    anger\n",
              "2046    i also feel disillusioned that someone who cla...  sadness\n",
              "98659   i mean is on this stupid trip of making the gr...      joy\n",
              "50434   i woke up feeling particularly vile tried to i...    anger\n",
              "9280    i could feel the vile moth burrowing its way i...    anger\n",
              "92846   i know its just doing its job and doesnt actua...      joy\n",
              "106363  i wish you knew every word i write i write for...  sadness\n",
              "23395   i feel weird knowing mine died when i wasn t a...     fear\n",
              "31583   i feel assured that there is no such thing as ...      joy\n",
              "8271    i feel blessed everyday for our little man and...     love"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G_MN24JE0bUx",
        "colab_type": "code",
        "outputId": "62440e79-baf0-4a0f-aaf0-37e733feba5b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 106
        }
      },
      "source": [
        "data[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([   1,  459,  264,   12, 1267, 4070, 1386,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 136
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OAM3BN_JQ9Kf",
        "colab_type": "code",
        "outputId": "6d2a4851-f45d-4b6b-9a7d-6cb9ee4c5843",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 314
        }
      },
      "source": [
        "data.emotions.value_counts().plot.bar()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7fcb022af390>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "[encoded data, 9708 characters]\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BwBcnRE3RQSC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MAX_NB_WORDS = 100000    # max no. of words for tokenizer\n",
        "MAX_SEQUENCE_LENGTH = 170 # max length of each entry (sentence), including padding\n",
        "VALIDATION_SPLIT = 0.2   # data for validation (not used in training)\n",
        "EMBEDDING_DIM = 100      # embedding dimensions for word vectors (word2vec/GloVe)\n",
        "\n",
        "train = data\n",
        "\n",
        "#labels = emotions\n",
        "y = train['emotions'].values\n",
        "comments_train = train['text']\n",
        "comments_train = list(comments_train)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dPx48IedSTUt",
        "colab_type": "code",
        "outputId": "3b0d657c-af3e-4d11-dc0a-eb3ae5448351",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "y"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['sadness', 'sadness', 'joy', ..., 'sadness', 'joy', 'sadness'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ivFJmfwSTxS",
        "colab_type": "code",
        "outputId": "1159e381-cd62-478d-83bb-ee4194287aa7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(comments_train)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "416809"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KSEVIbcVSiGh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "max_text = (max(comments_train, key=len))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PSM4yn70S6xJ",
        "colab_type": "code",
        "outputId": "d027dfb7-9a26-4e96-a396-4469a2d7155f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(max_text.split())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "178"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c5rchMd6VWZs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def num_words(sentence):\n",
        "  words = sentence.split()\n",
        "  return len(words)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4wIeBGPLTOLc",
        "colab_type": "code",
        "outputId": "c798993f-a07d-44f4-8408-23b243165c5b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "total_avg_words = sum( map(num_words, comments_train) ) / len(comments_train)\n",
        "total_avg_words"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "19.211015117235952"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LeVLS2zpYGzM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MAX_SEQUENCE_LENGTH = 50"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b2Wwti9DYZ05",
        "colab_type": "text"
      },
      "source": [
        "## Text pre-proccessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f2xf7CpDoo7r",
        "colab_type": "code",
        "outputId": "97dabf51-54f7-4014-fbcc-bc6aeaae2545",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "import re\n",
        "from tqdm import tqdm_notebook\n",
        "\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "\n",
        "from nltk.corpus import stopwords"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_X_v8bTmqgq1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def clean_text(text, remove_stopwords = True):\n",
        "    output = \"\"\n",
        "    text = str(text).replace(\"\\n\", \"\")\n",
        "    text = re.sub(r'[^\\w\\s]','',text).lower()\n",
        "    if remove_stopwords:\n",
        "        text = text.split(\" \")\n",
        "        for word in text:\n",
        "            if word not in stopwords.words(\"english\"):\n",
        "                output = output + \" \" + word\n",
        "    else:\n",
        "        output = text\n",
        "    return str(output.strip())[1:-3].replace(\"  \", \" \")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vzRNXxd6YRZh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "texts = [] \n",
        "\n",
        "for line in tqdm_notebook(comments_train, total=159571): \n",
        "    texts.append(clean_text(line))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ETFF5Eb7dbcw",
        "colab_type": "code",
        "outputId": "25dcfb9f-8d34-400f-8f19-6003a1bc1043",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(texts)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "416809"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EgRSRXEddeNU",
        "colab_type": "code",
        "outputId": "5ac23fa6-f3a3-4479-b412-6021ea5ba118",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "print('Sample data:', texts[1], y[1])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sample data: m alone feel aw sadness\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OnRb9xWSdvWt",
        "colab_type": "text"
      },
      "source": [
        "## Tokenize the texts"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MKTxpal-o2DS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.preprocessing.text import Tokenizer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MToEWKBedmHU",
        "colab_type": "code",
        "outputId": "9c3e812a-eb54-480c-ddfd-6ced48db9f6a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "tokenizer = Tokenizer(num_words=MAX_NB_WORDS)\n",
        "tokenizer.fit_on_texts(texts)\n",
        "sequences = tokenizer.texts_to_sequences(texts)\n",
        "word_index = tokenizer.word_index\n",
        "print('Vocabulary size:', len(word_index))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Vocabulary size: 86474\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1jQj6lESd4EP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import json\n",
        "with open('word_index.json', 'w') as f:\n",
        "    json.dump(word_index, f)\n",
        "with open('index_word.json', 'w') as f2:\n",
        "    json.dump(tokenizer.index_word, f2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YWUFMe5venYt",
        "colab_type": "code",
        "outputId": "5f57dd63-87f1-4ec1-8e06-deb1f119a1a9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "data = pad_sequences(sequences, padding = 'post', maxlen = MAX_SEQUENCE_LENGTH)\n",
        "print('Shape of data tensor:', data.shape)\n",
        "print('Shape of label tensor:', y.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape of data tensor: (416809, 50)\n",
            "Shape of label tensor: (416809,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6wYmHzuVfJPZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "indices = np.arange(data.shape[0])\n",
        "np.random.shuffle(indices)\n",
        "data = data[indices]\n",
        "labels = y[indices]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LOczYxDefPK-",
        "colab_type": "code",
        "outputId": "a47b56fc-2094-46d7-a51e-8268de313b6d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 106
        }
      },
      "source": [
        "data[25]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([2562,  119,    3,   71,  120,  714,  602, 1546,  186,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0], dtype=int32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 125
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zm7BnNXqfRxm",
        "colab_type": "code",
        "outputId": "9bdd9ba1-3af7-4a92-e06e-c43bb4d42945",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "labels[25]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1, 0, 0, 0, 0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 128
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ow5WhdR9-vqm",
        "colab_type": "text"
      },
      "source": [
        "## One-hot encoding labels"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Lzq7JFcf2DA",
        "colab_type": "code",
        "outputId": "d9bcc70b-8323-40e7-bc9c-386d3c33c57a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "from sklearn import preprocessing\n",
        "lb = preprocessing.LabelBinarizer()\n",
        "lb.fit(labels)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LabelBinarizer(neg_label=0, pos_label=1, sparse_output=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OTffnsiMf8IU",
        "colab_type": "code",
        "outputId": "f1f2edd3-7ffb-44db-f855-f05a66dd0532",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "lb.classes_"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['anger', 'fear', 'joy', 'love', 'sadness', 'surprise'], dtype='<U8')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UISjirthgHbR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "labels = lb.transform(labels)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GUMSiashgNaG",
        "colab_type": "code",
        "outputId": "1994606c-24cb-440c-ac62-63cb3305e87b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "labels[25]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1, 0, 0, 0, 0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 127
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NuXPGlQAfYiZ",
        "colab_type": "code",
        "outputId": "9fe22061-3048-47df-a361-f07333fdb5f3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "num_validation_samples = int(VALIDATION_SPLIT*data.shape[0])\n",
        "x_train = data[: -num_validation_samples]\n",
        "y_train = labels[: -num_validation_samples]\n",
        "x_val = data[-num_validation_samples: ]\n",
        "y_val = labels[-num_validation_samples: ]\n",
        "print('Number of entries in each category:')\n",
        "print('training: ', y_train.sum(axis=0))\n",
        "print('validation: ', y_val.sum(axis=0))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of entries in each category:\n",
            "training:  [ 45948  38184 113044  27588  96741  11943]\n",
            "validation:  [11369  9528 28023  6966 24446  3029]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ofk5fpxXglFW",
        "colab_type": "code",
        "outputId": "700c3dd0-1d68-4731-995c-43bb4b303582",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "x_train.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(333448, 50)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 216
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J-QgdzHhgpIi",
        "colab_type": "code",
        "outputId": "33165b88-871e-4213-b2a3-08ea7a49ad85",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "y_train.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(333448, 6)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 217
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6I6piGBqsCtH",
        "colab_type": "code",
        "outputId": "0a5e6abb-b0d8-4bb9-bcbd-c573deb8136f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "x_val.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(83361, 50)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 218
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H7Ea8hUbtvSq",
        "colab_type": "code",
        "outputId": "100021da-d7e6-405d-caeb-f814847739b1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "y_val.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(83361, 6)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 219
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JKBBdhBa-m3V",
        "colab_type": "text"
      },
      "source": [
        "## Create test set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C4-MO5HlsL7D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x_val = x_val[: -40000]\n",
        "y_val = y_val[: -40000]\n",
        "x_test = x_val[-40000: ]\n",
        "y_test = y_val[-40000: ]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F1xA-v7EhDdg",
        "colab_type": "code",
        "outputId": "a42a5d02-04f8-4cc0-9d5f-8ae33edbfd06",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        }
      },
      "source": [
        "print('Tokenized sentences: \\n', data[10])\n",
        "print('One hot label: \\n', labels[10])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tokenized sentences: \n",
            " [ 433  917  120 1164 4537  215  171 2252    2 1865  496  136    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0    0    0    0    0    0    0\n",
            "    0    0    0    0    0    0    0    0]\n",
            "One hot label: \n",
            " [0 0 0 0 1 0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RzpqsYFv_5-T",
        "colab_type": "text"
      },
      "source": [
        "## Create the embedding matrix for our model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hABB26ndhIl8",
        "colab_type": "code",
        "outputId": "cf2383f0-e4d4-4270-b725-e75ff78148ea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "embeddings_index = {}\n",
        "f = open('/content/glove.6B.100d.txt')\n",
        "print('Loading GloVe from:', '/content/glove.6B.100d.txt','...', end='')\n",
        "for line in f:\n",
        "    values = line.split()\n",
        "    word = values[0]\n",
        "    embeddings_index[word] = np.asarray(values[1:], dtype='float32')\n",
        "f.close()\n",
        "print(\"Done.\\n Proceeding with Embedding Matrix...\", end=\"\")\n",
        "\n",
        "embedding_matrix = np.random.random((len(word_index) + 1, EMBEDDING_DIM))\n",
        "for word, i in word_index.items():\n",
        "    embedding_vector = embeddings_index.get(word)\n",
        "    if embedding_vector is not None:\n",
        "        embedding_matrix[i] = embedding_vector\n",
        "print(\" Completed!\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading GloVe from: /content/glove.6B.100d.txt ...Done.\n",
            " Proceeding with Embedding Matrix... Completed!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C0GL1iv6hW9h",
        "colab_type": "code",
        "outputId": "91ce7066-9a36-4a8d-9796-41695a819ce8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "embedding_matrix.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(86475, 100)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 90
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UQqqqPakhfzm",
        "colab_type": "text"
      },
      "source": [
        "## Create the model (function API)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "874LqeJFtTkg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras import regularizers, initializers, optimizers, callbacks\n",
        "from tensorflow.keras.layers import *\n",
        "from tensorflow.keras.models import Model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EiGxU1NthiQ9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
        "embedding_layer = Embedding(len(word_index) + 1,\n",
        "                           EMBEDDING_DIM,\n",
        "                           weights = [embedding_matrix],\n",
        "                           input_length = MAX_SEQUENCE_LENGTH,\n",
        "                           trainable=False,\n",
        "                           name = 'embeddings')\n",
        "embedded_sequences = embedding_layer(sequence_input)\n",
        "x = LSTM(60, return_sequences=True,name='lstm_layer')(embedded_sequences)\n",
        "x = GlobalMaxPool1D()(x)\n",
        "x = Dropout(0.1)(x)\n",
        "x = Dense(50, activation=\"relu\")(x)\n",
        "x = Dropout(0.1)(x)\n",
        "preds = Dense(6, activation=\"softmax\")(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GHp7DbdJhuMq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = Model(sequence_input, preds)\n",
        "model.compile(loss = 'categorical_crossentropy',\n",
        "             optimizer='adam',\n",
        "             metrics = ['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "baScL0nUh2az",
        "colab_type": "code",
        "outputId": "f3f135cb-b8dc-47ce-cf11-5a47151f6210",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 758
        }
      },
      "source": [
        "import tensorflow as tf\n",
        "tf.keras.utils.plot_model(model)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "image/png": "[encoded data, 28224 characters]\n",
            "text/plain": [
              "<IPython.core.display.Image object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 94
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "whiw6yd6iA-Y",
        "colab_type": "code",
        "outputId": "2e9b75a4-3478-49b3-e7c2-00b468074704",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 212
        }
      },
      "source": [
        "print('Training progress:')\n",
        "history = model.fit(x_train, y_train, epochs = 5, batch_size=128, validation_data=(x_val, y_val))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training progress:\n",
            "Epoch 1/5\n",
            "2606/2606 [==============================] - 16s 6ms/step - loss: 0.5899 - accuracy: 0.7805 - val_loss: 0.2674 - val_accuracy: 0.8910\n",
            "Epoch 2/5\n",
            "2606/2606 [==============================] - 15s 6ms/step - loss: 0.2450 - accuracy: 0.9001 - val_loss: 0.1784 - val_accuracy: 0.9136\n",
            "Epoch 3/5\n",
            "2606/2606 [==============================] - 15s 6ms/step - loss: 0.1852 - accuracy: 0.9156 - val_loss: 0.1560 - val_accuracy: 0.9189\n",
            "Epoch 4/5\n",
            "2606/2606 [==============================] - 15s 6ms/step - loss: 0.1593 - accuracy: 0.9211 - val_loss: 0.1423 - val_accuracy: 0.9227\n",
            "Epoch 5/5\n",
            "2606/2606 [==============================] - 15s 6ms/step - loss: 0.1450 - accuracy: 0.9238 - val_loss: 0.1426 - val_accuracy: 0.9214\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q7pVZVQOibnL",
        "colab_type": "code",
        "outputId": "a5f3e88d-e263-4f37-b267-a36ac67e4d52",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "loss = history.history['loss']\n",
        "val_loss = history.history['val_loss']\n",
        "epochs = range(1, len(loss)+1)\n",
        "plt.plot(epochs, loss, label='Training loss')\n",
        "plt.plot(epochs, val_loss, label='Validation loss')\n",
        "plt.title('Training and validation loss')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.legend()\n",
        "plt.show();"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "[encoded data, 16544 characters]\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ZwhCBSTi0g2",
        "colab_type": "code",
        "outputId": "6b9a9820-6c4a-44c4-fa7f-14c35fa19a25",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        }
      },
      "source": [
        "accuracy = history.history['accuracy']\n",
        "val_accuracy = history.history['val_accuracy']\n",
        "plt.plot(epochs, accuracy, label='Training accuracy')\n",
        "plt.plot(epochs, val_accuracy, label='Validation accuracy')\n",
        "plt.title('Training and validation accuracy')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epochs')\n",
        "plt.legend()\n",
        "plt.show();"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "[encoded data, 20904 characters]\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9g5O1Nap4B0b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(\"Accuracy in the test set:\")\n",
        "model.evaluate(x_test, y_test)[1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BLPvRne5lymD",
        "colab_type": "text"
      },
      "source": [
        "## Test the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wCpK7qyol1dU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "samples = ['i feel like, i do not know...', 'love you woman', 'that is so funny', 'mamma, i just killed a man', 'i want to ride my bicycle', 'im alone i feel awful', 'i beleive that i am much more sensitive to oth...']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QHoN-zl1vXVx",
        "colab_type": "code",
        "outputId": "213434d8-19d8-40a0-9a35-0ea27ada2fe2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 266
        }
      },
      "source": [
        "cleaned_samples = []\n",
        "for sentence in samples:\n",
        "  print(sentence)\n",
        "  cleaned = clean_text(sentence)\n",
        "  print(cleaned)\n",
        "  cleaned_samples.append(cleaned)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "i feel like, i do not know...\n",
            "eel like k\n",
            "love you woman\n",
            "ove wo\n",
            "that is so funny\n",
            "u\n",
            "mamma, i just killed a man\n",
            "amma killed \n",
            "i want to ride my bicycle\n",
            "ant ride bicy\n",
            "im alone i feel awful\n",
            "m alone feel aw\n",
            "i beleive that i am much more sensitive to oth...\n",
            "eleive much sensitive \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6TErUPivwgEK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenized_seq = tokenizer.texts_to_sequences(cleaned_samples)\n",
        "padded_seq =  pad_sequences(tokenized_seq, padding = 'post', maxlen = MAX_SEQUENCE_LENGTH)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z863mKHHysRt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "int2label = {\n",
        "    0: 'anger',\n",
        "    1: 'fear',\n",
        "    2: 'joy',\n",
        "    3: 'love',\n",
        "    4: 'sadness',\n",
        "    5: 'surprise'\n",
        "}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r5zLcDjCli8R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "predictions = model.predict(padded_seq)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9uXrH6FSxsgN",
        "colab_type": "code",
        "outputId": "8bca96dd-35ce-49fa-ac25-377727cdad69",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        }
      },
      "source": [
        "for i, prediction in enumerate(predictions):\n",
        "  print(samples[i] +\" => \" +int2label[(np.argmax(prediction))])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "i feel like, i do not know... => joy\n",
            "love you woman => anger\n",
            "that is so funny => anger\n",
            "mamma, i just killed a man => anger\n",
            "i want to ride my bicycle => fear\n",
            "im alone i feel awful => sadness\n",
            "i beleive that i am much more sensitive to oth... => anger\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2fiMeU0Dxtay",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.save('model.h5')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dOcnrWhz2tik",
        "colab_type": "code",
        "outputId": "18835cc2-f66e-4602-aadb-4a185825dca5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 266
        }
      },
      "source": [
        "!pip install -q tensorflowjs"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 61kB 3.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 92kB 6.8MB/s \n",
            "\u001b[K     |████████████████████████████████| 104.6MB 118kB/s \n",
            "\u001b[K     |████████████████████████████████| 256kB 54.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 921kB 56.9MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.9MB 59.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 450kB 50.2MB/s \n",
            "\u001b[?25h  Building wheel for PyInquirer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for gast (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: tensorflow 2.2.0rc3 has requirement gast==0.3.3, but you'll have gast 0.2.2 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: tensorflow 2.2.0rc3 has requirement tensorboard<2.3.0,>=2.2.0, but you'll have tensorboard 2.1.1 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: tensorflow 2.2.0rc3 has requirement tensorflow-estimator<2.3.0,>=2.2.0rc0, but you'll have tensorflow-estimator 2.1.0 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: tensorflow-probability 0.10.0rc0 has requirement gast>=0.3.2, but you'll have gast 0.2.2 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J3DsWNAg-Pgs",
        "colab_type": "text"
      },
      "source": [
        "## Convert the model to TFjs format, zip and download it"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NEiE1SWp2r8O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!tensorflowjs_converter --input_format=keras /content/model.h5 /content/tfjs_model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZEzEo4yg9uUp",
        "colab_type": "code",
        "outputId": "bca0ac72-8afa-425d-c88b-10e74bbe8838",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 212
        }
      },
      "source": [
        "!zip -r tfjs_model /content/tfjs_model"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  adding: content/tfjs_model/ (stored 0%)\n",
            "  adding: content/tfjs_model/group1-shard7of9.bin (deflated 8%)\n",
            "  adding: content/tfjs_model/group1-shard9of9.bin (deflated 8%)\n",
            "  adding: content/tfjs_model/group1-shard4of9.bin (deflated 8%)\n",
            "  adding: content/tfjs_model/model.json (deflated 76%)\n",
            "  adding: content/tfjs_model/group1-shard2of9.bin (deflated 8%)\n",
            "  adding: content/tfjs_model/group1-shard6of9.bin (deflated 8%)\n",
            "  adding: content/tfjs_model/group1-shard5of9.bin (deflated 8%)\n",
            "  adding: content/tfjs_model/group1-shard1of9.bin (deflated 8%)\n",
            "  adding: content/tfjs_model/group1-shard3of9.bin (deflated 8%)\n",
            "  adding: content/tfjs_model/group1-shard8of9.bin (deflated 8%)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "41Nkip4Z9kTT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import files\n",
        "files.download('/content/tfjs_model.zip')"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}