{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Transformer positional encoding graph.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mrm8488/shared_colab_notebooks/blob/master/Transformer_positional_encoding_graph.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JaLFyOLDKr56",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pMvYBo3RKfmA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "# Code from https://www.tensorflow.org/tutorials/text/transformer\n",
        "def get_angles(pos, i, d_model):\n",
        "  angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n",
        "  return pos * angle_rates\n",
        "\n",
        "def positional_encoding(position, d_model):\n",
        "  angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n",
        "                          np.arange(d_model)[np.newaxis, :],\n",
        "                          d_model)\n",
        "  \n",
        "  # apply sin to even indices in the array; 2i\n",
        "  angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
        "  \n",
        "  # apply cos to odd indices in the array; 2i+1\n",
        "  angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
        "    \n",
        "  pos_encoding = angle_rads[np.newaxis, ...]\n",
        "    \n",
        "  return pos_encoding\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SB9-56JWKv6T",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 518
        },
        "outputId": "379c3f5a-07c7-49c3-e7ad-36de8080b2a1"
      },
      "source": [
        "tokens = 10\n",
        "dimensions = 64\n",
        "\n",
        "pos_encoding = positional_encoding(tokens, dimensions)\n",
        "print (pos_encoding.shape)\n",
        "\n",
        "plt.figure(figsize=(12,8))\n",
        "plt.pcolormesh(pos_encoding[0], cmap='viridis')\n",
        "plt.xlabel('Embedding Dimensions')\n",
        "plt.xlim((0, dimensions))\n",
        "plt.ylim((tokens,0))\n",
        "plt.ylabel('Token Position')\n",
        "plt.colorbar()\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1, 10, 64)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "[encoded data, 16684 characters]\n",
            "text/plain": [
              "<Figure size 864x576 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GdyrTa8vMZLs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}