{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.contrib.slim as slim\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# [Inception-ResNet-v1](https://arxiv.org/pdf/1602.07261.pdf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overall of network\n",
    "![image](images/overall.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inception_resnet_v1(inputs, is_training=True,\n",
    "                        dropout_keep_prob=0.8,\n",
    "                        bottleneck_layer_size=128,\n",
    "                        reuse=None, \n",
    "                        scope='InceptionResnetV1'):\n",
    "    \"\"\"Creates the Inception Resnet V1 model.\n",
    "    Args:\n",
    "      inputs: a 4-D tensor of size [batch_size, height, width, 3].\n",
    "      num_classes: number of predicted classes.\n",
    "      is_training: whether is training or not.\n",
    "      dropout_keep_prob: float, the fraction to keep before final layer.\n",
    "      reuse: whether or not the network and its variables should be reused. To be\n",
    "        able to reuse 'scope' must be given.\n",
    "      scope: Optional variable_scope.\n",
    "    Returns:\n",
    "      logits: the logits outputs of the model.\n",
    "      end_points: the set of end_points from the inception model.\n",
    "    \"\"\"\n",
    "    end_points = {}\n",
    "  \n",
    "    with tf.variable_scope(scope, 'InceptionResnetV1', [inputs], reuse=reuse):\n",
    "        with slim.arg_scope([slim.batch_norm, slim.dropout],\n",
    "                            is_training=is_training):\n",
    "            with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d],\n",
    "                                stride=1, padding='SAME'):\n",
    "      \n",
    "                # 149 x 149 x 32\n",
    "                net = slim.conv2d(inputs, 32, 3, stride=2, padding='VALID',\n",
    "                                  scope='Conv2d_1a_3x3')\n",
    "                end_points['Conv2d_1a_3x3'] = net\n",
    "                # 147 x 147 x 32\n",
    "                net = slim.conv2d(net, 32, 3, padding='VALID',\n",
    "                                  scope='Conv2d_2a_3x3')\n",
    "                end_points['Conv2d_2a_3x3'] = net\n",
    "                # 147 x 147 x 64\n",
    "                net = slim.conv2d(net, 64, 3, scope='Conv2d_2b_3x3')\n",
    "                end_points['Conv2d_2b_3x3'] = net\n",
    "                # 73 x 73 x 64\n",
    "                net = slim.max_pool2d(net, 3, stride=2, padding='VALID',\n",
    "                                      scope='MaxPool_3a_3x3')\n",
    "                end_points['MaxPool_3a_3x3'] = net\n",
    "                # 73 x 73 x 80\n",
    "                net = slim.conv2d(net, 80, 1, padding='VALID',\n",
    "                                  scope='Conv2d_3b_1x1')\n",
    "                end_points['Conv2d_3b_1x1'] = net\n",
    "                # 71 x 71 x 192\n",
    "                net = slim.conv2d(net, 192, 3, padding='VALID',\n",
    "                                  scope='Conv2d_4a_3x3')\n",
    "                end_points['Conv2d_4a_3x3'] = net\n",
    "                # 35 x 35 x 256\n",
    "                net = slim.conv2d(net, 256, 3, stride=2, padding='VALID',\n",
    "                                  scope='Conv2d_4b_3x3')\n",
    "                end_points['Conv2d_4b_3x3'] = net\n",
    "                \n",
    "                # 5 x Inception-resnet-A\n",
    "                net = slim.repeat(net, 5, block35, scale=0.17)\n",
    "                end_points['Mixed_5a'] = net\n",
    "        \n",
    "                # Reduction-A\n",
    "                with tf.variable_scope('Mixed_6a'):\n",
    "                    net = reduction_a(net, 192, 192, 256, 384)\n",
    "                end_points['Mixed_6a'] = net\n",
    "                \n",
    "                # 10 x Inception-Resnet-B\n",
    "                net = slim.repeat(net, 10, block17, scale=0.10)\n",
    "                end_points['Mixed_6b'] = net\n",
    "                \n",
    "                # Reduction-B\n",
    "                with tf.variable_scope('Mixed_7a'):\n",
    "                    net = reduction_b(net)\n",
    "                end_points['Mixed_7a'] = net\n",
    "                \n",
    "                # 5 x Inception-Resnet-C\n",
    "                net = slim.repeat(net, 5, block8, scale=0.20)\n",
    "                end_points['Mixed_8a'] = net\n",
    "                \n",
    "                net = block8(net, activation_fn=None)\n",
    "                end_points['Mixed_8b'] = net\n",
    "                \n",
    "                with tf.variable_scope('Logits'):\n",
    "                    end_points['PrePool'] = net\n",
    "                    #pylint: disable=no-member\n",
    "                    net = slim.avg_pool2d(net, net.get_shape()[1:3], padding='VALID',\n",
    "                                          scope='AvgPool_1a_8x8')\n",
    "                    net = slim.flatten(net)\n",
    "          \n",
    "                    net = slim.dropout(net, dropout_keep_prob, is_training=is_training,\n",
    "                                       scope='Dropout')\n",
    "          \n",
    "                    end_points['PreLogitsFlatten'] = net\n",
    "                \n",
    "                net = slim.fully_connected(net, bottleneck_layer_size, activation_fn=None, \n",
    "                        scope='Bottleneck', reuse=False)\n",
    "  \n",
    "    return net, end_points"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stem\n",
    "![image](images/stem.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stem(inputs):\n",
    "                # 149 x 149 x 32\n",
    "                net = slim.conv2d(inputs, 32, 3, stride=2, padding='VALID',\n",
    "                                  scope='Conv2d_1a_3x3')\n",
    "                end_points['Conv2d_1a_3x3'] = net\n",
    "                # 147 x 147 x 32\n",
    "                net = slim.conv2d(net, 32, 3, padding='VALID',\n",
    "                                  scope='Conv2d_2a_3x3')\n",
    "                end_points['Conv2d_2a_3x3'] = net\n",
    "                # 147 x 147 x 64\n",
    "                net = slim.conv2d(net, 64, 3, scope='Conv2d_2b_3x3')\n",
    "                end_points['Conv2d_2b_3x3'] = net\n",
    "                # 73 x 73 x 64\n",
    "                net = slim.max_pool2d(net, 3, stride=2, padding='VALID',\n",
    "                                      scope='MaxPool_3a_3x3')\n",
    "                end_points['MaxPool_3a_3x3'] = net\n",
    "                # 73 x 73 x 80\n",
    "                net = slim.conv2d(net, 80, 1, padding='VALID',\n",
    "                                  scope='Conv2d_3b_1x1')\n",
    "                end_points['Conv2d_3b_1x1'] = net\n",
    "                # 71 x 71 x 192\n",
    "                net = slim.conv2d(net, 192, 3, padding='VALID',\n",
    "                                  scope='Conv2d_4a_3x3')\n",
    "                end_points['Conv2d_4a_3x3'] = net\n",
    "                # 35 x 35 x 256\n",
    "                net = slim.conv2d(net, 256, 3, stride=2, padding='VALID',\n",
    "                                  scope='Conv2d_4b_3x3')\n",
    "                end_points['Conv2d_4b_3x3'] = net"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inception-Renset-A\n",
    "![images](images/a.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inception-Renset-A\n",
    "def block35(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None):\n",
    "    \"\"\"Builds the 35x35 resnet block.\"\"\"\n",
    "    with tf.variable_scope(scope, 'Block35', [net], reuse=reuse):\n",
    "        with tf.variable_scope('Branch_0'):\n",
    "            tower_conv = slim.conv2d(net, 32, 1, scope='Conv2d_1x1')\n",
    "        with tf.variable_scope('Branch_1'):\n",
    "            tower_conv1_0 = slim.conv2d(net, 32, 1, scope='Conv2d_0a_1x1')\n",
    "            tower_conv1_1 = slim.conv2d(tower_conv1_0, 32, 3, scope='Conv2d_0b_3x3')\n",
    "        with tf.variable_scope('Branch_2'):\n",
    "            tower_conv2_0 = slim.conv2d(net, 32, 1, scope='Conv2d_0a_1x1')\n",
    "            tower_conv2_1 = slim.conv2d(tower_conv2_0, 32, 3, scope='Conv2d_0b_3x3')\n",
    "            tower_conv2_2 = slim.conv2d(tower_conv2_1, 32, 3, scope='Conv2d_0c_3x3')\n",
    "        mixed = tf.concat([tower_conv, tower_conv1_1, tower_conv2_2], 3)\n",
    "        up = slim.conv2d(mixed, net.get_shape()[3], 1, normalizer_fn=None,\n",
    "                         activation_fn=None, scope='Conv2d_1x1')\n",
    "        net += scale * up\n",
    "        if activation_fn:\n",
    "            net = activation_fn(net)\n",
    "    return net"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inception-Renset-B\n",
    "![images](images/b.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inception-Renset-B\n",
    "def block17(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None):\n",
    "    \"\"\"Builds the 17x17 resnet block.\"\"\"\n",
    "    with tf.variable_scope(scope, 'Block17', [net], reuse=reuse):\n",
    "        with tf.variable_scope('Branch_0'):\n",
    "            tower_conv = slim.conv2d(net, 128, 1, scope='Conv2d_1x1')\n",
    "        with tf.variable_scope('Branch_1'):\n",
    "            tower_conv1_0 = slim.conv2d(net, 128, 1, scope='Conv2d_0a_1x1')\n",
    "            tower_conv1_1 = slim.conv2d(tower_conv1_0, 128, [1, 7],\n",
    "                                        scope='Conv2d_0b_1x7')\n",
    "            tower_conv1_2 = slim.conv2d(tower_conv1_1, 128, [7, 1],\n",
    "                                        scope='Conv2d_0c_7x1')\n",
    "        mixed = tf.concat([tower_conv, tower_conv1_2], 3)\n",
    "        up = slim.conv2d(mixed, net.get_shape()[3], 1, normalizer_fn=None,\n",
    "                         activation_fn=None, scope='Conv2d_1x1')\n",
    "        net += scale * up\n",
    "        if activation_fn:\n",
    "            net = activation_fn(net)\n",
    "    return net\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inception-Renset-C\n",
    "![image](images/c.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inception-Resnet-C\n",
    "def block8(net, scale=1.0, activation_fn=tf.nn.relu, scope=None, reuse=None):\n",
    "    \"\"\"Builds the 8x8 resnet block.\"\"\"\n",
    "    with tf.variable_scope(scope, 'Block8', [net], reuse=reuse):\n",
    "        with tf.variable_scope('Branch_0'):\n",
    "            tower_conv = slim.conv2d(net, 192, 1, scope='Conv2d_1x1')\n",
    "        with tf.variable_scope('Branch_1'):\n",
    "            tower_conv1_0 = slim.conv2d(net, 192, 1, scope='Conv2d_0a_1x1')\n",
    "            tower_conv1_1 = slim.conv2d(tower_conv1_0, 192, [1, 3],\n",
    "                                        scope='Conv2d_0b_1x3')\n",
    "            tower_conv1_2 = slim.conv2d(tower_conv1_1, 192, [3, 1],\n",
    "                                        scope='Conv2d_0c_3x1')\n",
    "        mixed = tf.concat([tower_conv, tower_conv1_2], 3)\n",
    "        up = slim.conv2d(mixed, net.get_shape()[3], 1, normalizer_fn=None,\n",
    "                         activation_fn=None, scope='Conv2d_1x1')\n",
    "        net += scale * up\n",
    "        if activation_fn:\n",
    "            net = activation_fn(net)\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
