{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic layers\n",
    "### Convolutional layer\n",
    "### Pooling layer\n",
    "### Activation layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convolutional layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ```tf.nn``` implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "image = tf.random_normal([1,112,96,3])\n",
    "in_channels = 3\n",
    "out_channels = 32\n",
    "kernel_size = 5\n",
    "#[filter_height,filter_width,in_channels,out_channels]\n",
    "conv_weight = tf.Variable(tf.truncated_normal([kernel_size,kernel_size,in_channels,out_channels],\n",
    "                          stddev=0.1,dtype=tf.float32))\n",
    "\n",
    "bias = tf.Variable(tf.zeros([out_channels],dtype=tf.float32))\n",
    "#data_format='NHWC'\n",
    "#input_data with shape [batch,in_height,in_width,in_channels]\n",
    "#output_data with shape [batch,out_height,out_width,out_channels]\n",
    "conv = tf.nn.conv2d(image,conv_weight,strides=[1,2,2,1],padding='SAME')\n",
    "conv = tf.nn.bias_add(conv,bias)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ```tf.layers``` implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "image = tf.random_normal([1,112,96,3])\n",
    "in_channels = 3\n",
    "out_channels = 32\n",
    "kernel_size = 5\n",
    "truncated_norm_init = tf.truncated_normal_initializer(\n",
    "                          stddev=0.1,dtype=tf.float32)\n",
    "\n",
    "\n",
    "zero_init = tf.zeros_initializer(dtype=tf.float32)\n",
    "\n",
    "l2_regularizer= tf.contrib.layers.l2_regularizer(1.0)\n",
    "conv = tf.layers.conv2d(image,out_channels,[kernel_size,kernel_size],strides=[2,2],padding='SAME',\n",
    "                 kernel_initializer=truncated_norm_init,bias_initializer=zero_init,\n",
    "                        kernel_regularizer=l2_regularizer,bias_regularizer=l2_regularizer)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [The difference between 'SAME' and 'VALID' in tensorflow](https://stackoverflow.com/questions/37674306/what-is-the-difference-between-same-and-valid-padding-in-tf-nn-max-pool-of-t)\n",
    "\n",
    "- For the SAME padding, the output height and width are computed as:\n",
    "out_height = ceil(float(in_height) / float(strides[1]))\n",
    "\n",
    "out_width = ceil(float(in_width) / float(strides[2]))\n",
    "\n",
    "And\n",
    "\n",
    "- For the VALID padding, the output height and width are computed as:\n",
    "out_height = ceil(float(in_height - filter_height + 1) / float(strides[1]))\n",
    "\n",
    "out_width = ceil(float(in_width - filter_width + 1) / float(strides[2]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "image shape (1, 112, 96, 3)\n",
      "conv weight shape (5, 5, 3, 32)\n",
      "conv output shape with SAME padded (1, 56, 48, 32)\n",
      "conv output shape with VALID padded (1, 54, 46, 32)\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "image = tf.random_normal([1,112,96,3])\n",
    "in_channels = 3\n",
    "out_channels = 32\n",
    "kernel_size = 5\n",
    "conv_weight = tf.Variable(tf.truncated_normal([kernel_size,kernel_size,in_channels,out_channels],\n",
    "                          stddev=0.1,dtype=tf.float32))\n",
    "print 'image shape',image.get_shape()\n",
    "print 'conv weight shape',conv_weight.get_shape()\n",
    "bias = tf.Variable(tf.zeros([out_channels],dtype=tf.float32))\n",
    "conv = tf.nn.conv2d(image,conv_weight,strides=[1,2,2,1],padding='SAME')\n",
    "conv = tf.nn.bias_add(conv,bias)\n",
    "print 'conv output shape with SAME padded',conv.get_shape()\n",
    "conv = tf.nn.conv2d(image,conv_weight,strides=[1,2,2,1],padding='VALID')\n",
    "conv = tf.nn.bias_add(conv,bias)\n",
    "print 'conv output shape with VALID padded',conv.get_shape()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pooling layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 27, 23, 32)\n",
      "(1, 27, 23, 32)\n"
     ]
    }
   ],
   "source": [
    "pool_size = 3\n",
    "pool = tf.nn.max_pool(conv,ksize=[1,pool_size,pool_size,1],strides=[1,2,2,1],padding='SAME')\n",
    "print pool.get_shape()\n",
    "pool = tf.nn.avg_pool(conv,ksize=[1,pool_size,pool_size,1],strides=[1,2,2,1],padding='SAME')\n",
    "print pool.get_shape()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Activation layer¶"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tf.nn.relu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "relu = tf.nn.relu(pool)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### prelu\n",
    "```\n",
    "y = prelu(x) \n",
    "if x > 0: \n",
    "    y = x\n",
    "else:\n",
    "    y = alpha*x #alpha >0\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 27, 23, 32)\n"
     ]
    }
   ],
   "source": [
    "def prelu(x, name = 'prelu'):\n",
    "    with tf.variable_scope(name):\n",
    "        alphas = tf.get_variable('alpha', x.get_shape()[-1], initializer=tf.constant_initializer(0.25), regularizer = l2_regularizer, dtype = tf.float32)\n",
    "    pos = tf.nn.relu(x)\n",
    "    neg = tf.multiply(alphas,(x - abs(x)) * 0.5)\n",
    "    return pos + neg\n",
    "prelu_out = prelu(pool)\n",
    "print prelu_out.get_shape()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
