{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear Regression Example\n",
    "\n",
    "A linear regression learning algorithm example using TensorFlow library.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]='PCI_BUS_ID'\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\"\n",
    "rng = numpy.random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters\n",
    "learning_rate = 0.01\n",
    "training_epochs = 1000\n",
    "display_step = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training Data\n",
    "train_X = numpy.asarray([3.3,4.4,5.5,6.71,6.93,4.168,9.779,6.182,7.59,2.167,\n",
    "                         7.042,10.791,5.313,7.997,5.654,9.27,3.1])\n",
    "train_Y = numpy.asarray([1.7,2.76,2.09,3.19,1.694,1.573,3.366,2.596,2.53,1.221,\n",
    "                         2.827,3.465,1.65,2.904,2.42,2.94,1.3])\n",
    "n_samples = train_X.shape[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f12504f5650>]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 5632 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1250804cd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#show data\n",
    "plt.plot(train_X, train_Y, 'ro', label='Original data')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learn a linear regression to fit the dataset\n",
    "y_pred = w*x+b\n",
    "\n",
    "We learn w and b from the dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Two phase to write your tf program\n",
    "### Build a model (assemble a graph)\n",
    "1. Define placeholders for input and output (label)\n",
    "2. Define the weight\n",
    "3. Define the inference model\n",
    "4. Define loss function\n",
    "5. Define optimizer\n",
    "\n",
    "### Training (Use a session to execute operations in the graph)\n",
    "1. Initialize model parameters\n",
    "2. Input training data\n",
    "3. Execute inference model on training data\n",
    "4. Compute loss\n",
    "5. Adust mode parameters to minimize the loss\n",
    "6. Repeat 2->5 util loss convergence\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### linear regression v1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.summary.writer.writer.FileWriter at 0x7f12a490bb50>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "# define the input by placeholder\n",
    "with tf.name_scope('inputs'):\n",
    "    x = tf.placeholder(tf.float32,name='x')\n",
    "    y = tf.placeholder(tf.float32,name='y')\n",
    "# define the weight\n",
    "# represent the learnable paramters by tf.placeholder\n",
    "with tf.variable_scope('params'):\n",
    "    w = tf.placeholder(tf.float32,name='w')\n",
    "    b = tf.placeholder(tf.float32,name='b')\n",
    "# construct inference model\n",
    "with tf.name_scope('inference'):\n",
    "    y_pred = tf.multiply(w,x) + b\n",
    "# Mean squared error as loss function\n",
    "with tf.name_scope('loss'):\n",
    "    cost = tf.reduce_sum(tf.pow(y_pred-y, 2))/(2*n_samples)\n",
    "with tf.name_scope('grads'):\n",
    "    grad_w,grad_b = tf.gradients(cost,[w,b])\n",
    "tf.summary.FileWriter('linear_model_v1',tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0050 cost= 0.001796215 W= 0.377180755138 b= -0.116386435926\n",
      "Epoch: 0100 cost= 0.001373004 W= 0.369592547417 b= -0.0617973804474\n",
      "Epoch: 0150 cost= 0.001026769 W= 0.362455636263 b= -0.0104549890384\n",
      "Epoch: 0200 cost= 0.000746951 W= 0.355743199587 b= 0.0378338359296\n",
      "Epoch: 0250 cost= 0.000524312 W= 0.349429965019 b= 0.0832507014275\n",
      "Epoch: 0300 cost= 0.000350774 W= 0.343492209911 b= 0.125966414809\n",
      "Epoch: 0350 cost= 0.000219277 W= 0.337907612324 b= 0.166141614318\n",
      "Epoch: 0400 cost= 0.000123660 W= 0.332655161619 b= 0.20392742753\n",
      "Epoch: 0450 cost= 0.000058550 W= 0.327715098858 b= 0.239465937018\n",
      "Epoch: 0500 cost= 0.000019269 W= 0.323068827391 b= 0.272890806198\n",
      "Epoch: 0550 cost= 0.000001746 W= 0.318698912859 b= 0.304327726364\n",
      "Epoch: 0600 cost= 0.000002447 W= 0.314588874578 b= 0.333894968033\n",
      "Epoch: 0650 cost= 0.000018303 W= 0.310723304749 b= 0.361703693867\n",
      "Epoch: 0700 cost= 0.000046660 W= 0.307087630033 b= 0.387858510017\n",
      "Epoch: 0750 cost= 0.000085223 W= 0.303668171167 b= 0.412457793951\n",
      "Epoch: 0800 cost= 0.000132012 W= 0.300452083349 b= 0.435594022274\n",
      "Epoch: 0850 cost= 0.000185324 W= 0.297427296638 b= 0.45735424757\n",
      "Epoch: 0900 cost= 0.000243696 W= 0.294582396746 b= 0.477820277214\n",
      "Epoch: 0950 cost= 0.000305879 W= 0.291906684637 b= 0.497069120407\n",
      "Epoch: 1000 cost= 0.000370805 W= 0.28939011693 b= 0.515173137188\n",
      "Optimization Finished, spend time: 11.0523588657!\n",
      "{<tf.Tensor 'params/w:0' shape=<unknown> dtype=float32>: 0.28939011265340536, <tf.Tensor 'params/b:0' shape=<unknown> dtype=float32>: 0.51517313003748655, <tf.Tensor 'inputs/x:0' shape=<unknown> dtype=float32>: array([  3.3  ,   4.4  ,   5.5  ,   6.71 ,   6.93 ,   4.168,   9.779,\n",
      "         6.182,   7.59 ,   2.167,   7.042,  10.791,   5.313,   7.997,\n",
      "         5.654,   9.27 ,   3.1  ]), <tf.Tensor 'inputs/y:0' shape=<unknown> dtype=float32>: array([ 1.7  ,  2.76 ,  2.09 ,  3.19 ,  1.694,  1.573,  3.366,  2.596,\n",
      "        2.53 ,  1.221,  2.827,  3.465,  1.65 ,  2.904,  2.42 ,  2.94 ,  1.3  ])}\n",
      "Training cost= 0.0820096 w= 0.28939011693 b= 0.515173137188 \n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 12304 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f12a197f550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# compute\n",
    "# Start training\n",
    "with tf.Session() as sess:\n",
    "    #initialize the w,b by random\n",
    "    values = {w:rng.randn(),b:rng.randn()}\n",
    "    # Fit all training data\n",
    "    t_start = time.time()\n",
    "    for epoch in range(training_epochs):\n",
    "        for (X, Y) in zip(train_X, train_Y):\n",
    "            values[x] = X\n",
    "            values[y] = Y\n",
    "            #compute the gradients\n",
    "            cost_np,grad_w_np,grad_b_np = sess.run([cost,grad_w,grad_b], feed_dict=values)\n",
    "            #update the parameters\n",
    "            values[w] -= learning_rate * grad_w_np\n",
    "            values[b] -= learning_rate * grad_b_np\n",
    "\n",
    "        #Display logs per epoch step\n",
    "        #print (epoch+1) % display_step\n",
    "        #print values\n",
    "        if (epoch+1) % display_step == 0:\n",
    "            c = sess.run(cost, feed_dict=values)\n",
    "            print \"Epoch:\", '%04d' % (epoch+1), \"cost=\", \"{:.9f}\".format(c), \\\n",
    "                \"W=\", sess.run(w,feed_dict=values), \"b=\", sess.run(b,feed_dict=values)\n",
    "\n",
    "    print \"Optimization Finished, spend time: {}!\".format(time.time()-t_start)\n",
    "    values[x] = train_X\n",
    "    values[y] = train_Y\n",
    "    print values\n",
    "    training_cost = sess.run(cost, feed_dict=values)\n",
    "    print \"Training cost=\", training_cost, \"w=\", sess.run(w,feed_dict=values), \"b=\", sess.run(b,feed_dict=values), '\\n'\n",
    "\n",
    "    #Graphic display\n",
    "    plt.plot(train_X, train_Y, 'ro', label='Original data')\n",
    "    plt.plot(train_X, sess.run(w,feed_dict=values) * train_X + sess.run(b,feed_dict=values), label='Fitted line')\n",
    "    plt.legend()\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## linear regression v2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.summary.writer.writer.FileWriter at 0x7f1250457610>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "# define the input by placeholder\n",
    "with tf.name_scope('inputs'):\n",
    "    x = tf.placeholder(tf.float32,name='x')\n",
    "    y = tf.placeholder(tf.float32,name='y')\n",
    "# define the weight\n",
    "# represent the learnable paramters by tf.placeholder\n",
    "with tf.variable_scope('params'):\n",
    "    w = tf.Variable(rng.randn(),name='w')\n",
    "    b = tf.Variable(rng.randn(),name='b')\n",
    "# construct inference model\n",
    "with tf.name_scope('inference'):\n",
    "    y_pred = tf.multiply(w,x) + b\n",
    "# Mean squared error as loss function\n",
    "with tf.name_scope('loss'):\n",
    "    cost = tf.reduce_sum(tf.pow(y_pred-y, 2))/(2*n_samples)\n",
    "with tf.name_scope('grads'):\n",
    "    grad_w,grad_b = tf.gradients(cost,[w,b])\n",
    "with tf.name_scope('update_params'):\n",
    "    new_w = w.assign(w-learning_rate*grad_w)\n",
    "    new_b = w.assign(b-learning_rate*grad_b)\n",
    "init = tf.global_variables_initializer()\n",
    "tf.summary.FileWriter('linear_model_v2',tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "v3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0050 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0100 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0150 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0200 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0250 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0300 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0350 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0400 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0450 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0500 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0550 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0600 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0650 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0700 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0750 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0800 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0850 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0900 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 0950 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Epoch: 1000 cost= 0.071842253 W= 0.508991 b= 1.28502\n",
      "Optimization Finished, spend time: 7.58594489098!\n",
      "Training cost= 2.44103 w= 0.508991 b= 1.28502 \n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 11140 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f125014b110>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# compute\n",
    "# Start training\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    # Fit all training data\n",
    "    t_start = time.time()\n",
    "    values = {}\n",
    "    for epoch in range(training_epochs):\n",
    "        for (X, Y) in zip(train_X, train_Y):\n",
    "            values[x] = X\n",
    "            values[y] = Y\n",
    "            #compute the gradients\n",
    "            cost_np = sess.run([cost], feed_dict=values)\n",
    "            #update the parameters\n",
    "            #values[w] -= learning_rate * grad_w_np\n",
    "            #values[b] -= learning_rate * grad_b_np\n",
    "\n",
    "        #Display logs per epoch step\n",
    "        #print (epoch+1) % display_step\n",
    "        #print values\n",
    "        if (epoch+1) % display_step == 0:\n",
    "            c = sess.run(cost, feed_dict=values)\n",
    "            print \"Epoch:\", '%04d' % (epoch+1), \"cost=\", \"{:.9f}\".format(c), \\\n",
    "                \"W=\", sess.run(w), \"b=\", sess.run(b)\n",
    "\n",
    "    print \"Optimization Finished, spend time: {}!\".format(time.time()-t_start)\n",
    "    values[x] = train_X\n",
    "    values[y] = train_Y\n",
    "  \n",
    "    training_cost = sess.run(cost, feed_dict=values)\n",
    "    print \"Training cost=\", training_cost, \"w=\", sess.run(w,feed_dict=values), \"b=\", sess.run(b,feed_dict=values), '\\n'\n",
    "\n",
    "    #Graphic display\n",
    "    plt.plot(train_X, train_Y, 'ro', label='Original data')\n",
    "    plt.plot(train_X, sess.run(w,feed_dict=values) * train_X + sess.run(b,feed_dict=values), label='Fitted line')\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"update_params/Assign:0\", shape=(), dtype=float32_ref) Tensor(\"update_params/Assign_1:0\", shape=(), dtype=float32_ref)\n",
      "name: \"update_params/group_deps\"\n",
      "op: \"NoOp\"\n",
      "input: \"^update_params/Assign\"\n",
      "input: \"^update_params/Assign_1\"\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.summary.writer.writer.FileWriter at 0x7f125015bb90>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "# define the input by placeholder\n",
    "with tf.name_scope('inputs'):\n",
    "    x = tf.placeholder(tf.float32,name='x')\n",
    "    y = tf.placeholder(tf.float32,name='y')\n",
    "# define the weight\n",
    "# represent the learnable paramters by tf.placeholder\n",
    "with tf.variable_scope('params'):\n",
    "    w = tf.Variable(rng.randn(),name='w')\n",
    "    b = tf.Variable(rng.randn(),name='b')\n",
    "# construct inference model\n",
    "with tf.name_scope('inference'):\n",
    "    y_pred = tf.multiply(w,x) + b\n",
    "# Mean squared error as loss function\n",
    "with tf.name_scope('loss'):\n",
    "    cost = tf.reduce_sum(tf.pow(y_pred-y, 2))/(2*n_samples)\n",
    "with tf.name_scope('grads'):\n",
    "    grad_w,grad_b = tf.gradients(cost,[w,b])\n",
    "with tf.name_scope('update_params'):\n",
    "    new_w = w.assign(w-learning_rate*grad_w)\n",
    "    new_b = b.assign(b-learning_rate*grad_b)\n",
    "    print new_w,new_b\n",
    "    updates = tf.group(new_w,new_b)\n",
    "    print updates\n",
    "init = tf.global_variables_initializer()\n",
    "tf.summary.FileWriter('linear_model_v2',tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0055483\n",
      "Epoch: 0050 cost= 0.005548301 W= 0.210726 b= 1.08108\n",
      "0.00530915\n",
      "Epoch: 0100 cost= 0.005309150 W= 0.213037 b= 1.06445\n",
      "0.00508904\n",
      "Epoch: 0150 cost= 0.005089037 W= 0.215211 b= 1.04881\n",
      "0.00488626\n",
      "Epoch: 0200 cost= 0.004886260 W= 0.217256 b= 1.0341\n",
      "0.0046993\n",
      "Epoch: 0250 cost= 0.004699301 W= 0.219179 b= 1.02026\n",
      "0.00452679\n",
      "Epoch: 0300 cost= 0.004526786 W= 0.220988 b= 1.00725\n",
      "0.00436747\n",
      "Epoch: 0350 cost= 0.004367469 W= 0.22269 b= 0.995011\n",
      "0.00422023\n",
      "Epoch: 0400 cost= 0.004220233 W= 0.22429 b= 0.983499\n",
      "0.00408405\n",
      "Epoch: 0450 cost= 0.004084054 W= 0.225795 b= 0.972672\n",
      "0.00395801\n",
      "Epoch: 0500 cost= 0.003958011 W= 0.227211 b= 0.962488\n",
      "0.0038413\n",
      "Epoch: 0550 cost= 0.003841298 W= 0.228542 b= 0.952913\n",
      "0.00373312\n",
      "Epoch: 0600 cost= 0.003733117 W= 0.229793 b= 0.943907\n",
      "0.00363279\n",
      "Epoch: 0650 cost= 0.003632791 W= 0.230971 b= 0.935437\n",
      "0.00353967\n",
      "Epoch: 0700 cost= 0.003539667 W= 0.232078 b= 0.92747\n",
      "0.00345319\n",
      "Epoch: 0750 cost= 0.003453186 W= 0.23312 b= 0.919978\n",
      "0.00337283\n",
      "Epoch: 0800 cost= 0.003372825 W= 0.234099 b= 0.91293\n",
      "0.00329811\n",
      "Epoch: 0850 cost= 0.003298106 W= 0.235021 b= 0.906302\n",
      "0.00322859\n",
      "Epoch: 0900 cost= 0.003228594 W= 0.235887 b= 0.900068\n",
      "0.00316388\n",
      "Epoch: 0950 cost= 0.003163884 W= 0.236702 b= 0.894204\n",
      "0.00310362\n",
      "Epoch: 1000 cost= 0.003103620 W= 0.237469 b= 0.888689\n",
      "Optimization Finished, spend time: 9.24333190918!\n",
      "Training cost= 0.0774802 w= 0.237469 b= 0.888689 \n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 12264 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f124c03db10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# compute\n",
    "# Start training\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    # Fit all training data\n",
    "    t_start = time.time()\n",
    "    values = {}\n",
    "    for epoch in range(training_epochs):\n",
    "        for (X, Y) in zip(train_X, train_Y):\n",
    "            values[x] = X\n",
    "            values[y] = Y\n",
    "            #compute the gradients\n",
    "            cost_np,_ = sess.run([cost,updates], feed_dict=values)\n",
    "            #print(w_np,b_np)\n",
    "            #print cost_np\n",
    "            #update the parameters\n",
    "            #values[w] -= learning_rate * grad_w_np\n",
    "            #values[b] -= learning_rate * grad_b_np\n",
    "\n",
    "        #Display logs per epoch step\n",
    "        #print (epoch+1) % display_step\n",
    "        #print values\n",
    "        if (epoch+1) % display_step == 0:\n",
    "            c = sess.run(cost, feed_dict=values)\n",
    "            print(c)\n",
    "            print \"Epoch:\", '%04d' % (epoch+1), \"cost=\", \"{:.9f}\".format(c), \\\n",
    "                \"W=\", sess.run(w), \"b=\", sess.run(b)\n",
    "\n",
    "    print \"Optimization Finished, spend time: {}!\".format(time.time()-t_start)\n",
    "    values[x] = train_X\n",
    "    values[y] = train_Y\n",
    "  \n",
    "    training_cost = sess.run(cost, feed_dict=values)\n",
    "    print \"Training cost=\", training_cost, \"w=\", sess.run(w,feed_dict=values), \"b=\", sess.run(b,feed_dict=values), '\\n'\n",
    "\n",
    "    #Graphic display\n",
    "    plt.plot(train_X, train_Y, 'ro', label='Original data')\n",
    "    plt.plot(train_X, sess.run(w,feed_dict=values) * train_X + sess.run(b,feed_dict=values), label='Fitted line')\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "# tf Graph Input\n",
    "X = tf.placeholder(\"float\")\n",
    "Y = tf.placeholder(\"float\")\n",
    "\n",
    "# Set model weights\n",
    "W = tf.Variable(rng.randn(), name=\"weight\")\n",
    "b = tf.Variable(rng.randn(), name=\"bias\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Construct a linear model\n",
    "pred = tf.add(tf.multiply(X, W), b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17\n"
     ]
    }
   ],
   "source": [
    "# Mean squared error\n",
    "cost = tf.reduce_sum(tf.pow(pred-Y, 2))/(2.*n_samples)\n",
    "print n_samples\n",
    "#cost = tf.reduce_sum(tf.pow(y_pred-y, 2))/(2*n_samples)\n",
    "# Gradient descent\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the variables (i.e. assign their default value)\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0050 cost= 0.079245120 W= 0.277222 b= 0.609393\n",
      "Epoch: 0100 cost= 0.078977935 W= 0.275593 b= 0.621037\n",
      "Epoch: 0150 cost= 0.078742325 W= 0.274062 b= 0.631974\n",
      "Epoch: 0200 cost= 0.078534484 W= 0.272625 b= 0.642249\n",
      "Epoch: 0250 cost= 0.078351215 W= 0.271274 b= 0.651901\n",
      "Epoch: 0300 cost= 0.078189649 W= 0.270006 b= 0.660967\n",
      "Epoch: 0350 cost= 0.078047112 W= 0.268814 b= 0.669485\n",
      "Epoch: 0400 cost= 0.077921450 W= 0.267694 b= 0.677487\n",
      "Epoch: 0450 cost= 0.077810638 W= 0.266642 b= 0.685003\n",
      "Epoch: 0500 cost= 0.077712923 W= 0.265654 b= 0.692063\n",
      "Epoch: 0550 cost= 0.077626802 W= 0.264726 b= 0.698696\n",
      "Epoch: 0600 cost= 0.077550873 W= 0.263854 b= 0.704927\n",
      "Epoch: 0650 cost= 0.077483937 W= 0.263035 b= 0.710781\n",
      "Epoch: 0700 cost= 0.077424921 W= 0.262266 b= 0.716279\n",
      "Epoch: 0750 cost= 0.077372931 W= 0.261543 b= 0.721446\n",
      "Epoch: 0800 cost= 0.077327058 W= 0.260864 b= 0.726299\n",
      "Epoch: 0850 cost= 0.077286676 W= 0.260226 b= 0.730858\n",
      "Epoch: 0900 cost= 0.077251084 W= 0.259627 b= 0.73514\n",
      "Epoch: 0950 cost= 0.077219732 W= 0.259064 b= 0.739163\n",
      "Epoch: 1000 cost= 0.077192098 W= 0.258535 b= 0.742941\n",
      "Optimization Finished, spend time: 8.76383304596!\n",
      "Training cost= 0.0771423 W= 0.258535 b= 0.742941 \n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 12392 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f129c313310>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Start training\n",
    "with tf.Session() as sess:\n",
    "    t_start = time.time()\n",
    "    sess.run(init)\n",
    "\n",
    "    # Fit all training data\n",
    "    for epoch in range(training_epochs):\n",
    "        for (x, y) in zip(train_X, train_Y):\n",
    "            #print x\n",
    "            sess.run(optimizer, feed_dict={X: x, Y: y})\n",
    "\n",
    "        #Display logs per epoch step\n",
    "        if (epoch+1) % display_step == 0:\n",
    "            c,_ = sess.run([cost,optimizer], feed_dict={X: train_X, Y:train_Y})\n",
    "            print \"Epoch:\", '%04d' % (epoch+1), \"cost=\", \"{:.9f}\".format(c), \\\n",
    "                \"W=\", sess.run(W), \"b=\", sess.run(b)\n",
    "\n",
    "    print \"Optimization Finished, spend time: {}!\".format(time.time()-t_start)\n",
    "    training_cost = sess.run(cost, feed_dict={X: train_X, Y: train_Y})\n",
    "    print \"Training cost=\", training_cost, \"W=\", sess.run(W), \"b=\", sess.run(b), '\\n'\n",
    "\n",
    "    #Graphic display\n",
    "    plt.plot(train_X, train_Y, 'ro', label='Original data')\n",
    "    plt.plot(train_X, sess.run(W) * train_X + sess.run(b), label='Fitted line')\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Regression result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
