<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import keras as ks\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\Han\\Anaconda3\\envs\\tf\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "42\n"
     ]
    }
   ],
   "source": [
    "x = tf.Variable(3, name='x')\n",
    "y = tf.Variable(4, name='y')\n",
    "f = x*x*y + y + 2\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(x.initializer)\n",
    "sess.run(y.initializer)\n",
    "result = sess.run(f)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42\n"
     ]
    }
   ],
   "source": [
    "x = tf.Variable(3, name='x')\n",
    "y = tf.Variable(4, name='y')\n",
    "f = x*x*y + y + 2\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    result = f.eval()\n",
    "    print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "\n",
    "m, n = housing.data.shape\n",
    "\n",
    "housing_data_plus_bias = np.c_[np.ones((m,1)), housing.data]\n",
    "\n",
    "X = tf.constant(housing_data_plus_bias, dtype=tf.float32, name='X')\n",
    "y = tf.constant(housing.target.reshape(-1,1), dtype=tf.float32, name='y')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-3.6957924e+01]\n",
      " [ 4.3679234e-01]\n",
      " [ 9.4300564e-03]\n",
      " [-1.0747863e-01]\n",
      " [ 6.4558423e-01]\n",
      " [-3.9862407e-06]\n",
      " [-3.7863622e-03]\n",
      " [-4.2142743e-01]\n",
      " [-4.3466288e-01]]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "XT = tf.transpose(X)\n",
    "\n",
    "theta = tf.matmul(tf.matmul(tf.matrix_inverse(tf.matmul(XT, X)), XT), y)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    theta_value = theta.eval()\n",
    "    print(theta_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "theta = tf.Variable(tf.random_uniform([n+1,1],-1.0,1.0), name='theta')\n",
    "y_pred = tf.matmul(X, theta, name='y_pred')\n",
    "error = y_pred -y\n",
    "mse = tf.reduce_mean(tf.square(error), name='mse')\n",
    "gradients = 2/m * tf.matmul(tf.transpose(X), error)\n",
    "training_op = tf.assign(theta, theta-learning_rate * gradients)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for epoch in range(n_epochs):\n",
    "        if epoch % 100 == 0:\n",
    "            print('Epoch', epoch, \"MSE =\", mse.eval())\n",
    "        sess.run(training_op)\n",
    "    best_theta = theta.eval()\n",
    "print(best_theta)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2<commit_msg>handson-ml chapter 9<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import keras as ks\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/han/anaconda3/envs/tf-gpu/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "42\n"
     ]
    }
   ],
   "source": [
    "x = tf.Variable(3, name='x')\n",
    "y = tf.Variable(4, name='y')\n",
    "f = x*x*y + y + 2\n",
    "\n",
    "sess = tf.Session()\n",
    "sess.run(x.initializer)\n",
    "sess.run(y.initializer)\n",
    "result = sess.run(f)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42\n"
     ]
    }
   ],
   "source": [
    "x = tf.Variable(3, name='x')\n",
    "y = tf.Variable(4, name='y')\n",
    "f = x*x*y + y + 2\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    result = f.eval()\n",
    "    print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "from sklearn.preprocessing import StandardScaler \n",
    "housing = fetch_california_housing()\n",
    "\n",
    "m, n = housing.data.shape\n",
    "\n",
    "std_scaler = StandardScaler()\n",
    "\n",
    "housing_data_scaled = std_scaler.fit_transform(housing.data)\n",
    "\n",
    "housing_data_plus_bias = np.c_[np.ones((m,1)), housing_data_scaled]\n",
    "\n",
    "X = tf.constant(housing_data_plus_bias, dtype=tf.float32, name='X')\n",
    "y = tf.constant(housing.target.reshape(-1,1), dtype=tf.float32, name='y')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 2.0685625 ]\n",
      " [ 0.82961977]\n",
      " [ 0.11875165]\n",
      " [-0.26552737]\n",
      " [ 0.30569723]\n",
      " [-0.00450288]\n",
      " [-0.03932627]\n",
      " [-0.8998867 ]\n",
      " [-0.87054163]]\n",
      "0.524321\n"
     ]
    }
   ],
   "source": [
    "\n",
    "XT = tf.transpose(X)\n",
    "\n",
    "theta = tf.matmul(tf.matmul(tf.matrix_inverse(tf.matmul(XT, X)), XT), y)\n",
    "y_pred = tf.matmul(X, theta, name='y_pred')\n",
    "error = y_pred -y\n",
    "mse = tf.reduce_mean(tf.square(error), name='mse')\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    theta_value = theta.eval()\n",
    "    print(theta_value)\n",
    "    print(mse.eval())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 MSE = 10.26011\n",
      "Epoch 100 MSE = 0.8208547\n",
      "Epoch 200 MSE = 0.6618923\n",
      "Epoch 300 MSE = 0.63001823\n",
      "Epoch 400 MSE = 0.60719544\n",
      "Epoch 500 MSE = 0.5896548\n",
      "Epoch 600 MSE = 0.576077\n",
      "Epoch 700 MSE = 0.56551117\n",
      "Epoch 800 MSE = 0.5572465\n",
      "Epoch 900 MSE = 0.5507492\n",
      "[[ 2.0685523 ]\n",
      " [ 0.9353447 ]\n",
      " [ 0.1652334 ]\n",
      " [-0.41541597]\n",
      " [ 0.40771294]\n",
      " [ 0.01103384]\n",
      " [-0.0455125 ]\n",
      " [-0.46176472]\n",
      " [-0.44231313]]\n"
     ]
    }
   ],
   "source": [
    "# Manually Computing the Gradients\n",
    "\n",
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "theta = tf.Variable(tf.random_uniform([n+1,1],-1.0,1.0), name='theta')\n",
    "y_pred = tf.matmul(X, theta, name='y_pred')\n",
    "error = y_pred -y\n",
    "mse = tf.reduce_mean(tf.square(error), name='mse')\n",
    "gradients = 2/m * tf.matmul(tf.transpose(X), error)\n",
    "training_op = tf.assign(theta, theta-learning_rate * gradients)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for epoch in range(n_epochs):\n",
    "        if epoch % 100 == 0:\n",
    "            print('Epoch', epoch, \"MSE =\", mse.eval())\n",
    "        sess.run(training_op)\n",
    "    best_theta = theta.eval()\n",
    "print(best_theta)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 MSE = 2.4434247\n",
      "Epoch 100 MSE = 0.33161554\n",
      "Epoch 200 MSE = 0.33421266\n",
      "Epoch 300 MSE = 0.3269612\n",
      "Epoch 400 MSE = 0.31481284\n",
      "Epoch 500 MSE = 0.30427787\n",
      "Epoch 600 MSE = 0.295708\n",
      "Epoch 700 MSE = 0.2887715\n",
      "Epoch 800 MSE = 0.2831533\n",
      "Epoch 900 MSE = 0.27859667\n"
     ]
    }
   ],
   "source": [
    "# Autodiff\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None,n+1), name='X')\n",
    "y = tf.placeholder(tf.float32, shape=(None, 1), name='y')\n",
    "theta = tf.Variable(tf.random_uniform([n+1,1],-1.0,1.0), name='theta')\n",
    "y_pred = tf.matmul(X, theta, name='y_pred')\n",
    "error = y_pred - y\n",
    "mse = tf.reduce_mean(tf.square(error), name='mse')\n",
    "gradients = 2 / m * tf.matmul(tf.transpose(X), error)\n",
    "training_op = tf.assign(theta, theta - learning_rate * gradients)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "batch_size = 1024\n",
    "n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "def fetch_batch(epoch, batch_index, batch_size):\n",
    "    a = batch_index * batch_size\n",
    "    b = min(batch_index * batch_size + batch_size, m)\n",
    "    return housing_data_plus_bias[a:b,:],housing.target.reshape(-1,1)[a:b,:]\n",
    "    \n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for epoch in range(n_epochs):\n",
    "        for batch_index in range(n_batches):\n",
    "            X_batch, y_batch = fetch_batch(epoch, batch_index, batch_size)\n",
    "            mse_value, _ = sess.run([mse,training_op], feed_dict={X:X_batch, y: y_batch})\n",
    "        if epoch % 100 == 0:\n",
    "            print('Epoch', epoch, \"MSE =\", mse_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 MSE = 3.037168\n",
      "Epoch 100 MSE = 0.25105837\n",
      "Epoch 200 MSE = 0.2581487\n"
     ]
    }
   ],
   "source": [
    "# Saving and Restoring Models\n",
    "\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "n_epochs = 201\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for epoch in range(n_epochs):\n",
    "        for batch_index in range(n_batches):\n",
    "            X_batch, y_batch = fetch_batch(epoch, batch_index, batch_size)\n",
    "            mse_value, _ = sess.run([mse,training_op], feed_dict={X:X_batch, y: y_batch})\n",
    "        if epoch % 100 == 0:\n",
    "            print('Epoch', epoch, \"MSE =\", mse_value)\n",
    "            saver.save(sess, '/tmp/my_model_final.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /tmp/my_model_final.ckpt\n",
      "Epoch 0 MSE = 0.26196417\n",
      "Epoch 100 MSE = 0.26132464\n",
      "Epoch 200 MSE = 0.26083633\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, '/tmp/my_model_final.ckpt')\n",
    "    for epoch in range(n_epochs):\n",
    "        for batch_index in range(n_batches):\n",
    "            X_batch, y_batch = fetch_batch(epoch, batch_index, batch_size)\n",
    "            mse_value, _ = sess.run([mse,training_op], feed_dict={X:X_batch, y: y_batch})\n",
    "        if epoch % 100 == 0:\n",
    "            print('Epoch', epoch, \"MSE =\", mse_value)\n",
    "            saver.save(sess, '/tmp/my_model_final.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 MSE = 1.8571377\n",
      "Epoch 100 MSE = 0.27233684\n",
      "Epoch 200 MSE = 0.27052534\n"
     ]
    }
   ],
   "source": [
    "# Visualizing the Graph\n",
    "\n",
    "from datetime import datetime\n",
    "\n",
    "now = datetime.utcnow().strftime('%Y%m%d%H%M%S')\n",
    "\n",
    "root_logdir = 'tf_logs'\n",
    "\n",
    "logdir = '{}/run-{}'.format(root_logdir, now)\n",
    "\n",
    "mse_summary = tf.summary.scalar('MSE', mse)\n",
    "\n",
    "file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for epoch in range(n_epochs):\n",
    "        for batch_index in range(n_batches):\n",
    "            X_batch, y_batch = fetch_batch(epoch, batch_index, batch_size)\n",
    "            mse_value, _ = sess.run([mse,training_op], feed_dict={X:X_batch, y: y_batch})\n",
    "            if batch_index % 10 == 0:\n",
    "                summary_str = mse_summary.eval(feed_dict={X:X_batch, y: y_batch})\n",
    "                file_writer.add_summary(summary_str, epoch * n_batches + batch_index)\n",
    "        if epoch % 100 == 0:\n",
    "            print('Epoch', epoch, \"MSE =\", mse_value)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
{'fraction_non_alphanumeric': 0.21689046866771985, 'fraction_numerical': 0.048249078462348605, 'mean_word_length': 2.276471856803968, 'pattern_counts': {'":': 165, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5263013', 'n_tokens_mistral': 6700, 'n_tokens_neox': 6162, 'n_words': 1018}