<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TensorFlow Model !\n",
    "import tensorflow as tf\n",
    "from cell import ConvLSTMCell\n",
    "\n",
    "# Run when your in trouble ... !\n",
    "tf.reset_default_graph()\n",
    "\n",
    "class conv_lstm_model():\n",
    "    def __init__(self):\n",
    "        \"\"\"Parameter initialization\"\"\"\n",
    "        self.batch_size = 128\n",
    "        self.timesteps = 32\n",
    "        self.shape = [64, 64] # Image shape\n",
    "        self.kernel = [3, 3]\n",
    "        self.channels = 3\n",
    "        self.filters = [32,128,32,3] # 4 stacked conv lstm filters\n",
    "        \n",
    "        # Create a placeholder for videos.\n",
    "        self.inputs = tf.placeholder(tf.float32, [batch_size, timesteps] + shape + [channels]) # (batch_size, timestep, H, W, C)\n",
    "        self.outputs_exp = tf.placeholder(tf.float32, [batch_size, timesteps] + shape + [channels] ) # (batch_size, timestep, H, W, C)\n",
    "        \n",
    "        # model output\n",
    "        self.model_output = None\n",
    "        \n",
    "        # loss\n",
    "        self.l2_loss = None\n",
    "        \n",
    "        # optimizer\n",
    "        self.optimizer = None\n",
    "        \n",
    "    def create_model(self):\n",
    "        cells = []\n",
    "        for i, each_filter in enumerate(self.filters):\n",
    "            cell = ConvLSTMCell(self.shape, each_filter, self.kernel)\n",
    "            cells.append(cell)\n",
    "            \n",
    "        cell = tf.nn.rnn_cell.MultiRNNCell(cells, state_is_tuple=True)        \n",
    "        states_series, current_state = tf.nn.dynamic_rnn(cell, self.inputs, dtype=self.inputs.dtype)\n",
    "        # current_state => Not used ... \n",
    "        self.model_output = states_series\n",
    "    \n",
    "    def loss(self):\n",
    "        frames_difference = tf.subtract(self.outputs_exp, self.model_output)\n",
    "        batch_l2_loss = tf.nn.l2_loss(frames_difference)\n",
    "        # divide by batch size ... \n",
    "        l2_loss = tf.divide(batch_l2_loss, float(batch_size))\n",
    "        self.l2_loss = l2_loss\n",
    "    \n",
    "    def optimize(self):\n",
    "        train_step = tf.train.AdamOptimizer().minimize(self.l2_loss)\n",
    "        self.optimizer = train_step\n",
    "        \n",
    "model = conv_lstm_model()\n",
    "model.create_model()\n",
    "model.loss()\n",
    "model.optimize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LSTMStateTuple(c=TensorShape([Dimension(64), Dimension(64), Dimension(12)]), h=TensorShape([Dimension(64), Dimension(64), Dimension(12)]))"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2<commit_msg>TensorBoard and CheckPoint<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class read_data():\n",
    "    def __init__(self,data_directory, batch_size):\n",
    "        self.data_directory = data_directory\n",
    "        self.batch_size = batch_size\n",
    "        \n",
    "    def next_batch(self):\n",
    "        return np.random.rand(4,32,64,64,3), np.random.rand(4,32,64,64,3)\n",
    "        # return X, y as per batch size ... \n",
    "        # infinite batch generation\n",
    "    \n",
    "    def val_batch_init(self):\n",
    "        pass\n",
    "        # reset validation iterator\n",
    "    \n",
    "    def val_next_batch(self):\n",
    "        return np.random.rand(4,32,64,64,3), np.random.rand(4,32,64,64,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TensorFlow Model !\n",
    "import os\n",
    "import shutil\n",
    "import tensorflow as tf\n",
    "from cell import ConvLSTMCell\n",
    "\n",
    "class conv_lstm_model():\n",
    "    def __init__(self):\n",
    "        # Run when your in trouble ... !\n",
    "        tf.reset_default_graph()\n",
    "\n",
    "        \"\"\"Parameter initialization\"\"\"\n",
    "        self.batch_size = 4 #128\n",
    "        self.timesteps = 32\n",
    "        self.shape = [64, 64] # Image shape\n",
    "        self.kernel = [3, 3]\n",
    "        self.channels = 3\n",
    "        self.filters = [32,128,32,3] # 4 stacked conv lstm filters\n",
    "        \n",
    "        # Create a placeholder for videos.\n",
    "        self.inputs = tf.placeholder(tf.float32, [self.batch_size, self.timesteps] + self.shape + [self.channels]) # (batch_size, timestep, H, W, C)\n",
    "        self.outputs_exp = tf.placeholder(tf.float32, [self.batch_size, self.timesteps] + self.shape + [self.channels] ) # (batch_size, timestep, H, W, C)\n",
    "        \n",
    "        # model output\n",
    "        self.model_output = None\n",
    "        \n",
    "        # loss\n",
    "        self.l2_loss = None\n",
    "        \n",
    "        # optimizer\n",
    "        self.optimizer = None\n",
    "        \n",
    "    def create_model(self):\n",
    "        cells = []\n",
    "        for i, each_filter in enumerate(self.filters):\n",
    "            cell = ConvLSTMCell(self.shape, each_filter, self.kernel)\n",
    "            cells.append(cell)\n",
    "            \n",
    "        cell = tf.nn.rnn_cell.MultiRNNCell(cells, state_is_tuple=True)        \n",
    "        states_series, current_state = tf.nn.dynamic_rnn(cell, self.inputs, dtype=self.inputs.dtype)\n",
    "        # current_state => Not used ... \n",
    "        self.model_output = states_series\n",
    "    \n",
    "    def loss(self):\n",
    "        frames_difference = tf.subtract(self.outputs_exp, self.model_output)\n",
    "        batch_l2_loss = tf.nn.l2_loss(frames_difference)\n",
    "        # divide by batch size ... \n",
    "        l2_loss = tf.divide(batch_l2_loss, float(self.batch_size))\n",
    "        self.l2_loss = l2_loss\n",
    "    \n",
    "    def optimize(self):\n",
    "        train_step = tf.train.AdamOptimizer().minimize(self.l2_loss)\n",
    "        self.optimizer = train_step\n",
    "        \n",
    "    def build_model(self):\n",
    "        self.create_model()\n",
    "        self.loss()\n",
    "        self.optimize()\n",
    "\n",
    "\n",
    "log_dir_file_path = \"../logs/\" \n",
    "model_save_file_path = \"../checkpoint/\"\n",
    "checkpoint_iterations = 100\n",
    "best_model_iterations = 25\n",
    "best_l2_loss = float(\"inf\")\n",
    "iterations=\"iterations/\"\n",
    "best = \"best/\"\n",
    "\n",
    "def log_directory_creation():\n",
    "    if tf.gfile.Exists(log_dir_file_path):\n",
    "        tf.gfile.DeleteRecursively(log_dir_file_path)\n",
    "    tf.gfile.MakeDirs(log_dir_file_path)\n",
    "    \n",
    "    # model save directory\n",
    "    if os.path.exists(model_save_file_path):\n",
    "        shutil.rmtree(model_save_file_path)\n",
    "    os.makedirs(model_save_file_path+iterations)\n",
    "    os.makedirs(model_save_file_path+best)\n",
    "    \n",
    "def save_model_session(sess,file_name):\n",
    "    saver = tf.train.Saver()\n",
    "    save_path = saver.save(sess, model_save_file_path+file_name+\".ckpt\")\n",
    "    \n",
    "def restore_model_session(sess,file_name):\n",
    "    saver = tf.train.Saver()\n",
    "    saver.restore(sess, model_save_file_path+file_name+\".ckpt\")\n",
    "    \n",
    "def train():\n",
    "    global best_l2_loss\n",
    "    # clear logs !\n",
    "    log_directory_creation()\n",
    "    \n",
    "    # data read iterator\n",
    "    data = read_data(\"../data/UCF-101/\",128)\n",
    "    # conv lstm model\n",
    "    model = conv_lstm_model()\n",
    "    model.build_model()\n",
    "    \n",
    "    # Initialize the variables (i.e. assign their default value)\n",
    "    init = tf.global_variables_initializer()\n",
    "    \n",
    "    # Start training\n",
    "    sess =  tf.InteractiveSession()\n",
    "    sess.run(init)\n",
    "    \n",
    "    # Tensorflow Summary\n",
    "    tf.summary.scalar(\"train_l2_loss\",model.l2_loss)\n",
    "    summary_merged = tf.summary.merge_all()\n",
    "    train_writer = tf.summary.FileWriter(log_dir_file_path+\"/train\", sess.graph)\n",
    "    test_writer = tf.summary.FileWriter(log_dir_file_path+\"/test\", sess.graph)\n",
    "\n",
    "    global_step=0\n",
    "    while True:\n",
    "        X_batch, y_batch = data.next_batch()\n",
    "        _, summary = sess.run([model.optimizer, summary_merged], feed_dict={model.inputs: X_batch, model.outputs_exp: y_batch})\n",
    "        train_writer.add_summary(summary,global_step)\n",
    "        global_step += 1\n",
    "        \n",
    "        if global_step%checkpoint_iterations==0:\n",
    "            save_model_session(sess,iterations+\"conv_lstm_model\")\n",
    "        \n",
    "        if global_step%best_model_iterations==0:\n",
    "            data.val_batch_init()\n",
    "            \n",
    "            val_l2_loss_history = list()\n",
    "            # iterate on validation batch ...\n",
    "            # for X_val, y_val in data.val_next_batch():\n",
    "            X_val, y_val = data.val_next_batch()\n",
    "            test_summary, val_l2_loss = sess.run([summary_merged, model.l2_loss], feed_dict={model.inputs: X_val, model.outputs_exp: y_val})\n",
    "            test_writer.add_summary(test_summary,global_step)\n",
    "            val_l2_loss_history.append(val_l2_loss)\n",
    "            temp_loss = sum(val_l2_loss_history) * 1.0 /len(val_l2_loss_history)\n",
    "            \n",
    "            # save if better !\n",
    "            if best_l2_loss > temp_loss:\n",
    "                best_l2_loss = temp_loss \n",
    "                save_model_session(sess,best+\"conv_lstm_model\")\n",
    "        \n",
    "        if global_step%100==0:\n",
    "            print (\"Iteration \",global_step, \" best_l2_loss \", best_l2_loss)\n",
    "        \n",
    "    train_writer.close()\n",
    "    test_writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "128"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
{'fraction_non_alphanumeric': 0.18575261836752358, 'fraction_numerical': 0.014628235090452696, 'mean_word_length': 1.983987603305785, 'pattern_counts': {'":': 75, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6516669', 'n_tokens_mistral': 4404, 'n_tokens_neox': 4269, 'n_words': 647}