<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Datasets Test\n",
    "\n",
    "Test the various kind of data sets that come with TensorTools.\n",
    "\n",
    "*Remarks: The order of the image outputs can change between single executions!*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Force matplotlib to use inline rendering\n",
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "import sys\n",
    "\n",
    "# add path to libraries for ipython\n",
    "sys.path.append(os.path.expanduser(\"~/libs\"))\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import tensortools as tt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mnist_train = tt.datasets.mnist.MNISTTrainDataset()\n",
    "mnist_valid = tt.datasets.mnist.MNISTValidDataset()\n",
    "mnist_test = tt.datasets.mnist.MNISTTestDataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def display_mnist(dataset):\n",
    "    x, y = dataset.get_batch(1)\n",
    "    tt.visualization.display_array(x[0] * 255)\n",
    "    print('Label: {}'.format(y))\n",
    "    \n",
    "display_mnist(mnist_train)\n",
    "display_mnist(mnist_valid)\n",
    "display_mnist(mnist_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MovingMNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "moving_train = tt.datasets.moving_mnist.MovingMNISTTrainDataset(input_seq_length=5, target_seq_length=5)\n",
    "moving_valid = tt.datasets.moving_mnist.MovingMNISTValidDataset(input_seq_length=5, target_seq_length=5)\n",
    "moving_test = tt.datasets.moving_mnist.MovingMNISTTestDataset(input_seq_length=5, target_seq_length=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def display_moving(dataset, title):\n",
    "    x, y = dataset.get_batch(1)\n",
    "    full_seq = np.concatenate((x[0], y[0]))\n",
    "    tt.visualization.display_batch(full_seq, nrows=2, ncols=5, title=title)\n",
    "    \n",
    "display_moving(moving_train, 'Train')\n",
    "display_moving(moving_valid, 'Validation')\n",
    "display_moving(moving_test, 'Test')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },<commit_msg>Implemented datasets example for UCF11 queue<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Datasets Test\n",
    "\n",
    "Test the various kind of data sets that come with TensorTools.\n",
    "\n",
    "*Remarks: The order of the image outputs can change between single executions!*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Force matplotlib to use inline rendering\n",
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "import sys\n",
    "\n",
    "# add path to libraries for ipython\n",
    "sys.path.append(os.path.expanduser(\"~/libs\"))\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import tensortools as tt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mnist_train = tt.datasets.mnist.MNISTTrainDataset()\n",
    "mnist_valid = tt.datasets.mnist.MNISTValidDataset()\n",
    "mnist_test = tt.datasets.mnist.MNISTTestDataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def display_mnist(dataset):\n",
    "    x, y = dataset.get_batch(1)\n",
    "    tt.visualization.display_array(x[0] * 255)\n",
    "    print('Label: {}'.format(y))\n",
    "    \n",
    "display_mnist(mnist_train)\n",
    "display_mnist(mnist_valid)\n",
    "display_mnist(mnist_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MovingMNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "moving_train = tt.datasets.moving_mnist.MovingMNISTTrainDataset(input_seq_length=5, target_seq_length=5)\n",
    "moving_valid = tt.datasets.moving_mnist.MovingMNISTValidDataset(input_seq_length=5, target_seq_length=5)\n",
    "moving_test = tt.datasets.moving_mnist.MovingMNISTTestDataset(input_seq_length=5, target_seq_length=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def display_moving(dataset, title):\n",
    "    x, y = dataset.get_batch(1)\n",
    "    full_seq = np.concatenate((x[0], y[0]))\n",
    "    tt.visualization.display_batch(full_seq, nrows=2, ncols=5, title=title)\n",
    "    \n",
    "display_moving(moving_train, 'Train')\n",
    "display_moving(moving_valid, 'Validation')\n",
    "display_moving(moving_test, 'Test')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## UCF11\n",
    "\n",
    "*Remarks: This example uses an input queue.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ucf_train = tt.datasets.ucf11.UCF11TrainDataset(input_seq_length=3, target_seq_length=3,\n",
    "                                                image_size=(60, 80, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "batch_x, batch_y = ucf_train.get_batch(32)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    coord = tf.train.Coordinator()\n",
    "    threads = tf.train.start_queue_runners(sess=sess, coord=coord)\n",
    "    \n",
    "    try:\n",
    "        print('Starting queue runners...')\n",
    "        x, y = sess.run([batch_x, batch_y])       \n",
    "    except tf.errors.OutOfRangeError:\n",
    "        print(\"Done training -- epoch limit reached\")\n",
    "    finally:\n",
    "        # When done, ask the threads to stop\n",
    "        coord.request_stop()\n",
    "\n",
    "    coord.join(threads)\n",
    "    \n",
    "def display_ucf(x, y, title):\n",
    "    x = (x + 1) * 127.5\n",
    "    y = (y + 1) * 127.5\n",
    "    print(x.min())\n",
    "    print(x.max())\n",
    "    print(y.min())\n",
    "    print(y.max())\n",
    "    full_seq = np.concatenate((x[0], y[0]))\n",
    "    tt.visualization.display_batch(full_seq, nrows=2, ncols=3, title=title)\n",
    "\n",
    "display_ucf(x, y, 'Train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
{'fraction_non_alphanumeric': 0.1982027591444121, 'fraction_numerical': 0.008733071763067967, 'mean_word_length': 2.687354176388241, 'pattern_counts': {'":': 124, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4598164', 'n_tokens_mistral': 3130, 'n_tokens_neox': 3021, 'n_words': 498}