<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from __future__ import print_function\n",
    "import gc\n",
    "import os\n",
    "import random\n",
    "import sys\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import Image\n",
    "from scipy import ndimage\n",
    "\n",
    "import improc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reload (improc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# http://stackoverflow.com/questions/29772158/make-ipython-notebook-print-in-real-time\n",
    "oldsysstdout = sys.stdout\n",
    "class flushfile():\n",
    "    def __init__(self, f):\n",
    "        self.f = f\n",
    "    def __getattr__(self,name): \n",
    "        return object.__getattribute__(self.f, name)\n",
    "    def write(self, x):\n",
    "        self.f.write(x)\n",
    "        self.f.flush()\n",
    "    def flush(self):\n",
    "        self.f.flush()\n",
    "sys.stdout = flushfile(sys.stdout)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Image Processing\n",
    "Each image file contains a color image (top half), and an encoded depth image (bottom half)\n",
    "<img src=\"testing/IMG_2114.PNG\">\n",
    "* Note: The image may also contain the orientation data. If so it is encoded in the first two pixels of the depth image. If the first pixel of the depth image is red, the second has the x, y, z, w quaternion components encoded in the r,g,b,a values.\n",
    "\n",
    "The improc module contains functions for splitting the image, decoding the depth back into floating point millimeters, and for filling in gaps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "COLOR_CHANNELS = 3\n",
    "\n",
    "def prepare_image(image_path, strategy=np.mean):\n",
    "    combined_image = ndimage.imread(image_path).astype(np.float32)\n",
    "    color_image, depth_image = improc.split(combined_image)\n",
    "    depths, attitude = improc.decode_depth(depth_image)\n",
    "    color_image = color_image[:,:,0:COLOR_CHANNELS] / improc.BYTE_MAX # Discard alpha and normalize\n",
    "    cleaned = improc.mipmap_imputer(depths, strategy) / improc.MAX_DEPTH # fill holes and normalize\n",
    "    return (color_image, depths, attitude, cleaned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "prepared_image = prepare_image(\"testing/IMG_2114.PNG\")\n",
    "plt.imshow(prepared_image[0])\n",
    "print(prepared_image[0].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.imshow(prepared_image[1])\n",
    "print(prepared_image[1].shape)\n",
    "print(prepared_image[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.imshow(prepared_image[3])\n",
    "print(prepared_image[3].shape)\n",
    "print(prepared_image[3].dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "image_height = 480\n",
    "image_width = 640\n",
    "\n",
    "def prepare_images(paths, height=image_height, width=image_width, channels=COLOR_CHANNELS):\n",
    "    inputs = np.empty(shape=(len(paths), height, width, channels), dtype=np.float32)\n",
    "    targets = np.empty(shape=(len(paths), height * width), dtype=np.float32)\n",
    "    for i, path in enumerate(paths):\n",
    "        example = prepare_image(path)\n",
    "        inputs[i] = example[0]\n",
    "        targets[i] = example[3].reshape((height * width))\n",
    "    return (inputs, targets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Enumerate Images\n",
    "Image names are sequential, so add every tenth image to the validation set based on filename."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "training = []\n",
    "test = []\n",
    "\n",<commit_msg>Reorder redux.<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from __future__ import print_function\n",
    "import gc\n",
    "import os\n",
    "import random\n",
    "import sys\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython.display import Image\n",
    "from scipy import ndimage\n",
    "\n",
    "import improc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reload (improc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# http://stackoverflow.com/questions/29772158/make-ipython-notebook-print-in-real-time\n",
    "oldsysstdout = sys.stdout\n",
    "class flushfile():\n",
    "    def __init__(self, f):\n",
    "        self.f = f\n",
    "    def __getattr__(self,name): \n",
    "        return object.__getattribute__(self.f, name)\n",
    "    def write(self, x):\n",
    "        self.f.write(x)\n",
    "        self.f.flush()\n",
    "    def flush(self):\n",
    "        self.f.flush()\n",
    "sys.stdout = flushfile(sys.stdout)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Image Processing\n",
    "Each image file contains a color image (top half), and an encoded depth image (bottom half)\n",
    "<img src=\"testing/IMG_2114.PNG\">\n",
    "* Note: The image may also contain the orientation data. If so it is encoded in the first two pixels of the depth image. If the first pixel of the depth image is red, the second has the x, y, z, w quaternion components encoded in the r,g,b,a values.\n",
    "\n",
    "The improc module contains functions for splitting the image, decoding the depth back into floating point millimeters, and for filling in gaps."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "COLOR_CHANNELS = 3\n",
    "\n",
    "def prepare_image(image_path, strategy=np.mean):\n",
    "    combined_image = ndimage.imread(image_path).astype(np.float32)\n",
    "    color_image, depth_image = improc.split(combined_image)\n",
    "    depths, attitude = improc.decode_depth(depth_image)\n",
    "    color_image = color_image[:,:,0:COLOR_CHANNELS] / improc.BYTE_MAX # Discard alpha and normalize\n",
    "    cleaned = improc.mipmap_imputer(depths, strategy) / improc.MAX_DEPTH # fill holes and normalize\n",
    "    return (color_image, depths, attitude, cleaned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "image_height = 480\n",
    "image_width = 640\n",
    "\n",
    "def prepare_images(paths, height=image_height, width=image_width, channels=COLOR_CHANNELS):\n",
    "    inputs = np.empty(shape=(len(paths), height, width, channels), dtype=np.float32)\n",
    "    targets = np.empty(shape=(len(paths), height * width), dtype=np.float32)\n",
    "    for i, path in enumerate(paths):\n",
    "        example = prepare_image(path)\n",
    "        inputs[i] = example[0]\n",
    "        targets[i] = example[3].reshape((height * width))\n",
    "    return (inputs, targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "prepared_image = prepare_image(\"testing/IMG_2114.PNG\")\n",
    "plt.imshow(prepared_image[0])\n",
    "print(prepared_image[0].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.imshow(prepared_image[1])\n",
    "print(prepared_image[1].shape)\n",
    "print(prepared_image[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.imshow(prepared_image[3])\n",
    "print(prepared_image[3].shape)\n",
    "print(prepared_image[3].dtype)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Enumerate Images\n",
    "Image names are sequential, so add every tenth image to the validation set based on filename."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "training = []\n",
    "test = []\n",
    "\n",
{'fraction_non_alphanumeric': 0.18188582970128628, 'fraction_numerical': 0.008716912937174444, 'mean_word_length': 2.770741482965932, 'pattern_counts': {'":': 122, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6760999', 'n_tokens_mistral': 3450, 'n_tokens_neox': 3361, 'n_words': 700}