<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Flickr30k to Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#'./data/Flickr30k'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#import tensorflow as tf\n",
    "import tensorflow.contrib.keras as keras\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib.keras.api.keras.applications.inception_v3 import decode_predictions\n",
    "from tensorflow.contrib.keras.api.keras.preprocessing import image as keras_preprocessing_image\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def image_to_input(model, preprocess_input_fn, img_path):\n",
    "    target_size=model.input_shape[1:]\n",
    "    img = keras_preprocessing_image.load_img(img_path, target_size=target_size)\n",
    "    \n",
    "    x = keras.preprocessing.image.img_to_array(img)\n",
    "    x = np.expand_dims(x, axis=0)\n",
    "    x = preprocess_input_fn(x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib.keras.api.keras.applications.inception_v3 import InceptionV3, preprocess_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = InceptionV3(weights='imagenet', include_top=False, pooling='avg')\n",
    "print(\"InceptionV3 loaded\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plan \n",
    "\n",
    "*  Have a look inside the captions ```flickr30k.tar.gz```\n",
    "*  Extract contents of ```flickr30k.tar.gz``` to ```dict( photo_id -> [capitions] )```\n",
    "*  Filter out a subset of those ```photo_id``` to convert\n",
    "*  Run InceptionV3 over the list\n",
    "*  Save off features to an easy-to-load filetype\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "#    weight_count=[ float(np.sum([keras.backend.count_params(p) for p in set(w)]))/1000./1000. \n",
    "#                   for w in [m.trainable_weights, m.non_trainable_weights] ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [<commit_msg>Predict on a batch of images<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Flickr30k to Features\n",
    "\n",
    "*   <NAME>, <NAME>, <NAME>, and <NAME>. _From image description to visual denotations: New similarity metrics for semantic inference over event descriptions._ Transactions of the Association for Computational Linguistics (to appear).\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "#import tensorflow as tf\n",
    "import tensorflow.contrib.keras as keras\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_path = './data/Flickr30k'\n",
    "image_path = os.path.join(data_path, 'flickr30k-images')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib.keras.api.keras.applications.inception_v3 import decode_predictions\n",
    "from tensorflow.contrib.keras.api.keras.preprocessing import image as keras_preprocessing_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib.keras.api.keras.applications.inception_v3 import InceptionV3, preprocess_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model = InceptionV3(weights='imagenet', include_top=False, pooling='avg')\n",
    "print(\"InceptionV3 loaded\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plan \n",
    "\n",
    "*  Have a look inside the captions ```flickr30k.tar.gz``` : includes ```results_20130124.token```\n",
    "*  Extract contents of ```flickr30k.tar.gz``` to ```dict( photo_id -> [capitions] )```\n",
    "*  Filter out a subset of those ```photo_id``` to convert\n",
    "*  Run InceptionV3 over the list\n",
    "*  Save off features to an easy-to-load filetype\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#from datasets import dataset_utils\n",
    "\n",
    "img_to_captions=dict()\n",
    "\n",
    "tarfilepath = os.path.join(data_path, 'flickr30k.tar.gz')\n",
    "if os.path.isfile(tarfilepath):\n",
    "    import tarfile\n",
    "    with tarfile.open(tarfilepath, 'r:gz').extractfile('results_20130124.token') as tokenized:\n",
    "        n_captions = 0\n",
    "        for l in tokenized.readlines():\n",
    "            #print(l)  # This is bytes\n",
    "            img_num, caption = l.decode(\"utf-8\").strip().split(\"\\t\")\n",
    "            img, num = img_num.split(\"#\")\n",
    "            #print(img, caption); break\n",
    "            if img not in img_to_captions:  img_to_captions[img]=[]\n",
    "            img_to_captions[img].append(caption)\n",
    "            n_captions += 1\n",
    "            \n",
    "print(\"Found %d images, with a total of %d captions\" % (len(img_to_captions),n_captions, ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Filter for the images that we care about\n",
    "img_to_captions = { img:captions for img, captions in img_to_captions.items() if img=='3947306345.jpg' }\n",
    "img_to_captions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "img_arr = sorted(img_to_captions.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Create a generator for preprocessed images\n",
    "def preprocesed_image_gen():\n",
    "    #target_size=model.input_shape[1:]\n",
    "    target_size=(299, 299, 3)\n",
    "    print(\"target_size\", target_size)\n",
    "    for img_name in img_arr:\n",
    "        print(\"img_name\", img_name)\n",
    "        img_path = os.path.join(image_path, img_name)\n",
    "        img = keras_preprocessing_image.load_img(img_path, target_size=target_size)\n",
    "        yield keras.preprocessing.image.img_to_array(img)\n",
    "        #x = np.expand_dims(x, axis=0)  # This is to make a single image \n",
    "\n",
    "def image_batch(batchsize=16):\n",
    "    preprocesed_image_generator = preprocesed_image_gen()\n",
    "    try:\n",
    "        while True:\n",
    "            arr = []\n",
    "            for _ in range(batchsize):\n",
    "                img = next(preprocesed_image_generator)\n",
    "                print(img.shape)\n",
    "                arr.append(img)\n",
    "            yield preprocess_input( np.stack( arr, axis=0 ) )\n",
    "    except StopIteration as e:  # Last set\n",
    "        print(\"Last batch of %d elements\" % ( len(arr),))\n",
    "        #s = np.stack( arr, axis=0 )\n",
    "        #p = preprocess_input( s )\n",
    "        #print(\"s,p\", s.shape, p.shape)\n",
    "    finally:\n",
    "        yield preprocess_input( np.stack( arr, axis=0 ) )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#preprocesed_image_generator = preprocesed_image_gen()\n",
    "#next(preprocesed_image_generator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "image_batcher = image_batch()\n",
    "batch = next(image_batcher)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#features = model.predict_generator(image_batcher, steps=len(img_arr))  #, verbose=1\n",
    "features = model.predict_on_batch(batch)\n",
    "features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "#    weight_count=[ float(np.sum([keras.backend.count_params(p) for p in set(w)]))/1000./1000. \n",
    "#                   for w in [m.trainable_weights, m.non_trainable_weights] ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
{'fraction_non_alphanumeric': 0.19295558958652373, 'fraction_numerical': 0.008996937212863706, 'mean_word_length': 2.5710868079289133, 'pattern_counts': {'":': 182, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2650144', 'n_tokens_mistral': 3969, 'n_tokens_neox': 3861, 'n_words': 702}