<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import PIL\n",
    "import glob\n",
    "import rarfile\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import namedtuple\n",
    "from itertools import product\n",
    "\n",
    "offset = namedtuple('offset', ['width', 'height',\n",
    "                               'convertMode',\n",
    "                               'xmin', 'xmax', 'xoffsets',  \n",
    "                               'ymin', 'ymax', 'yoffsets' ])\n",
    "offsets = [\n",
    "    # 1989\n",
    "    offset(600, 406, 'LA', 1, 125, [20,165, 370, 468], 1, 125, [142,278]), # Sunday 1989\n",
    "    offset(640, 189, 'LA', 2, 185, [10,230,445], 2, 185, [0]),    # Weekday 1989\n",
    "    \n",
    "    offset(640, 186, 'LA', 2, 185, [10,230,445], 2, 185, [0]),    # Weekday 1990\n",
    "    offset(640, 196, 'LA', 5, 197, [0,218,436], 3, 195, [0]),      # Weekday 1990\n",
    "    \n",
    "    offset(640, 199, 'L', 5, 199, [0,218,436], 3, 197, [0]), # Weekday 2012\n",
    "    offset(640, 288, 'L', 9, 145, [0, 160, 322, 486], 1,137, [0, 149]), # Weekend 2012\n",
    "    \n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pattern = '/Users/ajmendez/data/dilbert/raw/*.rar'\n",
    "sizes = []\n",
    "for filename in sorted(glob.iglob(pattern)):\n",
    "    if any([x in filename for x in ['1989']]):\n",
    "        continue\n",
    "        \n",
    "    print(filename)\n",
    "    with rarfile.RarFile(filename) as rf:\n",
    "        files = sorted(rf.namelist())\n",
    "        files = [f for f in files\n",
    "                 if not (f.replace('.jpg','-colour.jpg') in files)]\n",
    "        for f in files:\n",
    "            img = PIL.Image.open(rf.open(f))\n",
    "            sizes.append([img.width, img.height])\n",
    "            \n",
    "            hasoffset = any([(np.abs(img.width-off.width) < 2) &\n",
    "                             (np.abs(img.height-off.height) < 2) \n",
    "                             for off in offsets])\n",
    "            if hasoffset:\n",
    "                continue\n",
    "            break\n",
    "    print('Done')\n",
    "    break\n",
    "\n",
    "print(img.width, img.height)\n",
    "plt.imshow(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def convert(image, off):\n",
    "    plt.figure(figsize=(12,12))\n",
    "    plt.imshow(image.convert('LA'))\n",
    "    print(off.xmax-off.xmin)\n",
    "    print(off.ymax-off.ymin)\n",
    "    for xoffset in off.xoffsets:\n",
    "        plt.axvline(xoffset+off.xmin, color='r', lw=2)\n",
    "        plt.axvline(xoffset+off.xmax, color='r', lw=2)\n",
    "    for yoffset in off.yoffsets:\n",
    "        plt.axhline(yoffset+off.ymin, color='r', lw=2)\n",
    "        plt.axhline(yoffset+off.ymax, color='r', lw=2)\n",
    "#     for j,(yoffset, xoffset) in enumerate(product(offset.yoffsets,offset.xoffsets)):\n",
    "#         plt.axvline(xoffset, color='r')\n",
    "#         plt.axhline(yoffset, color='r')\n",
    "        \n",
    "\n",
    "off = offset(640, 196, 'LA', 5, 197, [0,218,436], 3, 195, [0])\n",
    "\n",
    "convert(img, off)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "NX=NY=128\n",
    "def carveImages(image, off, outpattern=None, convertMode=None):\n",
    "    if convertMode is None:\n",
    "        convertMode = off.convertMode\n",
    "    for k,(yoffset, xoffset) in enumerate(product(off.yoffsets, off.xoffsets)):\n",
    "        img = image.crop((xoffset + off.xmin, \n",
    "                          yoffset + off.ymin, \n",
    "                          xoffset + off.xmax - off.xmin, \n",
    "                          yoffset + off.ymax - off.ymin))\n",
    "        img.thumbnail((NX,NY), PIL.Image.ANTIALIAS)\n",
    "        if outpattern:\n",
    "            img.convert(convertMode).save(outpattern.format(j=j))\n",
    "        else:\n",
    "            yield img.convert(convertMode)\n",
    "\n",
    "def plotImages(image, off):\n",
    "    nx,ny = len(off.xoffsets), len(off.yoffsets)\n",
    "    fig, axes = plt.subplots(ny, nx, figsize=(nx*3, \n",
    "                                      ny*3))\n",
    "    \n",
    "    for (img,ax) in zip(carveImages(image, off), axes.flatten()):\n",
    "        plt.sca(ax)\n",
    "        plt.imshow(img)\n",
    "\n",
    "plotImages(img, off)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "    for xoffset in \n",
    "    xmin,xmax = [9,145]\n",
    "    ymin,ymax = [1,137]\n",
    "    xoffsets = [0, 160, 322, 486]\n",
    "    yoffsets = [0, 149]\n",
    "    # Remember people read left to right -- top to bottom\n",
    "    for j,(yoffset, xoffset) in enumerate(product(yoffsets, xoffsets)):\n",
    "        img = image.crop((xoffset + xmin, \n",
    "                          yoffset + ymin, \n",
    "                          xoffset + xmax - xmin, \n",
    "                          yoffset + ymax - ymin))\n",
    "        img.thumbnail((NX,NY), PIL.Image.ANTIALIAS)\n",
    "        img.convert('L').save(outpattern.format(j=j))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}<commit_msg>update full archive<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import PIL.Image\n",
    "import glob\n",
    "import rarfile\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import namedtuple\n",
    "from itertools import product\n",
    "\n",
    "offset = namedtuple('offset', ['width', 'height',\n",
    "                               'convertMode',\n",
    "                               'xmin', 'xmax', 'xoffsets',  \n",
    "                               'ymin', 'ymax', 'yoffsets' ])\n",
    "offsets = [\n",
    "    # 1989\n",
    "    offset(600, 406, 'LA', 1, 125, [20,165, 370, 468], 1, 125, [142,278]), # Sunday 1989\n",
    "    offset(640, 189, 'LA', 2, 185, [10,230,445], 2, 185, [0]),    # Weekday 1989\n",
    "    \n",
    "    offset(640, 186, 'LA', 2, 185, [10,230,445], 2, 185, [0]),    # Weekday 1990\n",
    "    offset(640, 196, 'LA', 5, 197, [0,218,436], 3, 195, [0]),      # Weekday 1990\n",
    "    offset(600, 271, 'L', 9, 125, [0, 160, 322, 466], 4, 120, [8, 140]), # weekend 1990\n",
    "    offset(600, 189, 'L', 5, 182, [7,208,406],  5, 182, [0]),\n",
    "    offset(600, 191, 'L', 5, 182, [7,208,406], 7, 184, [0]),\n",
    "    offset(600, 273, 'L', 9, 125, [10, 150, 310, 446], 4, 120, [8, 140]),\n",
    "    \n",
    "    # 1991\n",
    "    offset(640, 304, 'L', 9, 145, [0, 170, 332, 489], 1,137, [10, 162]),\n",
    "    offset(640, 284, 'L', 9, 140, [0, 170, 322, 489], 1,132, [2, 148]),\n",
    "    offset(640, 279, 'L', 9, 140, [0, 170, 322, 489], 1,132, [2, 148]),\n",
    "    \n",
    "    #1992\n",
    "    offset(640, 274, 'L', 9, 140, [0, 170, 322, 489], 1,132, [2, 148]),\n",
    "    offset(600, 196, 'L', 5, 182, [7,208,406], 7, 184, [0]),\n",
    "    \n",
    "    #1993\n",
    "    offset(600, 403, 'L', 1, 125, [20,165, 370, 468], 1, 125, [142,278]), # Sunday 1989\n",
    "    offset(600, 400, 'L', 1, 125, [20,165, 370, 468], 1, 125, [142,278]), # Sunday 1989\n",
    "    offset(600, 264, 'L', 9, 125, [0, 160, 322, 466], 4, 120, [8, 140]), # weekend 1990\n",
    "    \n",
    "    #1994\n",
    "    offset(780, 281, 'L', 5, 225, [10, 270, 542],5,225, [10]),\n",
    "    offset(780, 360, 'L', 10, 160, [10, 205, 400, 590], 10, 160, [5, 180]), # Weekend\n",
    "    offset(780, 292, 'L', 5, 230, [10, 280, 542],5,230, [15]),\n",
    "    offset(600, 222, 'L', 9, 180, [10, 205, 405], 9, 180, [5]),\n",
    "    offset(780, 287, 'L', 5, 225, [10, 270, 532], 10,225, [10]),\n",
    "    \n",
    "    offset(640, 199, 'L', 5, 199, [0,218,436], 3, 197, [0]), # Weekday 2012\n",
    "    offset(640, 288, 'L', 9, 145, [0, 160, 322, 486], 1,137, [0, 149]), # Weekend 2012\n",
    "    \n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pattern = '/Users/ajmendez/Dropbox/tmp/dilbert/raw/*.rar'\n",
    "img = None\n",
    "sizes = []\n",
    "years = ['1989', '1990', '1991', '1992', '1993']\n",
    "for filename in sorted(glob.iglob(pattern)):\n",
    "    if any([x in filename for x in years]):\n",
    "        continue\n",
    "        \n",
    "    print(filename)\n",
    "    with rarfile.RarFile(filename) as rf:\n",
    "        files = sorted(rf.namelist())\n",
    "        print len(files)\n",
    "        goodfiles = [f for f in files\n",
    "                     if ('.gif' in f) or (not (f.replace('.jpg','-colour.jpg') in files) )]\n",
    "        for f in goodfiles:\n",
    "            img = PIL.Image.open(rf.open(f))\n",
    "            sizes.append([img.width, img.height])\n",
    "            \n",
    "            hasoffset = any([(np.abs(img.width-off.width) <= 3) &\n",
    "                             (np.abs(img.height-off.height) <= 3) \n",
    "                             for off in offsets])\n",
    "            if hasoffset:\n",
    "                continue\n",
    "            else:\n",
    "                print(img.width, img.height)\n",
    "                plt.imshow(np.array(img.convert('L')), cmap=plt.cm.gray)\n",
    "                break\n",
    "    break\n",
    "print len(files), len(goodfiles), len(sizes)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def convert(image, off):\n",
    "    plt.figure(figsize=(12,12))\n",
    "    plt.imshow(np.array(image.convert(off.convertMode)),\n",
    "               cmap=plt.cm.gray)\n",
    "    print(off.xmax-off.xmin)\n",
    "    print(off.ymax-off.ymin)\n",
    "    for xoffset in off.xoffsets:\n",
    "        plt.axvline(xoffset+off.xmin, color='r', lw=2)\n",
    "        plt.axvline(xoffset+off.xmax, color='r', lw=2)\n",
    "    for yoffset in off.yoffsets:\n",
    "        plt.axhline(yoffset+off.ymin, color='r', lw=2)\n",
    "        plt.axhline(yoffset+off.ymax, color='r', lw=2)\n",
    "#     for j,(yoffset, xoffset) in enumerate(product(offset.yoffsets,offset.xoffsets)):\n",
    "#         plt.axvline(xoffset, color='r')\n",
    "#         plt.axhline(yoffset, color='r')\n",
    "        \n",
    "\n",
    "off = offset(780, 287, 'L', 5, 225, [10, 270, 532], 10,225, [10])\n",
    "\n",
    "\n",
    "convert(img, off)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "NX=NY=128\n",
    "def carveImages(image, off, outpattern=None, convertMode=None):\n",
    "    if convertMode is None:\n",
    "        convertMode = off.convertMode\n",
    "    for k,(yoffset, xoffset) in enumerate(product(off.yoffsets, off.xoffsets)):\n",
    "        img = image.crop((xoffset + off.xmin, \n",
    "                          yoffset + off.ymin, \n",
    "                          xoffset + off.xmax - off.xmin, \n",
    "                          yoffset + off.ymax - off.ymin))\n",
    "        img.thumbnail((NX,NY), PIL.Image.ANTIALIAS)\n",
    "        if outpattern:\n",
    "            img.convert(convertMode).save(outpattern.format(j=j))\n",
    "        else:\n",
    "            yield img.convert(convertMode)\n",
    "\n",
    "def plotImages(image, off):\n",
    "    nx,ny = len(off.xoffsets), len(off.yoffsets)\n",
    "    fig, axes = plt.subplots(ny, nx, figsize=(nx*3, \n",
    "                                      ny*3))\n",
    "    \n",
    "    for (img,ax) in zip(carveImages(image, off), axes.flatten()):\n",
    "        plt.sca(ax)\n",
    "        plt.imshow(np.array(img), interpolation='nearest')\n",
    "\n",
    "plotImages(img, off)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import cv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.array(img.convert('L'))/255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.imshow(np.array(img.convert('L')), cmap=plt.cm.gray)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ret, thresh = cv2.threshold(np.array(img.convert('L')), 127, 255, 0)\n",
    "plt.imshow(thresh, cmap=plt.cm.gray, vmin=0, vmax=255)\n",
    "contours,hierarchy = cv2.findContours(thresh, 3, 1)\n",
    "cnt = contours[0]\n",
    "rect = cv2.minAreaRect(cnt)\n",
    "box = cv2.boundingRect(rect)\n",
    "box = np.int0(box)\n",
    "cv2.drawContours(img,[box],0,(0,0,255),2)\n",
    "\n",
    "# area = cv2.contourArea(cnt)\n",
    "# print area\n",
    "\n",
    "# contours, hierarchy = cv2.findContours(thres)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#     for xoffset in \n",
    "#     xmin,xmax = [9,145]\n",
    "#     ymin,ymax = [1,137]\n",
    "#     xoffsets = [0, 160, 322, 486]\n",
    "#     yoffsets = [0, 149]\n",
    "#     # Remember people read left to right -- top to bottom\n",
    "#     for j,(yoffset, xoffset) in enumerate(product(yoffsets, xoffsets)):\n",
    "#         img = image.crop((xoffset + xmin, \n",
    "#                           yoffset + ymin, \n",
    "#                           xoffset + xmax - xmin, \n",
    "#                           yoffset + ymax - ymin))\n",
    "#         img.thumbnail((NX,NY), PIL.Image.ANTIALIAS)\n",
    "#         img.convert('L').save(outpattern.format(j=j))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:py27]",
   "language": "python",
   "name": "conda-env-py27-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
{'fraction_non_alphanumeric': 0.22086799509898755, 'fraction_numerical': 0.06700199909718192, 'mean_word_length': 1.951655881233346, 'pattern_counts': {'":': 121, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4921741', 'n_tokens_mistral': 6940, 'n_tokens_neox': 6030, 'n_words': 1161}