<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import sys\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Padding\n",
    "def pad_one_im(direc_in, name, direc_out, pad_n):\n",
    "    img_file = direc_in + name\n",
    "    img = Image.open(img_file)\n",
    "    old_size = img.size\n",
    "    new_size = (old_size[0] + pad_n, old_size[1]+pad_n)\n",
    "    new_im = Image.new(\"RGB\", new_size)\n",
    "    new_im.paste(img, ((new_size[0] - old_size[0])/2, (new_size[1]-old_size[1])/2))\n",
    "    out_file = img_file[0:(len(img_file)-4)] + \"padded.png\"\n",
    "    new_im.save(direc_out + out_file)\n",
    "    \n",
    "def pad_directory(direc_in, direc_out, pad_n):\n",
    "    imgs = os.listdir(direc_in)\n",
    "    for i in imgs:\n",
    "        if '.DS_Store' not in i:\n",
    "            pad_one_im(direc_in, i, direc_out, pad_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Black & White\n",
    "def makeItGray(direc_in, name, direc_out):\n",
    "    img_file = direc_in + name\n",
    "    img = Image.open(img_file)\n",
    "    gray = img.convert('LA')\n",
    "    out_file = img_file[0:(len(img_file)-4)] + \"_gray.png\" \n",
    "    gray.save(direc_out + out_file)\n",
    "    \n",
    "def makeDirGray(direc_in, direc_out):\n",
    "    imgs = os.listdir(direc_in)\n",
    "    for i in imgs:\n",
    "        if '.DS_Store' not in i:\n",
    "            makeItGray(direc_in, i, direc_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Color Balancing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Luminance Balancing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "direc_in_list = ['../data/raw/train' + d for d in os.listdir('../data/raw/train')\n",
    "                if '.DS_Store' not in d]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['../data/processed/gray/ALB', '../data/processed/gray/BET', '../data/processed/gray/DOL', '../data/processed/gray/LAG', '../data/processed/gray/NoF', '../data/processed/gray/OTHER', '../data/processed/gray/SHARK', '../data/processed/gray/YFT']\n"
     ]
    }
   ],
   "source": [
    "direc_gray_out_list = ['../data/processed/gray/'+ d for d in os.listdir('../data/raw/train')\n",
    "                        if '.DS_Store' not in d]\n",
    "\n",
    "makeDirGray(direc_in_list, direc_gray_out_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "padded_out_list = ['../data/processed/padded/'+ d for d in os.listdir('../data/raw/train')\n",
    "                        if '.DS_Store' not in d]\n",
    "\n",
    "pad_directory(direc_in_list, padded_out_list, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gray_in_list = direc_gray_out_list\n",
    "\n",
    "padded_gray_out_list = ['../data/processed/gray_padded/'+ d for d in os.listdir('../data/raw/train')\n",
    "                        if '.DS_Store' not in d]\n",
    "\n",
    "pad_directory(gray_in_list, padded_gray_out_list, 50)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
<commit_msg>run for gray, padded, and gray and padded<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import sys\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Padding\n",
    "def pad_one_im(direc_in, name, direc_out, pad_n):\n",
    "    img_file = direc_in + name\n",
    "    img = Image.open(img_file)\n",
    "    old_size = img.size\n",
    "    new_size = (old_size[0] + pad_n, old_size[1]+pad_n)\n",
    "    new_im = Image.new(\"RGB\", new_size)\n",
    "    new_im.paste(img, ((new_size[0] - old_size[0])/2, (new_size[1]-old_size[1])/2))\n",
    "    out_file = name[0:(len(name)-4)] + \"padded.png\"\n",
    "    new_im.save(direc_out + out_file)\n",
    "    \n",
    "def pad_directory(direc_in, direc_out, pad_n):\n",
    "    imgs = os.listdir(direc_in)\n",
    "    for i in imgs:\n",
    "        if '.DS_Store' not in i:\n",
    "            pad_one_im(direc_in, i, direc_out, pad_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Black & White\n",
    "def makeItGray(direc_in, name, direc_out):\n",
    "    img_file = direc_in + name\n",
    "    img = Image.open(img_file)\n",
    "    gray = img.convert('LA')\n",
    "    out_file = name[0:(len(name)-4)] + \"_gray.png\"\n",
    "    gray.save(direc_out + out_file)\n",
    "    \n",
    "def makeDirGray(direc_in, direc_out):\n",
    "    imgs = os.listdir(direc_in)\n",
    "    for i in imgs:\n",
    "        if '.DS_Store' not in i:\n",
    "            makeItGray(direc_in, i, direc_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Color Balancing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Luminance Balancing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "direc_in_list = ['../data/raw/train/' + d + '/' for d in os.listdir('../data/raw/train')\n",
    "                if '.DS_Store' not in d]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "direc_gray_out_list = ['../data/processed/gray/'+ d + '/'  for d in os.listdir('../data/raw/train')\n",
    "                        if '.DS_Store' not in d]\n",
    "#for d in range(0, len(direc_in_list)):\n",
    "#    makeDirGray(direc_in_list, direc_gray_out_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "padded_out_list = ['../data/processed/padded/'+ d + '/'  for d in os.listdir('../data/raw/train')\n",
    "                        if '.DS_Store' not in d]\n",
    "\n",
    "#pad_directory(direc_in_list, padded_out_list, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gray_in_list = direc_gray_out_list\n",
    "\n",
    "padded_gray_out_list = ['../data/processed/gray_padded/'+ d + '/'  for d in os.listdir('../data/raw/train')\n",
    "                        if '.DS_Store' not in d]\n",
    "\n",
    "#pad_directory(gray_in_list, padded_gray_out_list, 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for d in range(0, len(direc_in_list)):\n",
    "    makeDirGray(direc_in_list[d], direc_gray_out_list[d])\n",
    "    pad_directory(direc_in_list[d], padded_out_list[d], 50)\n",
    "    pad_directory(gray_in_list[d], padded_gray_out_list[d], 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}

{'fraction_non_alphanumeric': 0.21224000871174997, 'fraction_numerical': 0.0079494718501579, 'mean_word_length': 2.3579524680073125, 'pattern_counts': {'":': 159, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3512414', 'n_tokens_mistral': 3809, 'n_tokens_neox': 3737, 'n_words': 614}