<commit_before><commit_msg>Add classification benchmarks<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_table(\"tagged.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_comments</th>\n",
       "      <th>year</th>\n",
       "      <th>title</th>\n",
       "      <th>speaker_info</th>\n",
       "      <th>section</th>\n",
       "      <th>target_audience</th>\n",
       "      <th>type</th>\n",
       "      <th>prerequisites</th>\n",
       "      <th>description</th>\n",
       "      <th>speaker_link_present</th>\n",
       "      <th>content_url_present</th>\n",
       "      <th>deadlinediff</th>\n",
       "      <th>selected</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2015</td>\n",
       "      <td>consuming government data python d</td>\n",
       "      <td>pratap vardhan data scientist gramenercom data...</td>\n",
       "      <td>data visualization analytics</td>\n",
       "      <td>intermediate</td>\n",
       "      <td>talks</td>\n",
       "      <td>NaN</td>\n",
       "      <td>the explosion open data especially government ...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>-94</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2015</td>\n",
       "      <td>dont get scared get started</td>\n",
       "      <td>tapasweni pathaki done bachelors it igdtuw i w...</td>\n",
       "      <td>others</td>\n",
       "      <td>beginner</td>\n",
       "      <td>talks</td>\n",
       "      <td>nothingother passion coding</td>\n",
       "      <td>opensource world full excitement knowledge enc...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2015</td>\n",
       "      <td>distributed scheduling leveraging multiple nod...</td>\n",
       "      <td>i software engineer red hat inc working gluste...</td>\n",
       "      <td>concurrency</td>\n",
       "      <td>beginner</td>\n",
       "      <td>talks</td>\n",
       "      <td>a basic understanding distributed system works...</td>\n",
       "      <td>setting cron job machine perhaps easiest way s...</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2015</td>\n",
       "      <td>analyzing python code pylint</td>\n",
       "      <td>im open source enthusiast coming romania lead ...</td>\n",
       "      <td>others</td>\n",
       "      <td>intermediate</td>\n",
       "      <td>talks</td>\n",
       "      <td>the participants basic understanding python no...</td>\n",
       "      <td>given dynamic nature python bugs tend creep co...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>-112</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2015</td>\n",
       "      <td>python  metaprogramming macros madness  more</td>\n",
       "      <td>suhas data scientist gramener previously engin...</td>\n",
       "      <td>core python</td>\n",
       "      <td>intermediate</td>\n",
       "      <td>talks</td>\n",
       "      <td>NaN</td>\n",
       "      <td>summaryever wanted conquer world fell short kn...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>-94</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   n_comments  year                                              title  \\\n",
       "0           0  2015                 consuming government data python d   \n",
       "1           1  2015                        dont get scared get started   \n",
       "2           0  2015  distributed scheduling leveraging multiple nod...   \n",
       "3           0  2015                       analyzing python code pylint   \n",
       "4           0  2015       python  metaprogramming macros madness  more   \n",
       "\n",
       "                                        speaker_info  \\\n",
       "0  pratap vardhan data scientist gramenercom data...   \n",
       "1  tapasweni pathaki done bachelors it igdtuw i w...   \n",
       "2  i software engineer red hat inc working gluste...   \n",
       "3  im open source enthusiast coming romania lead ...   \n",
       "4  suhas data scientist gramener previously engin...   \n",
       "\n",
       "                        section target_audience   type  \\\n",
       "0  data visualization analytics    intermediate  talks   \n",
       "1                        others        beginner  talks   \n",
       "2                   concurrency        beginner  talks   \n",
       "3                        others    intermediate  talks   \n",
       "4                   core python    intermediate  talks   \n",
       "\n",
       "                                       prerequisites  \\\n",
       "0                                                NaN   \n",
       "1                        nothingother passion coding   \n",
       "2  a basic understanding distributed system works...   \n",
       "3  the participants basic understanding python no...   \n",
       "4                                                NaN   \n",
       "\n",
       "                                         description speaker_link_present  \\\n",
       "0  the explosion open data especially government ...                False   \n",
       "1  opensource world full excitement knowledge enc...                False   \n",
       "2  setting cron job machine perhaps easiest way s...                 True   \n",
       "3  given dynamic nature python bugs tend creep co...                False   \n",
       "4  summaryever wanted conquer world fell short kn...                False   \n",
       "\n",
       "  content_url_present  deadlinediff selected  \n",
       "0               False           -94     True  \n",
       "1               False             1    False  \n",
       "2                True            35    False  \n",
       "3               False          -112    False  \n",
       "4               False           -94     True  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'n_comments', u'year', u'title', u'speaker_info', u'section',\n",
       "       u'target_audience', u'type', u'prerequisites', u'description',\n",
       "       u'speaker_link_present', u'content_url_present', u'deadlinediff',\n",
       "       u'selected'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "categoricals = []\n",
    "for col in \"section target_audience type\".split():\n",
    "    categoricals.append(pd.get_dummies(df[col]).values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(290, 13), (290, 3), (290, 2)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[x.shape for x in categoricals]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xCat = np.hstack(categoricals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(290, 18)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xCat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xBool = df['speaker_link_present content_url_present'.split()].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xText = df['title speaker_info prerequisites description'.split()].copy()\n",
    "xText.fillna(value=\" \", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "xText = xText.apply(lambda x: \" \".join(x), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xText = TfidfVectorizer().fit_transform(xText.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xNum = df['n_comments deadlinediff'.split()].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "all the input arrays must have same number of dimensions",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-25-6b5f024e8a43>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mxCat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxBool\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxText\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxNum\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'selected'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/jaidevd/anaconda/lib/python2.7/site-packages/numpy/core/shape_base.pyc\u001b[0m in \u001b[0;36mhstack\u001b[0;34m(tup)\u001b[0m\n\u001b[1;32m    278\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0m_nx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    279\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 280\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_nx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    281\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    282\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: all the input arrays must have same number of dimensions"
     ]
    }
   ],
   "source": [
    "X = np.hstack([xCat, xBool, xText, xNum])\n",
    "y = df['selected'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "feats = [xCat, xBool.values, xText.toarray(), xNum]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = np.hstack(feats)\n",
    "y = df['selected'].astype(int).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from scipy.io import savemat\n",
    "savemat(\"trainData.mat\", dict(X=X, y=y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xScaled = StandardScaler().fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from jeeves.cross_validation import report_stratifiedKfold_cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy_score</th>\n",
       "      <th>recall_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.813559</td>\n",
       "      <td>0.384615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.844828</td>\n",
       "      <td>0.615385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.793103</td>\n",
       "      <td>0.461538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.844828</td>\n",
       "      <td>0.307692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.859649</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   accuracy_score  recall_score\n",
       "0        0.813559      0.384615\n",
       "1        0.844828      0.615385\n",
       "2        0.793103      0.461538\n",
       "3        0.844828      0.307692\n",
       "4        0.859649      0.500000"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report_stratifiedKfold_cv(LogisticRegression(), X, y, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy_score</th>\n",
       "      <th>recall_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.847458</td>\n",
       "      <td>0.384615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.879310</td>\n",
       "      <td>0.615385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.810345</td>\n",
       "      <td>0.538462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.538462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.859649</td>\n",
       "      <td>0.583333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   accuracy_score  recall_score\n",
       "0        0.847458      0.384615\n",
       "1        0.879310      0.615385\n",
       "2        0.810345      0.538462\n",
       "3        0.862069      0.538462\n",
       "4        0.859649      0.583333"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report_stratifiedKfold_cv(SVC(), X, y, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_non_text = [xCat, xBool.values, xNum]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_non_text = np.hstack(x_non_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_non_text_scaled = StandardScaler().fit_transform(x_non_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "pca = PCA(n_components=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x114531710>"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "<KEY>//+<KEY>MDW<KEY>",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x114243a50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x_red = pca.fit_transform(x_non_text_scaled)\n",
    "plt.figure(figsize=(10, 8))\n",
    "plt.scatter(x_red[:, 0], x_red[:, 1], c=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy_score</th>\n",
       "      <th>recall_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.881356</td>\n",
       "      <td>0.692308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.758621</td>\n",
       "      <td>0.384615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.724138</td>\n",
       "      <td>0.307692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.896552</td>\n",
       "      <td>0.615385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.859649</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   accuracy_score  recall_score\n",
       "0        0.881356      0.692308\n",
       "1        0.758621      0.384615\n",
       "2        0.724138      0.307692\n",
       "3        0.896552      0.615385\n",
       "4        0.859649      0.333333"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report_stratifiedKfold_cv(LogisticRegression(), x_non_text_scaled, y, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy_score</th>\n",
       "      <th>recall_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.866667</td>\n",
       "      <td>0.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.857143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.866667</td>\n",
       "      <td>0.571429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.833333</td>\n",
       "      <td>0.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.827586</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.793103</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.821429</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.892857</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.892857</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   accuracy_score  recall_score\n",
       "0        0.866667      0.428571\n",
       "1        0.900000      0.857143\n",
       "2        0.866667      0.571429\n",
       "3        0.833333      0.285714\n",
       "4        0.827586      0.500000\n",
       "5        0.793103      0.666667\n",
       "6        0.821429      0.333333\n",
       "7        0.892857      0.500000\n",
       "8        0.857143      0.500000\n",
       "9        0.892857      0.666667"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report_stratifiedKfold_cv(RandomForestClassifier(), x_non_text, y, shuffle=True, n_folds=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy_score</th>\n",
       "      <th>recall_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.714286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.714286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.966667</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.857143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.896552</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.896552</td>\n",
       "      <td>0.833333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.964286</td>\n",
       "      <td>0.833333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.928571</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.892857</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   accuracy_score  recall_score\n",
       "0        0.933333      0.714286\n",
       "1        0.933333      0.714286\n",
       "2        0.966667      1.000000\n",
       "3        0.933333      0.857143\n",
       "4        0.896552      0.666667\n",
       "5        0.896552      0.833333\n",
       "6        0.964286      0.833333\n",
       "7        0.857143      0.666667\n",
       "8        0.928571      0.666667\n",
       "9        0.892857      0.666667"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "report_stratifiedKfold_cv(DecisionTreeClassifier(), x_non_text, y, shuffle=True, n_folds=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy_score    0.878834\n",
      "recall_score      0.621429\n",
      "dtype: float64\n",
      "accuracy_score    0.779573\n",
      "recall_score      0.016667\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print report_stratifiedKfold_cv(RandomForestClassifier(), x_non_text, y, shuffle=True, n_folds=10).mean()\n",
    "print report_stratifiedKfold_cv(RandomForestClassifier(), X, y, shuffle=True, n_folds=10).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy_score    0.916921\n",
      "recall_score      0.780952\n",
      "dtype: float64\n",
      "accuracy_score    0.879204\n",
      "recall_score      0.766667\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print report_stratifiedKfold_cv(DecisionTreeClassifier(), x_non_text, y, shuffle=True, n_folds=10).mean()\n",
    "print report_stratifiedKfold_cv(DecisionTreeClassifier(), X, y, shuffle=True, n_folds=10).mean()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
{'fraction_non_alphanumeric': 0.21392814575174526, 'fraction_numerical': 0.07481695896475396, 'mean_word_length': 1.546700199462319, 'pattern_counts': {'":': 305, '<': 558, '<?xml version=': 0, '>': 560, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6187903', 'n_tokens_mistral': 12819, 'n_tokens_neox': 11629, 'n_words': 1569}