<commit_before>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>urlid</th>\n",
       "      <th>boilerplate</th>\n",
       "      <th>alchemy_category</th>\n",
       "      <th>alchemy_category_score</th>\n",
       "      <th>avglinksize</th>\n",
       "      <th>commonlinkratio_1</th>\n",
       "      <th>commonlinkratio_2</th>\n",
       "      <th>commonlinkratio_3</th>\n",
       "      <th>commonlinkratio_4</th>\n",
       "      <th>...</th>\n",
       "      <th>linkwordscore</th>\n",
       "      <th>news_front_page</th>\n",
       "      <th>non_markup_alphanum_characters</th>\n",
       "      <th>numberOfLinks</th>\n",
       "      <th>numwords_in_url</th>\n",
       "      <th>parametrizedLinkRatio</th>\n",
       "      <th>spelling_errors_ratio</th>\n",
       "      <th>label</th>\n",
       "      <th>title</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://www.bloomberg.com/news/2010-12-23/ibm-p...</td>\n",
       "      <td>4042</td>\n",
       "      <td>{\"title\":\"IBM Sees Holographic Calls Air Breat...</td>\n",
       "      <td>business</td>\n",
       "      <td>0.789131</td>\n",
       "      <td>2.055556</td>\n",
       "      <td>0.676471</td>\n",
       "      <td>0.205882</td>\n",
       "      <td>0.047059</td>\n",
       "      <td>0.023529</td>\n",
       "      <td>...</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>5424</td>\n",
       "      <td>170</td>\n",
       "      <td>8</td>\n",
       "      <td>0.152941</td>\n",
       "      <td>0.079130</td>\n",
       "      <td>0</td>\n",
       "      <td>IBM Sees Holographic Calls Air Breathing Batte...</td>\n",
       "      <td>A sign stands outside the International Busine...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://www.popsci.com/technology/article/2012-...</td>\n",
       "      <td>8471</td>\n",
       "      <td>{\"title\":\"The Fully Electronic Futuristic Star...</td>\n",
       "      <td>recreation</td>\n",
       "      <td>0.574147</td>\n",
       "      <td>3.677966</td>\n",
       "      <td>0.508021</td>\n",
       "      <td>0.288770</td>\n",
       "      <td>0.213904</td>\n",
       "      <td>0.144385</td>\n",
       "      <td>...</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>4973</td>\n",
       "      <td>187</td>\n",
       "      <td>9</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.125448</td>\n",
       "      <td>1</td>\n",
       "      <td>The Fully Electronic Futuristic Starting Gun T...</td>\n",
       "      <td>And that can be carried on a plane without the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://www.menshealth.com/health/flu-fighting-...</td>\n",
       "      <td>1164</td>\n",
       "      <td>{\"title\":\"Fruits that Fight the Flu fruits tha...</td>\n",
       "      <td>health</td>\n",
       "      <td>0.996526</td>\n",
       "      <td>2.382883</td>\n",
       "      <td>0.562016</td>\n",
       "      <td>0.321705</td>\n",
       "      <td>0.120155</td>\n",
       "      <td>0.042636</td>\n",
       "      <td>...</td>\n",
       "      <td>55</td>\n",
       "      <td>0</td>\n",
       "      <td>2240</td>\n",
       "      <td>258</td>\n",
       "      <td>11</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.057613</td>\n",
       "      <td>1</td>\n",
       "      <td>Fruits that Fight the Flu fruits that fight th...</td>\n",
       "      <td>Apples The most popular source of antioxidants...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://www.dumblittleman.com/2007/12/10-foolpr...</td>\n",
       "      <td>6684</td>\n",
       "      <td>{\"title\":\"10 Foolproof Tips for Better Sleep \"...</td>\n",
       "      <td>health</td>\n",
       "      <td>0.801248</td>\n",
       "      <td>1.543103</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.016667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>2737</td>\n",
       "      <td>120</td>\n",
       "      <td>5</td>\n",
       "      <td>0.041667</td>\n",
       "      <td>0.100858</td>\n",
       "      <td>1</td>\n",
       "      <td>10 Foolproof Tips for Better Sleep</td>\n",
       "      <td>There was a period in my life when I had a lot...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://bleacherreport.com/articles/1205138-the...</td>\n",
       "      <td>9006</td>\n",
       "      <td>{\"title\":\"The 50 Coolest Jerseys You Didn t Kn...</td>\n",
       "      <td>sports</td>\n",
       "      <td>0.719157</td>\n",
       "      <td>2.676471</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0.123457</td>\n",
       "      <td>0.043210</td>\n",
       "      <td>...</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>12032</td>\n",
       "      <td>162</td>\n",
       "      <td>10</td>\n",
       "      <td>0.098765</td>\n",
       "      <td>0.082569</td>\n",
       "      <td>0</td>\n",
       "      <td>The 50 Coolest Jerseys You Didn t Know Existed...</td>\n",
       "      <td>Jersey sales is a curious business Whether you...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 url  urlid  \\\n",
       "0  http://www.bloomberg.com/news/2010-12-23/ibm-p...   4042   \n",
       "1  http://www.popsci.com/technology/article/2012-...   8471   \n",
       "2  http://www.menshealth.com/health/flu-fighting-...   1164   \n",
       "3  http://www.dumblittleman.com/2007/12/10-foolpr...   6684   \n",
       "4  http://bleacherreport.com/articles/1205138-the...   9006   \n",
       "\n",
       "                                         boilerplate alchemy_category  \\\n",
       "0  {\"title\":\"IBM Sees Holographic Calls Air Breat...         business   \n",
       "1  {\"title\":\"The Fully Electronic Futuristic Star...       recreation   \n",
       "2  {\"title\":\"Fruits that Fight the Flu fruits tha...           health   \n",
       "3  {\"title\":\"10 Foolproof Tips for Better Sleep \"...           health   \n",
       "4  {\"title\":\"The 50 Coolest Jerseys You Didn t Kn...           sports   \n",
       "\n",
       "  alchemy_category_score  avglinksize  commonlinkratio_1  commonlinkratio_2  \\\n",
       "0               0.789131     2.055556           0.676471           0.205882   \n",
       "1               0.574147     3.677966           0.508021           0.288770   \n",
       "2               0.996526     2.382883           0.562016           0.321705   \n",
       "3               0.801248     1.543103           0.400000           0.100000   \n",
       "4               0.719157     2.676471           0.500000           0.222222   \n",
       "\n",
       "   commonlinkratio_3  commonlinkratio_4  \\\n",
       "0           0.047059           0.023529   \n",
       "1           0.213904           0.144385   \n",
       "2           0.120155           0.042636   \n",
       "3           0.016667           0.000000   \n",
       "4           0.123457           0.043210   \n",
       "\n",
       "                         ...                          linkwordscore  \\\n",
       "0                        ...                                     24   \n",
       "1                        ...                                     40   \n",
       "2                        ...                                     55   \n",
       "3                        ...                                     24   \n",
       "4                        ...                                     14   \n",
       "\n",
       "   news_front_page  non_markup_alphanum_characters  numberOfLinks  \\\n",
       "0                0                            5424            170   \n",
       "1                0                            4973            187   \n",
       "2                0                            2240            258   \n",
       "3                0                            2737            120   \n",
       "4                0                           12032            162   \n",
       "\n",
       "   numwords_in_url  parametrizedLinkRatio  spelling_errors_ratio label  \\\n",
       "0                8               0.152941               0.079130     0   \n",
       "1                9               0.181818               0.125448     1   \n",
       "2               11               0.166667               0.057613     1   \n",
       "3                5               0.041667               0.100858     1   \n",
       "4               10               0.098765               0.082569     0   \n",
       "\n",
       "                                               title  \\\n",
       "0  IBM Sees Holographic Calls Air Breathing Batte...   \n",
       "1  The Fully Electronic Futuristic Starting Gun T...   \n",
       "2  Fruits that Fight the Flu fruits that fight th...   \n",
       "3                10 Foolproof Tips for Better Sleep    \n",
       "4  The 50 Coolest Jerseys You Didn t Know Existed...   \n",
       "\n",
       "                                                body  \n",
       "0  A sign stands outside the International Busine...  \n",
       "1  And that can be carried on a plane without the...  \n",
       "2  Apples The most popular source of antioxidants...  \n",
       "3  There was a period in my life when I had a lot...  \n",
       "4  Jersey sales is a curious business Whether you...  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "data = pd.read_csv(\"../datasets/train.tsv\", sep='\\t')\n",
    "data['title'] = data.boilerplate.map(lambda x: json.loads(x).get('title', ''))\n",
    "data['body'] = data.boilerplate.map(lambda x: json.loads(x).get('body', ''))\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predicting \"Greenness\" Of Content"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This dataset comes from [stumbleupon](https://www.stumbleupon.com/), a web page recommender.  \n",
    "\n",
    "A description of the columns is below"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "FieldName|Type|Description\n",
    "---------|----|-----------\n",
    "url|string|Url of the webpage to be classified\n",
    "title|string|Title of the article\n",
    "body|string|Body text of article\n",
    "urlid|integer| StumbleUpon's unique identifier for each url\n",
    "boilerplate|json|Boilerplate text\n",
    "alchemy_category|string|Alchemy category (per the publicly available Alchemy API found at www.alchemyapi.com)\n",
    "alchemy_category_score|double|Alchemy category score (per the publicly available Alchemy API found at www.alchemyapi.com)\n",
    "avglinksize| double|Average number of words in each link\n",
    "commonlinkratio_1|double|# of links sharing at least 1 word with 1 other links / # of links\n",
    "commonlinkratio_2|double|# of links sharing at least 1 word with 2 other links / # of links\n",
    "commonlinkratio_3|double|# of links sharing at least 1 word with 3 other links / # of links\n",
    "commonlinkratio_4|double|# of links sharing at least 1 word with 4 other links / # of links\n",
    "compression_ratio|double|Compression achieved on this page via gzip (measure of redundancy)\n",
    "embed_ratio|double|Count of number of <embed> usage\n",
    "frameBased|integer (0 or 1)|A page is frame-based (1) if it has no body markup but have a frameset markup\n",
    "frameTagRatio|double|Ratio of iframe markups over total number of markups\n",
    "hasDomainLink|integer (0 or 1)|True (1) if it contains an <a> with an url with domain\n",
    "html_ratio|double|Ratio of tags vs text in the page\n",
    "image_ratio|double|Ratio of <img> tags vs text in the page\n",
    "is_news|integer (0 or 1) | True (1) if StumbleUpon's news classifier determines that this webpage is news\n",
    "lengthyLinkDomain| integer (0 or 1)|True (1) if at least 3 <a> 's text contains more than 30 alphanumeric characters\n",
    "linkwordscore|double|Percentage of words on the page that are in hyperlink's text\n",
    "news_front_page| integer (0 or 1)|True (1) if StumbleUpon's news classifier determines that this webpage is front-page news\n",
    "non_markup_alphanum_characters|integer| Page's text's number of alphanumeric characters\n",
    "numberOfLinks|integer Number of <a>|markups\n",
    "numwords_in_url| double|Number of words in url\n",
    "parametrizedLinkRatio|double|A link is parametrized if it's url contains parameters or has an attached onClick event\n",
    "spelling_errors_ratio|double|Ratio of words not found in wiki (considered to be a spelling mistake)\n",
    "label|integer (0 or 1)|User-determined label. Either evergreen (1) or non-evergreen (0); available for train.tsv only"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Let's try extracting some of the text content.\n",
    "> ### Create a feature for the title containing 'recipe'. Is the % of evegreen websites higher or lower on pages that have recipe in the the title?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Option 1: Create a function to check for this\n",
    "\n",
    "def has_recipe(text_in):\n",
    "    if 'recipe' in str(text_in).lower():\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "data['recipe'] = data['title'].map(has_recipe)\n",
    "\n",
    "# Option 2: lambda functions\n",
    "\n",
    "data['recipe'] = data['title'].map(lambda t: 1 if 'recipe' in str(t).lower() else 0)\n",
    "\n",
    "\n",
    "# Option 3: string functions\n",
    "data['recipe'] = data['title'].str.contains('recipe')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Demo: Use of the Count Vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "titles = data['title'].fillna('')\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "vectorizer = CountVectorizer(max_features = 1000, \n",
    "                             ngram_range=(1, 2), \n",
    "                             stop_words='english',\n",
    "                             binary=True)\n",
    "\n",
    "# Use `fit` to learn the vocabulary of the titles\n",
    "vectorizer.fit(titles)\n",
    "\n",
    "# Use `tranform` to generate the sample X word matrix - one column per feature (word or n-grams)\n",
    "X = vectorizer.transform(titles)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Demo: Build a random forest model to predict evergreeness of a website using the title features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV AUC [ 0.78917062  0.80180599  0.80089666], Average AUC 0.7972910912470859\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "model = RandomForestClassifier(n_estimators = 20)\n",
    "    \n",
    "# Use `fit` to learn the vocabulary of the titles\n",
    "vectorizer.fit(titles)\n",
    "\n",
    "# Use `tranform` to generate the sample X word matrix - one column per feature (word or n-grams)\n",
    "X = vectorizer.transform(titles)\n",
    "y = data['label']\n",
    "\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "\n",
    "scores = cross_val_score(model, X, y, scoring='roc_auc')\n",
    "print('CV AUC {}, Average AUC {}'.format(scores, scores.mean()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exercise: Build a random forest model to predict evergreeness of a website using the title features and quantitative features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV AUC [ 0.78708893  0.79673682  0.7896231 ], Average AUC 0.7911496193486233\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Features</th>\n",
       "      <th>Importance Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>html_ratio</td>\n",
       "      <td>0.153023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1001</th>\n",
       "      <td>image_ratio</td>\n",
       "      <td>0.096890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>712</th>\n",
       "      <td>recipe</td>\n",
       "      <td>0.037589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>721</th>\n",
       "      <td>recipes</td>\n",
       "      <td>0.018967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>chocolate</td>\n",
       "      <td>0.013562</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Features  Importance Score\n",
       "1000   html_ratio          0.153023\n",
       "1001  image_ratio          0.096890\n",
       "712        recipe          0.037589\n",
       "721       recipes          0.018967\n",
       "190     chocolate          0.013562"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use `tranform` to generate the sample X word matrix - one column per feature (word or n-grams)\n",
    "X_text_features = vectorizer.transform(titles)\n",
    "\n",
    "# Identify the features you want from the original dataset\n",
    "other_features_columns = ['html_ratio', 'image_ratio']\n",
    "other_features = data[other_features_columns]\n",
    "\n",
    "# Stack them horizontally together\n",
    "# This takes all of the word/n-gram columns and appends on two more columns for `html_ratio` and `image_ratio`\n",
    "from scipy.sparse import hstack\n",
    "X = hstack((X_text_features, other_features))\n",
    "\n",
    "scores = cross_val_score(model, X, y, scoring='roc_auc')\n",
    "print('CV AUC {}, Average AUC {}'.format(scores, scores.mean()))\n",
    "\n",
    "# What features of these are most important?\n",
    "model.fit(X, y)\n",
    "\n",
    "all_feature_names = vectorizer.get_feature_names() + other_features_columns\n",
    "feature_importances = pd.DataFrame({'Features' : all_feature_names, 'Importance Score': model.feature_importances_})\n",
    "feature_importances.sort('Importance Score', ascending=False).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Exercise: Build a random forest model to predict evergreeness of a website using the body features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV AUC [ 0.83957379  0.85196607  0.84562597], Average AUC 0.8457219429733792\n"
     ]
    }
   ],
   "source": [
    "body_text = data['body'].fillna('')\n",
    "\n",
    "# Use `fit` to learn the vocabulary\n",
    "vectorizer.fit(body_text)\n",
    "\n",
    "# Use `tranform` to generate the sample X word matrix - one column per feature (word or n-grams)\n",
    "X = vectorizer.transform(body_text)\n",
    "\n",
    "scores = cross_val_score(model, X, y, scoring='roc_auc')\n",
    "print('CV AUC {}, Average AUC {}'.format(scores, scores.mean()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Exercise: Use `TfIdfVectorizer` instead of `CountVectorizer` - is this an improvement?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV AUC [ 0.84163803  0.85402602  0.84591013], Average AUC 0.8471913945265227\n"
     ]
    }
   ],
   "source": [
    "titles = data['title'].fillna('')\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "vectorizer = TfidfVectorizer(max_features = 1000, \n",
    "                             ngram_range=(1, 2), \n",
    "                             stop_words='english')\n",
    "\n",
    "\n",
    "# Use `fit` to learn the vocabulary\n",
    "vectorizer.fit(body_text)\n",
    "\n",
    "# Use `tranform` to generate the sample X word matrix - one column per feature (word or n-grams)\n",
    "X = vectorizer.transform(body_text)\n",
    "\n",
    "scores = cross_val_score(model, X, y, scoring='roc_auc')\n",
    "print('CV AUC {}, Average AUC {}'.format(scores, scores.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}<commit_msg>Updating file path and adding toarray method<commit_after>{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>urlid</th>\n",
       "      <th>boilerplate</th>\n",
       "      <th>alchemy_category</th>\n",
       "      <th>alchemy_category_score</th>\n",
       "      <th>avglinksize</th>\n",
       "      <th>commonlinkratio_1</th>\n",
       "      <th>commonlinkratio_2</th>\n",
       "      <th>commonlinkratio_3</th>\n",
       "      <th>commonlinkratio_4</th>\n",
       "      <th>...</th>\n",
       "      <th>linkwordscore</th>\n",
       "      <th>news_front_page</th>\n",
       "      <th>non_markup_alphanum_characters</th>\n",
       "      <th>numberOfLinks</th>\n",
       "      <th>numwords_in_url</th>\n",
       "      <th>parametrizedLinkRatio</th>\n",
       "      <th>spelling_errors_ratio</th>\n",
       "      <th>label</th>\n",
       "      <th>title</th>\n",
       "      <th>body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://www.bloomberg.com/news/2010-12-23/ibm-p...</td>\n",
       "      <td>4042</td>\n",
       "      <td>{\"title\":\"IBM Sees Holographic Calls Air Breat...</td>\n",
       "      <td>business</td>\n",
       "      <td>0.789131</td>\n",
       "      <td>2.055556</td>\n",
       "      <td>0.676471</td>\n",
       "      <td>0.205882</td>\n",
       "      <td>0.047059</td>\n",
       "      <td>0.023529</td>\n",
       "      <td>...</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>5424</td>\n",
       "      <td>170</td>\n",
       "      <td>8</td>\n",
       "      <td>0.152941</td>\n",
       "      <td>0.079130</td>\n",
       "      <td>0</td>\n",
       "      <td>IBM Sees Holographic Calls Air Breathing Batte...</td>\n",
       "      <td>A sign stands outside the International Busine...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://www.popsci.com/technology/article/2012-...</td>\n",
       "      <td>8471</td>\n",
       "      <td>{\"title\":\"The Fully Electronic Futuristic Star...</td>\n",
       "      <td>recreation</td>\n",
       "      <td>0.574147</td>\n",
       "      <td>3.677966</td>\n",
       "      <td>0.508021</td>\n",
       "      <td>0.288770</td>\n",
       "      <td>0.213904</td>\n",
       "      <td>0.144385</td>\n",
       "      <td>...</td>\n",
       "      <td>40</td>\n",
       "      <td>0</td>\n",
       "      <td>4973</td>\n",
       "      <td>187</td>\n",
       "      <td>9</td>\n",
       "      <td>0.181818</td>\n",
       "      <td>0.125448</td>\n",
       "      <td>1</td>\n",
       "      <td>The Fully Electronic Futuristic Starting Gun T...</td>\n",
       "      <td>And that can be carried on a plane without the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://www.menshealth.com/health/flu-fighting-...</td>\n",
       "      <td>1164</td>\n",
       "      <td>{\"title\":\"Fruits that Fight the Flu fruits tha...</td>\n",
       "      <td>health</td>\n",
       "      <td>0.996526</td>\n",
       "      <td>2.382883</td>\n",
       "      <td>0.562016</td>\n",
       "      <td>0.321705</td>\n",
       "      <td>0.120155</td>\n",
       "      <td>0.042636</td>\n",
       "      <td>...</td>\n",
       "      <td>55</td>\n",
       "      <td>0</td>\n",
       "      <td>2240</td>\n",
       "      <td>258</td>\n",
       "      <td>11</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.057613</td>\n",
       "      <td>1</td>\n",
       "      <td>Fruits that Fight the Flu fruits that fight th...</td>\n",
       "      <td>Apples The most popular source of antioxidants...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://www.dumblittleman.com/2007/12/10-foolpr...</td>\n",
       "      <td>6684</td>\n",
       "      <td>{\"title\":\"10 Foolproof Tips for Better Sleep \"...</td>\n",
       "      <td>health</td>\n",
       "      <td>0.801248</td>\n",
       "      <td>1.543103</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.016667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>2737</td>\n",
       "      <td>120</td>\n",
       "      <td>5</td>\n",
       "      <td>0.041667</td>\n",
       "      <td>0.100858</td>\n",
       "      <td>1</td>\n",
       "      <td>10 Foolproof Tips for Better Sleep</td>\n",
       "      <td>There was a period in my life when I had a lot...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://bleacherreport.com/articles/1205138-the...</td>\n",
       "      <td>9006</td>\n",
       "      <td>{\"title\":\"The 50 Coolest Jerseys You Didn t Kn...</td>\n",
       "      <td>sports</td>\n",
       "      <td>0.719157</td>\n",
       "      <td>2.676471</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>0.123457</td>\n",
       "      <td>0.043210</td>\n",
       "      <td>...</td>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>12032</td>\n",
       "      <td>162</td>\n",
       "      <td>10</td>\n",
       "      <td>0.098765</td>\n",
       "      <td>0.082569</td>\n",
       "      <td>0</td>\n",
       "      <td>The 50 Coolest Jerseys You Didn t Know Existed...</td>\n",
       "      <td>Jersey sales is a curious business Whether you...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 url  urlid  \\\n",
       "0  http://www.bloomberg.com/news/2010-12-23/ibm-p...   4042   \n",
       "1  http://www.popsci.com/technology/article/2012-...   8471   \n",
       "2  http://www.menshealth.com/health/flu-fighting-...   1164   \n",
       "3  http://www.dumblittleman.com/2007/12/10-foolpr...   6684   \n",
       "4  http://bleacherreport.com/articles/1205138-the...   9006   \n",
       "\n",
       "                                         boilerplate alchemy_category  \\\n",
       "0  {\"title\":\"IBM Sees Holographic Calls Air Breat...         business   \n",
       "1  {\"title\":\"The Fully Electronic Futuristic Star...       recreation   \n",
       "2  {\"title\":\"Fruits that Fight the Flu fruits tha...           health   \n",
       "3  {\"title\":\"10 Foolproof Tips for Better Sleep \"...           health   \n",
       "4  {\"title\":\"The 50 Coolest Jerseys You Didn t Kn...           sports   \n",
       "\n",
       "  alchemy_category_score  avglinksize  commonlinkratio_1  commonlinkratio_2  \\\n",
       "0               0.789131     2.055556           0.676471           0.205882   \n",
       "1               0.574147     3.677966           0.508021           0.288770   \n",
       "2               0.996526     2.382883           0.562016           0.321705   \n",
       "3               0.801248     1.543103           0.400000           0.100000   \n",
       "4               0.719157     2.676471           0.500000           0.222222   \n",
       "\n",
       "   commonlinkratio_3  commonlinkratio_4  \\\n",
       "0           0.047059           0.023529   \n",
       "1           0.213904           0.144385   \n",
       "2           0.120155           0.042636   \n",
       "3           0.016667           0.000000   \n",
       "4           0.123457           0.043210   \n",
       "\n",
       "                         ...                          linkwordscore  \\\n",
       "0                        ...                                     24   \n",
       "1                        ...                                     40   \n",
       "2                        ...                                     55   \n",
       "3                        ...                                     24   \n",
       "4                        ...                                     14   \n",
       "\n",
       "   news_front_page  non_markup_alphanum_characters  numberOfLinks  \\\n",
       "0                0                            5424            170   \n",
       "1                0                            4973            187   \n",
       "2                0                            2240            258   \n",
       "3                0                            2737            120   \n",
       "4                0                           12032            162   \n",
       "\n",
       "   numwords_in_url  parametrizedLinkRatio  spelling_errors_ratio label  \\\n",
       "0                8               0.152941               0.079130     0   \n",
       "1                9               0.181818               0.125448     1   \n",
       "2               11               0.166667               0.057613     1   \n",
       "3                5               0.041667               0.100858     1   \n",
       "4               10               0.098765               0.082569     0   \n",
       "\n",
       "                                               title  \\\n",
       "0  IBM Sees Holographic Calls Air Breathing Batte...   \n",
       "1  The Fully Electronic Futuristic Starting Gun T...   \n",
       "2  Fruits that Fight the Flu fruits that fight th...   \n",
       "3                10 Foolproof Tips for Better Sleep    \n",
       "4  The 50 Coolest Jerseys You Didn t Know Existed...   \n",
       "\n",
       "                                                body  \n",
       "0  A sign stands outside the International Busine...  \n",
       "1  And that can be carried on a plane without the...  \n",
       "2  Apples The most popular source of antioxidants...  \n",
       "3  There was a period in my life when I had a lot...  \n",
       "4  Jersey sales is a curious business Whether you...  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "\n",
    "data = pd.read_csv(\"../../assets/dataset/stumbleupon.tsv\", sep='\\t')\n",
    "data['title'] = data.boilerplate.map(lambda x: json.loads(x).get('title', ''))\n",
    "data['body'] = data.boilerplate.map(lambda x: json.loads(x).get('body', ''))\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predicting \"Greenness\" Of Content"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This dataset comes from [stumbleupon](https://www.stumbleupon.com/), a web page recommender.  \n",
    "\n",
    "A description of the columns is below"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "FieldName|Type|Description\n",
    "---------|----|-----------\n",
    "url|string|Url of the webpage to be classified\n",
    "title|string|Title of the article\n",
    "body|string|Body text of article\n",
    "urlid|integer| StumbleUpon's unique identifier for each url\n",
    "boilerplate|json|Boilerplate text\n",
    "alchemy_category|string|Alchemy category (per the publicly available Alchemy API found at www.alchemyapi.com)\n",
    "alchemy_category_score|double|Alchemy category score (per the publicly available Alchemy API found at www.alchemyapi.com)\n",
    "avglinksize| double|Average number of words in each link\n",
    "commonlinkratio_1|double|# of links sharing at least 1 word with 1 other links / # of links\n",
    "commonlinkratio_2|double|# of links sharing at least 1 word with 2 other links / # of links\n",
    "commonlinkratio_3|double|# of links sharing at least 1 word with 3 other links / # of links\n",
    "commonlinkratio_4|double|# of links sharing at least 1 word with 4 other links / # of links\n",
    "compression_ratio|double|Compression achieved on this page via gzip (measure of redundancy)\n",
    "embed_ratio|double|Count of number of <embed> usage\n",
    "frameBased|integer (0 or 1)|A page is frame-based (1) if it has no body markup but have a frameset markup\n",
    "frameTagRatio|double|Ratio of iframe markups over total number of markups\n",
    "hasDomainLink|integer (0 or 1)|True (1) if it contains an <a> with an url with domain\n",
    "html_ratio|double|Ratio of tags vs text in the page\n",
    "image_ratio|double|Ratio of <img> tags vs text in the page\n",
    "is_news|integer (0 or 1) | True (1) if StumbleUpon's news classifier determines that this webpage is news\n",
    "lengthyLinkDomain| integer (0 or 1)|True (1) if at least 3 <a> 's text contains more than 30 alphanumeric characters\n",
    "linkwordscore|double|Percentage of words on the page that are in hyperlink's text\n",
    "news_front_page| integer (0 or 1)|True (1) if StumbleUpon's news classifier determines that this webpage is front-page news\n",
    "non_markup_alphanum_characters|integer| Page's text's number of alphanumeric characters\n",
    "numberOfLinks|integer Number of <a>|markups\n",
    "numwords_in_url| double|Number of words in url\n",
    "parametrizedLinkRatio|double|A link is parametrized if it's url contains parameters or has an attached onClick event\n",
    "spelling_errors_ratio|double|Ratio of words not found in wiki (considered to be a spelling mistake)\n",
    "label|integer (0 or 1)|User-determined label. Either evergreen (1) or non-evergreen (0); available for train.tsv only"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> ### Let's try extracting some of the text content.\n",
    "> ### Create a feature for the title containing 'recipe'. Is the % of evegreen websites higher or lower on pages that have recipe in the the title?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Option 1: Create a function to check for this\n",
    "\n",
    "def has_recipe(text_in):\n",
    "    try:\n",
    "        if 'recipe' in str(text_in).lower():\n",
    "            return 1\n",
    "        else:\n",
    "            return 0\n",
    "    except: \n",
    "        return 0\n",
    "        \n",
    "data['recipe'] = data['title'].map(has_recipe)\n",
    "\n",
    "# Option 2: lambda functions\n",
    "\n",
    "#data['recipe'] = data['title'].map(lambda t: 1 if 'recipe' in str(t).lower() else 0)\n",
    "\n",
    "\n",
    "# Option 3: string functions\n",
    "data['recipe'] = data['title'].str.contains('recipe')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Demo: Use of the Count Vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "titles = data['title'].fillna('')\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "vectorizer = CountVectorizer(max_features = 1000, \n",
    "                             ngram_range=(1, 2), \n",
    "                             stop_words='english',\n",
    "                             binary=True)\n",
    "\n",
    "# Use `fit` to learn the vocabulary of the titles\n",
    "vectorizer.fit(titles)\n",
    "\n",
    "# Use `tranform` to generate the sample X word matrix - one column per feature (word or n-grams)\n",
    "X = vectorizer.transform(titles)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Demo: Build a random forest model to predict evergreeness of a website using the title features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV AUC [ 0.78368418  0.80482115  0.80396444], Average AUC 0.797489919631\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "model = RandomForestClassifier(n_estimators = 20)\n",
    "    \n",
    "# Use `fit` to learn the vocabulary of the titles\n",
    "vectorizer.fit(titles)\n",
    "\n",
    "# Use `tranform` to generate the sample X word matrix - one column per feature (word or n-grams)\n",
    "X = vectorizer.transform(titles).toarray()\n",
    "y = data['label']\n",
    "\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "\n",
    "scores = cross_val_score(model, X, y, scoring='roc_auc')\n",
    "print('CV AUC {}, Average AUC {}'.format(scores, scores.mean()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exercise: Build a random forest model to predict evergreeness of a website using the title features and quantitative features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV AUC [ 0.78106108  0.79495949  0.79300504], Average AUC 0.789675203793\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Features</th>\n",
       "      <th>Importance Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>html_ratio</td>\n",
       "      <td>0.151065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1001</th>\n",
       "      <td>image_ratio</td>\n",
       "      <td>0.095441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>712</th>\n",
       "      <td>recipe</td>\n",
       "      <td>0.039477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>721</th>\n",
       "      <td>recipes</td>\n",
       "      <td>0.019451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>chocolate</td>\n",
       "      <td>0.014175</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Features  Importance Score\n",
       "1000   html_ratio          0.151065\n",
       "1001  image_ratio          0.095441\n",
       "712        recipe          0.039477\n",
       "721       recipes          0.019451\n",
       "190     chocolate          0.014175"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use `tranform` to generate the sample X word matrix - one column per feature (word or n-grams)\n",
    "X_text_features = vectorizer.transform(titles)\n",
    "\n",
    "# Identify the features you want from the original dataset\n",
    "other_features_columns = ['html_ratio', 'image_ratio']\n",
    "other_features = data[other_features_columns]\n",
    "\n",
    "# Stack them horizontally together\n",
    "# This takes all of the word/n-gram columns and appends on two more columns for `html_ratio` and `image_ratio`\n",
    "from scipy.sparse import hstack\n",
    "X = hstack((X_text_features, other_features)).toarray()\n",
    "\n",
    "scores = cross_val_score(model, X, y, scoring='roc_auc')\n",
    "print('CV AUC {}, Average AUC {}'.format(scores, scores.mean()))\n",
    "\n",
    "# What features of these are most important?\n",
    "model.fit(X, y)\n",
    "\n",
    "all_feature_names = vectorizer.get_feature_names() + other_features_columns\n",
    "feature_importances = pd.DataFrame({'Features' : all_feature_names, 'Importance Score': model.feature_importances_})\n",
    "feature_importances.sort_values('Importance Score', ascending=False).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Exercise: Build a random forest model to predict evergreeness of a website using the body features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV AUC [ 0.84035249  0.84910507  0.83544423], Average AUC 0.841633929285\n"
     ]
    }
   ],
   "source": [
    "body_text = data['body'].fillna('')\n",
    "\n",
    "# Use `fit` to learn the vocabulary\n",
    "vectorizer.fit(body_text)\n",
    "\n",
    "# Use `tranform` to generate the sample X word matrix - one column per feature (word or n-grams)\n",
    "X = vectorizer.transform(body_text).toarray()\n",
    "\n",
    "scores = cross_val_score(model, X, y, scoring='roc_auc')\n",
    "print('CV AUC {}, Average AUC {}'.format(scores, scores.mean()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Exercise: Use `TfIdfVectorizer` instead of `CountVectorizer` - is this an improvement?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV AUC [ 0.84249638  0.85254216  0.84371034], Average AUC 0.846249626246\n"
     ]
    }
   ],
   "source": [
    "titles = data['title'].fillna('')\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "vectorizer = TfidfVectorizer(max_features = 1000, \n",
    "                             ngram_range=(1, 2), \n",
    "                             stop_words='english')\n",
    "\n",
    "\n",
    "# Use `fit` to learn the vocabulary\n",
    "vectorizer.fit(body_text)\n",
    "\n",
    "# Use `tranform` to generate the sample X word matrix - one column per feature (word or n-grams)\n",
    "X = vectorizer.transform(body_text).toarray()\n",
    "\n",
    "scores = cross_val_score(model, X, y, scoring='roc_auc')\n",
    "print('CV AUC {}, Average AUC {}'.format(scores, scores.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
{'fraction_non_alphanumeric': 0.18658974026535285, 'fraction_numerical': 0.052206231902848, 'mean_word_length': 1.596442998204668, 'pattern_counts': {'":': 254, '<': 697, '<?xml version=': 0, '>': 701, 'https://': 2, 'lorem ipsum': 0, 'www.': 22, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 80, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2650584', 'n_tokens_mistral': 17692, 'n_tokens_neox': 16166, 'n_words': 3339}