<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Objective of the Notebook. **\n",
    "\n",
    "* Learn how to create new pipelines.\n",
    "* Feature Engineering.\n",
    "* Extract text for different tags and weight them differently to introduce some domain knowledge.\n",
    "* Text Mining.\n",
    "* Parse raw html to extract text content and extract features from it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os, sys\n",
    "import re, json\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.pipeline import FeatureUnion, Pipeline\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from sklearn.feature_selection import f_classif, chi2, SelectKBest\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.externals import joblib\n",
    "\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "from collections import defaultdict\n",
    "\n",
    "basepath = os.path.expanduser('~/Desktop/src/Stumbleupon_classification_challenge/')\n",
    "sys.path.append(os.path.join(basepath, 'src'))\n",
    "\n",
    "np.random.seed(2)\n",
    "\n",
    "from data import load_datasets\n",
    "from models import train_test_split, cross_val_scheme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# intialize Porter Stemmer\n",
    "\n",
    "sns = SnowballStemmer(language='english')\n",
    "por = PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# add some custom stopwords to the list of stopwords \n",
    "custom_stopwords = ['i', 'http', 'www']\n",
    "ENGLISH_STOP_WORDS = set(ENGLISH_STOP_WORDS) | set(custom_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load datasets\n",
    "train, test, sample_sub = load_datasets.load_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['boilerplate'] = list(map(json.loads, train.boilerplate))\n",
    "test['boilerplate'] = list(map(json.loads, test.boilerplate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def decompose_boilerplate(boilerplate_json, key='body'):\n",
    "    return [bp[key] if key in bp and bp[key] else u'' for bp in boilerplate_json]\n",
    "    \n",
    "train_body = decompose_boilerplate(train.boilerplate)\n",
    "train_title = decompose_boilerplate(train.boilerplate, key='title')\n",
    "\n",
    "test_body = decompose_boilerplate(test.boilerplate)\n",
    "test_title = decompose_boilerplate(test.boilerplate, 'title')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['body'] = train_body\n",
    "train['title'] = train_title\n",
    "\n",
    "test['body'] = test_body\n",
    "test['title'] = test_title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Parse():\n",
    "    TAGS = ['h1', 'h2', 'h3', 'h4', 'span',\\\n",
    "            'a', 'label_', 'meta-title', 'meta-description','li']\n",
    "    \n",
    "    @staticmethod\n",
    "    def read_html(urlid):\n",
    "        with open(os.path.join(basepath, 'data/raw/raw_content/'+str(urlid)), 'r', encoding='utf-8', errors='ignore') as infile:\n",
    "            html = infile.read()\n",
    "            infile.close()\n",
    "        return html\n",
    "    \n",
    "    @staticmethod\n",
    "    def parse_html(html):\n",
    "        return BeautifulSoup(html, 'lxml')\n",
    "    \n",
    "    @staticmethod\n",
    "    def remove_tags(html, tags):\n",
    "        for tag in tags:\n",
    "            for el in html.find_all(tag):\n",
    "                el.extract()\n",
    "\n",
    "        return html\n",
    " \n",
    "    @staticmethod\n",
    "    def tag_content(html, tag):\n",
    "        def process(s):\n",
    "            s = s.lower()\n",
    "            s = s.strip()\n",
    "            s = re.sub(r'[^a-z0-9]+', ' ', s)\n",
    "            return s\n",
    "\n",
    "        tags_component = tag.split('-')\n",
    "        attrs = {}\n",
    "        \n",
    "        if len(tags_component) > 1:\n",
    "            tag_name = tags_component[0]\n",
    "            attrs['name'] = tags_component[1]\n",
    "        else:\n",
    "            tag_name = tags_component[0]\n",
    "        \n",
    "        for el in html.find_all(tag_name, attrs):\n",
    "            if len(attrs.keys()) > 0:    \n",
    "                return process(el.get('content', ''))\n",
    "            else:\n",
    "                return process(el.text) if el.text else ''        \n",
    "        return '' # could not find the tag\n",
    "        \n",
    "    def __init__(self, key='urlid'):\n",
    "        self.key = key\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, df):\n",
    "        urlids = df[self.key]\n",
    "        tags_content_dict = defaultdict(list)\n",
    "        \n",
    "        for urlid in urlids.values:\n",
    "            html = self.read_html(urlid)\n",
    "            html = self.parse_html(html)\n",
    "            html = self.remove_tags(html, ['style', 'script'])\n",
    "            \n",
    "            for tag in self.TAGS:\n",
    "                tags_content_dict[tag].append(self.tag_content(html, tag))\n",
    "        \n",
    "        for tag in self.TAGS:\n",
    "            df[tag] = tags_content_dict[tag]\n",
    "        \n",
    "        return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# parse all the raw content\n",
    "parse_train = Parse()\n",
    "train = parse_train.transform(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "parse_test = Parse()\n",
    "test = parse_test.transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl',\n",
       " '/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl_01.npy',\n",
       " '/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl_02.npy',\n",
       " '/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl_03.npy',\n",
       " '/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl_04.npy',\n",
       " '/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl_05.npy',\n",
       " '/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl_06.npy',\n",
       " '/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl_07.npy']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dump parsed content to the disk\n",
    "joblib.dump(train, os.path.join(basepath, 'data/processed/train_raw_content.pkl'))\n",
    "joblib.dump(test, os.path.join(basepath, 'data/processed/test_raw_content.pkl'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load pickle from the disk\n",
    "train = joblib.load(os.path.join(basepath, 'data/processed/train_raw_content.pkl'))\n",
    "test = joblib.load(os.path.join(basepath, 'data/processed/test_raw_content.pkl'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "feature_df = train[train.columns[26:]]\n",
    "feature_df['label'] = train.label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = list(train.columns[27:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "    'test_size': 0.2,\n",
    "    'random_state': 2,\n",
    "    'stratify': train.is_news\n",
    "}\n",
    "\n",
    "itrain, itest = train_test_split.tr_ts_split(len(train), **params)\n",
    "\n",
    "X_train = feature_df.iloc[itrain][features]\n",
    "X_test = feature_df.iloc[itest][features]\n",
    "\n",
    "y_train = feature_df.iloc[itrain].label\n",
    "y_test = feature_df.iloc[itest].label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Task **\n",
    "\n",
    "* Text Preprocessing\n",
    "    * Lowercase all the string, remove stopwords, stem the words.\n",
    "* Decompose the boilerplate into body, title and url\n",
    "* Create text features for these parts and weigh them differently."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class LemmaTokenizer(object):\n",
    "    def __init__(self):\n",
    "        self.wnl = WordNetLemmatizer()\n",
    "    \n",
    "    def __call__(self, doc):\n",
    "        return [self.wnl.lemmatize(t) for t in word_tokenize(doc)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class VarSelect(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, keys):\n",
    "        self.keys = keys\n",
    "        \n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, df):\n",
    "        return df[self.keys]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Weights(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, weight):\n",
    "        self.weight = weight\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X):\n",
    "        return self.weight * X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stem_tokens(x):\n",
    "    return ' '.join([sns.stem(word) for word in word_tokenize(x)])\n",
    "\n",
    "def preprocess_string(s):\n",
    "    return stem_tokens(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "strip_non_words = FunctionTransformer(\n",
    "    lambda x: x.replace(r'[^A-Za-z0-9]+', ' ', regex=True), validate=False)\n",
    "\n",
    "pipeline = Pipeline([\n",
    "            ('strip', strip_non_words),\n",
    "            ('union', FeatureUnion([\n",
    "                    ('h1_', Pipeline([\n",
    "                        ('var', VarSelect(keys='h1')),\n",
    "                        ('tfidf', TfidfVectorizer(ngram_range=(1, 2), min_df=2, tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50))\n",
    "                    ])),\n",
    "                    ('h2_', Pipeline([\n",
    "                        ('var', VarSelect(keys='h2')),\n",
    "                        ('tfidf', TfidfVectorizer(ngram_range=(1, 2), min_df=2, tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50))\n",
    "                    ])),\n",
    "                    ('h3_', Pipeline([\n",
    "                        ('var', VarSelect(keys='h3')),\n",
    "                        ('tfidf', TfidfVectorizer(ngram_range=(1, 2), min_df=2, tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50))\n",
    "                    ])),\n",
    "                    ('h4_', Pipeline([\n",
    "                        ('var', VarSelect(keys='h4')),\n",
    "                        ('tfidf', TfidfVectorizer(ngram_range=(1, 2), min_df=2, tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50))\n",
    "                    ])),\n",
    "                    ('meta_title', Pipeline([\n",
    "                        ('var', VarSelect(keys='meta-title')),\n",
    "                        ('tfidf', TfidfVectorizer(ngram_range=(1, 2), min_df=2, tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50)),\n",
    "                        ('weight', Weights(weight=5))\n",
    "                    ])),\n",
    "                    ('meta_description', Pipeline([\n",
    "                        ('var', VarSelect(keys='meta-description')),\n",
    "                        ('tfidf', TfidfVectorizer(ngram_range=(1, 2), min_df=2, tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50)),\n",
    "                        ('weight', Weights(weight=3))\n",
    "                    ])),\n",
    "                    ('span_', Pipeline([\n",
    "                        ('var', VarSelect(keys='span')),\n",
    "                        ('tfidf', TfidfVectorizer(min_df=2, ngram_range=(1, 2), tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50))\n",
    "                    ])),\n",
    "                    ('lsa_body', Pipeline([\n",
    "                        ('var', VarSelect(keys='body')),\n",
    "                        ('tfidf', TfidfVectorizer(min_df=2, ngram_range=(1, 2), tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=100)),\n",
    "                        ('weight', Weights(weight=20))\n",
    "                    ])),\n",
    "                    ('lsa_title', Pipeline([\n",
    "                        ('var', VarSelect(keys='title')),\n",
    "                        ('tfidf', TfidfVectorizer(ngram_range=(1, 2), min_df=2, tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS,preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50)),\n",
    "                        ('weight', Weights(weight=5))\n",
    "                    ])),\n",
    "                ])),\n",
    "            ('scale', MinMaxScaler()),\n",
    "            ('feat', SelectKBest(chi2, k=100)),\n",
    "            ('model', LogisticRegression())\n",
    "        ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('strip', FunctionTransformer(accept_sparse=False,\n",
       "          func=<function <lambda> at 0x7efea02252f0>, pass_y=False,\n",
       "          validate=False)), ('union', FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('h1_', Pipeline(steps=[('var', VarSelect(keys='h1')), ('tfidf', TfidfVectorizer(analyze...ty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False))])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Private Leaderboard Score: 0.87247 -> 0.86875**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROC AUC score on unseen examples:  0.861660971934\n"
     ]
    }
   ],
   "source": [
    "y_preds = pipeline.predict_proba(X_test)[:, 1]\n",
    "print('ROC AUC score on unseen examples: ', roc_auc_score(y_test, y_preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = feature_df[features]\n",
    "y = feature_df.label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('strip', FunctionTransformer(accept_sparse=False,\n",
       "          func=<function <lambda> at 0x7efea02252f0>, pass_y=False,\n",
       "          validate=False)), ('union', FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('h1_', Pipeline(steps=[('var', VarSelect(keys='h1')), ('tfidf', TfidfVectorizer(analyze...ty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False))])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train on full dataset\n",
    "pipeline.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = pipeline.predict_proba(test[features])[:, 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],<commit_msg>Chi2 Feature Selection and Min Max Scaling<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Objective of the Notebook. **\n",
    "\n",
    "* Learn how to create new pipelines.\n",
    "* Feature Engineering.\n",
    "* Extract text for different tags and weight them differently to introduce some domain knowledge.\n",
    "* Text Mining.\n",
    "* Parse raw html to extract text content and extract features from it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os, sys\n",
    "import re, json\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.pipeline import FeatureUnion, Pipeline\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from sklearn.feature_selection import f_classif, chi2, SelectKBest\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.externals import joblib\n",
    "\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "from collections import defaultdict\n",
    "\n",
    "basepath = os.path.expanduser('~/Desktop/src/Stumbleupon_classification_challenge/')\n",
    "sys.path.append(os.path.join(basepath, 'src'))\n",
    "\n",
    "np.random.seed(2)\n",
    "\n",
    "from data import load_datasets\n",
    "from models import train_test_split, cross_val_scheme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# intialize Porter Stemmer\n",
    "\n",
    "sns = SnowballStemmer(language='english')\n",
    "por = PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# add some custom stopwords to the list of stopwords \n",
    "custom_stopwords = ['i', 'http', 'www']\n",
    "ENGLISH_STOP_WORDS = set(ENGLISH_STOP_WORDS) | set(custom_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load datasets\n",
    "train, test, sample_sub = load_datasets.load_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['boilerplate'] = list(map(json.loads, train.boilerplate))\n",
    "test['boilerplate'] = list(map(json.loads, test.boilerplate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def decompose_boilerplate(boilerplate_json, key='body'):\n",
    "    return [bp[key] if key in bp and bp[key] else u'' for bp in boilerplate_json]\n",
    "    \n",
    "train_body = decompose_boilerplate(train.boilerplate)\n",
    "train_title = decompose_boilerplate(train.boilerplate, key='title')\n",
    "\n",
    "test_body = decompose_boilerplate(test.boilerplate)\n",
    "test_title = decompose_boilerplate(test.boilerplate, 'title')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['body'] = train_body\n",
    "train['title'] = train_title\n",
    "\n",
    "test['body'] = test_body\n",
    "test['title'] = test_title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Parse():\n",
    "    TAGS = ['h1', 'h2', 'h3', 'h4', 'span',\\\n",
    "            'a', 'label_', 'meta-title', 'meta-description','li']\n",
    "    \n",
    "    @staticmethod\n",
    "    def read_html(urlid):\n",
    "        with open(os.path.join(basepath, 'data/raw/raw_content/'+str(urlid)), 'r', encoding='utf-8', errors='ignore') as infile:\n",
    "            html = infile.read()\n",
    "            infile.close()\n",
    "        return html\n",
    "    \n",
    "    @staticmethod\n",
    "    def parse_html(html):\n",
    "        return BeautifulSoup(html, 'lxml')\n",
    "    \n",
    "    @staticmethod\n",
    "    def remove_tags(html, tags):\n",
    "        for tag in tags:\n",
    "            for el in html.find_all(tag):\n",
    "                el.extract()\n",
    "\n",
    "        return html\n",
    " \n",
    "    @staticmethod\n",
    "    def tag_content(html, tag):\n",
    "        def process(s):\n",
    "            s = s.lower()\n",
    "            s = s.strip()\n",
    "            s = re.sub(r'[^a-z0-9]+', ' ', s)\n",
    "            return s\n",
    "\n",
    "        tags_component = tag.split('-')\n",
    "        attrs = {}\n",
    "        \n",
    "        if len(tags_component) > 1:\n",
    "            tag_name = tags_component[0]\n",
    "            attrs['name'] = tags_component[1]\n",
    "        else:\n",
    "            tag_name = tags_component[0]\n",
    "        \n",
    "        for el in html.find_all(tag_name, attrs):\n",
    "            if len(attrs.keys()) > 0:    \n",
    "                return process(el.get('content', ''))\n",
    "            else:\n",
    "                return process(el.text) if el.text else ''        \n",
    "        return '' # could not find the tag\n",
    "        \n",
    "    def __init__(self, key='urlid'):\n",
    "        self.key = key\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, df):\n",
    "        urlids = df[self.key]\n",
    "        tags_content_dict = defaultdict(list)\n",
    "        \n",
    "        for urlid in urlids.values:\n",
    "            html = self.read_html(urlid)\n",
    "            html = self.parse_html(html)\n",
    "            html = self.remove_tags(html, ['style', 'script'])\n",
    "            \n",
    "            for tag in self.TAGS:\n",
    "                tags_content_dict[tag].append(self.tag_content(html, tag))\n",
    "        \n",
    "        for tag in self.TAGS:\n",
    "            df[tag] = tags_content_dict[tag]\n",
    "        \n",
    "        return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# parse all the raw content\n",
    "parse_train = Parse()\n",
    "train = parse_train.transform(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "parse_test = Parse()\n",
    "test = parse_test.transform(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl',\n",
       " '/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl_01.npy',\n",
       " '/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl_02.npy',\n",
       " '/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl_03.npy',\n",
       " '/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl_04.npy',\n",
       " '/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl_05.npy',\n",
       " '/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl_06.npy',\n",
       " '/home/abhishek/Desktop/src/Stumbleupon_classification_challenge/data/processed/test_raw_content.pkl_07.npy']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dump parsed content to the disk\n",
    "joblib.dump(train, os.path.join(basepath, 'data/processed/train_raw_content.pkl'))\n",
    "joblib.dump(test, os.path.join(basepath, 'data/processed/test_raw_content.pkl'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load pickle from the disk\n",
    "train = joblib.load(os.path.join(basepath, 'data/processed/train_raw_content.pkl'))\n",
    "test = joblib.load(os.path.join(basepath, 'data/processed/test_raw_content.pkl'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "feature_df = train[train.columns[26:]]\n",
    "feature_df['label'] = train.label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = list(train.columns[27:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "params = {\n",
    "    'test_size': 0.2,\n",
    "    'random_state': 2,\n",
    "    'stratify': train.is_news\n",
    "}\n",
    "\n",
    "itrain, itest = train_test_split.tr_ts_split(len(train), **params)\n",
    "\n",
    "X_train = feature_df.iloc[itrain][features]\n",
    "X_test = feature_df.iloc[itest][features]\n",
    "\n",
    "y_train = feature_df.iloc[itrain].label\n",
    "y_test = feature_df.iloc[itest].label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Task **\n",
    "\n",
    "* Text Preprocessing\n",
    "    * Lowercase all the string, remove stopwords, stem the words.\n",
    "* Decompose the boilerplate into body, title and url\n",
    "* Create text features for these parts and weigh them differently."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class LemmaTokenizer(object):\n",
    "    def __init__(self):\n",
    "        self.wnl = WordNetLemmatizer()\n",
    "    \n",
    "    def __call__(self, doc):\n",
    "        return [self.wnl.lemmatize(t) for t in word_tokenize(doc)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class VarSelect(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, keys):\n",
    "        self.keys = keys\n",
    "        \n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, df):\n",
    "        return df[self.keys]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Weights(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, weight):\n",
    "        self.weight = weight\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X):\n",
    "        return self.weight * X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stem_tokens(x):\n",
    "    return ' '.join([sns.stem(word) for word in word_tokenize(x)])\n",
    "\n",
    "def preprocess_string(s):\n",
    "    return stem_tokens(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "strip_non_words = FunctionTransformer(\n",
    "    lambda x: x.replace(r'[^A-Za-z0-9]+', ' ', regex=True), validate=False)\n",
    "\n",
    "pipeline = Pipeline([\n",
    "            ('strip', strip_non_words),\n",
    "            ('union', FeatureUnion([\n",
    "                    ('h1_', Pipeline([\n",
    "                        ('var', VarSelect(keys='h1')),\n",
    "                        ('tfidf', TfidfVectorizer(ngram_range=(1, 2), min_df=2, tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50))\n",
    "                    ])),\n",
    "                    ('h2_', Pipeline([\n",
    "                        ('var', VarSelect(keys='h2')),\n",
    "                        ('tfidf', TfidfVectorizer(ngram_range=(1, 2), min_df=2, tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50))\n",
    "                    ])),\n",
    "                    ('h3_', Pipeline([\n",
    "                        ('var', VarSelect(keys='h3')),\n",
    "                        ('tfidf', TfidfVectorizer(ngram_range=(1, 2), min_df=2, tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50))\n",
    "                    ])),\n",
    "                    ('h4_', Pipeline([\n",
    "                        ('var', VarSelect(keys='h4')),\n",
    "                        ('tfidf', TfidfVectorizer(ngram_range=(1, 2), min_df=2, tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50))\n",
    "                    ])),\n",
    "                    ('meta_title', Pipeline([\n",
    "                        ('var', VarSelect(keys='meta-title')),\n",
    "                        ('tfidf', TfidfVectorizer(ngram_range=(1, 2), min_df=2, tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50)),\n",
    "                        ('weight', Weights(weight=5))\n",
    "                    ])),\n",
    "                    ('meta_description', Pipeline([\n",
    "                        ('var', VarSelect(keys='meta-description')),\n",
    "                        ('tfidf', TfidfVectorizer(ngram_range=(1, 2), min_df=2, tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50)),\n",
    "                        ('weight', Weights(weight=3))\n",
    "                    ])),\n",
    "                    ('span_', Pipeline([\n",
    "                        ('var', VarSelect(keys='span')),\n",
    "                        ('tfidf', TfidfVectorizer(min_df=2, ngram_range=(1, 2), tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50))\n",
    "                    ])),\n",
    "                    ('lsa_body', Pipeline([\n",
    "                        ('var', VarSelect(keys='body')),\n",
    "                        ('tfidf', TfidfVectorizer(min_df=2, ngram_range=(1, 2), tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS, preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=100)),\n",
    "                        ('weight', Weights(weight=20))\n",
    "                    ])),\n",
    "                    ('lsa_title', Pipeline([\n",
    "                        ('var', VarSelect(keys='title')),\n",
    "                        ('tfidf', TfidfVectorizer(ngram_range=(1, 2), min_df=2, tokenizer=LemmaTokenizer(), stop_words=ENGLISH_STOP_WORDS,preprocessor=preprocess_string, strip_accents='unicode', norm='l2', sublinear_tf=True)),\n",
    "                        ('svd', TruncatedSVD(n_components=50)),\n",
    "                        ('weight', Weights(weight=5))\n",
    "                    ])),\n",
    "                ])),\n",
    "            ('scale', MinMaxScaler()),\n",
    "            ('feat', SelectKBest(chi2, k=100)),\n",
    "            ('model', LogisticRegression())\n",
    "        ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('strip', FunctionTransformer(accept_sparse=False,\n",
       "          func=<function <lambda> at 0x7efea02252f0>, pass_y=False,\n",
       "          validate=False)), ('union', FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('h1_', Pipeline(steps=[('var', VarSelect(keys='h1')), ('tfidf', TfidfVectorizer(analyze...ty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False))])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Private Leaderboard Score: 0.87341 **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROC AUC score on unseen examples:  0.861660971934\n"
     ]
    }
   ],
   "source": [
    "y_preds = pipeline.predict_proba(X_test)[:, 1]\n",
    "print('ROC AUC score on unseen examples: ', roc_auc_score(y_test, y_preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = feature_df[features]\n",
    "y = feature_df.label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('strip', FunctionTransformer(accept_sparse=False,\n",
       "          func=<function <lambda> at 0x7efea02252f0>, pass_y=False,\n",
       "          validate=False)), ('union', FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('h1_', Pipeline(steps=[('var', VarSelect(keys='h1')), ('tfidf', TfidfVectorizer(analyze...ty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False))])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train on full dataset\n",
    "pipeline.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predictions = pipeline.predict_proba(test[features])[:, 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
{'fraction_non_alphanumeric': 0.188982478942238, 'fraction_numerical': 0.01087255367542303, 'mean_word_length': 2.278970660547451, 'pattern_counts': {'":': 372, '<': 11, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5421880', 'n_tokens_mistral': 14810, 'n_tokens_neox': 14116, 'n_words': 2217}