<jupyter_start><jupyter_text>RUN MODEL<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.pipeline import make_pipeline, make_union
from sklearn.cross_validation import train_test_split
from sklearn.decomposition import TruncatedSVD
from sklearn.base import TransformerMixin
from xgboost import XGBClassifier
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
import pandas as pd
import numpy as np
import warnings
import pickle
import re
% matplotlib inline<jupyter_output><empty_output><jupyter_text>TRANSFORMERS<jupyter_code>class HtmlTransformer(TransformerMixin):
    def transform(self, X, **transform_params):
        cleaner = re.compile(r'<.*?>')
        return pd.Series(X.apply(lambda x: re.sub(cleaner,' ',x)))
    def fit(self, X, y=None, **fit_params):
        return self
    
class RemoveCamelCaseTransformer(TransformerMixin):
    def transform(self, X, **transform_params):
        return pd.Series(X.apply(lambda x: re.sub(r'([a-z](?=[A-Z])|[A-Z](?=[A-Z][a-z]))', r'\1 ',x)))
    def fit(self, X, y=None, **fit_params):
        return self
    
class RemoveSymsTransformer(TransformerMixin):
    def transform(self, X, **transform_params):
        return pd.Series(X.apply(lambda x: re.sub(re.compile(r'[^A-za-z0-9\s\.]'),' ',x)))
    def fit(self, X, y=None, **fit_params):
        return self
    
class TokenizeTransformer(TransformerMixin):
    def transform(self, X, **transform_params):
        return pd.Series(X.apply(lambda x:word_tokenize(x)))
    def fit(self, X, y=None, **fit_params):
        return self

class LemmatizeTransformer(TransformerMixin):
    def transform(self, X, **transform_params):
        lmtzr = WordNetLemmatizer()
        return pd.Series(X.apply(lambda x:[lmtzr.lemmatize(token.lower()) for token in x]))
    def fit(self, X, y=None, **fit_params):
        return self

class RemoveStopWordsTransformer(TransformerMixin):
    def transform(self, X, **transform_params):
        stop = set(stopwords.words('english'))
        return pd.Series(X.apply(lambda x: ' '.join([token for token in x if token not in stop])))
    def fit(self, X, y=None, **fit_params):
        return self<jupyter_output><empty_output><jupyter_text>LOAD MODELS<jupyter_code>with open('models/MODEL_XG_Boost_Product_Long_Description.plk','rb') as f:
    XGB_MODEL_PLD = pickle.load(f)
    
with open('models/MODEL_XG_Boost_Product_Name.plk','rb') as f:
    XGB_MODEL_PN = pickle.load(f)<jupyter_output><empty_output><jupyter_text>LOAD TEST SET<jupyter_code>df = pd.read_csv('data/test.tsv',sep='\t')
keep = set(['item_id','Product Long Description','Product Name'])
drop = set(df.columns) - keep
df.drop(drop,axis=1,inplace=True)
df.head(7)
df.shape<jupyter_output><empty_output><jupyter_text>REPLACE NaNs<jupyter_code>text = 'This is a super fake text block that I am writing to replace the NaNs. \
I should have account for these 6 NaNs in the test set. Next time, I will account for this!'
nan_indx = [2335,4098,6132,8174,9421,10209]
labels = ['Product Long Description','Product Name']
for label in labels:
    for indx in nan_indx:
        df.loc[indx,label] = text<jupyter_output><empty_output><jupyter_text>GET PREDICTIONS<jupyter_code>X_test1 = df['Product Name']
X_test2 = df['Product Long Description']
y_pred1 = XGB_MODEL_PN.predict(X_test1)
y_pred2 = XGB_MODEL_PLD.predict(X_test2)
y_pred1[5]
y_pred2[5]<jupyter_output><empty_output><jupyter_text>ENSEMBLE MODEL PREDICTIONS<jupyter_code>pred_bin = []
for indx,ID in enumerate(df.item_id):
    if y_pred1[indx] == y_pred2[indx]:
        pred = y_pred1[indx]
    else:
        tag1 = re.findall('\d+\d',y_pred1[indx])
        tag2 = re.findall('\d+\d',y_pred2[indx])
        pred = '['+tag1[0]+', '+tag2[0]+']'
    pred_bin.append(pred)<jupyter_output><empty_output><jupyter_text>SAVE TO TSV<jupyter_code>d = {}
d['item_id'] = list(df.item_id)
d['tag'] = pred_bin
pred_df = pd.DataFrame(d)
pred_df.to_csv('tags.tsv',sep='\t',index=False)
pred_df.head()
pred_df.tag.value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13713592233009708, 'fraction_numerical': 0.01262135922330097, 'mean_word_length': 5.187687687687688, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '136719', 'license': 'MIT', 'n_tokens_mistral': 1554, 'n_tokens_neox': 1526, 'n_words': 299, 'path': 'walmart-hackerrank_competition/run_model.ipynb', 'repo_name': 'marwin-ko/projects', 'size': '4120'}