<jupyter_start><jupyter_text><NAME> Login<jupyter_code>from google.colab import drive

drive.mount("/content/drive",force_remount=True)

%cd /content/drive/MyDrive/DataDays2021/

pwd = %pwd
pwd<jupyter_output>Mounted at /content/drive
/content/drive/MyDrive/DataDays2021<jupyter_text>**-------------------------------------------------------------------------------------------------------------------------------**  installing packages<jupyter_code>!pip install "dask[complete]"
!pip install -Uqq fastai
!pip install parsivar<jupyter_output>Requirement already satisfied: dask[complete] in /usr/local/lib/python3.7/dist-packages (2.12.0)
Collecting partd>=0.3.10
  Downloading partd-1.2.0-py3-none-any.whl (19 kB)
Requirement already satisfied: bokeh>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from dask[complete]) (2.3.3)
Collecting distributed>=2.0
  Downloading distributed-2021.8.0-py3-none-any.whl (776 kB)
[K     |████████████████████████████████| 776 kB 7.5 MB/s 
[?25hRequirement already satisfied: toolz>=0.7.3 in /usr/local/lib/python3.7/dist-packages (from dask[complete]) (0.11.1)
Requirement already satisfied: PyYaml in /usr/local/lib/python3.7/dist-packages (from dask[complete]) (3.13)
Requirement already satisfied: numpy>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from dask[complete]) (1.19.5)
Collecting fsspec>=0.6.0
  Downloading fsspec-2021.7.0-py3-none-any.whl (118 kB)
[K     |████████████████████████████████| 118 kB 68.3 MB/s 
[?25hRequirement already satisfied: pandas>=0.23.0 in /usr/local/lib/[...]<jupyter_text>Imports<jupyter_code>import dask.dataframe as dd
import pandas as pd
import os
from fastai.text.all import *
from fastai.tabular.all import *
import pickle

from torch.utils.data import Dataset
from tqdm import tqdm
import torch as T
from fastai.data.core import DataLoaders
import torch.nn as nn
from torch.utils.data.dataloader import DataLoader

from helper import utils<jupyter_output><empty_output><jupyter_text>declaring paths<jupyter_code>data_folder = os.path.join(pwd,"data")
final_clicked = os.path.join(data_folder,"final","final_clicked.csv")
final_products = os.path.join(data_folder,"final","final_products.csv")
mix_model_path = os.path.join(data_folder,"models","mix_model")
category_classifier_path = os.path.join(data_folder,"models","category_classifier")
cdf = dd.read_csv(final_clicked)
cdf.head()<jupyter_output><empty_output><jupyter_text>loading vocab<jupyter_code>#if vocab exits:
vocab = pickle.load(open(os.path.join(category_classifier_path,"vocab"),'rb'))
vocab<jupyter_output><empty_output><jupyter_text>nlp trasnforms pipeline<jupyter_code>class nlp_pipeline:
    def __init__(self, vocab,):
        self.vocab = vocab
        self.tok = SpacyTokenizer(lang='fa')
        self.num = Numericalize(vocab=self.vocab)

    def encode(self,x):
        x = utils._normalize_text(x)
        x = tokenize1(x, self.tok)
        x = self.num.encodes(x)
        return x

    def decode(self,x):
        x = self.num.decodes(x)
        x = " ".join(x)
        return x
drop_cols =["Unnamed: 0",
            "product_showed",
            "DAY(datetime)",
            "HOUR(datetime)",
            "IS_WEEKEND(datetime)",
            "MINUTE(datetime)",
            "MONTH(datetime)",
            "WEEKDAY(datetime)",
            "YEAR(datetime)",
            "_id"]

cont_cols = ["rank",
             "NUM_WORDS(raw_query)",
             "page",
             "products.sellers_count",
             "products.availabilty_ratio",
             "products.mean_all_price",
             "products.max_all_price",
             "products.min_all_price",
             "products.std_all_price",
             "products.skew_all_price",
             "products.mean_available_price",
             "products.max_available_price",
             "products.min_available_price",
             "products.std_available_price",
             "products.skew_available_price",
             "products.COUNT(clicked_merged)",
             "products.PERCENT_TRUE(clicked_merged.is_clicked)",
             "products.NUM_WORDS(product_name_normalized)"]

cat_cols = ['products.DAY(first_added_date)',
            'products.DAY(last_added_date)',
            'products.HOUR(first_added_date)',
            'products.HOUR(last_added_date)',
            'products.IS_WEEKEND(first_added_date)',
            'products.IS_WEEKEND(last_added_date)',
            'products.MINUTE(first_added_date)',
            'products.MINUTE(last_added_date)',
            'products.MONTH(first_added_date)',
            'products.MONTH(last_added_date)',
            'products.WEEKDAY(first_added_date)',
            'products.WEEKDAY(last_added_date)',
            'products.YEAR(first_added_date)',
            'products.YEAR(last_added_date)']

text_col = ['raw_query',
            'products.category_name',
            'products.product_name_normalized',]

target = 'is_clicked'
df = cdf.drop(drop_cols,axis=1).compute()
df

class MixDataSet(Dataset):
    def __init__(self, df, cat_col_names, cont_col_names, query_col_name, category_col_name, product_col_name, vocab,target_col_name=None, test=False,normalize=True,):
        self.df = df.reset_index(drop=True).copy()
        self.cat_col_names = cat_col_names
        self.cont_col_names = cont_col_names
        self.query_col_name = query_col_name
        self.category_col_name = category_col_name
        self.product_col_name = product_col_name
        self.target_col_name = target_col_name


        self.test = test

        self.normalize = normalize

        self.vocab = vocab
        self.nlp_pipeline = nlp_pipeline(vocab=self.vocab)

        if self.test == False:
            self.target_col_name = target_col_name


    def __len__(self):
        return self.df.shape[0]
    
    def __getitem__(self, i):
        ndf = self.df.iloc[i]
        ndf = ndf.fillna(0.0,inplace=False)

        if self.normalize == True:
            if type(ndf) == pd.core.series.Series:
             for col in ndf.index:
                if "price" in col:
                    ndf[col] = np.log10(ndf[col]/1000 + 1)
                elif "NUM" in col or "COUNT" in col:
                    ndf[col] = np.log10(ndf[col] + 1)
           
            elif type(ndf) == pd.core.frame.DataFrame:
             for col in ndf.columns:
                if "price" in col:
                    ndf[col] = np.log10(ndf[col]/1000 + 1)
                elif "NUM" in col or "COUNT" in col:
                    ndf[col] = np.log10(ndf[col] + 1)

        for col in self.cat_col_names:
            if "YEAR" in col:
                ndf[col] = ndf[col] - 2017

            
        cat = ndf[self.cat_col_names].values

        cont = ndf[self.cont_col_names].values

        query = self.nlp_pipeline.encode(ndf[self.query_col_name])
        category = self.nlp_pipeline.encode(ndf[self.category_col_name])
        product = self.nlp_pipeline.encode(ndf[self.product_col_name])
        
        if self.test == False:
            target = ndf[self.target_col_name]
            return (T.tensor(cat.astype(np.int32)),T.tensor(cont.astype(np.float32)),query,category,product), T.tensor(target)
        else:
            return (T.tensor(cat.astype(np.int32)),T.tensor(cont.astype(np.float32)),query,category,product)

stratified_df = pd.concat([df[df["is_clicked"] == True].reset_index(drop=True)[0:10000],
                           df[df["is_clicked"] == False].reset_index(drop=True)[0:10000]]).reset_index(drop=True).sample(frac=1)
stratified_df.groupby(by="is_clicked").count()
b = MixDataSet(stratified_df,cat_cols,cont_cols,"raw_query","products.category_name","products.product_name_normalized",target_col_name="is_clicked",vocab=vocab[0])
b[0:10]
len(b)
stratified_df["is_valid"] = False
stratified_df["is_valid"] = stratified_df["is_valid"].apply(lambda x: True if random.random() < 0.08 else False)
def my_collate(batch):
    b = list(zip(*batch))
    x,y = b
    x1,x2,x3,x4,x5 = list(zip(*x))

    return (T.stack(x1),T.stack(x2), nn.utils.rnn.pad_sequence(x3).T, nn.utils.rnn.pad_sequence(x4).T, nn.utils.rnn.pad_sequence(x5).T), T.stack(y).to(T.long)
train_ds = MixDataSet(stratified_df[stratified_df["is_valid"]==False],
                      cat_cols,
                      cont_cols,
                      "raw_query",
                      "products.category_name",
                      "products.product_name_normalized",
                      target_col_name="is_clicked",
                      vocab=vocab[0])

valid_ds = MixDataSet(stratified_df[stratified_df["is_valid"]==True],
                      cat_cols,
                      cont_cols,
                      "raw_query",
                      "products.category_name",
                      "products.product_name_normalized",
                      target_col_name="is_clicked",
                      vocab = vocab[0])
len(train_ds), len(valid_ds)
train_dl = DataLoader(dataset=train_ds, batch_size=150, shuffle=True, collate_fn=my_collate)
valid_dl = DataLoader(dataset=valid_ds, batch_size=150, shuffle=True, collate_fn=my_collate)
next(iter(train_dl))
next(iter(valid_dl))
class MixModel(nn.Module):
    def __init__(self, embed_sz, n_cont, layers, awd_config, vocab_sz, lin_ftrs,joint_layers):
        super(MixModel, self).__init__()

        self.embed_sz  = embed_sz
        self.n_cont = n_cont
        self.layers = layers
        self.awd_config = awd_config
        self.joint_layers = joint_layers

        self.tab_model = TabularModel(emb_szs=self.embed_sz, n_cont=n_cont, out_sz=2,layers=layers,)
        self.tab_model.layers = self.tab_model.layers[:-1]

        self.awd1 = get_text_classifier(AWD_LSTM,vocab_sz=vocab_sz, n_class=2, config=awd_config,lin_ftrs=lin_ftrs)
        self.awd1[-1].layers = self.awd1[-1].layers[:-1]

        self.awd2 = get_text_classifier(AWD_LSTM,vocab_sz=vocab_sz, n_class=2, config=awd_config,lin_ftrs=lin_ftrs)
        self.awd2[-1].layers = self.awd2[-1].layers[:-1]

        self.awd3 = get_text_classifier(AWD_LSTM,vocab_sz=vocab_sz, n_class=2, config=awd_config,lin_ftrs=lin_ftrs)
        self.awd3[-1].layers = self.awd3[-1].layers[:-1]

        self.joint_layers = [lin_ftrs[-1]*3 + layers[-1]] + joint_layers + [2]
        linBins = []
        for i in range(0,len(self.joint_layers)-1):
            linBins.append(LinBnDrop(self.joint_layers[i],self.joint_layers[i+1]))

        self.LinBins = nn.Sequential(*linBins)
        self.Softmax = nn.Softmax(dim=1)

    def forward(self, x):
        xtab = self.tab_model(x[0],x[1])
        xnlp1 = self.awd1(x[2])[0]
        xnlp2 = self.awd2(x[3])[0]
        xnlp3 = self.awd3(x[4])[0]

        X = T.cat([xtab, xnlp1, xnlp2, xnlp3],dim=1)
        X = self.LinBins(X)
        return self.Softmax(X)

    def reset(self):
        self.awd1.reset()
        self.awd2.reset()
        self.awd3.reset()
for col in cat_cols:
    print(col,"-->", df[col].nunique())
len(cont_cols)
emb_sz = [
          (32, 20),
          (32, 20),
          (24, 20),
          (24, 20),
          (3, 10),
          (3, 10),
          (61, 30),
          (61, 30),
          (13, 10),
          (13, 10),
          (8, 10),
          (8, 10),
          (6, 20),
          (6, 20)
]
awd_conf = {'bidir': True,
            'emb_sz': 1000,
            'embed_p': 0.05,
            'hidden_p': 0.3,
            'input_p': 0.4,
            'n_hid': 1000,
            'n_layers': 3,
            'output_p': 0.4,
            'pad_token': 1,
            'weight_p': 0.5}
mix_model = MixModel(embed_sz = emb_sz, n_cont= len(cont_cols), layers=[200], awd_config=awd_conf, vocab_sz=len(vocab[0]), lin_ftrs=[500],joint_layers=[500])
state_dict = T.load(open(os.path.join(mix_model_path,"models","mix_model.pth"),'rb'))
mix_model.load_state_dict(state_dict)
mix_model
sum(p.numel() for p in mix_model.parameters())
a = next(iter(train_dl))
tab = mix_model.tab_model(a[0][0],a[0][1])
nlp1 = mix_model.awd1(a[0][2])
nlp2 = mix_model.awd2(a[0][3])
nlp3 = mix_model.awd3(a[0][4])

print(tab.shape, nlp1[0].shape, nlp2[0].shape, nlp3[0].shape)
mix_model(a[0])
fast_mix_dl = DataLoaders(train_dl,valid_dl)
loss_func = nn.CrossEntropyLoss()

learn = Learner(fast_mix_dl,
                mix_model,
                loss_func = loss_func,
                path = mix_model_path,
                metrics=[accuracy,error_rate,Recall(),Precision(),F1Score()]).to_fp16()

grp = ShowGraphCallback
svm = SaveModelCallback(at_end=False,every_epoch=False,reset_on_fit=False,monitor='f1_score',fname="mix_model",)
esc = EarlyStoppingCallback(patience=3)
rlr = ReduceLROnPlateau(monitor="valid_loss",patience=2,factor=10,)
learn.add_cbs([grp,svm,esc,rlr,ModelResetter])
learn.cbs
learn.lr_find()
learn.fine_tune(10,1e-04,10)
stratified_df = pd.concat([df[df["is_clicked"] == True].reset_index(drop=True)[0:30000],
                           df[df["is_clicked"] == False].reset_index(drop=True)[0:30000]]).reset_index(drop=True).sample(frac=1)

stratified_df["is_valid"] = False
stratified_df["is_valid"] = stratified_df["is_valid"].apply(lambda x: True if random.random() < 0.08 else False)


train_ds1 = MixDataSet(stratified_df[stratified_df["is_valid"]==False],
                      cat_cols,
                      cont_cols,
                      "raw_query",
                      "products.category_name",
                      "products.product_name_normalized",
                      target_col_name="is_clicked",
                      vocab=vocab[0])

valid_ds1 = MixDataSet(stratified_df[stratified_df["is_valid"]==True],
                      cat_cols,
                      cont_cols,
                      "raw_query",
                      "products.category_name",
                      "products.product_name_normalized",
                      target_col_name="is_clicked",
                      vocab = vocab[0])

train_dl1 = DataLoader(dataset=train_ds1, batch_size=200, shuffle=True, collate_fn=my_collate)
valid_dl1 = DataLoader(dataset=valid_ds1, batch_size=200, shuffle=True, collate_fn=my_collate)


fast_mix_dl1 = DataLoaders(train_dl1,valid_dl1)

loss_func = nn.CrossEntropyLoss()

learn = Learner(fast_mix_dl1,
                mix_model,
                loss_func = loss_func,
                path = mix_model_path,
                metrics=[accuracy,error_rate,Recall(),Precision(),F1Score()]).to_fp16()

grp = ShowGraphCallback
svm = SaveModelCallback(at_end=False,every_epoch=False,reset_on_fit=False,monitor='f1_score',fname="mix_model",)
esc = EarlyStoppingCallback(patience=3)
rlr = ReduceLROnPlateau(monitor="valid_loss",patience=2,factor=10,)
learn.add_cbs([grp,svm,esc,rlr,ModelResetter])
learn.cbs
learn.fine_tune(10,1e-04,10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1468245864815193, 'fraction_numerical': 0.02695527874208699, 'mean_word_length': 2.77880658436214, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '53118', 'license': 'MIT', 'n_tokens_mistral': 5487, 'n_tokens_neox': 5203, 'n_words': 722, 'path': 'train_mix_model.ipynb', 'repo_name': 'DaraSamii/DataDays2021', 'size': '14697'}