<jupyter_start><jupyter_text>RecSys RetailRocket  Setup<jupyter_code># !pip install -q -U kaggle
# !pip install --upgrade --force-reinstall --no-deps kaggle
# !mkdir ~/.kaggle
# !cp /content/drive/MyDrive/kaggle.json ~/.kaggle/
# !chmod 600 ~/.kaggle/kaggle.json
# # !kaggle datasets list
!kaggle datasets download -d retailrocket/ecommerce-dataset
!mkdir -p ./data && unzip ecommerce-dataset.zip
!mv ./*.csv ./data && rm ecommerce-dataset.zip
import os
import re
import time
import datetime
from tqdm import tqdm

import numpy as np
import pandas as pd

import bz2
import csv
import json
import operator

from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split<jupyter_output><empty_output><jupyter_text>Data Loading<jupyter_code>events_df = pd.read_csv('./data/events.csv')
category_tree_df = pd.read_csv('./data/category_tree.csv')
item_properties_1_df = pd.read_csv('./data/item_properties_part1.csv')
item_properties_2_df = pd.read_csv('./data/item_properties_part2.csv')
item_prop_df = pd.concat([item_properties_1_df, item_properties_2_df])
item_prop_df.reset_index(drop=True, inplace=True)
del item_properties_1_df
del item_properties_2_df
events_df.head()
item_prop_df.head()<jupyter_output><empty_output><jupyter_text>- Property is the Item's attributes such as category id and availability while the rest are hashed for confidentiality purposes- Value is the item's property value e.g. availability is 1 if there is stock and 0 otherwise- Note: Values that start with "n" indicate that the value preceeding it is a number e.g. n277.200 is equal to `277.2`<jupyter_code>category_tree_df.head()<jupyter_output><empty_output><jupyter_text>EDA Q: what are the items under category id `1016`?<jupyter_code>item_prop_df.loc[(item_prop_df.property == 'categoryid') & (item_prop_df.value == '1016')].sort_values('timestamp').head()<jupyter_output><empty_output><jupyter_text>Q: What is the parent category of `1016`?<jupyter_code>category_tree_df[category_tree_df.categoryid==1016]<jupyter_output><empty_output><jupyter_text>Q: What are items under category `213`?<jupyter_code>item_prop_df.loc[(item_prop_df.property == 'categoryid') & (item_prop_df.value == '213')].sort_values('timestamp').head()<jupyter_output><empty_output><jupyter_text>visitors who bought something, assuming that there were no repeat users with different visitor IDs<jupyter_code>customer_purchased = events_df[events_df.transactionid.notnull()].visitorid.unique()
all_customers = events_df.visitorid.unique()
customer_browsed = [x for x in all_customers if x not in customer_purchased]
print("%d out of %d"%(len(all_customers)-len(customer_browsed), len(all_customers)))<jupyter_output>11719 out of 1407580<jupyter_text>Snapshot of a random session with visitor id 102019<jupyter_code>events_df[events_df.visitorid == 102019].sort_values('timestamp')
def _todatetime(dt):
  return datetime.datetime.fromtimestamp(int(dt/1000)).strftime('%Y-%m-%d %H:%M:%S')

print('Range of transaction dates = ', _todatetime(events_df['timestamp'].min()), 'to', _todatetime(events_df['timestamp'].max()))<jupyter_output>Range of transaction dates =  2015-05-03 03:00:04 to 2015-09-18 02:59:47<jupyter_text>Preprocessing<jupyter_code>def preprocess_events(df):

  # convert unix time to pandas datetime
  df['date'] = pd.to_datetime(df['timestamp'], unit='ms', origin='unix')
  
  # label the events
  # events.event.replace(to_replace=dict(view=1, addtocart=2, transaction=3), inplace=True)

  # convert event to categorical
  df['event_type'] = df['event'].astype('category')

  # # drop the transcationid and timestamp columns
  # df.drop(['transactionid', 'timestamp'], axis=1, inplace=True)

  # # label encode
  # le_users = LabelEncoder()
  # le_items = LabelEncoder()
  # events['visitorid'] = le_users.fit_transform(events['visitorid'])
  # events['itemid'] = le_items.fit_transform(events['itemid'])
  
  # return train, valid, test
  return df
events_processed = preprocess_events(events_df)
events_processed.head()
dfx = events_df.sample(frac=0.01)
def sessionize(events_df: pd.DataFrame):

  session_duration = datetime.timedelta(minutes=30)
  gpby_visitorid = events_df.groupby('visitorid')

  session_list = []
  for a_visitorid in gpby_visitorid.groups:

    visitor_df = events_df.loc[gpby_visitorid.groups[a_visitorid], :].sort_values('date')
    if not visitor_df.empty:
        visitor_df.sort_values('date', inplace=True)

        # Initialise first session
        startdate = visitor_df.iloc[0, :]['date']
        visitorid = a_visitorid
        items_dict = dict([ (i, []) for i in events_df['event_type'].cat.categories ])
        for index, row in visitor_df.iterrows():

            # Check if current event date is within session duration
            if row['date'] - startdate <= session_duration:
            # Add itemid to the list according to event type (i.e. view, addtocart or transaction)
                items_dict[row['event']].append(row['itemid'])
                enddate = row['date']
            else:
                # Complete current session
                session_list.append([visitorid, startdate, enddate] + [ value for key, value in items_dict.items() ])
                # Start a new session
                startdate = row['date']
                items_dict = dict([ (i, []) for i in events_df['event_type'].cat.categories ])
                # Add current itemid
                items_dict[row['event']].append(row['itemid'])

        # If dict if not empty, add item data as last session.
        incomplete_session = False
        for key, value in items_dict.items():
            if value:
                incomplete_session = True
                break
        if incomplete_session:
            session_list.append([visitorid, startdate, enddate] + [value for key, value in items_dict.items()])

  return session_list
session_list = sessionize(dfx)
sessions_df = pd.DataFrame(session_list, columns=['visitorid', 'startdate', 'enddate', 'addtocart', 'transaction', 'view'])
sessions_df.head()
class BaseDataset(object):
    def __init__(self, input_path, output_path):
        super(BaseDataset, self).__init__()

        self.dataset_name = ''
        self.input_path = input_path
        self.output_path = output_path
        self.check_output_path()

        # input file
        self.inter_file = os.path.join(self.input_path, 'inters.dat')
        self.item_file = os.path.join(self.input_path, 'items.dat')
        self.user_file = os.path.join(self.input_path, 'users.dat')
        self.sep = '\t'

        # output file
        self.output_inter_file, self.output_item_file, self.output_user_file = self.get_output_files()

        # selected feature fields
        self.inter_fields = {}
        self.item_fields = {}
        self.user_fields = {}

    def check_output_path(self):
        if not os.path.isdir(self.output_path):
            os.makedirs(self.output_path)

    def get_output_files(self):
        output_inter_file = os.path.join(self.output_path, self.dataset_name + '.inter')
        output_item_file = os.path.join(self.output_path, self.dataset_name + '.item')
        output_user_file = os.path.join(self.output_path, self.dataset_name + '.user')
        return output_inter_file, output_item_file, output_user_file

    def load_inter_data(self) -> pd.DataFrame():
        raise NotImplementedError

    def load_item_data(self) -> pd.DataFrame():
        raise NotImplementedError

    def load_user_data(self) -> pd.DataFrame():
        raise NotImplementedError

    def convert_inter(self):
        try:
            input_inter_data = self.load_inter_data()
            self.convert(input_inter_data, self.inter_fields, self.output_inter_file)
        except NotImplementedError:
            print('This dataset can\'t be converted to inter file\n')

    def convert_item(self):
        try:
            input_item_data = self.load_item_data()
            self.convert(input_item_data, self.item_fields, self.output_item_file)
        except NotImplementedError:
            print('This dataset can\'t be converted to item file\n')

    def convert_user(self):
        try:
            input_user_data = self.load_user_data()
            self.convert(input_user_data, self.user_fields, self.output_user_file)
        except NotImplementedError:
            print('This dataset can\'t be converted to user file\n')

    @staticmethod
    def convert(input_data, selected_fields, output_file):
        output_data = pd.DataFrame()
        for column in selected_fields:
            output_data[column] = input_data.iloc[:, column]
        with open(output_file, 'w') as fp:
            fp.write('\t'.join([selected_fields[column] for column in output_data.columns]) + '\n')
            for i in tqdm(range(output_data.shape[0])):
                fp.write('\t'.join([str(output_data.iloc[i, j])
                                    for j in range(output_data.shape[1])]) + '\n')

    def parse_json(self, data_path):
        with open(data_path, 'rb') as g:
            for l in g:
                yield eval(l)

    def getDF(self, data_path):
        i = 0
        df = {}
        for d in self.parse_json(data_path):
            df[i] = d
            i += 1
        data = pd.DataFrame.from_dict(df, orient='index')
        
        return data
class RETAILROCKETDataset(BaseDataset):
    def __init__(self, input_path, output_path, interaction_type, duplicate_removal):
        super(RETAILROCKETDataset, self).__init__(input_path, output_path)
        self.dataset_name = 'retailrocket'
        self.interaction_type = interaction_type
        assert self.interaction_type in ['view', 'addtocart',
                                         'transaction'], 'interaction_type must be in [view, addtocart, transaction]'
        self.duplicate_removal = duplicate_removal

        # input file
        self.inter_file = os.path.join(self.input_path, 'events.csv')
        self.item_file1 = os.path.join(self.input_path, 'item_properties_part1.csv')
        self.item_file2 = os.path.join(self.input_path, 'item_properties_part2.csv')
        self.sep = ','

        # output file
        if self.interaction_type == 'view':
            self.output_inter_file = os.path.join(self.output_path, 'retailrocket-view.inter')
        elif self.interaction_type == 'addtocart':
            self.output_inter_file = os.path.join(self.output_path, 'retailrocket-addtocart.inter')
        elif self.interaction_type == 'transaction':
            self.output_inter_file = os.path.join(self.output_path, 'retailrocket-transaction.inter')
        self.output_item_file = os.path.join(self.output_path, 'retailrocket.item')

        # selected feature fields
        if self.duplicate_removal:
            if self.interaction_type == 'view':
                self.inter_fields = {0: 'timestamp:float',
                                     1: 'visitor_id:token',
                                     2: 'item_id:token',
                                     3: 'count:float'}
            elif self.interaction_type == 'addtocart':
                self.inter_fields = {0: 'timestamp:float',
                                     1: 'visitor_id:token',
                                     2: 'item_id:token',
                                     3: 'count:float'}
            elif self.interaction_type == 'transaction':
                self.inter_fields = {0: 'timestamp:float',
                                     1: 'visitor_id:token',
                                     2: 'item_id:token',
                                     3: 'count:float'}
        else:
            if self.interaction_type == 'view':
                self.inter_fields = {0: 'timestamp:float',
                                     1: 'visitor_id:token',
                                     2: 'item_id:token'}
            elif self.interaction_type == 'addtocart':
                self.inter_fields = {0: 'timestamp:float',
                                     1: 'visitor_id:token',
                                     2: 'item_id:token'}
            elif self.interaction_type == 'transaction':
                self.inter_fields = {0: 'timestamp:float',
                                     1: 'visitor_id:token',
                                     2: 'item_id:token',
                                     3: 'transaction_id:token'}
        self.item_fields = {0: 'item_timestamp:float',
                            1: 'item_id:token',
                            2: 'property:token',
                            3: 'value:token_seq'}

    def convert_inter(self):
        if self.duplicate_removal:
            fin = open(self.inter_file, "r")
            fout = open(self.output_inter_file, "w")

            lines_count = 0
            for _ in fin:
                lines_count += 1
            fin.seek(0, 0)

            fout.write('\t'.join([self.inter_fields[column] for column in self.inter_fields.keys()]) + '\n')
            dic = {}

            for i in tqdm(range(lines_count)):
                if i == 0:
                    fin.readline()
                    continue
                line = fin.readline()
                line_list = line.split(',')
                key = (line_list[1], line_list[3])
                if line_list[2] == self.interaction_type:
                    if key not in dic:
                        dic[key] = (line_list[0], 1)
                    else:
                        if line_list[0] > dic[key][0]:
                            dic[key] = (line_list[0], dic[key][1] + 1)
                        else:
                            dic[key] = (dic[key][0], dic[key][1] + 1)

            for key in dic.keys():
                fout.write(dic[key][0] + '\t' + key[0] + '\t' + key[1] + '\t' + str(dic[key][1]) + '\n')

            fin.close()
            fout.close()
        else:
            fin = open(self.inter_file, "r")
            fout = open(self.output_inter_file, "w")

            lines_count = 0
            for _ in fin:
                lines_count += 1
            fin.seek(0, 0)

            fout.write('\t'.join([self.inter_fields[column] for column in self.inter_fields.keys()]) + '\n')

            for i in tqdm(range(lines_count)):
                if i == 0:
                    fin.readline()
                    continue
                line = fin.readline()
                line_list = line.split(',')
                if line_list[2] == self.interaction_type:
                    if self.interaction_type != 'transaction':
                        del line_list[4]
                    else:
                        line_list[4] = line_list[4].strip()
                    del line_list[2]
                    fout.write('\t'.join([str(line_list[i]) for i in range(len(line_list))]) + '\n')

            fin.close()
            fout.close()

    def convert_item(self):
        fin1 = open(self.item_file1, "r")
        fin2 = open(self.item_file2, "r")
        fout = open(self.output_item_file, "w")

        lines_count1 = 0
        for _ in fin1:
            lines_count1 += 1
        fin1.seek(0, 0)

        lines_count2 = 0
        for _ in fin2:
            lines_count2 += 1
        fin2.seek(0, 0)

        fout.write('\t'.join([self.item_fields[column] for column in self.item_fields.keys()]) + '\n')

        for i in tqdm(range(lines_count1)):
            if i == 0:
                line = fin1.readline()
                continue
            line = fin1.readline()
            line_list = line.split(',')
            fout.write('\t'.join([str(line_list[i]) for i in range(len(line_list))]))

        for i in tqdm(range(lines_count2)):
            if i == 0:
                line = fin2.readline()
                continue
            line = fin2.readline()
            line_list = line.split(',')
            fout.write('\t'.join([str(line_list[i]) for i in range(len(line_list))]))

        fin1.close()
        fin2.close()
        fout.close()
# data_object = RETAILROCKETDataset('./data', '.', 'view', True)
# data_object.convert_inter()
# data_object.convert_item()<jupyter_output>100%|██████████| 11000000/11000000 [00:23<00:00, 472602.35it/s]
100%|██████████| 9275904/9275904 [00:19<00:00, 478121.32it/s]<jupyter_text>Feature Engineering Page Time<jupyter_code>sessions_df['pages'] = sessions_df['view'].apply(lambda x: len(x))
pages_more_than1 = sessions_df['pages'] > 1
pages_less_than1 = pages_more_than1.apply(lambda x: not x)
sessions_df.loc[pages_more_than1, 'pagetime'] = (sessions_df.loc[pages_more_than1, 'enddate'] - sessions_df.loc[pages_more_than1, 'startdate']) /\
                                                (sessions_df.loc[pages_more_than1, 'pages'] - 1)
sessions_df.loc[pages_less_than1, 'pagetime'] = pd.Timedelta(0)
sessions_df.head(10)<jupyter_output><empty_output><jupyter_text>The rule of thumb on creating a simple yet effective recommender system is to downsample the data without losing quality. It means, you can take only maybe 50 latest transactions for each user and you still get the quality you want because behavior changes over-time.<jupyter_code>trans = events_df[events_df['event'] == 'transaction']
trans2 = trans.groupby(['visitorid']).head(50)
trans2
visitors = trans['visitorid'].unique()
items = trans['itemid'].unique()

trans2['visitors'] = trans2['visitorid'].apply(lambda x : np.argwhere(visitors == x)[0][0])
trans2['items'] = trans2['itemid'].apply(lambda x : np.argwhere(items == x)[0][0])
trans2.head()<jupyter_output><empty_output><jupyter_text>Create the user-item matrix<jupyter_code>from scipy.sparse import csr_matrix

occurences = csr_matrix((visitors.shape[0], items.shape[0]), dtype='int8')

def set_occurences(visitor, item):
    occurences[visitor, item] += 1

trans2.apply(lambda row: set_occurences(row['visitors'], row['items']), axis=1)

occurences<jupyter_output>/usr/local/lib/python3.6/dist-packages/scipy/sparse/_index.py:84: SparseEfficiencyWarning: Changing the sparsity structure of a csr_matrix is expensive. lil_matrix is more efficient.
  self._set_intXint(row, col, x.flat[0])<jupyter_text>Co-occurrence is a better occurrenceLet’s construct an item-item matrix where each element means how many times both items bought together by a user. Call it the co-occurrence matrix.<jupyter_code>cooc = occurences.transpose().dot(occurences)
cooc.setdiag(0)
  # split into train, test and valid
  train, test = train_test_split(events, train_size=0.9)
  train, valid = train_test_split(train, train_size=0.9)
  print('Train:{}, Valid:{}, Test:{}'.format(train.shape,
                                            valid.shape,
                                            test.shape))<jupyter_output><empty_output><jupyter_text>https://nbviewer.jupyter.org/github/tkokkeng/EB5202-RetailRocket/blob/master/retailrocket-features.ipynb ![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAwQAAACoCAYAAABezSnqAAAgAElEQVR4Ae2dTW7cuNOH+4YGDOQu/03gF4Hv4Wwyx0h2mTvMIptZzAm81guSKrKqRFJSf0qtZ4CBWy1+VD31Y5Gl7tingf8gAAEIQAACEIAABCAAgcMSOB3WcxyHAAQgAAEIQAACEIAABAYKAkQAAQhAAAIQgAAEIACBAxOgIDhw8HEdAhCAAAQgAAEIQAACFARoAAIQgAAEIAABCEAAAgcmQEFw4ODjOgQgAAEIQAACEIAABCgI0AAEIAABCEAAAhCAAAQOTICC4MDBx3UI<KEY>AQhQEKABCEAAAhCAAAQgAAEIHJjAgwuC/4YfX96H33sIwL8/htdvo6Xh9ek0vP713x4sx0YIQOCZCfz9PpxOp+H972d2Et9uS0DvxeH1aTh9+TFsZoczGvf2jden03A6vQ4//vXXZ5LTe/6ZQ+hu//31Oto3DIM/Q4z+hXV8CucMf60HuvJrY9eVx758uN/DuzC5ZLArx/ISU7bc96EFwe9vYQHvoSC4kii3rARsgwAEIACBQxLY/F5sCgIXovFwHQ/S4Za/ds2XXV5/z+8dvBP/UMwk6/z1MpvPa9Wz67wRr9nrGnG4xhjX9Gm7Yz2sIEgiDAWBq+q/vA/v4elErPSHYdpuhCkJ4q/0tD6Mo5/Y23626EiLTebW9/SThbFSH9x74amJr+6HYTBj6icrM3ZuVxpY<KEY>)  Matrix factorization model<jupyter_code># store the number of visitors and items in a variable
n_users = events.visitorid.nunique()
n_items = events.itemid.nunique()

# set the number of latent factors
n_latent_factors = 5

# import the required layers
from tensorflow import keras
from tensorflow.keras.layers import Input, Embedding, Flatten

# create input layer for items
item_input = Input(shape=[1],name='Items')

# create embedding layer for items
item_embed = Embedding(n_items,
                       n_latent_factors,
                       name='ItemsEmbedding')(item_input)
item_vec = Flatten(name='ItemsFlatten')(item_embed)

# create the input and embedding layer for users also
user_input = Input(shape=[1],name='Users')
user_embed = Embedding(n_users,
                       n_latent_factors, 
                       name='UsersEmbedding')(user_input)
user_vec = Flatten(name='UsersFlatten')(user_embed)

# create a layer for the dot product of both vector space representations
dot_prod = keras.layers.dot([item_vec, user_vec],axes=[1,1],
                             name='DotProduct')

# build and compile the model
model = keras.Model([item_input, user_input], dot_prod)
model.compile('adam', 'mse')
model.summary()
keras.utils.plot_model(model, 
                       to_file='model.png', 
                       show_shapes=True, 
                       show_layer_names=True)
from IPython import display
display.display(display.Image('model.png'))
# train and evaluate the model
model.fit([train.visitorid.values, train.itemid.values], train.event.values, epochs=50)
score = model.evaluate([test.visitorid, test.itemid], test.event)
print('mean squared error:', score)<jupyter_output><empty_output><jupyter_text>Neural net model<jupyter_code>n_lf_visitor = 5
n_lf_item = 5

item_input = Input(shape=[1],name='Items')
item_embed = Embedding(n_items + 1,
                           n_lf_visitor, 
                           name='ItemsEmbedding')(item_input)
item_vec = Flatten(name='ItemsFlatten')(item_embed)

visitor_input = Input(shape=[1],name='Visitors')
visitor_embed = Embedding(n_visitors + 1, 
                              n_lf_item,
                              name='VisitorsEmbedding')(visitor_input)
visitor_vec = Flatten(name='VisitorsFlatten')(visitor_embed)

concat = keras.layers.concatenate([item_vec, visitor_vec], name='Concat')
fc_1 = Dense(80,name='FC-1')(concat)
fc_2 = Dense(40,name='FC-2')(fc_1)
fc_3 = Dense(20,name='FC-3', activation='relu')(fc_2)

output = Dense(1, activation='relu',name='Output')(fc_3)

optimizer = keras.optimizers.Adam(lr=0.001)
model = keras.Model([item_input, visitor_input], output)
model.compile(optimizer=optimizer,loss= 'mse')

model.fit([train.visitorid, train.itemid], train.event, epochs=50)
score = model.evaluate([test.visitorid, test.itemid], test.event)
print('mean squared error:', score)<jupyter_output><empty_output><jupyter_text>---<jupyter_code>user_activity_count = dict()
for row in events.itertuples():
    if row.visitorid not in user_activity_count:
        user_activity_count[row.visitorid] = {'view':0 , 'addtocart':0, 'transaction':0};
    if row.event == 'addtocart':
        user_activity_count[row.visitorid]['addtocart'] += 1 
    elif row.event == 'transaction':
        user_activity_count[row.visitorid]['transaction'] += 1
    elif row.event == 'view':
        user_activity_count[row.visitorid]['view'] += 1 

d = pd.DataFrame(user_activity_count)
dataframe = d.transpose()

# Activity range
dataframe['activity'] = dataframe['view'] + dataframe['addtocart'] + dataframe['transaction']

# removing users with only a single view
cleaned_data = dataframe[dataframe['activity']!=1]

cleaned_data.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11979969183359014, 'fraction_numerical': 0.020330422872795753, 'mean_word_length': 2.843559796019082, 'pattern_counts': {'":': 0, '<': 66, '<?xml version=': 0, '>': 68, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '99905', 'license': 'Apache-2.0', 'n_tokens_mistral': 7959, 'n_tokens_neox': 7565, 'n_words': 1434, 'path': '_notebooks/2022-01-16-retailrocket.ipynb', 'repo_name': 'recohut/notebook', 'size': '41462'}