<jupyter_start><jupyter_text>Loading Libraries<jupyter_code>#loading libraries
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)

import pandas as pd
import numpy as np
import pickle
from datetime import datetime
from f1optimization_faron import get_best_prediction<jupyter_output><empty_output><jupyter_text>Final function to return product recommendation<jupyter_code>def final(X = None):
    
    """
    Function returns the predicted product recommendations if its existing user, else returns 
    most frequently purchased product at that hour and on that day.
    
    user_id           : provided
    order_hour_of_day : calculated by seeing current time
    order_dow         : calulated by seeing current day of week
    
    """

    #get current time
    start_time = datetime.now()

    # datetime object containing current date and time
    now = datetime.now()

    # dd/mm/YY H:M:S
    dt_string = now.strftime("%d/%m/%Y %H:%M:%S")
    today = int(dt_string.split("/")[0])

    #get data from user end
    user_id = int(X['user_id']) #user_id
    order_hour_of_day = int(dt_string.split(" ")[1].split(":")[0]) #current date
    order_dow = datetime.today().weekday() #current day of week

    """
    check if the user is new or existing,
    this file contains user's last order date
    
    Handling: since in our model days since prior order is a feature, which cannot be calculated without user's last order date.
              assuming that this model is deployed on 21-march-2021, all users last order are taken as 21-march-2021.
              But this assumption likely introduces new bugs.
              Ex- user A , never made a purchase with days_prior_order = 3, if he makes he does this in future , the below handles,
              such exceptions
    """
    
    ulp = pd.read_pickle("user_last_purchase.pkl")
    
    #if user is new -> cold start -> recommend products which are most frequently purchased based on time and day of week
    if user_id not in ulp['user_id'].values:

        #get top 10 products based on hour of day and day of week
        top= pd.read_pickle('top10_products.pkl')
        top_products = top[(top['order_dow']==order_dow) & (top['order_hour_of_day']==order_hour_of_day)]['product_name'].values.tolist()
        top_products = {i: value for i,value in enumerate(top_products)}
        #paths = get_image_paths(top5_products)
        predictions={}
        predictions['top'] =  top_products

        del ulp, top,now, today, dt_string, order_dow, order_hour_of_day

        end_time = datetime.now()
        difference = end_time - start_time
        #print("Total Time : {} seconds".format(difference))
        time = "{}".format(difference)
        
        return predictions,time
    
    #if user is existing one
    user_last_order_date = ulp[ulp['user_id']==user_id]['date'].values.tolist()[0]

    days_since_prior_order = today - int(user_last_order_date.split('-')[-1])
    del ulp, now, today, dt_string, user_last_order_date
    
    #build features
    hour_rate = pd.read_pickle("hour_reorder_rate.pkl")
    day_rate = pd.read_pickle("day_reorder_rate.pkl")
    p_days_rate = pd.read_pickle("p_days_since_prior_order_reorder_rate.pkl")
    u_days_rate = pd.read_pickle("u_days_since_prior_order_reorder_rate.pkl")
    up_days_rate = pd.read_pickle("days_since_prior_reorder_rate.pkl")
    merged_up_features = pd.read_pickle("merged_user_product_features.pkl")

    #collect features based on hour of day , day of week, and user id
    featurized_data = merged_up_features[merged_up_features['user_id']==user_id]
    hour_r = hour_rate[hour_rate['order_hour_of_day']==order_hour_of_day]
    day_r = day_rate[day_rate['order_dow'] == order_dow]
    p_days = p_days_rate[p_days_rate['days_since_prior_order']==days_since_prior_order]
    u_days = u_days_rate[(u_days_rate['user_id']==user_id) & (u_days_rate['days_since_prior_order']==days_since_prior_order)]
    
    """
    if user never made a purchase in the gap of (days_since_prior_order) between 2 orders, make a new dataframe, 
    with the given number of days , and with reorder_rate = 0.0
    """
    
    if p_days.empty:
        #handle
        p_days = pd.DataFrame(columns = p_days.columns)
        products_x = pd.read_pickle('product_mappings.pkl')
        p_days['product_id'] = products_x['product_id']
        p_days['days_since_prior_order'] = days_since_prior_order
        p_days['p_days_since_prior_order_reorder_rate']=0.0
    
    up_days = up_days_rate[(up_days_rate['user_id']==user_id) & (up_days_rate['days_since_prior_order']==days_since_prior_order)]

    if up_days.empty:
        #handle
        up_days = pd.DataFrame(columns = up_days_rate.columns)
        products_x = pd.read_pickle('product_mappings.pkl')
        up_days['product_id'] = products_x['product_id']
        up_days['user_id'] = user_id
        up_days['days_since_prior_order'] = days_since_prior_order
        up_days['days_since_prior_reorder_rate']=0
        del products_x
    
    if u_days.empty:
        #handle
        u_days = pd.DataFrame(columns = u_days.columns)
        df2 = {'user_id': user_id, 'days_since_prior_order': days_since_prior_order, 'u_days_since_prior_order_reorder_rate': 0}
        u_days = u_days.append(df2, ignore_index = True)
        del df2

    up_days = up_days_rate[(up_days_rate['user_id']==user_id) & (up_days_rate['days_since_prior_order']==days_since_prior_order)]
    
    """
    if user never made a purchase in the gap of (days_since_prior_order) between 2 orders, make a new dataframe, 
    with the given number of days , and with reorder_rate = 0.0
    """
    if up_days.empty:
        #handle
        up_days = pd.DataFrame(columns = up_days_rate.columns)
        products_x = pd.read_pickle('product_mappings.pkl')
        up_days['product_id'] = products_x['product_id']
        up_days['user_id'] = user_id
        up_days['days_since_prior_order'] = days_since_prior_order
        up_days['days_since_prior_reorder_rate']=0
        del products_x


    #print(up_days_rate[up_days_rate['user_id']==user_id])
    #print(u_days_rate[u_days_rate['user_id']==user_id])
    #print(day_rate)
    del merged_up_features, hour_rate, day_rate, p_days_rate, u_days_rate, up_days_rate

    featurized_data = pd.merge(featurized_data, up_days, on = ['user_id', 'product_id'])

    featurized_data = pd.merge(featurized_data, hour_r, on = 'product_id')
    featurized_data = pd.merge(featurized_data, day_r, on = 'product_id')
    featurized_data = pd.merge(featurized_data, p_days, on = ['product_id','days_since_prior_order'])
    featurized_data = pd.merge(featurized_data, u_days, on = ['user_id','days_since_prior_order'])
    featurized_data = featurized_data[['user_id', 'product_id', 'u_p_order_rate', 'u_p_reorder_rate',\
                                       'u_p_avg_position', 'u_p_orders_since_last', 'max_streak', 'user_reorder_rate',\
                                       'user_unique_products', 'user_total_products', 'user_avg_cart_size', \
                                       'user_avg_days_between_orders', 'user_reordered_products_ratio', \
                                       'product_reorder_rate', 'avg_pos_incart', 'p_reduced_feat_1', 'p_reduced_feat_2', \
                                       'p_reduced_feat_3', 'aisle_id', 'department_id', 'aisle_reorder_rate', \
                                       'dept_reorder_rate', 'order_dow', 'order_hour_of_day', 'days_since_prior_order',\
                                       'hour_reorder_rate', 'day_reorder_rate', 'p_days_since_prior_order_reorder_rate',\
                                       'u_days_since_prior_order_reorder_rate', 'days_since_prior_reorder_rate']]

    del up_days,u_days,p_days,day_r,hour_r

    #model
    with open("catboost_v3.pkl", "rb") as f:
        model = pickle.load(f)
    #model = pickle.load(open("catboost_v3.pkl", "rb"))
    
    data = featurized_data.drop(['user_id', 'product_id'], axis = 1)
    ypred = model.predict_proba(data)
    ypred = ypred[:,-1] #get probabilities of class 1
    del data,model

    #run faron's optimization code to get most probable set of products which might be reordered
    
    #set showThreshold = True , while debugging to print the threshold
    recommended_products = get_best_prediction(featurized_data['product_id'].tolist(), ypred.tolist(), None, showThreshold = True)
    recommended_products = recommended_products.replace("None", "")
    recommended_products = list(map(int, recommended_products.split()))
    products_x = pd.read_pickle('product_mappings.pkl')
    recommended_products = products_x.loc[products_x['product_id'].isin(recommended_products)]['product_name'].values.tolist()
    recommended_products = {i: value for i,value in enumerate(recommended_products)}

    predictions= {}
    predictions['recommend'] = recommended_products

    end_time = datetime.now()
    difference = end_time - start_time
    #print("Total Time : {} seconds".format(difference))
    time = "{}".format(difference)

    del featurized_data, products_x
    return predictions, time
#existing user

X = {}
X['user_id'] = 5
recommended_products = final(X)

print()
print("="*20)
print("Recommended products")
print("="*20)
for _,value in recommended_products[0]['recommend'].items():
    print(value)
#existing user

X = {}
X['user_id'] = 55
recommended_products = final(X)

print()
print("="*20)
print("Recommended products")
print("="*20)
for _,value in recommended_products[0]['recommend'].items():
    print(value)
#new user

X = {}
X['user_id'] = 225000
recommended_products = final(X)
print("Recommended products")
print("="*20)
for _,value in recommended_products[0]['top'].items():
    print(value)
print("Time taken :", recommended_products[1])<jupyter_output>Recommended products
====================
Banana
Bag of Organic Bananas
Organic Strawberries
Organic Baby Spinach
Organic Hass Avocado
Organic Avocado
Strawberries
Organic Whole Milk
Large Lemon
Limes
Time taken : 0:00:00.010989
{'fraction_non_alphanumeric': 0.11002882417254746, 'fraction_numerical': 0.007951495875161516, 'mean_word_length': 3.68, 'pattern_counts': {'":': 1, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '345845', 'license': 'MIT', 'n_tokens_mistral': 3410, 'n_tokens_neox': 3264, 'n_words': 692, 'path': 'final.ipynb', 'repo_name': 'asagar60/Instacart-Market-Basket-Analysis', 'size': '10061'}