<jupyter_start><jupyter_text>Content based pipe<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import pickle
import joblib
import sys
import jdc

from sklearn.ensemble import RandomForestRegressor

%load_ext autoreload
%autoreload 2 
from resype.preprocessing import *
from resype.cb_model import *
from resype.evaluate import *
# import nltk
# nltk.download('stopwords')
# nltk.download('punkt')
user_df
item_df
_
df
_, user_df = create_user_feature()
item_df = create_item_feature(num_features = 300)
df = get_augmented_table()
df = df.fillna(0)


df = df.sample(500)

item_ids = item_df['movieId'].unique()
user_ids = df['userId'].unique()
# # df, item_df, user_df, item_ids, user_ids = load_data('sample_data/augmented_transaction_table.parquet',
# #                                                     'sample_data/item_feature.parquet',
# #                                                     'sample_data/user_feature.parquet')


# df = df
# item_df = item_df
# user_df = user_df
# item_ids = item_ids
# user_ids = user_ids<jupyter_output><empty_output><jupyter_text>Fit model<jupyter_code>#Declare your model
rs_model1 = RandomForestRegressor(random_state=202109, n_jobs=-1)

#Do your train and test split
df_train, df_test, df_test_um, indx_train, indx_test = split_train_test(df, 0.7) #To split the data

# df_train = df_train.sample(100)

#Fit your model to the train data
model_fit = fit_ml_cb(df_train, rs_model1) #To fit the model

#Predict on the test data
preds_array = reco_ml_cb_tt(df_test, model_fit) #To make predictions as an array
preds_array<jupyter_output><empty_output><jupyter_text>Predict<jupyter_code>from resype import *
def get_rec(utility_matrix_preds, utility_matrix, user_list, top_n, uc_assignment=None):
    
    """Returns the top N item cluster recommendations for each user in the user list
    
            Parameters:
                    utility_matrix (numpy.ndarray): Matrix of utilities for each user-item pairing
                    utility_matrix_o (numpy.ndarray): Original utility matrix, before imputation
                    user_list (array-like): List of users
                    uc_assignment (array-like): List containing the cluster assignment of each user
                    top_n (int): Number of item clusters to recommend

            Returns:
                    df_rec (pandas.DataFrame): Table containing the top N item cluster recommendations for each user in the user list
                    
    """
    
    # Class stuff
    utility_matrix_o = utility_matrix.fillna(0).values
    utility_matrix = utility_matrix_preds.values
    
    # Don't recommend items that are already rated
    utility_matrix[np.where(utility_matrix_o != 0)] = -np.inf
    
    # Get top N per user cluster
    cluster_rec = utility_matrix.argsort()[:, -top_n:]

    # Create recommendation table
    df_rec = pd.DataFrame()
    df_rec['user_id'] = user_list
                
    for i in range(top_n):
        df_rec['rank_'+str(i+1)] = np.zeros(df_rec.shape[0])
        for j in range(df_rec.shape[0]):
            if uc_assignment is None:
                df_rec.iloc[j, i+1] = cluster_rec[user_list[j], top_n-i-1]
            else:
                df_rec.iloc[j, i+1] = cluster_rec[uc_assignment.iloc[user_list[j], 0], top_n-i-1]
    
    # look-up tables
    if uc_assignment is None:
        user_id_lookup = utility_matrix_preds.index
        item_id_lookup = utility_matrix_preds.columns
        for j in range(df_rec.shape[0]):
            df_rec.iloc[j, 0] = user_id_lookup[df_rec.iloc[j, 0].astype('int32')]
            for i in range(top_n):
                df_rec.iloc[j, i+1] = item_id_lookup[df_rec.iloc[j, i+1].astype('int32')]
    
    return df_rec
from resype.resype import Resype
user_ = user_df.drop(['userId'], axis=1)
user_ = user_.sample(10)
item_ = item_df.drop(['movieId'], axis=1)
reco_mtx = reco_ml_cb(user_, item_, item_ids, model_fit)
reco_mtx
reco_inds = np.array(reco_mtx.index)
reco_inds
um = _.drop('timestamp', axis=1)
um.columns = ['user_id', 'item_id', 'rating']

rs = Resype(um)
original_um = rs.construct_utility_matrix()
original_um = original_um.loc[reco_inds[:], :]
original_um = original_um[reco_mtx.columns]
reco_mtx.shape, original_um.shape
original_um = original_um.fillna(0)
original_um
reco_mtx = reco_mtx.fillna(0)
reco_mtx<jupyter_output><empty_output><jupyter_text>Get recommendations<jupyter_code>get_rec(reco_mtx, original_um, user_list = [0, 2], top_n = 3)<jupyter_output><empty_output><jupyter_text>Evaluate<jupyter_code>evaluate_arrays(preds_array, df, indx_test) #MSE and MAE<jupyter_output>proceed<jupyter_text>Cross Validation<jupyter_code>df
mse, mae = cross_val(df, 5, rs_model1, split_method='random')
mse, mae<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1005644992682417, 'fraction_numerical': 0.01108091156178131, 'mean_word_length': 3.5736137667304013, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '577939', 'license': 'MIT', 'n_tokens_mistral': 1728, 'n_tokens_neox': 1671, 'n_words': 362, 'path': 'combined_pipe_content_based_backup.ipynb', 'repo_name': 'phdinds-aim/resype', 'size': '4783'}