<jupyter_start><jupyter_text>Content-based Filtering Content-based filtering lijkt voor de CELP database een zinvolle optie, aangezien er sprake is van een lange tail en de rating density slechts een paar procent is in vele gevallen.<jupyter_code>import sklearn.metrics.pairwise as pw
import pandas as pd
import numpy as np
def load_city(city_name):
    lst=[]
    for file in ['business', 'checkin', 'review', 'tip', 'user']:
        f = open(f'yelp-all/{city_name}/{file}.json')
        lst.append(pd.read_json(f, lines = True))

    business = lst[0]
    checkin = lst[1]
    review = lst[2]
    tip = lst[3]
    user = lst[4]
    return business, checkin, review, tip, user
def number_of_businesses(reviews):
    """
    returns the number of unique businesses in a set of reviews
    """
    return len(reviews['business_id'].unique())

def number_of_users(reviews):
    """
    returns the number of unique users in a set of reviews
    """
    return len(reviews['user_id'].unique())

def number_of_ratings(reviews):
    """
    returns the number of ratings of a set of reviews
    """
    return reviews.shape[0]

def rating_density(reviews):
    """
    Compute the density of a dataset
    """
    return number_of_ratings(reviews)/(number_of_businesses(reviews) * number_of_users(reviews))

def split_data(data, d=0.75):
    """
    Split data in a training and test set
    """
    data = data[['user_id', 'business_id', 'stars']]
    np.random.seed(seed=5)
    mask_test = np.random.rand(data.shape[0]) < d
    mask_test = mask_test
    return data[mask_test], data[~mask_test]
business, checkin, review, tip, user = load_city('stouffville')
number_of_businesses(review)
number_of_users(review)
number_of_ratings(review)
rating_density(review)
review_business = review.set_index('business_id').join(business.set_index('business_id'), rsuffix='review').sort_values(by='address').reset_index()
def split_categories(review_business):
    genres_m = review_business.apply(lambda row: pd.Series([row['review_id']] + row['categories'].lower().split(",")), axis=1)
    stack_genres = genres_m.set_index(0).stack()
    df_stack_genres = stack_genres.to_frame()
    df_stack_genres['review_id'] = stack_genres.index.droplevel(1)
    df_stack_genres.columns = ['category', 'review_id']
    temp =  df_stack_genres.reset_index()[['review_id', 'category']].set_index('review_id')
    review_business = review_business.set_index('review_id')
    return temp.join(review_business, rsuffix='r')[['category', 'business_id', 'user_id', 'stars']].reset_index()

def just_categories(business):
    genres_m = review_business.apply(lambda row: pd.Series([row['business_id']] + row['categories'].lower().split(",")), axis=1)
    stack_genres = genres_m.set_index(0).stack()
    df_stack_genres = stack_genres.to_frame()
    df_stack_genres['business_id'] = stack_genres.index.droplevel(1)
    df_stack_genres.columns = ['category', 'business_id']
    return df_stack_genres.reset_index()[['business_id', 'category']]
    

def pivot_categories(df):
    """
    Create a one-hot encoded matrix for genres
    """
    return df.pivot_table(index='business_id', columns='category', aggfunc='size', fill_value=0)

def pivot_ratings(df):
    """
    Creates a utility matrix for user ratings for businesses
    """
    return df.pivot(values='stars', columns='user_id', index='business_id')
    
df_categories = split_categories(review_business).sort_values(['review_id', 'business_id'])
df_utility_categories = pivot_categories(df_categories)
df_categories.head()
df_utility_categories.head()
def create_similarity_matrix_categories(matrix):
    """
    Create a similarity matrix based on categories
    """
    npu = matrix.values
    m1 = npu @ npu.T
    diag = np.diag(m1)
    m2 = m1 / diag
    m3 = np.minimum(m2, m2.T)
    return pd.DataFrame(m3, index = matrix.index, columns = matrix.index)
df_similarity_categories = create_similarity_matrix_categories(df_utility_categories)
df_similarity_categories.head()
def predict_ratings(similarity, utility, to_predict):
    """Predicts the predicted rating for the input test data.
    
    Arguments:
    similarity -- a dataFrame that describes the similarity between items
    utility    -- a dataFrame that contains a rating for each user (columns) and each movie (rows). 
                  If a user did not rate an item the value np.nan is assumed. 
    to_predict -- A dataFrame containing at least the columns movieId and userId for which to do the predictions
    """
    # copy input (don't overwrite)
    ratings_test_c = to_predict.copy()
    # apply prediction to each row
    ratings_test_c['predicted rating'] = to_predict.apply(lambda row: predict_ids(similarity, utility, row['user_id'], row['business_id']), axis=1)
    return ratings_test_c[['business_id', 'user_id', 'stars', 'predicted rating']]

### Helper functions for predict_ratings_item_based ###

def predict_ids(similarity, utility, user_id, business_id):
    # select right series from matrices and compute
    if user_id in utility.columns and business_id in similarity.index:
        return predict_vectors(utility.loc[:,user_id], similarity[business_id])
    return 0

def predict_vectors(user_ratings, similarities):
    # select only movies actually rated by user
    relevant_ratings = user_ratings.dropna()
    
    # select corresponding similairties
    similarities_s = similarities[relevant_ratings.index]
    
    # select neighborhood
    similarities_s = similarities_s[similarities_s > 0.0]
    relevant_ratings = relevant_ratings[similarities_s.index]
    
    # if there's nothing left return a prediction of 0
    norm = similarities_s.sum()
    if(norm == 0):
        return 0
    
    # compute a weighted average (i.e. neighborhood is all) 
    return np.dot(relevant_ratings, similarities_s)/norm

def mse(predicted_ratings):
    """
    Computes the means square error betweeen actual ratings 
    """
    diff = predicted_ratings['stars'] - predicted_ratings['predicted rating']
    return (diff**2).mean()<jupyter_output><empty_output><jupyter_text>Nieuwe Test, vanaf hier proberen<jupyter_code>df = just_categories(business).drop_duplicates()
df_reviews_training, df_reviews_test = split_data(review, d=0.9)
df_reviews_training = df_reviews_training.groupby(['user_id', 'business_id'])['user_id', 'business_id', 'stars'].mean().reset_index()
df_utility_ratings = pivot_ratings(df_reviews_training)
df_utility_genres = pivot_categories(df)
df_similarity_genres = create_similarity_matrix_categories(df_utility_genres)
df_predicted_new = predict_ratings(df_similarity_genres, df_utility_ratings, df_reviews_test[['user_id', 'business_id', 'stars']])
mse_content = mse(df_predicted_new)
mse_content
def business_id_to_name(ids, city):
    business = pd.read_json(open(f'yelp-all/{city}/business.json'), lines = True)
    outcome = [list(business[business['business_id'] == i]['name'].values)[0] for i in ids]
    return outcome
    
def top_recommendations(df_predicted, city, amount = 10, name=False):
    temp = list(df_predicted.sort_values('predicted rating', ascending=False)[:10]['business_id'].values)
    if name:
        return business_id_to_name(temp, city)
    return temp
def total_run(city, name=False):
    business, check_in, review, tip, user = load_city(city)
    df = just_categories(business).drop_duplicates()
    df_reviews_training, df_reviews_test = split_data(review, d=0.9)
    df_reviews_training = df_reviews_training.groupby(['user_id', 'business_id'])['user_id', 'business_id', 'stars'].mean().reset_index()
    df_utility_ratings = pivot_ratings(df_reviews_training)
    df_utility_genres = pivot_categories(df)
    df_similarity_genres = create_similarity_matrix_categories(df_utility_genres)
    df_predicted_new = predict_ratings(df_similarity_genres, df_utility_ratings, df_reviews_test[['user_id', 'business_id', 'stars']])
    print(mse(df_predicted_new))
    return top_recommendations(df_predicted_new, city, name=name)
total_run('westlake', name=True)
for i in user.index:
    id_chosen = user.loc[i]['user_id']
    already_rated = list(review[review['user_id'] == id_chosen].values)
    print('\n')<jupyter_output>[...]
{'fraction_non_alphanumeric': 0.11884446611409069, 'fraction_numerical': 0.005241345685031692, 'mean_word_length': 4.9499637418419145, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '298439', 'license': 'MIT', 'n_tokens_mistral': 2743, 'n_tokens_neox': 2668, 'n_words': 582, 'path': 'content_based_collaborative_filtering_celp.ipynb', 'repo_name': 'corne12345/celp', 'size': '8204'}