<jupyter_start><jupyter_text>Reading and preparing  dataset<jupyter_code>import pandas as pd
from pathlib import Path

DATASET_FOLDER = Path('data/the_movies_dataset')

SMALL_DATASET = True

RATINGS_FILEPATH = DATASET_FOLDER / 'ratings_small.csv' if SMALL_DATASET else DATASET_FOLDER / 'ratings.csv'
MOVIES_FILEPATH = DATASET_FOLDER / 'movies_metadata.csv'
ratings = pd.read_csv(RATINGS_FILEPATH)
movies = pd.read_csv(MOVIES_FILEPATH)
movies["popularity"] = pd.to_numeric(movies["popularity"], errors ='coerce').fillna(0.0).astype('float')
popular_movies=movies[movies["popularity"]> 3]["id"].astype('int').tolist()
ratings=ratings[ratings["movieId"].isin(popular_movies) ].reset_index(drop=True)
movie_embeddings = ratings.pivot_table(columns='userId',index='movieId',values='rating',fill_value=0.0)<jupyter_output><empty_output><jupyter_text>Algorithms Implementation<jupyter_code>class HashTable:
    def __init__(self, hash_size, inp_dimensions):
        self.hash_size = hash_size
        self.inp_dimensions = inp_dimensions
        self.hash_table = dict()
        self.projections = np.random.randn(self.hash_size, inp_dimensions)
        
    def generate_hash(self, inp_vector):
        bools = (np.dot(inp_vector, self.projections.T) > 0).astype('int')
        return ''.join(bools.astype('str'))
    
    def __setitem__(self, inp_vec, label):
        #print("generating hash value for hash table generation")
        hash_value = self.generate_hash(inp_vec)
        #print("completed generating hash value for hash table generation")
        self.hash_table[hash_value] = self.hash_table.get(hash_value, list()) + [label]
        #print("completed Setting values to hash table")
    def __getitem__(self, inp_vec):
        #print("generating hash value for searching hash table")
        hash_value = self.generate_hash(inp_vec)
        return self.hash_table.get(hash_value, [])
        
class LSH:
    def __init__(self, num_tables, hash_size, inp_dimensions):
        self.num_tables = num_tables
        self.hash_size = hash_size
        self.inp_dimensions = inp_dimensions
        self.hash_tables = list()
        for i in range(self.num_tables):
            self.hash_tables.append(HashTable(self.hash_size, self.inp_dimensions))
            
    def __setitem__(self, inp_vec, label):
        #print("generating hash tables")
        for table in self.hash_tables:
            table[inp_vec] = label
        #print("hash table generation complete")
    
    def __getitem__(self, inp_vec):
        results = list()
        for table in self.hash_tables:
            #print("looping through hash tables")
            results.extend(table[inp_vec])
            #print("suggestions from table ",results)
          
        return list(set(results))
    


nusers=movie_embeddings.columns
nmovies=movie_embeddings.index
hash_table = LSH(num_tables=20,hash_size=10, inp_dimensions=len(nusers))
for i in range(len(nmovies)):
    hash_table[movie_embeddings.loc[nmovies[i]]]=nmovies[i]
# for hashtab  in hash_table.hash_tables:
#     print(hashtab.hash_table)
    
movies.head()
def getJaccardSim(movie1,movie2):
    movie1=''.join((movie1 > 3).astype(int).astype('str'))
    movie2=''.join((movie2 > 3).astype(int).astype('str'))
    N = 0
    D = 0
    for i in range(len(movie1)):
        sum_ = int(movie1[i]) + int(movie2[i])
        if(sum_ >= 1):
            flag = 1
            D += 1
            if(sum_ == 2):
                N += 1
    if D == 0:
        return 0
    return(float(N)/D)
def getCosineSim(movie1,movie2):

    return np.dot(movie1,movie2)/(np.linalg.norm(movie1)*np.linalg.norm(movie2))
from sklearn.neighbors import NearestNeighbors
from scipy.sparse import csr_matrix
def lsh(m,k,sim_measure):
    inp_vec=movie_embeddings.loc[m]
    #print("Movie_id" ,m)
    #print("Query movie name",movies[movies["id"]==str(m)]["original_title"].values[0])
    similar_movies = hash_table[inp_vec]
    
    similarity_values =[]
    for a in similar_movies:
        if a== m:
            continue
        out_vec = movie_embeddings.loc[a]
        similarity=0
        if sim_measure == "cos":
            similarity = getCosineSim (inp_vec,out_vec)
        elif sim_measure == "jac":
            similarity = getJaccardSim (inp_vec,out_vec)
        similarity_values.append(similarity)
        
    ranked_sim = np.argsort(np.array(similarity_values))[::-1][:k]
    movie_sugg=[]
    sim_values=[]
    for i in range (0,k):
        movie_id = similar_movies[ranked_sim[i]]
        movie_sugg.append(movie_id)
        sim_values.append(similarity_values[ranked_sim[i]])
    return movie_sugg,sim_values
    
def knn(m,k):
    embeddings_sparse = csr_matrix(movie_embeddings)

    model = NearestNeighbors(n_neighbors=k,algorithm='brute',metric='cosine')
    model.fit(embeddings_sparse)
    movie_emb = movie_embeddings.loc[m,:]
    distances,suggestions=model.kneighbors(movie_emb.values.reshape(1,-1),k+1)
    suggestions= suggestions.flatten()
    distances= distances.flatten()
    movies = []
    similarity =[]
    for i in range(1,len(suggestions)):
        movies.append(movie_embeddings.index[suggestions[i]])
        similarity.append(1-distances[i])   

    return movies,similarity<jupyter_output><empty_output><jupyter_text>Benchmarking algorithms<jupyter_code>def benchmark_rating_pred(test_movies,k,model):
    predicted_ratings=pd.DataFrame(columns=['movie_id', 'predicted_rating','user_id'])
    if model=="lsh_cos":
        sim_indicator="cos"
    elif  model=="lsh_jac":
        sim_indicator="jac"
    for index,m in enumerate(test_movies):
        if model == "knn":
            similar_movies,sim_values = knn(m,k)
        else:
            similar_movies,sim_values=lsh(m,k,sim_indicator)
            #print(similar_movies)
        count_similar=len(similar_movies)
        selected_user=ratings[ratings["movieId"].isin(similar_movies) & ratings["rating"]!=0]["userId"].value_counts().nlargest(1).index[0]
        rating_N=0
        rating_D=0
            #print("selected user",selected_user)
        for idx,i in enumerate(similar_movies):
            rating_val=movie_embeddings.loc[i][selected_user]
            sim_val=sim_values[idx]
                #print("sim val", sim_val,"rating val",rating_val)
            rating_N=rating_N+sim_val*rating_val
                #print("numerator",rating_N)
            rating_D=rating_D+sim_val
                #print("denominator",rating_D)
            #print("final numerator",rating_N,"final denominator",rating_D)
        predicted_ratings.loc[index] = [int(m), float(rating_N/rating_D),int(selected_user)]
    return predicted_ratings
    
    
    
def actual_rating(x):
    y=ratings[(ratings["userId"]==x.user_id) & (ratings["movieId"]==x.movie_id)]["rating"]
    if len(y) != 0:
        return y.values[0]  
    else:
        return 0
def get_rmse(ratings_pred):
    ratings_pred["actual_rating"]=ratings_pred.apply(actual_rating ,axis=1)
    ratings_pred["squared_diff"]=np.square(ratings_pred["actual_rating"]-ratings_pred["predicted_rating"])
    rmse = np.sqrt(ratings_pred["squared_diff"].mean())
    return rmse
    
test_movies=random.sample(list(ratings["movieId"].unique()),10)<jupyter_output><empty_output><jupyter_text>LSH cosine Performance<jupyter_code>start_time = time.time()
ratings_pred=benchmark_rating_pred(test_movies,5,"lsh_cos")
exec_time= time.time() - start_time
print("Performance of LSH with cosine rmse : ",get_rmse(ratings_pred),"execution time : ",exec_time)
ratings_pred<jupyter_output><empty_output><jupyter_text>LSH jacard Performance<jupyter_code>start_time = time.time()
ratings_pred=benchmark_rating_pred(test_movies,5,"lsh_jac")
exec_time= time.time() - start_time
print("Performance of LSH with jacard rmse : ",get_rmse(ratings_pred),"execution time : ",exec_time)
ratings_pred<jupyter_output><empty_output><jupyter_text>KNN Performance<jupyter_code>start_time = time.time()
ratings_pred=benchmark_rating_pred(test_movies,5,"knn")
exec_time= time.time() - start_time
print("Performance of KNN : ",get_rmse(ratings_pred),"execution time : ",exec_time)
ratings_pred<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12535332432100282, 'fraction_numerical': 0.007250829544058006, 'mean_word_length': 3.9261501210653753, 'pattern_counts': {'":': 5, '<': 25, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '351942', 'license': 'MIT', 'n_tokens_mistral': 2913, 'n_tokens_neox': 2819, 'n_words': 422, 'path': 'notebooks/Evaluation.ipynb', 'repo_name': 'noufanpmc/Movie-Recommendation-System', 'size': '8137'}