<jupyter_start><jupyter_text>1. Dependencies<jupyter_code>import os
import sys
import os.path as osp

import numpy as np
import pandas as pd
from tqdm.auto import tqdm

from datasketch import MinHashLSHForest, MinHash<jupyter_output><empty_output><jupyter_text>2. Data Loading and Preprocessing  2.1 Data Loading<jupyter_code>## data root
data_dir = 'archive'

## file names
file_names = ['combined_data_{}.txt'.format(i) for i in range(1, 5)]

## Read only the first file combine_data_1
df = pd.read_csv(osp.join(data_dir, file_names[0]), header = None, names = ['Cust_Id', 'Rating'], usecols = [0,1])
## Preview
df.head()<jupyter_output><empty_output><jupyter_text>2.2 Data PreprocessingConvert the dataframe with columns: 'Cust_Id', 'Rating', 'Movie_Id'<jupyter_code>## Find the index of movie id
df_nan = pd.DataFrame(pd.isnull(df.Rating))
df_nan = df_nan[df_nan['Rating'] == True]
df_nan = df_nan.reset_index()
print(df_nan.shape)
df_nan.head()


movie_np = []
movie_id = 1

for i,j in tqdm(zip(df_nan['index'][1:],df_nan['index'][:-1])):
    # numpy approach
    temp = np.full((1,i-j-1), movie_id)
    movie_np = np.append(movie_np, temp)
    movie_id += 1
    
# Account for last record and corresponding length
# numpy approach
last_record = np.full((1,len(df) - df_nan.iloc[-1, 0] - 1),movie_id)
movie_np = np.append(movie_np, last_record)

# remove those Movie ID rows
df = df[pd.notnull(df['Rating'])]

df['Movie_Id'] = movie_np.astype(int)
df['Cust_Id'] = df['Cust_Id'].astype(int)
print('-Dataset examples-')
df.head()<jupyter_output>(4499, 2)<jupyter_text>Create two new dataframes: group by 'Cust_Id' and 'Movie_id' separately.<jupyter_code>## group by user
df_user = df.groupby('Cust_Id')['Movie_Id'].apply(list)

print(df_user.shape)
df_user
## group by movie
df_movie = df.groupby('Movie_Id')['Cust_Id'].apply(list)

print(df_movie.shape)
df_movie<jupyter_output>(4499,)<jupyter_text>3. MinHash  Helper function: Calculate jaccard similarity<jupyter_code>def jaccard(list1, list2):
    intersection = len(list(set(list1).intersection(list2)))
    union = (len(list1) + len(list2)) - intersection
    return float(intersection) / union
la = [0,1,2]
lb = [1,5]
print(jaccard(la, lb))<jupyter_output>0.25<jupyter_text>3.1 User-User Similarity: Find similar users based on movies they have watched Build LSH<jupyter_code>random_seed = 100

# Initialize the LSH
start_time = time.time()
forest = MinHashLSHForest(num_perm=128)

M = {}

for i in tqdm(range(len(df_user))):
    
    user_id = df_user.index[i]

    users = df_user[user_id]
    M[user_id] = MinHash(seed = random_seed, num_perm = 128)
    
    for user in users:
        M[user_id].update(str(user).encode('utf8'))
    
    forest.add(user_id, M[user_id])

print("--- %s seconds ---" % (time.time() - start_time))

forest.index()<jupyter_output><empty_output><jupyter_text>Query from LSH. Given a user id, find similar users.<jupyter_code>## Query from lsh: Given a user id, find similar users

## user id
q = df_user.index[0]
print('User ID:', q, '\n')
## num of neighbors (including itself)
N = 11


result = forest.query(M[q], N)
print('Neighbor ids:', result)

jaccard_real = []
jaccard_predict = []

## find real jaccard similarity and predicted similarity
for idx, i in enumerate(result):   


        
    print('Actually jaccard similarity: ', jaccard(df_user[q], df_user[i]))
    jaccard_real.append(jaccard(df_user[q], df_user[i]))
    print('Predicted Similarity:', M[q].jaccard(M[i]) , '\n')
    jaccard_predict.append(M[q].jaccard(M[i]))<jupyter_output><empty_output><jupyter_text>Find real neighbors of this user<jupyter_code>q = df_user.index[0]

neighbors = []
for i in tqdm(range(len(df_user))):
    
    u_id = df_user.index[i]
    
    if u_id != q:
        sim = jaccard(df_user[q], df_user[u_id])
        
        neighbors.append((u_id, sim))
        
neighbors.sort(key = lambda x: -x[1])

print(neighbors[:10])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1489091831557585, 'fraction_numerical': 0.016489091831557584, 'mean_word_length': 5.001522070015221, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '283835', 'license': 'MIT', 'n_tokens_mistral': 1584, 'n_tokens_neox': 1524, 'n_words': 323, 'path': 'Netflix-minhash.ipynb', 'repo_name': 'sh2439/cs5112-algo', 'size': '3942'}