<jupyter_start><jupyter_text>Preprocessing<jupyter_code>raw_df = pd.read_csv("../movielens20M/rating.csv")
# Drop timestamp col
raw_df = raw_df.drop(['timestamp'], axis=1)<jupyter_output><empty_output><jupyter_text>Create subset by discarding low occurrence users<jupyter_code>df = discard_low_occurence(raw_df, 'userId')
plt.title(f'{len(df)} entries remaining')
plt.show()<jupyter_output><empty_output><jupyter_text>Further reduce subset by discarding low occurrence movies<jupyter_code>df = discard_low_occurence(df, 'movieId')
plt.title(f'{len(df)} entries remaining')
plt.show()<jupyter_output><empty_output><jupyter_text>Test/Train split<jupyter_code>split_mask = np.random.rand(len(df)) < 0.8
train_df = df[split_mask]
test_df = df[~split_mask]<jupyter_output><empty_output><jupyter_text>Item-Item Colab. Filtering<jupyter_code>KEY = 'movieId'
FEATURE = 'userId'
train_dict = df_to_relational_dict(train_df, key_col=KEY, feature_col=FEATURE)
test_dict = df_to_relational_dict(test_df, key_col=KEY, feature_col=FEATURE)
train_deviations = build_deviation_dict(train_dict, KEY, FEATURE)<jupyter_output><empty_output><jupyter_text>Precompute user-user weights<jupyter_code>PROCESSES = 8
MIN_OVERLAP = 5
# %%snakeviz -t

entries = list(train_dict.keys())
weights = build_weights_parallel(entries, train_dict, PROCESSES)<jupyter_output>100%|██████████| 852/852 [01:42<00:00,  8.31it/s]<jupyter_text>Prediction<jupyter_code># Limit to most important K neighbours
K = 25
limited_weights = {}
for key in weights:
    w_i = weights[key]
    limited_weights[key] = dict(sorted(w_i.items(), key=lambda x: abs(x[1]), reverse=True)[1:K+1])<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code># %%snakeviz -t
mae, mse, n = evaluate(train_dict)
print(f"Train MAE: {mae/n:.04f}, MSE: {mse/n:.04f}, RMSE: {np.sqrt(mse/n):.04f}")
mae, mse, n = evaluate(test_dict)
print(f"Test MAE: {mae/n:.04f}, MSE: {mse/n:.04f}, RMSE: {np.sqrt(mse/n):.04f}")<jupyter_output>100%|██████████| 852/852 [01:47<00:00,  7.93it/s]
  0%|          | 3/852 [00:00<00:32, 26.26it/s]<jupyter_text>User-User Colab. Filtering<jupyter_code>KEY = 'userId'
FEATURE = 'movieId'
train_dict = df_to_relational_dict(train_df, key_col=KEY, feature_col=FEATURE)
test_dict = df_to_relational_dict(test_df, key_col=KEY, feature_col=FEATURE)
train_deviations = build_deviation_dict(train_dict, KEY, FEATURE)<jupyter_output><empty_output><jupyter_text>Precompute user-user weights<jupyter_code># %%snakeviz -t
entries = list(train_dict.keys())
weights = build_weights_parallel(entries, train_dict, PROCESSES)<jupyter_output>100%|██████████| 1386/1386 [03:57<00:00,  5.83it/s]<jupyter_text>Prediction<jupyter_code># Limit to most important K neighbours
K = 25
limited_weights = {}
for key in weights:
    w_i = weights[key]
    limited_weights[key] = dict(sorted(w_i.items(), key=lambda x: abs(x[1]), reverse=True)[1:K+1])<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code># %%snakeviz -t
mae, mse, n = evaluate(train_dict)
print(f"Train MAE: {mae/n:.04f}, MSE: {mse/n:.04f}, RMSE: {np.sqrt(mse/n):.04f}")
mae, mse, n = evaluate(test_dict)
print(f"Test MAE: {mae/n:.04f}, MSE: {mse/n:.04f}, RMSE: {np.sqrt(mse/n):.04f}")<jupyter_output>100%|██████████| 1386/1386 [02:41<00:00,  8.57it/s]
  0%|          | 4/1386 [00:00<00:38, 35.93it/s]
{'fraction_non_alphanumeric': 0.17043740573152338, 'fraction_numerical': 0.048265460030165915, 'mean_word_length': 8.035422343324251, 'pattern_counts': {'":': 0, '<': 52, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '253010', 'license': 'MIT', 'n_tokens_mistral': 1548, 'n_tokens_neox': 1403, 'n_words': 207, 'path': 'ColabFiltering.ipynb', 'repo_name': 'glefundes/collaborative-filtering-python', 'size': '3315'}