<jupyter_start><jupyter_text>Data loading<jupyter_code>import pandas as pd
import torch
from sklearn.metrics import log_loss, roc_auc_score
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder, MinMaxScaler
import pandas as pd
import pickle
import numpy as np
from fastai.collab import *
from pprint import pprint
import matplotlib.pyplot as plt
from scipy import stats
%matplotlib inline
from operator import itemgetter
from urllib.request import urlopen
# the original csv from https://raw.githubusercontent.com/beefsack/bgg-ranking-historicals/master/
# The column ID is used in API calls to retrieve the game reviews
link="https://raw.githubusercontent.com/beefsack/bgg-ranking-historicals/master/2019-05-30.csv"
f = urlopen(link)
games = pd.read_csv(f)
games.describe()
games.sort_values('Users rated',ascending=False,inplace=True)
games.rename(index=str, columns={"Bayes average": "Geekscore",'Name':'name'}, inplace=True)
# load the file I composed with all the reviews

reviews = pd.read_csv('bgg-19m-reviews.csv', index_col=0)
print(len(reviews))
reviews.head()
reviews = reviews.sample(frac=0.002)
games_by_all_users = reviews.groupby('name')['rating'].agg(['mean','count']).sort_values('mean',ascending=False)
games_by_all_users['rank']=games_by_all_users.reset_index().index+1
print(len(games_by_all_users))
games_by_all_users.head()<jupyter_output>7724<jupyter_text>Collaborative Filtering using Fast.ai library<jupyter_code>dls = CollabDataLoaders.from_df(reviews, bs=64, item_name='name', user_name='user', rating_name='rating', seed = 112)
dls.show_batch()
learn = collab_learner(dls, y_range=(1,10.5))

learn.lr_find()
# learn.recorder.plot()
learn.fit_one_cycle(5, lr_max=0.0531763744354248, wd=0.1)
learn.recorder.plot_losses()
learn.model
g = reviews.groupby('name')['rating'].count()
top_movies = g.sort_values(ascending=False).index.values[:1000]
top_movies[:10]
movie_bias = learn.model.bias(top_movies, is_item=True)
movie_bias.shape
mean_ratings = reviews.groupby('name')['rating'].mean()
movie_ratings = [(b, i, mean_ratings.loc[i]) for i,b in zip(top_movies,movie_bias)]
item0 = lambda o:o[0]
sorted(movie_ratings, key=item0)[:15]
sorted(movie_ratings, key=lambda o: o[0], reverse=True)[:15]
movie_w = learn.model.weight(top_movies, is_item=True)
movie_w.shape
movie_pca = movie_w.pca(3)
movie_pca.shape
fac0,fac1,fac2 = movie_pca.t()
# first dimension
movie_comp = [(f, i) for f,i in zip(fac0, top_movies)]

sorted(movie_comp, key=itemgetter(0), reverse=True)[:10]
sorted(movie_comp, key=itemgetter(0))[:10]
movie_comp = [(f, i) for f,i in zip(fac1, top_movies)]
sorted(movie_comp, key=itemgetter(0), reverse=True)[:10]
sorted(movie_comp, key=itemgetter(0))[:10]
idxs = np.random.choice(len(top_movies), 50, replace=False)
idxs = list(range(50))
X = fac0[idxs]
Y = fac2[idxs]
plt.figure(figsize=(15,15))
plt.scatter(X, Y)
for i, x, y in zip(top_movies[idxs], X, Y):
    plt.text(x,y,i, color=np.random.rand(3)*0.7, fontsize=11)
plt.show()
data = CollabDataBunch.from_df(reviews, user_name='user',item_name='name',rating_name='rating',bs=100000, seed = 42)
data.show_batch()
learner = collab_learner(data, n_factors=50, y_range=(2.,10))
lr_find(learner)
learner.recorder.plot()
learner.fit_one_cycle(3, 1e-2, wd=0.15)
learner.recorder.plot_losses()
#learner.save('3cycles1e-2-bs100000factors50yrange2-10wd005')
#learner.load('3cycles1e-2-bs100000factors50yrange2-10wd005')
learner.model
CUTOFF = 5000
mean_ratings = reviews.groupby('name')['rating'].mean()
top_games = games_by_all_users[games_by_all_users['count']>CUTOFF].sort_values('mean',ascending=False).index
print(len(top_games))
game_bias = learner.bias(top_games, is_item=True)
game_bias.shapemean_ratings = reviews.groupby('name')['rating'].mean()
game_ratings = [(b, i, mean_ratings.loc[i]) for i,b in zip(top_games,game_bias)]
item0 = lambda o:o[0]
sorted(game_ratings, key=item0)[:10]
sorted(game_ratings, key=lambda o: o[0], reverse=True)[:25]
game_weights = learner.weight(top_games, is_item=True)
game_weights.shape
game_pca = game_weights.pca(3)
game_pca.shape
fac0,fac1,fac2 = game_pca.t()
game_comp = [(f, i) for f,i in zip(fac0, top_games)]
pprint(sorted(game_comp, key=itemgetter(0), reverse=True)[:10]) # pretty friendly worker placement games
pprint(sorted(game_comp, key=itemgetter(0), reverse=False)[:10]) # horror/zombie games?
#fantasy versus real world/abstract
game_comp = [(f, i) for f,i in zip(fac1, top_games)]
pprint(sorted(game_comp, key=itemgetter(0), reverse=True)[:10])
pprint(sorted(game_comp, key=itemgetter(0), reverse=False)[:10])
#this one i don't get
game_comp = [(f, i) for f,i in zip(fac2, top_games)]
pprint(sorted(game_comp, key=itemgetter(0), reverse=True)[:10])
pprint(sorted(game_comp, key=itemgetter(0), reverse=False)[:10])
idxs = np.random.choice(len(top_games), 50, replace=False)
idxs = list(range(50))
X = fac0[idxs]
Y = fac1[idxs]
plt.figure(figsize=(15,15))
plt.scatter(X, Y)
for i, x, y in zip(top_games[idxs], X, Y):
    plt.text(x,y,i, color=np.random.rand(3)*0.7, fontsize=11)
plt.show()<jupyter_output><empty_output><jupyter_text>Find similar games<jupyter_code>top_games = games_by_all_users[games_by_all_users['count']>500].sort_values('mean',ascending=False).index
print(len(top_games))
game_bias = learner.bias(top_games, is_item=True)
game_bias.shapemean_ratings = reviews.groupby('name')['rating'].mean()
game_ratings = [(b, i, mean_ratings.loc[i]) for i,b in zip(top_games,game_bias)]
item0 = lambda o:o[0]
sorted(game_ratings, key=lambda o: o[0], reverse=True)[:25]
from sklearn.neighbors import NearestNeighbors
import pickle
with open('games_by_all_users.pickle', 'wb') as handle:
    pickle.dump(games_by_all_users, handle, protocol=pickle.HIGHEST_PROTOCOL)
top_games = games_by_all_users[games_by_all_users['count']>1000].sort_values('mean',ascending=False).reset_index()
game_weights = learner.weight(top_games['name'], is_item=True)
game_bias = learner.bias(top_games['name'], is_item=True)
npweights = game_weights.numpy()
top_games['model_score']=game_bias.numpy()
top_games['weights_sum']=np.sum(np.abs(npweights),axis=1)

nn = NearestNeighbors(n_neighbors=10)
fitnn = nn.fit(npweights)
distances,indices = fitnn.kneighbors([npweights[1223]])
res = top_games[top_games['name'].str.contains('Coconuts')]
if len(res)==1:
    distances,indices = fitnn.kneighbors([npweights[res.index[0]]])
else:
    print(res.head())
top_games.iloc[indices[0][:500]].sort_values('model_score',ascending=False)
for d in distances[0]: print(d)
top_games.sort_values('weights_sum',ascending=False)[:100]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.157043530834341, 'fraction_numerical': 0.03476420798065296, 'mean_word_length': 8.716593245227607, 'pattern_counts': {'":': 1, '<': 12, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '596609', 'license': 'MIT', 'n_tokens_mistral': 2779, 'n_tokens_neox': 2591, 'n_words': 434, 'path': 'archive/4 bgg_ratings-collab copy.ipynb', 'repo_name': 'jvanelteren/boardgamegeek', 'size': '6616'}