<jupyter_start><jupyter_text>MyAnimeList Dataset: https://www.kaggle.com/azathoth42/myanimelist<jupyter_code>from google.colab import drive
drive.mount('/content/drive', force_remount=False)
DATA_PATH = './drive/MyDrive/Data/anime33/'
import pandas as pd
import numpy as np

anime_df = pd.read_csv('{}{}'.format(DATA_PATH, 'AnimeList.csv'))
anime_df.head()
print("anime_df: ", anime_df.shape)
anime_df.columns<jupyter_output><empty_output><jupyter_text>Basic Cosine SimilarityResults: Horrible **features that we want to consider:**type, episodes, rating, score, scored_by, rank, popularity, favorites, producer, studio, genre**features to potentially consider:**related (i.e. sequels), opening_theme, ending_theme<jupyter_code>anime_df.isnull().sum() # columns with null values
# cols = ['anime_id', "type", "episodes", "rating", "score", "scored_by", "rank", "popularity", "favorites", "producer", "studio", "genre"]
cols = ['anime_id', "type", "episodes", "rating", "score", "scored_by", "rank", "popularity", "favorites", "genre"]
anime = anime_df[cols]
anime.shape
anime.head()
anime.isnull().sum()
anime = anime.dropna(axis=0) # drow null rows
anime.isnull().sum()
anime.shape  
anime.head()
anime['type'].unique()
anime['rating'].unique()
OHE_type = anime['type'].str.get_dummies() # one hot encoding
OHE_genre = anime['genre'].str.get_dummies(sep=',')
OHE_rating = anime['rating'].str.get_dummies()

anime.drop(['rating', 'type','genre'], axis=1, inplace=True)

# data = pd.concat([data, OHE_rating, OHE_type, OHE_genre], axis=1)
anime = pd.concat([anime, OHE_type, OHE_genre], axis=1)
anime.head()
anime_features = anime.iloc[:, 7:].copy()
anime_features.head()
import matplotlib.pyplot as plt
from sklearn.metrics.pairwise import cosine_similarity
cosine_sim = cosine_similarity(anime_features.values, anime_features.values)
cosine_sim
anime_index = pd.Series(anime_df.index, index=anime_df.title).drop_duplicates()
anime_index
def get_recommendation(anime_name, similarity=cosine_sim):
    idx = anime_index[anime_name]
    
    # Get the pairwsie similarity scores of all anime with that anime
    sim_scores = list(enumerate(cosine_sim[idx]))

    # Sort the anime based on the similarity scores
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)

    # Get the scores of the 10 most similar anime
    sim_scores = sim_scores[0:25]

    # Get the anime indices
    anime_indices = [i[0] for i in sim_scores]

    # Return the top 10 most similar anime
    result = anime_df[['anime_id', 'title', 'genre']].iloc[anime_indices].drop(idx)
    return result
get_recommendation("Ookami to Koushinryou")<jupyter_output><empty_output><jupyter_text>KNN & Alternating Least Squares (ALS)  Create AnimeList Cleaned ListCreate username ids and filter out NaNs<jupyter_code>user_colnames = ['username', 'anime_id', 'my_score']
user_df = pd.read_csv('{}{}'.format(DATA_PATH, 'animelists_cleaned.csv'), usecols=user_colnames)
print("user_df: ", user_df.shape)
user_df.head()
anime = anime_df[['anime_id', 'title']]
anime.head()
anime.shape
print(user_df.isnull().sum())
print('shape before drop:', user_df.shape)

user_df = user_df.dropna(axis=0)

print(user_df.isnull().sum())
print('shape after drop:', user_df.shape)
# s = pd.Series(['a', 'a','b','c', 'a'])
# labels, levels = pd.factorize(s)
# labels

user_df['username'] = user_df['username'].factorize()[0]
user_df.rename(columns= {'username': 'uid'}, inplace=True)
user_df.head()
user_df.to_csv('{}{}'.format(DATA_PATH, 'users_cleaned.csv'), index=False, header=True)<jupyter_output><empty_output><jupyter_text>Creating Anime-User Matrix<jupyter_code>user_df = pd.read_csv('{}{}'.format(DATA_PATH, 'users_cleaned.csv'))
user_df['my_score'] += 1
user_df.shape<jupyter_output><empty_output><jupyter_text>Finding the largest uid from the user_df that we can slice<jupyter_code># user_df.iloc[25379534, :]

# id = user_df.iloc[20000000, 0]
# for i in range(20000000, 21000000):
#   if id == user_df.iloc[i, 0]:
#     print("@ idx: ", i)
#     id += 1
user_df.iloc[25379530:25379534, :]<jupyter_output><empty_output><jupyter_text>Slice the dataset (avoid mem errors)<jupyter_code>user_df = user_df.iloc[:25379534, :]
user_df.shape
print("Number of unique users: ", len(user_df['uid'].unique()))
print("Number of anime rated by users: ", len(user_df['anime_id'].unique()))
from scipy.sparse import csr_matrix

anime_user_mat = user_df.pivot(
    index='anime_id',
    columns='uid',
    values='my_score'
).fillna(0)

anime_user_sparse = csr_matrix(anime_user_mat.values)
anime_user_mat
anime_user_mat.iloc[0, 21] # (uid, anime_id)
# appended_data = []

# for chunk in user_df:
#   pivot_table = chunk.pivot_table(
#       index='anime_id',
#       columns='uid',
#       values='my_score',
#       fill_value=0)
#   appended_data.append(pivot_table)
# appended_data = pd.concat(appended_data, axis=0).reset_index()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1571021571021571, 'fraction_numerical': 0.017501017501017502, 'mean_word_length': 6.492378048780488, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '148647', 'license': 'MIT', 'n_tokens_mistral': 1908, 'n_tokens_neox': 1833, 'n_words': 356, 'path': 'utils/notebooks/anime33_experiments.ipynb', 'repo_name': 'Wingtail/thirty-three-anime', 'size': '4914'}