<jupyter_start><jupyter_text>3. для пользователей находим любимые категории4. фильтруем то что порекомендовать по тому чтобы они были в любимых категориях<jupyter_code>from lightfm import LightFM
from scipy.sparse import coo_matrix
import pandas as pd
import ml_metrics as metrics<jupyter_output>/Users/danil/Documents/github/sweet_RS/env/lib/python3.7/site-packages/lightfm/_lightfm_fast.py:9: UserWarning: LightFM was compiled without OpenMP support. Only a single thread will be used.
  warnings.warn('LightFM was compiled without OpenMP support. '<jupyter_text>validate<jupyter_code>user_hist_df = pd.read_csv('../data/raw/movies_dataset_10 months.csv')
user_hist_df['ts'] = pd.to_datetime(user_hist_df['ts'])
user_hist_df['month'] = user_hist_df['ts'].dt.month

test_1 = user_hist_df[user_hist_df['month'] == 6]
test_2 = user_hist_df[user_hist_df['month'] == 7]
train = user_hist_df[~user_hist_df['month'].isin([6, 7])]
correct_1 = test_1.drop_duplicates(['user_id', 'movie_id']).groupby('user_id')['movie_id'].apply(list).to_dict()
correct_2 = test_2.drop_duplicates(['user_id', 'movie_id']).groupby('user_id')['movie_id'].apply(list).to_dict()

train['event'] = 1
train = train.drop_duplicates(['user_id', 'movie_id', 'event'])
model = LightFM(no_components=10)

train_df = train[['user_id', 'movie_id', 'event']]
user_encode = {u: i for i, u in enumerate(train['user_id'].unique())}
item_encode = {u: i for i, u in enumerate(train['movie_id'].unique())}
user_decode = {v: k for k, v in user_encode.items()}
item_decode = {v: k for k, v in item_encode.items()}
n_users, n_items = len(user_encode), len(item_encode)

n_users, n_items
train['user_id'] = train['user_id'].apply(lambda x: user_encode[x])
train['movie_id'] = train['movie_id'].apply(lambda x: item_encode[x])
train_coo = coo_matrix(
    (train['event'], (train['user_id'], train['movie_id'])), shape=(n_users, n_items))
movies_database = pd.read_csv('../data/raw/movies.csv')
new_movies = movies_database[(movies_database['year']>= 2010) & 
                             (movies_database['id'].isin(list(item_encode.keys())))]['id'].values
movies_database = pd.read_csv('../data/raw/movies.csv')
new_movies = movies_database[(movies_database['year']>= 2010) & 
                             (movies_database['id'].isin(list(item_encode.keys()))) &
                             (movies_database['imdb_rating']>= 6)]['id'].values

moveis_views = train[train['movie_id'].isin([item_encode[x] for x in new_movies])]['movie_id'].value_counts()

items_to_recom = moveis_views[moveis_views>100].index
print(len(items_to_recom))<jupyter_output><empty_output><jupyter_text>preapre movie features and user prefs<jupyter_code>movies_database = pd.read_csv('../data/raw/movies.csv')
movies_to_recom = [item_decode[x] for x in items_to_recom]
movies_database = movies_database[movies_database['id'].isin(movies_to_recom)]
item_features = movies_database[['id', 'genres']]
item_features['id'] = item_features['id'].apply(lambda x: item_encode[x])
item_features['genres'] = item_features['genres'].str.split(',')
item_features = item_features.dropna()
movie_genres = {}

for i in item_features.iterrows():
    movie_genres[i[1][0]] = i[1][1]
users_to_predict = [user_encode[x] for x in list(test_1['user_id'])]
items_to_predict = set(items_to_recom.copy())
movie_user_hist = train[train['episode_id'] == 0].groupby('user_id')['movie_id'].apply(set).to_dict()
user_hist = train.groupby('user_id')['movie_id'].apply(set).to_dict()
#user_hist = {user_encode[k]: v for k, v in user_hist.items()}
from collections import Counter
user_pref = {}
for user in users_to_predict:
    #print(user)
    user_pref[user] = {}
    #print(user_hist[user])
    for film in user_hist[user]:
        if movie_genres.get(film):
            for genre in movie_genres[film]:
                if user_pref[user].get(genre):
                    user_pref[user][genre] += 1
                else:
                    user_pref[user][genre] = 1
                    
    user_pref[user] = [key for key, val in Counter(user_pref[user]).most_common(15)]                
    
items_for_user = {}

for user in user_pref:
    items_for_user[user] = []
    for item in items_to_predict:
        try:
            if len(set(movie_genres[item]).intersection(user_pref[user])) >= 1:
                items_for_user[user].append(item)
        except KeyError:
            #items_for_user[user].append(item)
            pass
        
model = LightFM(loss='logistic', no_components=10)
model.fit(train_coo, epochs=20)
recoms_genre = {}
num_to_recom = 5
for user in users_to_predict:
    #print(user)
    items_to_score = list(set(items_for_user[user]).difference(movie_user_hist[user]))
    if not items_to_score:
        items_to_score = list(items_to_predict)
    #print(len(items_to_score))
    predict = model.predict(user, items_to_score, num_threads=-1)
    top_recoms_id = sorted(range(len(predict)),
                           key=lambda i: predict[i])[-num_to_recom:]
    top_recoms_id.reverse()
    recoms_genre[user_decode[user]] = [item_decode[items_to_score[i]]
                                 for i in top_recoms_id]

metrics.mapk(list(recoms_genre.values()), list(correct_1.values()), 5)<jupyter_output><empty_output><jupyter_text>0.005921940928270043 > 100<jupyter_code>recoms = {}
num_to_recom = 5
for user in users_to_predict:
    #print(user)
    items_to_score = list(set(items_to_predict).difference(movie_user_hist[user]))
    #print(len(items_to_score))
    predict = model.predict(user, items_to_score, num_threads=-1)
    top_recoms_id = sorted(range(len(predict)),
                           key=lambda i: predict[i])[-num_to_recom:]
    top_recoms_id.reverse()
    recoms[user_decode[user]] = [item_decode[items_to_score[i]]
                                 for i in top_recoms_id]

metrics.mapk(list(recoms.values()), list(correct_1.values()), 5)<jupyter_output><empty_output><jupyter_text>0.0067932489451476 idbm > 6 0.006348101265822786 > 100<jupyter_code>movies_database = pd.read_csv('../data/raw/movies.csv')
N = 5

for ind, vals in enumerate(zip(recoms.items(), correct_1.values())):
    #print(vals)
    user = vals[0][0]
    pred, corr = set(vals[0][1]), set(vals[1])
    pred_genre = recoms_genre[user]
    error_rate = len(corr.intersection(pred)) / len(corr)
    print(error_rate)
    #if error_rate > 0:
    #    print(pred, corr)
    if (error_rate == 0) and (len(corr) >= 3):
        print(user)
        print('predict:')
        for movie in pred:
            film = movies_database[movies_database['id'] == movie]
            print(film['title'].values[0], film['genres'].values[0])
            
        print('predict with genre:')
        for movie in pred_genre:
            film = movies_database[movies_database['id'] == movie]
            try:
                print(film['title'].values[0], film['genres'].values[0])
            except:
                pass
            
        print('correct:')
        for movie in corr:
            film = movies_database[movies_database['id'] == movie]
            try:
                print(film['title'].values[0], film['genres'].values[0])
            except:
                pass
    print('--------------')
movie
corr
user_pref[user_encode[13233854463023521736]]
movie
users_to_predict = [user_encode[x] for x in list(test_2['user_id'])]
recoms = {}
num_to_recom = 5
for user in users_to_predict:
    #print(user)
    items_to_score = list(items_to_predict.difference(user_hist[user]))
    predict = model.predict(user, items_to_score, num_threads=-1)
    top_recoms_id = sorted(range(len(predict)),
                           key=lambda i: predict[i])[-num_to_recom:]
    top_recoms_id.reverse()
    recoms[user_decode[user]] = [item_decode[items_to_score[i]]
                                 for i in top_recoms_id]
metrics.mapk(list(recoms.values()), list(correct_1.values()), 5)
metrics.mapk(list(recoms.values()), list(correct_1.values()), 5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1482456140350877, 'fraction_numerical': 0.020050125313283207, 'mean_word_length': 3.902334152334152, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '320840', 'license': 'MIT', 'n_tokens_mistral': 2950, 'n_tokens_neox': 2819, 'n_words': 460, 'path': 'notebooks/lightFM_item_features.ipynb', 'repo_name': 'danilpolkov/sweet_RS', 'size': '7980'}