<jupyter_start><jupyter_text>validate<jupyter_code>user_hist_df = pd.read_csv('../data/raw/movies_dataset_10 months.csv')

user_hist_df['ts'] = pd.to_datetime(user_hist_df['ts'])
user_hist_df['month'] = user_hist_df['ts'].dt.month

test_1 = user_hist_df[user_hist_df['month'] == 6]
test_2 = user_hist_df[user_hist_df['month'] == 7]
user_hist_df = user_hist_df[user_hist_df['episode_id'] == 0]
train = user_hist_df[~user_hist_df['month'].isin([6, 7])]
correct_1 = test_1.drop_duplicates(['user_id', 'movie_id']).groupby('user_id')['movie_id'].apply(list).to_dict()
correct_2 = test_2.drop_duplicates(['user_id', 'movie_id']).groupby('user_id')['movie_id'].apply(list).to_dict()

train['event'] = 1
train = train.drop_duplicates(['user_id', 'movie_id', 'event'])
model = LightFM(loss='logistic', no_components=10, random_state = 42)

train_df = train[['user_id', 'movie_id', 'event']]
user_encode = {u: i for i, u in enumerate(train['user_id'].unique())}
item_encode = {u: i for i, u in enumerate(train['movie_id'].unique())}

user_decode = {v: k for k, v in user_encode.items()}
item_decode = {v: k for k, v in item_encode.items()}

n_users, n_items = len(user_encode), len(item_encode)

n_users, n_items
train['user_id'] = train['user_id'].apply(lambda x: user_encode[x])
train['movie_id'] = train['movie_id'].apply(lambda x: item_encode[x])
train_coo = coo_matrix(
    (train['event'], (train['user_id'], train['movie_id'])), shape=(n_users, n_items))
movies_database = pd.read_csv('../data/raw/movies.csv')
new_movies = movies_database[(movies_database['year']>= 2010) & 
                             (movies_database['id'].isin(list(item_encode.keys()))) &
                             (movies_database['imdb_rating']>= 6)]['id'].values

moveis_views = train[train['movie_id'].isin([item_encode[x] for x in new_movies])]['movie_id'].value_counts()

items_to_recom = moveis_views[moveis_views>20].index
print(len(items_to_recom))
users_to_predict = [user_encode[x] for x in list(test_1['user_id'])]
items_to_predict = set(items_to_recom.copy())
model.fit(train_coo, epochs=20)
user_hist = train_df[train_df['movie_id'] != 0].groupby('user_id')['movie_id'].apply(set).to_dict()
user_hist = {user_encode[k]: v for k, v in user_hist.items()}
recoms = {}
num_to_recom = 5

reco = []
corr = []


for user in users_to_predict:
    #print(user)
    items_to_score = list(items_to_predict.difference(user_hist[user]))
    #items_to_score = list(items_to_predict)
    #print(len(items_to_score))
    predict = model.predict(user, items_to_score, num_threads=-1)
    top_recoms_id = sorted(range(len(predict)),
                           key=lambda i: predict[i])[-num_to_recom:]
    top_recoms_id.reverse()
    recoms[user_decode[user]] = [item_decode[items_to_score[i]]
                                 for i in top_recoms_id]
    
    reco.append(recoms[user_decode[user]])
    corr.append(correct_1[user_decode[user]])

metrics.mapk(list(recoms.values()), list(correct_1.values()), 5)<jupyter_output><empty_output><jupyter_text>0.009470141150922911<jupyter_code>user_hist_df = pd.read_csv('../data/raw/movies_dataset_10 months.csv')
user_hist_df['ts'] = pd.to_datetime(user_hist_df['ts'])
user_hist_df['month'] = user_hist_df['ts'].dt.month

fav_serials = user_hist_df[(user_hist_df['episode_id'] != 0) & 
                           (user_hist_df['month'] == 5)].groupby(['user_id', 'movie_id'])['movie_id'].count()
fav_serials = fav_serials[fav_serials>=2].reset_index(name='cnt')

g = fav_serials.groupby(["user_id"]).apply(lambda x: x.sort_values(["cnt"], ascending = False)).reset_index(drop=True)

user_top_5_serials = g.groupby(["user_id"])['movie_id'].apply(lambda x: list(x[:5])).to_dict()


recoms_f = {}
num_to_recom = 5
for user in users_to_predict:
    user = user_decode[user]
    if user_top_5_serials.get(user):
    
        pref_serials = user_top_5_serials[user][:1]
        num_of_serials = len(pref_serials)

        if num_of_serials < 5:
            additional = 5 - num_of_serials
            light_fm_pred = recoms[user][:additional]

            pred = pref_serials + light_fm_pred
            
        else:
            pred = pref_serials
    else:
         pred = recoms[user]
            
    recoms_f[user] = pred

metrics.mapk(list(recoms_f.values()), list(correct_1.values()), 5)
users_to_predict = [user_encode[x] for x in list(test_2['user_id'])]
recoms = {}
num_to_recom = 5
for user in users_to_predict:
    #print(user)
    items_to_score = list(items_to_predict.difference(user_hist[user]))
    predict = model.predict(user, items_to_score, num_threads=-1)
    top_recoms_id = sorted(range(len(predict)),
                           key=lambda i: predict[i])[-num_to_recom:]
    top_recoms_id.reverse()
    recoms[user_decode[user]] = [item_decode[items_to_score[i]]
                                 for i in top_recoms_id]
metrics.mapk(list(recoms.values()), list(correct_1.values()), 5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.162020202020202, 'fraction_numerical': 0.014747474747474747, 'mean_word_length': 4.684270952927669, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '332815', 'license': 'MIT', 'n_tokens_mistral': 1943, 'n_tokens_neox': 1863, 'n_words': 263, 'path': 'notebooks/lightFM-fav_serial.ipynb', 'repo_name': 'danilpolkov/sweet_RS', 'size': '4950'}