<jupyter_start><jupyter_text>Load Data<jupyter_code># global variable
CATEGORY = "Grocery_and_Gourmet_Food"
DATA_PATH = "data/evaluation"
train = pd.read_csv(f"{DATA_PATH}/{CATEGORY}_train.csv")
train.head()<jupyter_output><empty_output><jupyter_text>Preparing Review Text for LDA<jupyter_code># tokenization
processed_reviews = train["processedReviewText"].progress_apply(lambda x: x.split())

# creating a bag-of-words
dictionary = gensim.corpora.Dictionary(processed_reviews)

# filtering out tokens that appear in less than 15 reviews
# or more than 0.5 of the corpus
dictionary.filter_extremes(no_below=5, no_above=0.75)

# creating dict how many words and time the word appear
bow_corpus = [dictionary.doc2bow(doc) for doc in processed_reviews]<jupyter_output>100%|█████████████████████████████████████████████████████| 47774/47774 [00:00<00:00, 139790.53it/s]
2021-08-24 16:16:41,209 : INFO : adding document #0 to Dictionary(0 unique tokens: [])
2021-08-24 16:16:41,568 : INFO : adding document #10000 to Dictionary(14898 unique tokens: ['add', 'chicken', 'coconut', 'curry', 'delicious']...)
2021-08-24 16:16:41,956 : INFO : adding document #20000 to Dictionary(21785 unique tokens: ['add', 'chicken', 'coconut', 'curry', 'delicious']...)
2021-08-24 16:16:42,360 : INFO : adding document #30000 to Dictionary(27143 unique tokens: ['add', 'chicken', 'coconut', 'curry', 'delicious']...)
2021-08-24 16:16:42,779 : INFO : adding document #40000 to Dictionary(31844 unique tokens: ['add', 'chicken', 'coconut', 'curry', 'delicious']...)
2021-08-24 16:16:43,143 : INFO : built Dictionary(34875 unique tokens: ['add', 'chicken', 'coconut', 'curry', 'delicious']...) from 47774 documents (total 1942617 corpus positions)
2021-08-24 16:16:43,164 : INFO : Dictionary[...]<jupyter_text>Training LDA Model<jupyter_code>lda_model = gensim.models.LdaMulticore(bow_corpus, num_topics=50, id2word=dictionary, passes=20, workers=8)
for idx, topic in lda_model.print_topics(-1):
    print(f"Topic: {idx}, Words: {topic}")
# Compute Coherence Score
coherence_model_lda = gensim.models.CoherenceModel(model=lda_model, corpus=bow_corpus, coherence='u_mass')
coherence_lda = coherence_model_lda.get_coherence()
print('\nCoherence Score: ', coherence_lda)<jupyter_output>2021-08-24 16:19:46,395 : INFO : CorpusAccumulator accumulated stats from 1000 documents
2021-08-24 16:19:46,412 : INFO : CorpusAccumulator accumulated stats from 2000 documents
2021-08-24 16:19:46,434 : INFO : CorpusAccumulator accumulated stats from 3000 documents
2021-08-24 16:19:46,453 : INFO : CorpusAccumulator accumulated stats from 4000 documents
2021-08-24 16:19:46,478 : INFO : CorpusAccumulator accumulated stats from 5000 documents
2021-08-24 16:19:46,497 : INFO : CorpusAccumulator accumulated stats from 6000 documents
2021-08-24 16:19:46,516 : INFO : CorpusAccumulator accumulated stats from 7000 documents
2021-08-24 16:19:46,538 : INFO : CorpusAccumulator accumulated stats from 8000 documents
2021-08-24 16:19:46,560 : INFO : CorpusAccumulator accumulated stats from 9000 documents
2021-08-24 16:19:46,579 : INFO : CorpusAccumulator accumulated stats from 10000 documents
2021-08-24 16:19:46,596 : INFO : CorpusAccumulator accumulated stats from 11000 documents
2021-08-24 16:19:46[...]<jupyter_text>Retrieving User/Item Topic Vectors<jupyter_code>user_reviews = train.groupby(["reviewerID"])['processedReviewText'].apply(lambda x: ' '.join(x))
item_reviews = train.groupby(["asin"])["processedReviewText"].apply(lambda x: ' '.join(x))

# get unique users and items
unique_users = user_reviews.index.tolist()
unique_items = item_reviews.index.tolist()

# tokenize reviews
user_reviews_list = user_reviews.apply(lambda x: x.split()).tolist()
item_reviews_list = item_reviews.apply(lambda x: x.split()).tolist()
print(f"Random user:\n{user_reviews[np.random.randint(0, user_reviews.shape[0])]}")
print(f"\nRandom item:\n{item_reviews[np.random.randint(0, item_reviews.shape[0])]}")
user_corpus = [dictionary.doc2bow(doc) for doc in user_reviews_list]
item_corpus = [dictionary.doc2bow(doc) for doc in item_reviews_list]
def get_topic_vectors(model, corpus, n_topics=50):
    """
    """
    topic_vecs = []
    for i in tqdm(range(len(corpus))):
        top_topics = lda_model.get_document_topics(corpus[i], minimum_probability=0.0)
        topic_vecs.append([top_topics[i][1] for i in range(n_topics)])
        
    return topic_vecs
user_vecs = get_topic_vectors(lda_model, user_corpus)
item_vecs = get_topic_vectors(lda_model, item_corpus)
# checking topic vector
pprint(item_vecs[:1])<jupyter_output>[[4.412026e-05,
  0.037068404,
  4.412026e-05,
  4.412026e-05,
  0.01118332,
  4.412026e-05,
  4.412026e-05,
  0.056985307,
  0.37342215,
  4.412026e-05,
  4.412026e-05,
  0.011684398,
  0.0029754133,
  4.412026e-05,
  4.412026e-05,
  0.015933922,
  0.03278494,
  4.412026e-05,
  0.018550407,
  4.412026e-05,
  0.021376897,
  0.05969913,
  4.412026e-05,
  0.041945748,
  0.013643295,
  4.412026e-05,
  4.412026e-05,
  4.412026e-05,
  4.412026e-05,
  4.412026e-05,
  4.412026e-05,
  4.412026e-05,
  4.412026e-05,
  4.412026e-05,
  4.412026e-05,
  0.040703386,
  4.412026e-05,
  4.412026e-05,
  0.17367385,
  0.0230405,
  4.412026e-05,
  4.412026e-05,
  0.016442873,
  4.412026e-05,
  0.0057348763,
  4.412026e-05,
  0.012069327,
  0.019473264,
  0.010329086,
  4.412026e-05]]<jupyter_text>Create Topic Vector Mappings<jupyter_code>user_idx_map = {k: unique_users[k] for k in range(len(unique_users))}
item_idx_map = {k: unique_items[k] for k in range(len(unique_items))}

user_vec_map = {k: v for k, v in zip(unique_users, user_vecs)}
item_vec_map = {k: v for k, v in zip(unique_items, item_vecs)}
# loading user topic vectors into DF
user_vecs = pd.DataFrame.from_dict(user_vec_map, orient='index')
user_vecs.index.name = 'reviewerID'
# loading item topic vectors into DF
item_vecs = pd.DataFrame.from_dict(item_vec_map, orient='index')
item_vecs.index.name = 'asin'
user_vecs
item_vecs
# converting factors into numpy obj
user_factors = user_vecs.to_numpy()
item_factors = item_vecs.to_numpy()
user_factors[0,:]
item_factors[0,:]<jupyter_output><empty_output><jupyter_text>Train `EmbeddedMF` Class w/ Initialized Topic Vectors<jupyter_code>from src.models import emf
ti_mf = emf.EmbeddedMF(user_map=user_idx_map,
                       item_map=item_idx_map,
                       user_factor=user_factors,
                       item_factor=item_factors,
                       learning_rate=.01,
                       beta=.02,
                       num_epochs=20,
                       num_factors=50)
# generate data required for surprise

# create reader
reader = Reader(rating_scale=(1,5))
# generate data required for surprise
data = Dataset.load_from_df(train[['reviewerID', 'asin', 'overall']], reader)
# generating training set
trainset = data.build_full_trainset()
%%time
# fitting algo to training set
ti_mf.fit(trainset)
%%time
# generate candidate items for user to predict rating
testset = trainset.build_anti_testset()
%%time
# predict ratings for all pairs (u, i) that are NOT in the training set
predictions = ti_mf.test(testset, verbose=False)<jupyter_output>CPU times: user 8min 36s, sys: 6min 18s, total: 14min 55s
Wall time: 18min 43s<jupyter_text>Evaluate Top-N Recommendations  Defining Evaluation Metrics<jupyter_code>def precision_at_k(asins, predicted_asins, k=10):
    # number of relevant items
    set_actual = set(asins)
    set_preds = set(predicted_asins)
    num_relevant = len(set_actual.intersection(set_preds))
    
    # calculating precision@K - relevant / total recommended
    precision_at_k = num_relevant / k
    
    return precision_at_k

def recall_at_k(asins, predicted_asins, k=10):
    # number of relevant items
    set_actual = set(asins)
    set_preds = set(predicted_asins)
    num_relevant = len(set_actual.intersection(set_preds))
    
    # calculating recall@K - relevant / total relevant items
    recall_at_k = num_relevant / len(asins)
    
    return recall_at_k
def get_top_n(predictions, n=10):
    """Return the top-N recommendation for each user from a set of predictions.

    Args:
        predictions(list of Prediction objects): The list of predictions, as
            returned by the test method of an algorithm.
        n(int): The number of recommendation to output for each user. Default
            is 10.

    Returns:
    A dict where keys are user (raw) ids and values are lists of tuples:
        [(raw item id, rating estimation), ...] of size n.
    """

    # First map the predictions to each user.
    top_n = defaultdict(list)
    for uid, iid, true_r, est, _ in predictions:
        top_n[uid].append((iid, est))

    # Then sort the predictions for each user and retrieve the k highest ones.
    for uid, user_ratings in top_n.items():
        user_ratings.sort(key=lambda x: x[1], reverse=True)
        top_n[uid] = user_ratings[:n]

    return top_n
# loading test dataset
test = pd.read_csv(f"{DATA_PATH}/{CATEGORY}_test.csv")

# generating test rating history
test_user_history = (pd.DataFrame(test.groupby(['reviewerID'])['asin']
                                  .apply(list).reset_index()))
top_ns = get_top_n(predictions)
# generating a random user
random_user = np.random.choice(list(train['reviewerID'].unique()), 1)[0]
print(f"For user: {random_user}:")
print(f"Purchase History:\n{train[train['reviewerID'] == random_user][['asin', 'title']]}")

# find the recommendations
# NOTE: This is not sorted in ranking
print(f"\nRecommending:\n")
print(f"{train[train['asin'].isin([i[0] for i in top_ns[random_user]])][['asin', 'title']].drop_duplicates(subset='asin')}")
print(f"\n{top_ns[random_user]}")<jupyter_output>For user: AWY8ZY16DPTVE:
Purchase History:
             asin                                              title
19630  B001E6K6B2  Kellogg's Raisin Bran Crunch, Breakfast Cereal...
33274  B0040J01KQ  Pop-Tarts Toaster Pastries Variety Pack, 48-Co...

Recommending:

             asin                                              title
344    B00014HS2S  Prince of Peace Oolong Tea - 100 Tea Bags net ...
1336   B00028PVA4  Tazo Refresh Mint Herbal Tea Filterbags (20 co...
2542   B000CMD64S  Yogi Rejuvenation Green Tea, 16 Tea Bags (Pack...
14852  B0013L0C6W                      Soda Stream Tonic Syrup 440ml
19011  B001E5E10K   White Gold Honey, 23-Ounce Container (Pack of 2)
19038  B001E5E12I   Maine Coast Sea Vegetables Kombu, Wild Atlant...
21967  B001EQ5IPQ  O.N.E. 100% Natural Coconut Water, 33.8 Ounce ...
21970  B001EQ5IBU  NOW Foods Macadamia Nuts Roasted-salted, 12-Ou...
25988  B001TNXSZG   Guayaki Traditional Organic Yerba Mate, Loose...
27609  B00286KM8E  Lipton Black Tea Bags, Am[...]<jupyter_text>N=10<jupyter_code>top_ns = get_top_n(predictions, n=10)

test_recommendations = pd.DataFrame(top_ns.items(), columns=['reviewerID', 'pred_asin'])
test_recommendations['pred_asin'] = test_recommendations['pred_asin'].apply(lambda x: [i[0] for i in x])

# combined test dataset and recommendations
test_merged = pd.merge(test_user_history, test_recommendations, on='reviewerID', how='inner')

k = 10
test_merged['precision@k'] = test_merged.progress_apply(lambda x: precision_at_k(x.asin, x.pred_asin, k=k), axis=1)
test_merged['recall@k'] = test_merged.progress_apply(lambda x: recall_at_k(x.asin, x.pred_asin, k=k), axis=1)

average_precision_at_k = test_merged["precision@k"].mean()
average_recall_at_k = test_merged["recall@k"].mean()

print(f"The MEM-ECF has a average precision@{k}: {average_precision_at_k:.5f}, average recall@{k}: {average_recall_at_k:.5f}.")<jupyter_output>100%|██████████████████████████████████████████████████████| 13279/13279 [00:00<00:00, 41617.66it/s]
100%|██████████████████████████████████████████████████████| 13279/13279 [00:00<00:00, 48716.33it/s]<jupyter_text>N=25<jupyter_code>top_ns = get_top_n(predictions, n=25)

test_recommendations = pd.DataFrame(top_ns.items(), columns=['reviewerID', 'pred_asin'])
test_recommendations['pred_asin'] = test_recommendations['pred_asin'].apply(lambda x: [i[0] for i in x])

# combined test dataset and recommendations
test_merged = pd.merge(test_user_history, test_recommendations, on='reviewerID', how='inner')

k = 25
test_merged['precision@k'] = test_merged.progress_apply(lambda x: precision_at_k(x.asin, x.pred_asin, k=k), axis=1)
test_merged['recall@k'] = test_merged.progress_apply(lambda x: recall_at_k(x.asin, x.pred_asin, k=k), axis=1)

average_precision_at_k = test_merged["precision@k"].mean()
average_recall_at_k = test_merged["recall@k"].mean()

print(f"The MEM-ECF has a average precision@{k}: {average_precision_at_k:.5f}, average recall@{k}: {average_recall_at_k:.5f}.")<jupyter_output>100%|██████████████████████████████████████████████████████| 13279/13279 [00:00<00:00, 45502.25it/s]
100%|██████████████████████████████████████████████████████| 13279/13279 [00:00<00:00, 45615.50it/s]<jupyter_text>N=30<jupyter_code>top_ns = get_top_n(predictions, n=30)

test_recommendations = pd.DataFrame(top_ns.items(), columns=['reviewerID', 'pred_asin'])
test_recommendations['pred_asin'] = test_recommendations['pred_asin'].apply(lambda x: [i[0] for i in x])

# combined test dataset and recommendations
test_merged = pd.merge(test_user_history, test_recommendations, on='reviewerID', how='inner')

k = 30
test_merged['precision@k'] = test_merged.progress_apply(lambda x: precision_at_k(x.asin, x.pred_asin, k=k), axis=1)
test_merged['recall@k'] = test_merged.progress_apply(lambda x: recall_at_k(x.asin, x.pred_asin, k=k), axis=1)

average_precision_at_k = test_merged["precision@k"].mean()
average_recall_at_k = test_merged["recall@k"].mean()

print(f"The MEM-ECF has a average precision@{k}: {average_precision_at_k:.5f}, average recall@{k}: {average_recall_at_k:.5f}.")<jupyter_output>100%|██████████████████████████████████████████████████████| 13279/13279 [00:00<00:00, 32080.60it/s]
100%|██████████████████████████████████████████████████████| 13279/13279 [00:00<00:00, 28881.46it/s]<jupyter_text>N=45<jupyter_code>top_ns = get_top_n(predictions, n=45)

test_recommendations = pd.DataFrame(top_ns.items(), columns=['reviewerID', 'pred_asin'])
test_recommendations['pred_asin'] = test_recommendations['pred_asin'].apply(lambda x: [i[0] for i in x])

# combined test dataset and recommendations
test_merged = pd.merge(test_user_history, test_recommendations, on='reviewerID', how='inner')

k = 45
test_merged['precision@k'] = test_merged.progress_apply(lambda x: precision_at_k(x.asin, x.pred_asin, k=k), axis=1)
test_merged['recall@k'] = test_merged.progress_apply(lambda x: recall_at_k(x.asin, x.pred_asin, k=k), axis=1)

average_precision_at_k = test_merged["precision@k"].mean()
average_recall_at_k = test_merged["recall@k"].mean()

print(f"The MEM-ECF has a average precision@{k}: {average_precision_at_k:.5f}, average recall@{k}: {average_recall_at_k:.5f}.")
# looking at how many get correct
test_merged[test_merged['recall@k'] > 0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1520170985840235, 'fraction_numerical': 0.08983435746727224, 'mean_word_length': 5.1516023007395235, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '195505', 'license': 'MIT', 'n_tokens_mistral': 6753, 'n_tokens_neox': 5611, 'n_words': 1141, 'path': 'notebooks/exploratory/09-lwt-topic-mf.ipynb', 'repo_name': 'wtlow003/recommendation-systems', 'size': '14972'}