<jupyter_start><jupyter_text>Load Data<jupyter_code># global variables
DATA_PATH = 'data/evaluation'
CATEGORY = 'Grocery_and_Gourmet_Food'

train = pd.read_csv(f"{DATA_PATH}/{CATEGORY}_train.csv")
train.head()<jupyter_output><empty_output><jupyter_text>Generate Training Corpus<jupyter_code>train["processedReviewText"] = train["processedReviewText"].progress_apply(lambda x: x.split())
train_corpus = [TaggedDocument(review, [asin]) for asin, review in list(zip(train["asin"], train["processedReviewText"]))]
train_corpus[:5]<jupyter_output><empty_output><jupyter_text>Training D2V Model<jupyter_code>params = yaml.safe_load(open("params.yaml"))["generate_vectors"]
MODEL_PARAMS = params["d2v_params"]
logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(message)s')
model = Doc2Vec(**MODEL_PARAMS)
model.build_vocab(train_corpus)
model.train(train_corpus, total_examples=model.corpus_count, epochs=model.epochs)
# save model
MODEL_PATH = Path("models/d2v")

model.save(f"{MODEL_PATH}/{CATEGORY}_{MODEL_PARAMS['vector_size']}_d2v.model")<jupyter_output>2021-08-21 17:02:33,784 Doc2Vec lifecycle event {'fname_or_handle': 'models/d2v/Grocery_and_Gourmet_Food_50_d2v.model', 'separately': 'None', 'sep_limit': 10485760, 'ignore': frozenset(), 'datetime': '2021-08-21T17:02:33.784912', 'gensim': '4.0.1', 'python': '3.9.6 (default, Jun 29 2021, 05:25:02) \n[Clang 12.0.5 (clang-1205.0.22.9)]', 'platform': 'macOS-11.4-x86_64-i386-64bit', 'event': 'saving'}
2021-08-21 17:02:33,787 not storing attribute cum_table
2021-08-21 17:02:33,788 {'uri': 'models/d2v/Grocery_and_Gourmet_Food_50_d2v.model', 'mode': 'wb', 'buffering': -1, 'encoding': None, 'errors': None, 'newline': None, 'closefd': True, 'opener': None, 'ignore_ext': False, 'compression': None, 'transport_params': None}
2021-08-21 17:02:33,804 saved models/d2v/Grocery_and_Gourmet_Food_50_d2v.model<jupyter_text>Evaluate D2V Model  Testing Retrieval of Vectors via Index<jupyter_code>model.dv[0]<jupyter_output><empty_output><jupyter_text>Testing Retrieval of Vectors via Tags<jupyter_code>model.dv['9742356831']<jupyter_output><empty_output><jupyter_text>Assessing Model<jupyter_code># let's try to generate a random item id and infer its vector and compare if we can get similar items back
random_asin = np.random.choice(list(train['asin'].unique()), 1)[0]

# combining all the words from the all reviews
asin_review = []
for review in train[train['asin'] == random_asin]["processedReviewText"]:
    asin_review.extend(review)

# inferring vector
print(f"For item {random_asin}, {train[train['asin'] == random_asin]['title'].unique()[0]}...\n")
top_n = model.dv.most_similar([model.infer_vector(asin_review, epochs=50)], topn=5)
print(f'Most similar D2V vectors: {top_n}')

# print item details
sim_asins = (pd.DataFrame(top_n, columns=['asin', 'similarities'])
             .merge(train[['asin', 'title']], how='left', on='asin')
             .drop_duplicates())
print(f"\n{sim_asins}")<jupyter_output>For item B001ELL67A,  Grocery &amp; Gourmet Food" />...

Most similar D2V vectors: [('B001ELL67A', 0.951135516166687), ('B0029XDZDK', 0.8652843832969666), ('B000TK6LBS', 0.8596549034118652), ('B003M7TTYG', 0.8523412942886353), ('B0029XDZFS', 0.840634286403656)]

          asin  similarities  \
0   B001ELL67A      0.951136   
14  B0029XDZDK      0.865284   
32  B000TK6LBS      0.859655   
46  B003M7TTYG      0.852341   
53  B0029XDZFS      0.840634   

                                                title  
0                      Grocery &amp; Gourmet Food" />  
14  Coffee People Black Tiger K-Cup Portion Pack f...  
32  Caribou Coffee Caribou Blend, Keurig Single-Se...  
46  Green Mountain Coffee Revv, 22-Count K-Cups fo...  
53  Coffee People Wake Up Call K-Cup Portion Pack ...
{'fraction_non_alphanumeric': 0.1501850872554204, 'fraction_numerical': 0.09598096245372818, 'mean_word_length': 5.315525876460768, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '403387', 'license': 'MIT', 'n_tokens_mistral': 1671, 'n_tokens_neox': 1448, 'n_words': 267, 'path': 'notebooks/exploratory/04-lwt-evaluate-d2v.ipynb', 'repo_name': 'wtlow003/recommendation-systems', 'size': '3782'}