<jupyter_start><jupyter_text>knn<jupyter_code>%%time
import nmslib
index = nmslib.init()
index.loadIndex('22a_sbert_neuralmind.nms')
embs_np = joblib.load("22a_embs_np.pkl.z")
item_emb_map = {t: embs_np[i] for i, t in enumerate(item_data['item_id'].values)} 
embs_search_np = joblib.load("22a_embs_search_np.pkl.z")
k=50<jupyter_output><empty_output><jupyter_text>train<jupyter_code>%%time
data = []
seq_index = 0
for hist, bought in tqdm.tqdm(train[['user_history', 'item_bought']].values):

    
    recall = False
    last_ts = None
    seq = 0
    ts = 0
    rep = dict()
    for item in json.loads(hist):
        i = item['event_info']
        item['bought_id'] = bought
        item['bought_title'] = item_title_map[bought]
        item['bought_price'] = item_price_map[bought]
        item['bought_domain'] = item_domain_map[bought]
        
        if item['event_type'] == 'view':
            item['item_title'] = item_title_map[i]
            item['item_price'] = item_price_map[i]
            item['item_domain'] = item_domain_map[i]
            item['has_bought'] = int(bought == i)
            item['pt'] = int('MLB' in item['item_domain']) if item['item_domain'] else np.nan
            item['viewed'] = 1
            rep[i] = item_emb_map[i]
        
        
        
        item['seq_pos'] = seq
        item['seq_index'] = seq_index

        seq += 1
        data.append(item)
    
    lrep = list(rep.values())
    if len(lrep) == 0:
        view_embedding_mean = embs_search_np[seq_index, :] #search para quem nao tem views
    else:
        view_embedding_mean = np.mean(lrep, axis=0)
    for neighbor in index.knnQuery(view_embedding_mean, k=k)[0]:
        item = dict()
        i = neighbor
        item['event_info'] = neighbor
        item['event_type'] = 'knn'
        item['bought_id'] = bought
        item['bought_title'] = item_title_map[bought]
        item['bought_price'] = item_price_map[bought]
        item['bought_domain'] = item_domain_map[bought]
        item['item_title'] = item_title_map[i]
        item['item_price'] = item_price_map[i]
        item['item_domain'] = item_domain_map[i]
        item['has_bought'] = int(bought == i)
        item['pt'] = int('MLB' in item['item_domain']) if item['item_domain'] else np.nan
        item['seq_pos'] = -1
        item['seq_index'] = seq_index
        item['viewed'] = 0
        
        data.append(item)
        
        
    seq_index += 1
        
df = pd.DataFrame(data)
del data, embs_search_np
gc.collect()
df['event_timestamp'] = pd.to_datetime(df['event_timestamp']).dt.tz_localize(None)
df[df['event_type'] != 'search'].to_parquet("./data/22_train_view_melted.parquet",engine='fastparquet', compression=None)
df[df['event_type'] == 'search'].to_parquet("./data/22_train_search_melted.parquet",engine='fastparquet', compression=None)
df.head()<jupyter_output><empty_output><jupyter_text>test<jupyter_code>embs_search_np = joblib.load("22a_embs_search_test_np.pkl.z")
# last k item matches bought item
data = []
seq_index = 0
for hist in tqdm.tqdm(test['user_history'].values):

    
    last_ts = None
    seq = 0
    ts = 0
    rep = dict()
    for item in json.loads(hist):
        i = item['event_info']
        
        if item['event_type'] == 'view':
            item['item_title'] = item_title_map[i]
            item['item_price'] = item_price_map[i]
            item['item_domain'] = item_domain_map[i]
            item['pt'] = int('MLB' in item['item_domain']) if item['item_domain'] else np.nan
            item['viewed'] = 1
            rep[i] = item_emb_map[i]
        
        item['seq_pos'] = seq
        item['seq_index'] = seq_index

        seq += 1
        data.append(item)
        
    lrep = list(rep.values())
    if len(lrep) == 0:
        view_embedding_mean = embs_search_np[seq_index, :]
    else:
        view_embedding_mean = np.mean(lrep, axis=0)
    for neighbor in index.knnQuery(view_embedding_mean, k=k)[0]:
        item = dict()
        i = neighbor
        item['event_info'] = neighbor
        item['event_type'] = 'knn'
        item['item_title'] = item_title_map[i]
        item['item_price'] = item_price_map[i]
        item['item_domain'] = item_domain_map[i]
        item['pt'] = int('MLB' in item['item_domain']) if item['item_domain'] else np.nan
        item['seq_pos'] = -1
        item['seq_index'] = seq_index
        item['viewed'] = 0
        
        data.append(item)
        
    seq_index += 1
        
df = pd.DataFrame(data)
del data, embs_search_np, embs_np, item_emb_map
gc.collect()
df['event_timestamp'] = pd.to_datetime(df['event_timestamp']).dt.tz_localize(None)
df[df['event_type'] != 'search'].to_parquet("./data/22_test_view_melted.parquet",engine='fastparquet', compression=None)
df[df['event_type'] == 'search'].to_parquet("./data/22_test_search_melted.parquet",engine='fastparquet', compression=None)
df.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14876706745465662, 'fraction_numerical': 0.008151620134501733, 'mean_word_length': 3.0098039215686274, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '529245', 'license': 'MIT', 'n_tokens_mistral': 1786, 'n_tokens_neox': 1758, 'n_words': 282, 'path': '1b_prep_ltr_knn_search.ipynb', 'repo_name': 'ledmaster/meli2020', 'size': '4907'}