<jupyter_start><jupyter_text>Models preperation  Loading data and trained song-vectors<jupyter_code>with open(data_path + pickle_file, 'rb') as handle:
    train, test, track_data_map, reverse_track_lookup = pickle.load(handle)
    
train = [val.split() for val in train]
test = [val.split() for val in test]
wv = KeyedVectors.load(data_path + vectors_file)<jupyter_output><empty_output><jupyter_text>Popularity calculation<jupyter_code>try:
    with open(data_path + kmeans_file, 'rb') as handle:
        POPULAR = pickle.load(data_path + song_counts_file)
except Exception as e:
    print('Loading failed: {}'.format(e))
    print('Creating file')
    
    songs_count = {}
    for pl in train:
        for s in pl:
            if s in songs_count:
                songs_count[s] += 1
            else:
                songs_count[s] = 1
    
    s_count = pd.DataFrame.from_dict(songs_count, orient='index')
    s_count.columns = ['count']
    s_count = s_count.sort_values('count', ascending=False)

    POPULAR = list(s_count.index)[:100]
    
    with open(data_path + song_counts_file, 'wb') as handle:
        pickle.dump(POPULAR, handle)<jupyter_output>Loading failed: file must have 'read' and 'readline' attributes
Creating file<jupyter_text>K-Means calculation<jupyter_code>vectors = []
for s in wv.index_to_key:
    vectors.append(wv.get_vector(s))
    
X = np.array(vectors)
try:
    with open(data_path + kmeans_file, 'rb') as handle:
        kmeans = pickle.load(handle)
except Exception as e:
    print('Loading failed: {}'.format(e))
    print('Creating file')
    kmeans = KMeans(n_clusters=NUM_CLUSTERS, random_state=42).fit(X)
    
    with open(data_path + kmeans_file, 'wb') as handle:
        pickle.dump(kmeans, handle)
cluster_map = {}

for i, s in enumerate(wv.key_to_index):
    cluster_map[s] = kmeans.labels_[i]
def pl_to_transition(pl):
    
    global cluster_map
    
    last = None
    plr = [s for s in pl if s in cluster_map]
    
    if len(plr) < 2:
        return {}
        
    ret = {}
    for s in plr:
        if last:
            last_cluster = cluster_map[last]
            current_cluster = cluster_map[s]
            
            transition = (last_cluster, current_cluster)
            if transition in ret:
                ret[transition] += 1
            else:
                ret[transition] = 1
            
        last = s
        
    return ret
%%time

try:
    with open(data_path + transitions_file, 'rb') as handle:
        transition_map = pickle.load(handle)
except Exception as e:
    print('Loading failed: {}'.format(e))
    print('Creating file')
    
    # Map every playlist to a transition_map
    with Pool(4) as p:
        transitions = p.map(pl_to_transition, train)
        
    # Reduce all transition maps to one transition map
    transition_map = {}

    for small_transition_map in transitions:
        for key, value in small_transition_map.items():
            if key[0] in transition_map:
                transition_map[key[0]][key[1]] += value
            else:
                transition_map[key[0]] = np.zeros(NUM_CLUSTERS)
                transition_map[key[0]][key[1]] += value
    
    with open(data_path + transitions_file, 'wb') as handle:
        pickle.dump(transition_map, handle)<jupyter_output>CPU times: user 1.01 ms, sys: 100 µs, total: 1.11 ms
Wall time: 863 µs<jupyter_text>Recommendation methods  Similarity directly from song-vectors<jupyter_code>def predict_using_similarity(s_sample, k, wv):
    song_in_dict = False
    for s in s_sample:
        if s in wv.key_to_index:
            song_in_dict = True
            break
    
    if song_in_dict:
        return wv.most_similar_cosmul(s, topn=k)
    else:
        return predict_using_popularity(k, POPULAR)<jupyter_output><empty_output><jupyter_text>Similarity using KMeans<jupyter_code>def predict_using_kmeans(s_sample, k, k_clusters, wv, cluster_map, transition_map):
    song_in_dict = False
    for s in s_sample:
        if s in wv.key_to_index:
            song_in_dict = True
            break
    
    if song_in_dict:
        s_cluster = cluster_map[s]

        best_clusters = np.argpartition(transition_map[s_cluster], -k_clusters)[-k_clusters:]
        best_clusters = set(best_clusters)

        best_candidates = wv.most_similar_cosmul(s, topn=k * 100)
        best_candidates = [t for t in best_candidates if cluster_map[t[0]] in best_clusters]

        return best_candidates[:k]
    
    else:
        return predict_using_popularity(k, POPULAR)<jupyter_output><empty_output><jupyter_text>Similarity using Popularity<jupyter_code>def predict_using_popularity(k, most_popular):
    best_candidates = most_popular[:k]
    scores = [1 for _ in best_candidates]
    
    ret = [(cand, score) for cand, score in zip(best_candidates, scores)]
    return ret<jupyter_output><empty_output><jupyter_text>Similarity using EmbtoEmb-Encoder-Decoder<jupyter_code>def predict_using_similarity_to_decoding(s_sample, k, wv, autoencoder):
    songs_in_dict = []
    all_results = []
    for s in s_sample:
        if s in wv.key_to_index:
            songs_in_dict.append(s)
    
    if songs_in_dict:
        for s in songs_in_dict:
            vec = wv.get_vector(s).reshape(1, 64, 1)
            encoded_vec = autoencoder.encoder(vec).numpy()
            decoded_vec = autoencoder.decoder(encoded_vec).numpy()
            temp_results = wv.similar_by_vector(decoded_vec[0], topn=int(k / len(songs_in_dict)))
            all_results.extend(temp_results)
        
        return all_results[:100]
    else:
        return predict_using_popularity(k, POPULAR)<jupyter_output><empty_output><jupyter_text>evaluation<jupyter_code>results_df = pd.DataFrame(columns=['Method', 'Time', 'R-Precision', 'Clicks'])
start = time.time()
prediction = [predict_using_popularity(100, POPULAR) for sample in test[:10000]]

results_df = results_df.append({'Method': 'Popularity', 
                                'Time': (time.time() - start) / 60, 
                                'R-Precision': r_precision(test[:10000], prediction), 
                                'Clicks': recommended_songs_clicks(test[:10000], prediction)},
                               ignore_index=True)
start = time.time()
prediction = [predict_using_similarity(sample[:3], 100, wv) for sample in test[:10000]]

results_df = results_df.append({'Method': 'Similarity', 
                                'Time': (time.time() - start) / 60, 
                                'R-Precision': r_precision(test[:10000], prediction), 
                                'Clicks': recommended_songs_clicks(test[:10000], prediction)},
                               ignore_index=True)
start = time.time()
prediction = [predict_using_kmeans(sample[:3], 100, 10, wv, cluster_map, transition_map) for sample in test[:10000]]
prediction_50 = [pred[:50] for pred in prediction]

results_df = results_df.append({'Method': 'Kmeans-Similarity', 
                                'Time': (time.time() - start) / 60, 
                                'R-Precision': r_precision(test[:10000], prediction), 
                                'Clicks': recommended_songs_clicks(test[:10000], prediction)},
                               ignore_index=True)
autoencoder = keras.models.load_model(data_path + 'emb_to_emb_encoder.model')

start = time.time()
prediction = [predict_using_similarity_to_decoding(sample[:3], 100, wv, autoencoder) for sample in test[:10000]]
prediction_50 = [pred[:50] for pred in prediction]

results_df = results_df.append({'Method': 'Emb-to-Emb Similarity', 
                                'Time': (time.time() - start) / 60, 
                                'R-Precision': r_precision(test[:10000], prediction), 
                                'Clicks': recommended_songs_clicks(test[:10000], prediction)},
                               ignore_index=True)
results_df
courses = results_df['Method']
values = results_df['R-Precision']
  
fig = plt.figure(figsize = (10, 5))
 
# creating the bar plot
plt.bar(courses, values, color ='maroon',
        width = 0.4)
 
plt.xlabel("Model")
plt.ylabel("r-precision")
plt.title("R-Precision for each model")
plt.show()
courses = results_df['Method']
values = results_df['Clicks']
  
fig = plt.figure(figsize = (10, 5))
 
# creating the bar plot
plt.bar(courses, values, color ='maroon',
        width = 0.4)
 
plt.xlabel("Model")
plt.ylabel("r-precision")
plt.title("Number of clicks to find song for each model")
plt.show()
courses = results_df['Method']
values = results_df['Time']
  
fig = plt.figure(figsize = (10, 5))
 
# creating the bar plot
plt.bar(courses, values, color ='maroon',
        width = 0.4)
 
plt.xlabel("Model")
plt.ylabel("r-precision")
plt.title("Runtime for each model")
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11353562904875554, 'fraction_numerical': 0.017047391749062394, 'mean_word_length': 2.78494623655914, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '542865', 'license': 'MIT', 'n_tokens_mistral': 3074, 'n_tokens_neox': 2840, 'n_words': 616, 'path': 'code/recommendation.ipynb', 'repo_name': 'raniz/song-rec-project', 'size': '8799'}