<jupyter_start><jupyter_text>**Tags recommendation (6/7)**.        This notebook's goal is to compare the performances of the 3 unsupervised approaches (NMF, LDA and LDA+Word2Vec) used in this project.    It is called in the notebook "6_Workflows_iteration.ipynb" in order to loop over it's hyperparameters             **PLAN**            **0) Libraries, functions and datasources import**        **I) Starting the program**        **II) Evaluation & Benchmark of the algorithms performances**<jupyter_code>print("Library, functions and datasources import")
import pandas as pd
import pickle
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import time
from sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer
from collections import Counter
from context import dir_path, datasources_path, enrichment_path, pickles_path, temp_files_path
from functions import aggregate_col_in_list
df_model = pickle.load(open(pickles_path+"df_model_unsupervised.p", "rb" ))
L_train_indexes = pickle.load(open(pickles_path+"L_train_indexes.p", "rb" ))
L_test_indexes = pickle.load(open(pickles_path+"L_test_indexes.p", "rb" ))
L_frequent_tags = pickle.load(open(pickles_path+"L_frequent_tags.p", "rb" ))
df_train = df_model[df_model['id'].isin(L_train_indexes)]
df_test = df_model[df_model['id'].isin(L_test_indexes)]
df_train_nmf = df_model[df_model['id'].isin(L_train_indexes)]
df_test_nmf = df_model[df_model['id'].isin(L_test_indexes)]

df_train_lda = df_model[df_model['id'].isin(L_train_indexes)]
df_test_lda = df_model[df_model['id'].isin(L_test_indexes)]
L_corpus = list(df_train_nmf['corpus'])
execution_params_workflows_iteration = pickle.load(open(pickles_path+"execution_params_workflows_iteration.p", "rb" ))
RUN = execution_params_workflows_iteration['RUN']
N_COMPONENTS = execution_params_workflows_iteration['N_COMPONENTS']
TOPICS_THRESHOLD = execution_params_workflows_iteration['TOPICS_THRESHOLD']
NEIGHBORS = execution_params_workflows_iteration['NEIGHBORS']
QUANTILE_THRESHOLD = execution_params_workflows_iteration['QUANTILE_THRESHOLD']<jupyter_output><empty_output><jupyter_text>**I) Starting the program**<jupyter_code>print("\n\n\nStarting run %s with params :\nN_COMPONENTS = %s \nTHRESHOLD = %s\nNEIGHBORS = %s\nQUANTILE_THRESHOLD = %s"%(RUN,N_COMPONENTS,TOPICS_THRESHOLD,NEIGHBORS,QUANTILE_THRESHOLD))
print("implementing NMF")
from sklearn.decomposition import NMF
N_FEATURES = 1000
N_TOP_WORDS = 20
# Use count (raw term count) features for NMF.
print("Extracting tf-idf features for NMF...")
tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=10,max_features=N_FEATURES)
start_time = time.time()
tfidf = tfidf_vectorizer.fit_transform(L_corpus)
print("--- %s seconds ---" % (time.time() - start_time))
print("Fitting the NMF model")
start_time = time.time()
nmf = NMF(n_components=N_COMPONENTS, random_state=0,alpha=.1, l1_ratio=.5).fit(tfidf)
print("--- %s seconds ---" % (time.time() - start_time))

print("\nTopics in NMF model (Frobenius norm):")
vectorizer_feature_names = tfidf_vectorizer.get_feature_names()
df_train_nmf_words_space_schema = pd.DataFrame(columns = vectorizer_feature_names)
df_train_nmf_words_space_schema
print("train_words_space_schema built!")
print("implementing LDA")
from sklearn.decomposition import LatentDirichletAllocation
print("Extracting count features for LDA...")
count_vectorizer = CountVectorizer(max_df=0.95, min_df=10, max_features=N_FEATURES)
start_time = time.time()
count = count_vectorizer.fit_transform(L_corpus)
print("--- %s seconds ---" % (time.time() - start_time))
print("Fitting the LDA model")
lda = LatentDirichletAllocation(n_components=N_COMPONENTS, max_iter=5,
                                learning_method='online',
                                learning_offset=50.,
                                random_state=0)
start_time = time.time()
lda.fit(count)

print("--- %s seconds ---" % (time.time() - start_time))
results_nmf = nmf.transform(tfidf)
results_lda = lda.transform(count)
L_main_topics_nmf = []
L_main_topics_lda = []
for sub_result_nmf,sub_result_lda in zip(results_nmf,results_lda):
    L_main_topics_nmf.append(np.argmax(sub_result_nmf))
    L_main_topics_lda.append(np.argmax(sub_result_lda))
df_corpus_topics_nmf = pd.DataFrame(results_nmf)
df_corpus_topics_lda = pd.DataFrame(results_lda)

df_corpus_topics_nmf['id'] = list(df_train_nmf['id'])
df_corpus_topics_lda['id'] = list(df_train_lda['id'])

df_corpus_topics_nmf['main_topic'] = L_main_topics_nmf
df_corpus_topics_lda['main_topic'] = L_main_topics_lda

df_corpus_topics_nmf['corpus'] = L_corpus
df_corpus_topics_lda['corpus'] = L_corpus
L_topics = []
for topic in range(N_COMPONENTS):
    L_topics.append(topic)
print("Topics associated to words management")
df_topics_words_nmf = pd.DataFrame(nmf.components_, columns=vectorizer_feature_names)
df_topics_words_lda = pd.DataFrame(lda.components_, columns=vectorizer_feature_names)
print("words/topics normalization")
for column in df_topics_words_nmf.columns:
    column_sum = np.sum(df_topics_words_nmf[column])
    df_topics_words_nmf[column] = df_topics_words_nmf[column].apply(lambda x : x/column_sum)
    column_sum = np.sum(df_topics_words_lda[column])
    df_topics_words_lda[column] = df_topics_words_lda[column].apply(lambda x : x/column_sum)
print("Words associated to Topics management")
df_words_topics_nmf = df_topics_words_nmf.T
df_words_topics_lda = df_topics_words_lda.T
L_matching_tags = [word for word in df_words_topics_nmf.index if word in L_frequent_tags]
L_missing_tags = [word for word in L_frequent_tags if not word in L_matching_tags]

print("The models has %s tags coordinates in the topics space | %s tags are missing"%(len(L_matching_tags),len(L_missing_tags)))
print("Tags associated to topics management")
print("Developping the train dataframe")
#We want to denormalize our entry dataframe thanks to the 'tag_list' column :
#We specify the excution params for  the python script responsible for the multiprocessing : 
execution_params = {'N_PROCS' : 6,
                    'dataframe':df_train[['tag_list','id']],
                   'column_to_develop':'tag_list'}
#We dump our execution params :
pickle.dump(execution_params,open(temp_files_path+"execution_params.p", "wb"))
#And we run the script responsible for the stopwords elimination on different process :  
%run -i multiprocessing_develop.py


#We load the result of the multiprocessing task : 
developped_dataframe = pickle.load(open(temp_files_path+"developped_dataframe.p", "rb" ))

#We clean the temp files dir :
import os
from functions import class_my_files
dict_extensions = class_my_files(temp_files_path)
pickles_temp_files  = dict_extensions['p']
for file_name in pickles_temp_files:
    os.remove(temp_files_path+file_name)

developped_dataframe.sort_values(by='id', inplace=True)
developped_dataframe.rename({'tag_list':'tag'}, axis=1, inplace=True)
print("train dataframe developped")
print("Join between the developped dataframe and the NMF and LDA corpus/topics distributions dataframes")
#Now the dataframe is denormalized on the 'tag' column, we can compute the average topic distribution for each tag :
global_df_corpus_topics_nmf = pd.merge(developped_dataframe,df_corpus_topics_nmf,how='left',on='id')
global_df_corpus_topics_lda = pd.merge(developped_dataframe,df_corpus_topics_lda,how='left',on='id')
print("Aggregation")
#We first parameter the list  of columns we need for the aggregation :
L_tags_topics = L_topics.copy()
L_tags_topics.append('tag')
#And we aggregate the topics distributions by the dataframe's tags
df_tags_topics_nmf = global_df_corpus_topics_nmf[L_tags_topics].groupby('tag').mean()
df_tags_topics_lda = global_df_corpus_topics_lda[L_tags_topics].groupby('tag').mean()
print("words/topics normalization")
for tag in df_tags_topics_nmf.index:
    row_sum = np.sum(df_tags_topics_nmf.loc[tag,])
    df_tags_topics_nmf.loc[tag,] = df_tags_topics_nmf.loc[tag,]/row_sum
    row_sum = np.sum(df_tags_topics_lda.loc[tag,])
    df_tags_topics_lda.loc[tag,] = df_tags_topics_lda.loc[tag,]/row_sum
#Finally, the dataframe mapping all tags to their topics distribution is :
df_tags_topics_nmf = df_tags_topics_nmf.loc[L_missing_tags,].append(df_words_topics_nmf.loc[L_matching_tags,])
df_tags_topics_lda = df_tags_topics_lda.loc[L_missing_tags,].append(df_words_topics_lda.loc[L_matching_tags,])
#And we can now append the missing tags in the dataframe which maps all words to their topics : 
df_words_topics_nmf = df_words_topics_nmf.append(df_tags_topics_nmf.loc[L_missing_tags,])
df_words_topics_lda = df_words_topics_lda.append(df_tags_topics_lda.loc[L_missing_tags,])
print("Building the NMF word/tags similatities matrices")
#With this two dataframes, we can compute the similarities between tags and words (with tags) in the topic's space:
#Because this space is sparse, we will choose manhattan distances
X_tags_topics_nmf = df_tags_topics_nmf.values
X_word_topics_nmf = df_words_topics_nmf.values
from sklearn.metrics.pairwise import manhattan_distances

df_nmf_tags_similarity = pd.DataFrame(manhattan_distances(X_tags_topics_nmf),columns=df_tags_topics_nmf.index, index=df_tags_topics_nmf.index)
df_nmf_words_similarity = pd.DataFrame(manhattan_distances(X_word_topics_nmf),columns=df_words_topics_nmf.index, index=df_words_topics_nmf.index)
print("Building the LDA word/tags similatities matrices")
#With this two dataframes, we can compute the similarities between tags and words (with tags) in the topic's space:
X_tags_topics = df_tags_topics_lda.values
X_word_topics = df_tags_topics_lda.values
from sklearn.metrics.pairwise import cosine_similarity

df_lda_tags_similarity = pd.DataFrame(cosine_similarity(X_tags_topics),columns=df_tags_topics_lda.index, index=df_tags_topics_lda.index)
df_lda_words_similarity = pd.DataFrame(cosine_similarity(X_word_topics),columns=df_tags_topics_lda.index, index=df_tags_topics_lda.index)
print("Topics management : building the topics/tags and topics/words dict for NMF and LDA")
dict_topics_tags_nmf = {}
dict_topics_words_nmf = {}
dict_topics_tags_lda = {}
dict_topics_words_lda = {}

for topic in L_topics:
    L_heavy_tags = list(df_tags_topics_nmf[df_tags_topics_nmf[topic]>=TOPICS_THRESHOLD].index)
    dict_topics_tags_nmf[topic] = L_heavy_tags
    L_heavy_tags = list(df_tags_topics_lda[df_tags_topics_lda[topic]>=TOPICS_THRESHOLD].index)
    dict_topics_tags_lda[topic] = L_heavy_tags
    
    L_heavy_words = list(df_words_topics_nmf[df_words_topics_nmf[topic]>=TOPICS_THRESHOLD].index)
    dict_topics_words_nmf[topic] = L_heavy_words
    L_heavy_words = list(df_words_topics_lda[df_words_topics_lda[topic]>=TOPICS_THRESHOLD].index)
    dict_topics_words_lda[topic] = L_heavy_words
print("Word2Vec Implementation")
df_train_word2vec = df_train_lda.copy()
start_time = time.time()
from gensim.models import word2vec
from nltk import word_tokenize

# tokenize sentences in corpus
corpus = df_train_word2vec['corpus'].copy()
tokenized_corpus = corpus.apply(word_tokenize)


# Set values for various parameters
feature_size = 100    # Word vector dimensionality  
window_context = 30          # Context window size                                                                                    
min_word_count = 1   # Minimum word count                        
sample = 1e-3   # Downsample setting for frequent words

w2v_model = word2vec.Word2Vec(tokenized_corpus, size=feature_size,
                              window=window_context, min_count=min_word_count,
                              sample=sample, iter=100,
                              workers=6)
print("--- %s seconds ---" % (time.time() - start_time))
vectors = w2v_model.wv
L_tags_word2vec = [tag for tag in L_frequent_tags if tag in vectors]
L_word2vec_words = [word for word in vectors.vocab.keys()]
print("Building the Word2Vec word/tags similatities matrices")
start_time = time.time()
df_word2vec_words_coordinates = pd.DataFrame(columns = [i for i in range(0,100)])
for word in L_word2vec_words:
    df_word_coordinates = pd.DataFrame(vectors[word]).T
    df_word_coordinates.index = [word]
    df_word2vec_words_coordinates = df_word2vec_words_coordinates.append(df_word_coordinates)
df_word2vec_tags_coordinates = df_word2vec_words_coordinates.loc[L_tags_word2vec]
X_tags_word2vec = df_word2vec_tags_coordinates.values
X_words_word2vec = df_word2vec_words_coordinates.values

df_word2vec_tags_similarity = pd.DataFrame(cosine_similarity(X_tags_word2vec),\
                                           columns=df_word2vec_tags_coordinates.index,\
                                           index=df_word2vec_tags_coordinates.index)

df_word2vec_words_similarity = pd.DataFrame(cosine_similarity(X_words_word2vec),\
                                            columns=df_word2vec_words_coordinates.index,\
                                            index=df_word2vec_words_coordinates.index)
print("--- %s seconds ---" % (time.time() - start_time))
print("Word2Vec moodel training on the pure corpus of tag lists")
start_time = time.time()
tags_corpus = df_train[['tag_list']].copy()
print("Managing the tag list for the train")
dict_tags_replacement = pickle.load(open(pickles_path+"dict_tags_replacement.p", "rb" ))
reverse_dict_tags_replacement = {v: k for k, v in dict_tags_replacement.items()}

tags_corpus['raw_tag_list'] = tags_corpus['tag_list'].apply(lambda x : [reverse_dict_tags_replacement[tag] for tag in x])

for col in ['raw_tag_list','tag_list']:
    tags_corpus[col] = tags_corpus[col].astype(str)
tags_corpus['tag_list'] = tags_corpus['tag_list']+tags_corpus['raw_tag_list']
tags_corpus.drop('raw_tag_list', axis=1, inplace=True)

dict_replacement = {'[-\. ]':' ','[cC] {0,1}#':'csharp','[\[\]\',]':' ',' +':' '}
for pattern, replacement in dict_replacement.items():
    tags_corpus['tag_list'] = tags_corpus['tag_list'].str.replace(pattern,replacement)

tags_corpus['tag_list'] = tags_corpus['tag_list'].apply(word_tokenize)
tokenized_tags_corpus = list(tags_corpus['tag_list'])
print("Starting the pure tags Word2Vec model")
feature_size = 50    # Word vector dimensionality  
window_context = 30          # Context window size                                                                                    
min_word_count = 1   # Minimum word count                        
sample = 1e-3   # Downsample setting for frequent words

w2v_pure_tags_model = word2vec.Word2Vec(tokenized_tags_corpus, size=feature_size,
                              window=window_context, min_count=min_word_count,
                              sample=sample, iter=100,
                              workers=6)
pure_tags_vectors = w2v_pure_tags_model.wv
print("Starting the pure tags similarity matrix")
L_word2vec_pure_tags_words = [word for word in pure_tags_vectors.vocab.keys()]

df_word2vec_pure_tags_coordinates = pd.DataFrame(columns = [i for i in range(0,50)])
for word in L_word2vec_pure_tags_words:
    df_word_coordinates = pd.DataFrame(pure_tags_vectors[word]).T
    df_word_coordinates.index = [word]
    df_word2vec_pure_tags_coordinates = df_word2vec_pure_tags_coordinates.append(df_word_coordinates)

X_pure_tags_word2vec = df_word2vec_pure_tags_coordinates.values

df_word2vec_pure_tags_similarity = pd.DataFrame(cosine_similarity(X_pure_tags_word2vec),\
                                           columns=df_word2vec_pure_tags_coordinates.index,\
                                           index=df_word2vec_pure_tags_coordinates.index)
df_word2vec_pure_tags_similarity
print("--- %s seconds ---" % (time.time() - start_time))
print("Prediction on the test data : Building the test datasets")
L_test_corpus = list(df_test['corpus'])
print("Extracting count features for NMF...")
test_tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=10, max_features=N_FEATURES)

start_time = time.time()
test_tfidf = test_tfidf_vectorizer.fit_transform(L_test_corpus)
test_vectorizer_feature_names = test_tfidf_vectorizer.get_feature_names()
test_tfidf_array = test_tfidf.toarray()
df_all_test_words_nmf = pd.DataFrame(test_tfidf_array, columns=test_vectorizer_feature_names)
print("--- %s seconds ---" % (time.time() - start_time))
print("Extracting count features for lda...")
test_tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=10, max_features=N_FEATURES)

start_time = time.time()
test_tfidf = test_tfidf_vectorizer.fit_transform(L_test_corpus)
test_vectorizer_feature_names = test_tfidf_vectorizer.get_feature_names()
test_tfidf_array = test_tfidf.toarray()
df_all_test_words_lda = pd.DataFrame(test_tfidf_array, columns=test_vectorizer_feature_names)
print("--- %s seconds ---" % (time.time() - start_time))
#We get back the train columns schema :
df_final_test_words = df_train_nmf_words_space_schema.copy()
#The original shape is :
test_words_dim = df_all_test_words_nmf.shape[1]
print("Original test space words dimension is %s"%test_words_dim)
#But we have to filter the data onto the train space perimeter
df_all_test_words_nmf = df_all_test_words_nmf[[col for col in df_all_test_words_nmf.columns if col in list(df_final_test_words.columns)]]
df_all_test_words_lda = df_all_test_words_lda[[col for col in df_all_test_words_lda.columns if col in list(df_final_test_words.columns)]]
test_words_dim = df_all_test_words_nmf.shape[1]
print("After filtering on the train columns perimeter, test space words dimension is %s"%test_words_dim)
#Finally, the test data looks like : 
df_final_test_words_nmf = df_final_test_words.append(df_all_test_words_nmf)
df_final_test_words_lda = df_final_test_words.append(df_all_test_words_lda)
df_final_test_words_nmf.fillna(0, inplace=True)
df_final_test_words_lda.fillna(0, inplace=True)
test_matrix_nmf = df_final_test_words_nmf.values
test_matrix_lda = df_final_test_words_lda.values
print("Applying NMF and LDA on the test data")
test_results_nmf = nmf.transform(test_matrix_nmf)
test_results_lda = lda.transform(test_matrix_lda)
print("Extracting the main topics")
L_test_main_topics_nmf = []
L_test_main_topics_lda = []
for sub_result_nmf, sub_result_lda in zip(test_results_nmf, test_results_lda):
    L_test_main_topics_nmf.append(np.argmax(sub_result_nmf))
    L_test_main_topics_lda.append(np.argmax(sub_result_lda))
df_test_nmf_corpus_topics = pd.DataFrame(test_results_nmf)
df_test_lda_corpus_topics = pd.DataFrame(test_results_lda)

df_test_nmf_corpus_topics['id'] = list(df_test_nmf['id'])
df_test_lda_corpus_topics['id'] = list(df_test_lda['id'])

df_test_nmf_corpus_topics['main_topic'] = L_test_main_topics_nmf
df_test_lda_corpus_topics['main_topic'] = L_test_main_topics_lda
df_test_nmf = pd.merge(df_test_nmf,df_test_nmf_corpus_topics,how='left',on='id')
df_test_lda = pd.merge(df_test_lda,df_test_lda_corpus_topics,how='left',on='id')
print("Tags generation on the test perimeter")
df_test_nmf['nmf_tags'] = df_test_nmf['main_topic'].apply(lambda x : dict_topics_tags_nmf[x])
df_test_lda['lda_tags'] = df_test_lda['main_topic'].apply(lambda x : dict_topics_tags_lda[x])
print("Generating nmf recommended tags")
#We specify the excution params for  the python script responsible for the multiprocessing : 
execution_params = {'N_PROCS' : 6,
                    'list_of_corpus':list(df_test_nmf['corpus']),
                    'nmf_word_sim_matrix':df_nmf_words_similarity,
                    'nmf_tag_sim_matrix':df_nmf_tags_similarity,
                    'existing_tags':L_frequent_tags,
                    'NEIGHBORS':NEIGHBORS,
                    'recommendation_filter':list(df_test_nmf['nmf_tags']),
                    'QUANTILE_THRESHOLD':QUANTILE_THRESHOLD,
                   }
#We dump our execution params :
pickle.dump(execution_params,open(temp_files_path+"execution_params.p", "wb"))
#And we run the script responsible for the tags generation on different process :  
%run -i multiprocessing_generate_tags_nmf.py


#Now we can load the result of the multiprocessing task : 
L_recommended_tags_nmf = pickle.load(open(temp_files_path+"L_recommended_tags.p", "rb" ))



#And we can clean the temp files dir :
import os
from functions import class_my_files
dict_extensions = class_my_files(temp_files_path)
pickles_temp_files  = dict_extensions['p']
for file_name in pickles_temp_files:
    os.remove(temp_files_path+file_name)

#And we assign the result of the multiprocessing :
df_test_nmf['recommendation_nmf'] = L_recommended_tags_nmf
print("Generating lda recommended tags")
#We specify the excution params for  the python script responsible for the multiprocessing : 
execution_params = {'N_PROCS' : 6,
                    'list_of_corpus':list(df_test_lda['corpus']),
                    'lda_word_sim_matrix':df_lda_words_similarity,
                    'lda_tag_sim_matrix':df_lda_tags_similarity,
                    'existing_tags':L_frequent_tags,
                    'NEIGHBORS':NEIGHBORS,
                    'recommendation_filter':list(df_test_lda['lda_tags']),
                    'QUANTILE_THRESHOLD':QUANTILE_THRESHOLD,
                   }
#We dump our execution params :
pickle.dump(execution_params,open(temp_files_path+"execution_params.p", "wb"))
#And we run the script responsible for the tags generation on different process :  
%run -i multiprocessing_generate_tags_lda.py


#Now we can load the result of the multiprocessing task : 
L_recommended_tags_lda = pickle.load(open(temp_files_path+"L_recommended_tags.p", "rb" ))



#And we can clean the temp files dir :
import os
from functions import class_my_files
dict_extensions = class_my_files(temp_files_path)
pickles_temp_files  = dict_extensions['p']
for file_name in pickles_temp_files:
    os.remove(temp_files_path+file_name)

#And we assign the result of the multiprocessing :
df_test_lda['recommendation_lda'] = L_recommended_tags_lda
print("Generating lda+vord2vec recommended tags")

df_test_lda_word2vec = df_test_lda.copy()

#We specify the excution params for  the python script responsible for the multiprocessing : 
execution_params = {'N_PROCS' : 6,
                    'list_of_corpus':list(df_test_lda_word2vec['corpus']),
                    'lda_word_sim_matrix':df_lda_words_similarity,
                    'lda_tag_sim_matrix':df_lda_tags_similarity,
                    
                    'w2v_word_sim_matrix':df_word2vec_words_similarity,
                    'w2v_tag_sim_matrix':df_word2vec_tags_similarity,
                    'w2v_pure_tag_sim_matrix':df_word2vec_pure_tags_similarity,
                    
                    'existing_tags':L_frequent_tags,
                    'NEIGHBORS':NEIGHBORS,
                    'recommendation_filter':list(df_test_lda_word2vec['lda_tags']),
                    'QUANTILE_THRESHOLD':QUANTILE_THRESHOLD,
                   }
#We dump our execution params :
pickle.dump(execution_params,open(temp_files_path+"execution_params.p", "wb"))
#And we run the script responsible for the tags generation on different process :  
%run -i multiprocessing_generate_tags_lda_word2vec.py


#Now we can load the result of the multiprocessing task : 
L_recommended_tags_lda_word2vec = pickle.load(open(temp_files_path+"L_recommended_tags.p", "rb" ))



#And we can clean the temp files dir :
import os
from functions import class_my_files
dict_extensions = class_my_files(temp_files_path)
pickles_temp_files  = dict_extensions['p']
for file_name in pickles_temp_files:
    os.remove(temp_files_path+file_name)

#And we assign the result of the multiprocessing :
df_test_lda_word2vec['recommendation_lda_w2v'] = L_recommended_tags_lda_word2vec
print("Evaluation on the test perimeter")
df_test_nmf['n_tags_recommended_nmf'] = df_test_nmf.apply(lambda x : len(x['recommendation_nmf']),axis=1)
df_test_lda['n_tags_recommended_lda'] = df_test_lda.apply(lambda x : len(x['recommendation_lda']),axis=1)
df_test_lda_word2vec['n_tags_recommended_lda_w2v'] = df_test_lda_word2vec.apply(lambda x : len(x['recommendation_lda_w2v']),axis=1)

df_test_nmf['matching_tags_test_nmf'] = df_test_nmf.apply(lambda x : set(x['recommendation_nmf']).intersection(set(x['tag_list'])), axis=1)
df_test_lda['matching_tags_test_lda'] = df_test_lda.apply(lambda x : set(x['recommendation_lda']).intersection(set(x['tag_list'])), axis=1)
df_test_lda_word2vec['matching_tags_test_lda_w2v'] = df_test_lda_word2vec.apply(lambda x : set(x['recommendation_lda_w2v']).intersection(set(x['tag_list'])), axis=1)

df_test_nmf['matching_score_test_nmf'] = df_test_nmf.apply(lambda x : len(x['matching_tags_test_nmf'])/x['n_tags'], axis=1)
df_test_lda['matching_score_test_lda'] = df_test_lda.apply(lambda x : len(x['matching_tags_test_lda'])/x['n_tags'], axis=1)
df_test_lda_word2vec['matching_score_test_lda_w2v'] = df_test_lda_word2vec.apply(lambda x : len(x['matching_tags_test_lda_w2v'])/x['n_tags'], axis=1)
df_evaluation = df_test.copy()
df_evaluation = df_evaluation[['id','n_tags','matching_score']]
df_evaluation = pd.merge(df_evaluation,df_test_nmf[['id','n_tags_recommended_nmf','matching_score_test_nmf']]\
                         ,how='left'\
                         ,on='id')

df_evaluation = pd.merge(df_evaluation,df_test_lda[['id','n_tags_recommended_lda','matching_score_test_lda']]\
                         ,how='left'\
                         ,on='id')

df_evaluation = pd.merge(df_evaluation,df_test_lda_word2vec[['id','n_tags_recommended_lda_w2v','matching_score_test_lda_w2v']]\
                         ,how='left'\
                         ,on='id')
df_evaluation<jupyter_output><empty_output><jupyter_text>**II) Evaluation & Benchmark of the algorithms performances**<jupyter_code>import math
df_evaluation['nmf_tags_number_accuracy'] = df_evaluation.apply(lambda x:
                                                                math.exp((x['n_tags']-x['n_tags_recommended_nmf'])/x['n_tags_recommended_nmf'])
                                                                if x['n_tags']< x['n_tags_recommended_nmf']
                                                                else
                                                                math.exp((x['n_tags_recommended_nmf']-x['n_tags'])/x['n_tags'])
                                                                ,axis=1)

df_evaluation['lda_tags_number_accuracy'] = df_evaluation.apply(lambda x:
                                                                math.exp((x['n_tags']-x['n_tags_recommended_lda'])/x['n_tags_recommended_lda'])
                                                                if x['n_tags']< x['n_tags_recommended_lda']
                                                                else
                                                                math.exp((x['n_tags_recommended_lda']-x['n_tags'])/x['n_tags'])
                                                                ,axis=1)

df_evaluation['lda_w2v_tags_number_accuracy'] = df_evaluation.apply(lambda x:
                                                                math.exp((x['n_tags']-x['n_tags_recommended_lda_w2v'])/x['n_tags_recommended_lda_w2v'])
                                                                if x['n_tags']< x['n_tags_recommended_lda_w2v']
                                                                else
                                                                math.exp((x['n_tags_recommended_lda_w2v']-x['n_tags'])/x['n_tags'])
                                                                ,axis=1)
orig_avg_accuracy = np.mean(df_evaluation['matching_score'])
nmf_avg_accuracy = np.mean(df_evaluation['matching_score_test_nmf'])
lda_avg_accuracy = np.mean(df_evaluation['matching_score_test_lda'])
lda_w2v_avg_accuracy = np.mean(df_evaluation['matching_score_test_lda_w2v'])
nmf_avg_tags_number_accuracy = np.mean(df_evaluation['nmf_tags_number_accuracy'])
lda_avg_tags_number_accuracy = np.mean(df_evaluation['lda_tags_number_accuracy'])
lda_w2v_avg_tags_number_accuracy = np.mean(df_evaluation['lda_w2v_tags_number_accuracy'])
nmf_global_accuracy = (2*nmf_avg_accuracy+nmf_avg_tags_number_accuracy)/3
lda_global_accuracy = (2*lda_avg_accuracy+lda_avg_tags_number_accuracy)/3
lda_global_avg_accuracy = (2*lda_w2v_avg_accuracy+lda_w2v_avg_tags_number_accuracy)/3
print("orig_avg_accuracy : %s"%orig_avg_accuracy)
print("nmf_avg_accuracy : %s"%nmf_avg_accuracy)
print("lda_avg_accuracy : %s"%lda_avg_accuracy)
print("lda_w2v_avg_accuracy : %s"%lda_w2v_avg_accuracy)

print()
print("nmf_avg_tags_number_accuracy : %s"%nmf_avg_tags_number_accuracy)
print("lda_avg_tags_number_accuracy : %s"%lda_avg_tags_number_accuracy)
print("lda_w2v_avg_tags_number_accuracy : %s"%lda_w2v_avg_tags_number_accuracy)

print()
print("nmf_global_accuracy : %s"%nmf_global_accuracy)
print("lda_global_accuracy : %s"%lda_global_accuracy)
print("lda_global_avg_accuracy : %s"%lda_global_avg_accuracy)
orig_med_accuracy = np.mean(df_evaluation['matching_score'])
nmf_med_accuracy = np.mean(df_evaluation['matching_score_test_nmf'])
lda_med_accuracy = np.mean(df_evaluation['matching_score_test_lda'])
lda_w2v_med_accuracy = np.mean(df_evaluation['matching_score_test_lda_w2v'])
orig_avg_tags_number = np.mean(df_evaluation['n_tags'])
nmf_avg_tags_number = np.mean(df_evaluation['n_tags_recommended_nmf'])
lda_avg_tags_number = np.mean(df_evaluation['n_tags_recommended_lda'])
lda_w2v_avg_tags_number = np.mean(df_evaluation['n_tags_recommended_lda_w2v'])
orig_med_tags_number = np.median(df_evaluation['n_tags'])
nmf_med_tags_number = np.median(df_evaluation['n_tags_recommended_nmf'])
lda_med_tags_number = np.median(df_evaluation['n_tags_recommended_lda'])
lda_w2v_med_tags_number = np.median(df_evaluation['n_tags_recommended_lda_w2v'])
RUN = execution_params_workflows_iteration['RUN']
N_COMPONENTS = execution_params_workflows_iteration['N_COMPONENTS']
TOPICS_THRESHOLD = execution_params_workflows_iteration['TOPICS_THRESHOLD']
NEIGHBORS = execution_params_workflows_iteration['NEIGHBORS']
QUANTILE_THRESHOLD = execution_params_workflows_iteration['QUANTILE_THRESHOLD']
dict_results = {'RUN':RUN,
                'N_COMPONENTS':N_COMPONENTS,
                'TOPICS_THRESHOLD':TOPICS_THRESHOLD,
                'NEIGHBORS':NEIGHBORS,
                'QUANTILE_THRESHOLD':QUANTILE_THRESHOLD,
                'orig_avg_accuracy':orig_avg_accuracy,
                'nmf_avg_accuracy':nmf_avg_accuracy,
                'lda_avg_accuracy':lda_avg_accuracy,
                'lda_w2v_avg_accuracy':lda_w2v_avg_accuracy,
                'nmf_avg_tags_number_accuracy':nmf_avg_tags_number_accuracy,
                'lda_avg_tags_number_accuracy':lda_avg_tags_number_accuracy,
                'lda_w2v_avg_tags_number_accuracy':lda_w2v_avg_tags_number_accuracy,
                'nmf_global_accuracy':nmf_global_accuracy,
                'lda_global_accuracy':lda_global_accuracy,
                'lda_global_avg_accuracy':lda_global_avg_accuracy,
                'orig_med_accuracy':orig_med_accuracy,
                'nmf_med_accuracy':nmf_med_accuracy,
                'lda_med_accuracy':lda_med_accuracy,
                'lda_w2v_med_accuracy':lda_w2v_med_accuracy,
                'orig_avg_tags_number':orig_avg_tags_number,
                'nmf_avg_tags_number':nmf_avg_tags_number,
                'lda_avg_tags_number':lda_avg_tags_number,
                'lda_w2v_avg_tags_number':lda_w2v_avg_tags_number,
                'orig_med_tags_number':orig_med_tags_number,
                'nmf_med_tags_number':nmf_med_tags_number,
                'lda_med_tags_number':lda_med_tags_number,
                'lda_w2v_med_tags_number':lda_w2v_med_tags_number
               }
results_columns = \
['RUN','N_COMPONENTS','TOPICS_THRESHOLD','NEIGHBORS','QUANTILE_THRESHOLD','orig_avg_accuracy','nmf_avg_accuracy',\
 'lda_avg_accuracy','lda_w2v_avg_accuracy','nmf_avg_tags_number_accuracy','lda_avg_tags_number_accuracy',\
 'lda_w2v_avg_tags_number_accuracy','nmf_global_accuracy','lda_global_accuracy','lda_global_avg_accuracy',\
 'orig_med_accuracy','nmf_med_accuracy','lda_med_accuracy','lda_w2v_med_accuracy','orig_avg_tags_number',\
 'nmf_avg_tags_number','lda_avg_tags_number','lda_w2v_avg_tags_number','orig_med_tags_number',\
 'nmf_med_tags_number','lda_med_tags_number','lda_w2v_med_tags_number']
results_df = pd.DataFrame(dict_results,columns = results_columns,index=[RUN])
results_df
pickle.dump(results_df,open(pickles_path+"1_workflows_iteration_results_df.p", "wb"))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09887967318643229, 'fraction_numerical': 0.006653874721465709, 'mean_word_length': 4.654068241469816, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 49, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '282004', 'license': 'MIT', 'n_tokens_mistral': 11723, 'n_tokens_neox': 11385, 'n_words': 1691, 'path': '5_Unsupervised_learning_benchmark_3_algorithms.ipynb', 'repo_name': 'E-tanok/NLP_stackoverflow_tags_recommender', 'size': '32312'}