<jupyter_start><jupyter_text>Database with additional treatment<jupyter_code>df = pd.read_csv("../2_Treatment_database/output/database_one_row_each_paper.csv")
print( "Loaded %d X %d dataframe with unique papers" % (len(df), len(df.columns) ))<jupyter_output>Loaded 4691 X 16 dataframe with unique papers<jupyter_text>Preprocessed abstracts<jupyter_code>abstracts_prepro = pd.read_csv("./interm/processed_abstracts.csv")
print( "%d preprocessed abstracts" % (len(abstracts_prepro)) )<jupyter_output>4691 preprocessed abstracts<jupyter_text>TF-IDF normalised document-term matrix and list of terms<jupyter_code>(tfidf, tfidf_feature_names) = joblib.load("./interm/tfidf_matrix-features_names.pkl" )
print( "Loaded %d X %d document-term matrix" % (tfidf.shape[0], tfidf.shape[1]) )<jupyter_output>Loaded 4691 X 1300 document-term matrix<jupyter_text>Model  with optimal parameters:<jupyter_code>k_topics,alpha,l1,W,H = joblib.load("./output/model_selected.pkl") 
param = [k_topics,alpha,l1,W,H]
param_names = ["k_topics", "alpha", "l1"]
for name in param_names:
    print(name, '=', repr(eval(name)))<jupyter_output>k_topics = 39
alpha = 0.1
l1 = 0.9<jupyter_text>Topics and how many abstracts are associated to each topic<jupyter_code>prop = pd.read_csv("./output/nb_papers_seuil_0.02.csv")
column_topics = prop['topic'].to_list()
number_of_abs = prop['seuil_0.02'].astype(int).to_list()<jupyter_output><empty_output><jupyter_text>Associate abstracts to topics  Function that return the 'n_top_abstracts' associated to 'topic_index' according to their weight in W<jupyter_code>def get_top_abstracts( data_samples, W, topic_index, n_top_abstracts ):
    # reverse sort the values to sort the indices
    top_indices = np.argsort( W[:,topic_index] )[::-1]
    # now get the abstracts corresponding to the top-ranked indices
    top_abstracts = []
    for doc_index in top_indices[0:n_top_abstracts]:
        top_abstracts.append( data_samples[doc_index] )
    return top_abstracts<jupyter_output><empty_output><jupyter_text>Add to the database the preprocessed abstracts and a column used further to contain the topic to which the paper is related<jupyter_code>df['stem_abstract'] = abstracts_prepro["abstracts_prepro"]
df['topic_charact'] = np.nan<jupyter_output><empty_output><jupyter_text>Create a dataframe that contains for each topic all the rows charactering papers from the main database Start y creating an empty dataframe<jupyter_code>df_topic_abs = pd.DataFrame()<jupyter_output><empty_output><jupyter_text>Double loop on all topics and te associated abstracts (with a non nul coefficient in W)<jupyter_code>for k in range(k_topics):
    topic_abstracts = get_top_abstracts(abstracts_prepro["abstracts_prepro"].to_list(), W, k, number_of_abs[k] )
    for i, abstract in enumerate(topic_abstracts):
        df_tempo = df[df['stem_abstract'].isin([str(abstract)])].copy()
        df_tempo['topic_charact'] = column_topics[k]
        df_tempo['number_of_abs'] = number_of_abs[k]
        df_topic_abs = df_topic_abs.append(df_tempo)<jupyter_output><empty_output><jupyter_text>Keep the index to memorize the weight order for papers<jupyter_code>df_topic_abs.reset_index(0,inplace = True)        
df_topic_abs['index1'] = df_topic_abs.index
col = ['index1','topic_charact','number_of_abs','title','abstract']
df_topic_abs = df_topic_abs.reindex(columns=col)
df_topic_abs.sort_values(by = ['number_of_abs','index1'], ascending = [True,True], inplace = True)
df_topic_abs.head(1)
df_topic_abs.to_csv("./output/df_topic_abs_classification.csv",index=False)<jupyter_output><empty_output><jupyter_text>Preprocess titles<jupyter_code>table = str.maketrans('','', string.punctuation)
df_topic_abs['stem_title'] = df_topic_abs['title'].str.lower()
df_topic_abs['stem_title'] = df_topic_abs['stem_title'].str.translate(table)
df_topic_abs['stem_title'] = df_topic_abs['stem_title'].apply(lambda x: " ".join([stem(y) for y in x.split(" ")]))<jupyter_output><empty_output><jupyter_text>Get the 5 weightest words for each topic<jupyter_code>def get_5_words( all_terms, H, topic_index):
    # reverse sort the values to sort the indices
    top_indices = np.argsort( H[topic_index,:] )[::-1]
    top_terms=[]
    for i in range(5):
        term = tfidf_feature_names[top_indices[i]]
        top_terms.append(term)
    return top_terms
list_words_topics = []
for k in range(k_topics):
    words_topics = get_5_words(tfidf_feature_names, H , k)
    list_words_topics.append(words_topics)<jupyter_output><empty_output><jupyter_text>Check stemmed titles by topic<jupyter_code>new_df = pd.DataFrame()
k=0
for topic_words in list_words_topics:
    column = column_topics[k]
    k+= 1
    for word in topic_words:
        df_tempo = df_topic_abs.loc[df_topic_abs['stem_title'].str.contains(word) & df_topic_abs['topic_charact'].str.contains(column)].copy()
        df_tempo['word'] = word
        new_df = new_df.append(df_tempo)       
print("%d X %d checked dataframe" % (len(new_df), len(new_df.columns) ))
df_sort = new_df.drop_duplicates(['topic_charact','title'], keep='first')
print("%d X %d dataframe" % (len(df_sort), len(df_sort.columns) ))<jupyter_output>10388 X 7 dataframe<jupyter_text>Count how many papers remain<jupyter_code>checked_topic_count = pd.DataFrame(df_sort['topic_charact'].value_counts(), columns = ['topic_charact'])
checked_topic_count.rename(columns ={'topic_charact': "checked_topic_count"}, inplace=True)
checked_topic_count.reset_index(inplace = True)
checked_topic_count.rename(columns={'index':'topic'},inplace=True)
tab = pd.merge(prop, checked_topic_count, on = 'topic')
tab
tab.to_csv("./output/Table_topics_count.csv",index=False)<jupyter_output><empty_output><jupyter_text>How many topics characterize how many papers<jupyter_code>checked_title_count = pd.DataFrame(df_sort['title'].value_counts(), columns = ['title'])
checked_title_count.rename(columns ={'title': "title_count"}, inplace=True)
checked_title_count.reset_index(inplace = True)
number_papers = pd.DataFrame(checked_title_count['title_count'].value_counts(), columns = ['title_count'])
number_papers.loc['Total'] = number_papers.sum() 
number_papers<jupyter_output><empty_output><jupyter_text>Add topics attribution with title check to the main database<jupyter_code>column_topics.append('Total_topics_in_paper')
topics = pd.crosstab(df_sort.title, df_sort.topic_charact)
topics =topics.reindex(columns= column_topics, fill_value=0)
topics.loc['Total',:]= topics.sum(axis=0)
topics.loc[:,'Total_topics_in_paper'] = topics.sum(axis=1)
df = df.reindex(columns= np.append(df.columns,column_topics), fill_value=0)
df.set_index('title', inplace = True)
df.update(topics)
df.reset_index(0, inplace = True)
df.to_csv('./output/database_titre_seuil0.02.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12876106194690265, 'fraction_numerical': 0.009587020648967551, 'mean_word_length': 7.371604938271605, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '171505', 'license': 'MIT', 'n_tokens_mistral': 2487, 'n_tokens_neox': 2421, 'n_words': 431, 'path': '3_Topic_modeling/6. Robustness_based_title.ipynb', 'repo_name': 'ClaireLepault/Mapping-country-mitig-pathways', 'size': '6780'}