<jupyter_start><jupyter_text>NMF based topic modelling - with tfidf<jupyter_code>data = pd.read_csv('../../data/processed/final_repo_english_whatwhy.csv', index_col='file_id')
data.columns
stop_words_extension = ['abstr_hyperlink', 'https', 'abstr_number', 'abstr_image',
                        'abstr mailto', 'abstr_code_section', 'abstr_image', 'abstr_number', 'http', 'www', 'please']
nmf_modeler = tm.topic_modeler(data['content_clean'].values.tolist(),
                               stopwords_extension= stop_words_extension)<jupyter_output><empty_output><jupyter_text>Plotting cohenrence for each K topics<jupyter_code>start= 40#125## 85 # 45
step=1
limit=146 # not included
#nmf_model_list=None
nmf_model_list, coherence_vals = nmf_modeler.compute_coherence_values(limit, start=start, step=step,
                                                                                      model_type='nmf',
                                                                                     corpus_type ='tfidf')


## Saving results 
n = nmf_modeler.num_topics
print('best model has {} topics'.format(n))

coherence_vals_df =  pd.DataFrame({'Number of Topics':range(start, limit, step), 'Coherence Value':coherence_vals} )
coherence_vals_df.to_csv('../../data/feature_extraction/nmf/nmf_coherence_values_{}_{}.csv'.format(start, (limit-1)))
def save_model_info(modeler):
    n = modeler.num_topics
    print('get_doc_dominant_topic')
    nmf_readable_document_topics_df =  modeler.get_doc_dominant_topic(save_path= "../../data/feature_extraction/nmf/nmf_readable_document_topics_{}.csv".format(n))
    
    print('get_topics_terms')
    nmf_topic_words_df = modeler.get_topics_terms(save_path= "../../data/feature_extraction/nmf/nmf_topic_words_{}.csv".format(n))
    
    print('get_doc_topic_matrix')
    nmf_document_topics_df = modeler.get_doc_topic_matrix(save_path= "../../data/feature_extraction/nmf/nmf_document_topics_{}.csv".format(n))
    
    return nmf_readable_document_topics_df, nmf_topic_words_df, nmf_document_topics_df
nmf_readable_document_topics_df, nmf_topic_words_df, nmf_document_topics_df = save_model_info(nmf_modeler)<jupyter_output><empty_output><jupyter_text>Ploting Topics Distributions<jupyter_code>topic_dstr = nmf_modeler.get_topic_distr()
topic_dstr['Dominant_Topic'].value_counts().plot(kind='bar')

topic_dstr['Topic_Keywords'].fillna('No Topic', inplace=True)
topic_dstr.to_csv('../../data/feature_extraction/nmf/nmf_topics_distribution_{}.csv'.format(n))<jupyter_output><empty_output><jupyter_text>Docs dominant Topics<jupyter_code>#doc_dominant_topic_df.reset_index().to_csv('../../data/feature_extraction/nmf_document_topics_25.csv')
nmf_readable_document_topics_df.head(10)<jupyter_output><empty_output><jupyter_text>Wordclouds for each topic<jupyter_code>from nltk.corpus import stopwords
from matplotlib import pyplot as plt
from wordcloud import WordCloud, STOPWORDS
import matplotlib.colors as mcolors


topics = nmf_modeler.model.show_topics(num_topics=35, num_words=10, formatted=False)



def show_word_cloud(topics, limit=10, stopwords_extension=[], topic_index=range(1, 11)):
    # 1. Wordcloud of Top N words in each topic

    cols = [color for name, color in mcolors.TABLEAU_COLORS.items()]  # more colors: 'mcolors.XKCD_COLORS'

    stop_words = stopwords.words('english')
    stop_words.extend(stopwords_extension)

    cloud = WordCloud(stopwords=stop_words,
                      background_color='white',
                      width=2500,
                      height=1800,
                      max_words=10,
                      colormap='tab10',
                      color_func=lambda *args, **kwargs: cols[i],
                      prefer_horizontal=1.0)

    fig, axes = plt.subplots(int(limit / 2), 2, figsize=(10, 10), sharex='all', sharey='all')

    for i, ax in enumerate(axes.flatten()):
        fig.add_subplot(ax)
        topic_words = dict(topics[i][1])
        cloud.generate_from_frequencies(topic_words, max_font_size=300)
        plt.gca().imshow(cloud)
        plt.gca().set_title('Topic ' + str(topic_index[i]), fontdict=dict(size=16))
        plt.gca().axis('off')
        if i > limit:
            break

    plt.subplots_adjust(wspace=0, hspace=0)
    plt.axis('off')
    plt.margins(x=0, y=0)
    plt.tight_layout()
    plt.show()

show_word_cloud(topics[:10],  limit=10, stopwords_extension=stop_words_extension, topic_index=range(0,10))
show_word_cloud(topics[10:20],  limit=10, stopwords_extension=stop_words_extension, topic_index=range(10,20))
show_word_cloud(topics[20:30],  limit=10, stopwords_extension=stop_words_extension, topic_index=range(20,30))
show_word_cloud(topics[30:35],  limit=5, stopwords_extension=stop_words_extension, topic_index=range(30,35))
show_word_cloud(topics_tfid[:10],  limit=10, stopwords_extension=stop, topic_index=range(0,10))<jupyter_output><empty_output><jupyter_text>1. Extract TF-IDF matrix for all documents<jupyter_code>stopwords_extension = ['abstr_hyperlink', 'https', 'abstr_number', 'abstr_image', 'http', '_', 'www']
stop_words  = stopwords.words('english')
stop_words.extend(stopwords_extension)
vectoriser = TfidfVectorizer(tokenizer=uf.tokenize, stop_words=stop_words)
tfidf_model = vectoriser.fit_transform(data.content_text_w_o_tags)<jupyter_output><empty_output><jupyter_text>2. Topic extraction **Estimating the number of topics**The number of components / topics can be estimated using singular value decomposition (SVD).The explained variance ratio gives a weight to each component. Low values indicate that one cannot gain much more information when the number of components increases.<jupyter_code>u, s, vt = svds(tfidf_model, k = 250)
from kneed import KneeLocator

fig, ax = plt.subplots()
x= [i for i in range(1,s.size + 1)]
y = [v for v in reversed(s)]
kn = KneeLocator(x, y, curve='convex', direction='decreasing')
print(kn.elbow)

ax.plot(x,y )
plt.xlabel('singular value index')
plt.ylabel('value')
plt.title("Singular values")<jupyter_output><empty_output><jupyter_text>The number of topics will be fixed to 35 since with more components there is not much more variance in the data. **NMF modelling**<jupyter_code>nmf_model = NMF(35, init="nndsvd")
document_topic_mat = nmf_model.fit_transform(tfidf_model)
topic_word_mat = nmf_model.components_
uf.print_top_words(nmf_model, vectoriser.get_feature_names(), 10)
#pd.DataFrame(document_topic_mat).to_csv("../../data/feature_extraction/nmf_document_topics_50.csv")
#pd.DataFrame(topic_word_mat, columns=vectoriser.get_feature_names()).to_csv("../../data/feature_extraction/nmf_topic_words_50.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12262443438914027, 'fraction_numerical': 0.01779788838612368, 'mean_word_length': 4.7912663755458516, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '218026', 'license': 'MIT', 'n_tokens_mistral': 2382, 'n_tokens_neox': 2258, 'n_words': 358, 'path': 'code/2_software-features-modeling/6_1_NMF_topic_extraction.ipynb', 'repo_name': 'DLR-SC/repository-synergy', 'size': '6630'}