<jupyter_start><jupyter_text>**Packages**<jupyter_code>import re
import numpy as np

import spacy
import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModel
import pyLDAvis
import pyLDAvis.gensim 
import matplotlib.pyplot as plt
%matplotlib inline

from nltk.corpus import stopwords
import imp

tools = imp.load_source('tools', '../lda_tools.py')
import tqdm<jupyter_output><empty_output><jupyter_text>**The data**<jupyter_code>import pandas as pd
df = pd.read_csv('../data/prisoners_dilemma_articles_meta_data_clean.csv')
data = df[['abstract', 'unique_key', 'title', 'date']]
data = data.drop_duplicates()
data = data.reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>We want to perform the analysis yearly. We will write a function to perform the analysis but also clean the dataand create the corpus each time.<jupyter_code>stop_words = stopwords.words('english')
mallet_path = '/Users/storm/rsc/mallet-2.0.8/bin/mallet'
def get_model_and_opt_num_of_topics(yearly_dataset, limit, step=1, start=2):
    
    # clean
    words = list(tools.sentences_to_words(yearly_data['abstract'].values))
#     bigram = gensim.models.Phrases(words, min_count=5, threshold=100)
#     bigram_mod = gensim.models.phrases.Phraser(bigram)
    lemmatized_words = tools.clean_words(words, stop_words)
    
    # corpus
    id2word = corpora.Dictionary(lemmatized_words)
    texts = lemmatized_words
    corpus = [id2word.doc2bow(text) for text in texts]
    
    model_list, coherence_values = tools.compute_coherence_values(limit=limit,
                                                                mallet_path=mallet_path,
                                                                dictionary=id2word,
                                                                corpus=corpus,
                                                                texts=lemmatized_words,
                                                                step=step, start=start)
    
    max_coherence_value = max(coherence_values)
    max_index = coherence_values.index(max_coherence_value)
    
#     num_of_topics = list(range(start, limit, step))

    return model_list, max_index, coherence_values
years = sorted(data.date.unique())
len(years)
results = []
for year in tqdm.tqdm_notebook(years[:5]):
    yearly_data = data[data['date'] <= year]
    
    result = get_model_and_opt_num_of_topics(yearly_data, limit=15, start=3)
    
    results.append(result)
len(yearly_data['title'].unique())
len(df['title'].unique())
num_of_topics = list(range(2, 7, 1))
models, max_indices, coh_val = zip(*results)
coh_val[1]
plt.plot([num_of_topics[index] for index in max_indices], '-o');<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10184182015167931, 'fraction_numerical': 0.007583965330444204, 'mean_word_length': 3.0855457227138645, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '501673', 'license': 'MIT', 'n_tokens_mistral': 911, 'n_tokens_neox': 879, 'n_words': 168, 'path': 'src/nbs/LDA Yearly analysis.ipynb', 'repo_name': 'Nikoleta-v3/bibliometric-study-of-the-prisoners-dilemma', 'size': '2769'}