<jupyter_start><jupyter_text>Load Data<jupyter_code>pharma_df = pd.concat([pd.read_csv(f, sep=',') for f in glob.glob(PHARMA_PATH + "/*.csv")],ignore_index=True)
govt_institutes_df = pd.concat([pd.read_csv(f, sep=',') for f in glob.glob(GOVT_INSTITUTES_PATH + "/*.csv")],ignore_index=True)
ngo_df = pd.concat([pd.read_csv(f, sep=',') for f in glob.glob(NGO_PATH + "/*.csv")],ignore_index=True)
print(pharma_df.shape, govt_institutes_df.shape, ngo_df.shape)
df = pd.concat([pharma_df,govt_institutes_df,ngo_df],ignore_index=True)
df.shape<jupyter_output><empty_output><jupyter_text>Divide dataset as per timelines<jupyter_code># Convert 'created_at' column to datetime
df['created_at'] = df['created_at'].str[:-6]
df['created_at'] = pd.to_datetime(df['created_at'])

# Sort by datetime ascending
df = df.sort_values(by='created_at')

# Pre-process the dataset
tqdm.tqdm.pandas()
df['tweet_tokenized'] = df['tweet'].progress_apply(lambda x:preprocess_text(str(x)))

# Divide as per dates
pre_covid_df = df.loc[df['created_at'] <= '2020-02-26 23:59:59']
print(pre_covid_df.shape)

post_covid_df = df.loc[df['created_at'] >= '2020-02-27 00:00:00']
print(post_covid_df.shape)<jupyter_output><empty_output><jupyter_text>Pre-COVID   Generate TF-IDF embedding<jupyter_code>documents = pre_covid_df['tweet_tokenized'].str.split()
dictionary = gensim.corpora.Dictionary(documents)
dictionary.filter_extremes(no_below=5, no_above=0.5, keep_n=20000)

tfidf_model = gensim.models.TfidfModel(dictionary=dictionary)

corpus = [dictionary.doc2bow(d) for d in documents]
corpus_tfidf = list(tfidf_model[corpus])
performance_metrics = pd.DataFrame(columns=['feature-extraction','clustering-algo','run#', 'state', 'c_v', 'c_umass','topics','time'])<jupyter_output><empty_output><jupyter_text>LDA<jupyter_code>for run, state in zip(range(2, 6, 1), range(4, 12, 2)):
    print('Run #', run)
    
    start = time.time()
    gensim_lda = gensim.models.ldamodel.LdaModel(corpus=corpus_tfidf, num_topics=TOPICS, id2word=dictionary, chunksize=CHUNK_SIZE, passes=EPOCHS, 
                                                     eval_every = EVAL_PERIOD, per_word_topics=True, random_state=state, alpha=ALPHA, eta=BETA)
    
    coherence_cv = gensim.models.CoherenceModel(model=gensim_lda, texts=documents, dictionary=dictionary, coherence='c_v').get_coherence()
    coherence_cumass = gensim.models.CoherenceModel(model=gensim_lda, texts=documents, dictionary=dictionary, coherence='u_mass').get_coherence()
    topics = gensim_lda.print_topics()
    
    stop = time.time()
    
    performance_metrics = performance_metrics.append({'feature-extraction':'tf-idf', 'clustering-algo':'LDA', 'run#':run, 'state':state,'c_v':coherence_cv,'c_umass':coherence_cumass,
                                                      'topics':topics,'time':(stop-start)}, ignore_index=True)<jupyter_output><empty_output><jupyter_text>Parallel LDA (LDA Multicore)<jupyter_code>for run, state in zip(range(1, 6, 1), range(2, 12, 2)):
    print('Run #', run)
    
    start = time.time()
    gensim_plda = gensim.models.ldamulticore.LdaMulticore(corpus=corpus_tfidf, num_topics=TOPICS, id2word=dictionary, chunksize=CHUNK_SIZE, workers=WORKERS, passes=EPOCHS, 
                                                     eval_every = EVAL_PERIOD, per_word_topics=True, random_state=state, alpha=ALPHA, eta=BETA)
    
    coherence_cv = gensim.models.CoherenceModel(model=gensim_plda, texts=documents, dictionary=dictionary, coherence='c_v').get_coherence()
    coherence_cumass = gensim.models.CoherenceModel(model=gensim_plda, texts=documents, dictionary=dictionary, coherence='u_mass').get_coherence()
    topics = gensim_plda.print_topics()
    
    stop = time.time()
    
    performance_metrics = performance_metrics.append({'feature-extraction':'tf-idf', 'clustering-algo':'Parallel LDA', 'run#':run, 'state':state,'c_v':coherence_cv,'c_umass':coherence_cumass,
                                                      'topics':topics,'time':(stop-start)}, ignore_index=True)
    break<jupyter_output><empty_output><jupyter_text>NMF<jupyter_code>for run, state in zip(range(1, 6, 1), range(2, 12, 2)):
    print('Run #', run)
    
    start = time.time()
    gensim_nmf = gensim.models.Nmf(corpus=corpus_tfidf, num_topics=TOPICS, id2word=dictionary, chunksize=CHUNK_SIZE, passes=EPOCHS, eval_every=EVAL_PERIOD, minimum_probability=0, 
                                   random_state=state, kappa=1)
    
    coherence_cv = gensim.models.CoherenceModel(model=gensim_nmf, texts=documents, dictionary=dictionary, coherence='c_v').get_coherence()
    coherence_cumass = gensim.models.CoherenceModel(model=gensim_nmf, texts=documents, dictionary=dictionary, coherence='u_mass').get_coherence()
    topics = gensim_nmf.print_topics()
    
    stop = time.time()
    
    performance_metrics = performance_metrics.append({'feature-extraction':'tf-idf', 'clustering-algo':'NMF', 'run#':run, 'state':state,'c_v':coherence_cv,'c_umass':coherence_cumass,
                                                      'topics':topics,'time':(stop-start)}, ignore_index=True)<jupyter_output><empty_output><jupyter_text>LSI<jupyter_code>for run, state in zip(range(1, 6, 1), range(2, 12, 2)):
    print('Run #', run)
    
    start = time.time()
    gensim_lsi = gensim.models.LsiModel(corpus=corpus_tfidf, num_topics=TOPICS, id2word=dictionary, chunksize=CHUNK_SIZE)
    
    coherence_cv = gensim.models.CoherenceModel(model=gensim_lsi, texts=documents, dictionary=dictionary, coherence='c_v').get_coherence()
    coherence_cumass = gensim.models.CoherenceModel(model=gensim_lsi, texts=documents, dictionary=dictionary, coherence='u_mass').get_coherence()
    topics = gensim_lsi.print_topics()
    stop = time.time()
      
    performance_metrics = performance_metrics.append({'feature-extraction':'tf-idf', 'clustering-algo':'LSI', 'run#':run, 'state':state,'c_v':coherence_cv,'c_umass':coherence_cumass,
                                                      'topics':topics,'time':(stop-start)}, ignore_index=True)<jupyter_output><empty_output><jupyter_text>HDP<jupyter_code>for run, state in zip(range(1, 6, 1), range(2, 12, 2)):
    print('Run #', run)
    
    start = time.time()
    gensim_hdp = gensim.models.hdpmodel.HdpModel(corpus=corpus_tfidf, id2word=dictionary, chunksize=CHUNK_SIZE, random_state=state, kappa=1, alpha=ALPHA)
    
    coherence_cv = gensim.models.CoherenceModel(model=gensim_hdp, texts=documents, dictionary=dictionary, coherence='c_v').get_coherence()
    coherence_cumass = gensim.models.CoherenceModel(model=gensim_hdp, texts=documents, dictionary=dictionary, coherence='u_mass').get_coherence()
    topics = gensim_hdp.print_topics()
    stop = time.time()
      
    performance_metrics = performance_metrics.append({'feature-extraction':'tf-idf', 'clustering-algo':'HDP', 'run#':run, 'state':state,'c_v':coherence_cv,'c_umass':coherence_cumass,
                                                      'topics':topics,'time':(stop-start)}, ignore_index=True)<jupyter_output><empty_output><jupyter_text>Post-COVID<jupyter_code>performance_metrics.to_csv('../../results/content-analysis/topic-modelling/pre-covid-nmf.csv',index=False)

mean_perf = performance_metrics.groupby('clustering-algo')[['c_v','c_umass','time']].mean()
mean_perf.to_csv('../../results/content-analysis/topic-modelling/pre-covid-mean-nmf.csv')
documents = post_covid_df['tweet_tokenized'].str.split()
dictionary = gensim.corpora.Dictionary(documents)
dictionary.filter_extremes(no_below=5, no_above=0.5, keep_n=20000)

tfidf_model = gensim.models.TfidfModel(dictionary=dictionary)

corpus = [dictionary.doc2bow(d) for d in documents]
corpus_tfidf = list(tfidf_model[corpus])
post_covid_performance_metrics = pd.DataFrame(columns=['feature-extraction','clustering-algo','run#', 'state', 'c_v', 'c_umass', 'topics','time'])<jupyter_output><empty_output><jupyter_text>LDA<jupyter_code>for run, state in zip(range(2, 6, 1), range(4, 12, 2)):
    print('Run #', run)
    
    start = time.time()
    gensim_lda = gensim.models.ldamodel.LdaModel(corpus=corpus_tfidf, num_topics=TOPICS, id2word=dictionary, chunksize=CHUNK_SIZE, passes=EPOCHS, 
                                                     eval_every = EVAL_PERIOD, per_word_topics=True, random_state=state, alpha=ALPHA, eta=BETA)
    
    coherence_cv = gensim.models.CoherenceModel(model=gensim_lda, texts=documents, dictionary=dictionary, coherence='c_v').get_coherence()
    coherence_cumass = gensim.models.CoherenceModel(model=gensim_lda, texts=documents, dictionary=dictionary, coherence='u_mass').get_coherence()
    topics = gensim_lda.print_topics()
    
    stop = time.time()
    
    performance_metrics = performance_metrics.append({'feature-extraction':'tf-idf', 'clustering-algo':'LDA', 'run#':run, 'state':state,'c_v':coherence_cv,'c_umass':coherence_cumass,
                                                      'topics':topics,'time':(stop-start)}, ignore_index=True)<jupyter_output><empty_output><jupyter_text>Parallel LDA (LDA Multicore)<jupyter_code>for run, state in zip(range(1, 6, 1), range(2, 12, 2)):
    print('Run #', run)
    
    start = time.time()
    gensim_plda = gensim.models.ldamulticore.LdaMulticore(corpus=corpus_tfidf, num_topics=TOPICS, id2word=dictionary, chunksize=CHUNK_SIZE, workers=WORKERS, passes=EPOCHS, 
                                                     eval_every = EVAL_PERIOD, per_word_topics=True, random_state=state, alpha=ALPHA, eta=BETA)
    
    coherence_cv = gensim.models.CoherenceModel(model=gensim_plda, texts=documents, dictionary=dictionary, coherence='c_v').get_coherence()
    coherence_cumass = gensim.models.CoherenceModel(model=gensim_plda, texts=documents, dictionary=dictionary, coherence='u_mass').get_coherence()
    topics = gensim_plda.print_topics()
    
    stop = time.time()
    
    performance_metrics = performance_metrics.append({'feature-extraction':'tf-idf', 'clustering-algo':'Parallel LDA', 'run#':run, 'state':state,'c_v':coherence_cv,'c_umass':coherence_cumass,
                                                      'topics':topics,'time':(stop-start)}, ignore_index=True)<jupyter_output><empty_output><jupyter_text>NMF<jupyter_code>for run, state in zip(range(1, 6, 1), range(2, 12, 2)):
    print('Run #', run)
    
    start = time.time()
    gensim_nmf = gensim.models.Nmf(corpus=corpus_tfidf, num_topics=TOPICS, id2word=dictionary, chunksize=CHUNK_SIZE, passes=EPOCHS, eval_every=EVAL_PERIOD, minimum_probability=0, 
                                   random_state=state, kappa=1)
    
    coherence_cv = gensim.models.CoherenceModel(model=gensim_nmf, texts=documents, dictionary=dictionary, coherence='c_v').get_coherence()
    coherence_cumass = gensim.models.CoherenceModel(model=gensim_nmf, texts=documents, dictionary=dictionary, coherence='u_mass').get_coherence()
    topics = gensim_nmf.print_topics()
    
    stop = time.time()
    
    post_covid_performance_metrics = post_covid_performance_metrics.append({'feature-extraction':'tf-idf', 'clustering-algo':'NMF', 'run#':run, 'state':state,'c_v':coherence_cv,'c_umass':coherence_cumass,
                                                      'topics':topics,'time':(stop-start)}, ignore_index=True)<jupyter_output><empty_output><jupyter_text>LSI<jupyter_code>for run, state in zip(range(1, 6, 1), range(2, 12, 2)):
    print('Run #', run)
    
    start = time.time()
    gensim_lsi = gensim.models.LsiModel(corpus=corpus_tfidf, num_topics=TOPICS, id2word=dictionary, chunksize=CHUNK_SIZE)
    
    coherence_cv = gensim.models.CoherenceModel(model=gensim_lsi, texts=documents, dictionary=dictionary, coherence='c_v').get_coherence()
    coherence_cumass = gensim.models.CoherenceModel(model=gensim_lsi, texts=documents, dictionary=dictionary, coherence='u_mass').get_coherence()
    topics = gensim_lsi.print_topics()
    stop = time.time()
      
    performance_metrics = performance_metrics.append({'feature-extraction':'tf-idf', 'clustering-algo':'LSI', 'run#':run, 'state':state,'c_v':coherence_cv,'c_umass':coherence_cumass,
                                                      'topics':topics,'time':(stop-start)}, ignore_index=True)<jupyter_output><empty_output><jupyter_text>HDP<jupyter_code>for run, state in zip(range(1, 6, 1), range(2, 12, 2)):
    print('Run #', run)
    
    start = time.time()
    gensim_hdp = gensim.models.hdpmodel.HdpModel(corpus=corpus_tfidf, id2word=dictionary, chunksize=CHUNK_SIZE, random_state=state, kappa=1, alpha=ALPHA)
    
    coherence_cv = gensim.models.CoherenceModel(model=gensim_hdp, texts=documents, dictionary=dictionary, coherence='c_v').get_coherence()
    coherence_cumass = gensim.models.CoherenceModel(model=gensim_hdp, texts=documents, dictionary=dictionary, coherence='u_mass').get_coherence()
    topics = gensim_hdp.print_topics()
    stop = time.time()
      
    performance_metrics = performance_metrics.append({'feature-extraction':'tf-idf', 'clustering-algo':'HDP', 'run#':run, 'state':state,'c_v':coherence_cv,'c_umass':coherence_cumass,
                                                      'topics':topics,'time':(stop-start)}, ignore_index=True)
post_covid_performance_metrics.to_csv('../../results/content-analysis/topic-modelling/post-covid-nmf.csv',index=False)

mean_perf = post_covid_performance_metrics.groupby('clustering-algo')[['c_v','c_umass','time']].mean()
mean_perf.to_csv('../../results/content-analysis/topic-modelling/post-covid-mean-nmf.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14369262659164939, 'fraction_numerical': 0.009846017175007403, 'mean_word_length': 4.790398628375482, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '641529', 'license': 'MIT', 'n_tokens_mistral': 5085, 'n_tokens_neox': 4742, 'n_words': 568, 'path': 'code/content-analysis/topic-modelling/topic_modelling.ipynb', 'repo_name': 'manmeetkaurbaxi/Sentiment-Forecasting-on-tweets', 'size': '13508'}