<jupyter_start><jupyter_text>Import libraries<jupyter_code>import os
import pandas as pd
import numpy as np
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
import plotly as py
import plotly.graph_objs as go
import gensim
from gensim import corpora, models, similarities
import logging
import tempfile
from nltk.corpus import stopwords
from string import punctuation
from collections import OrderedDict
import seaborn as sns
import pyLDAvis
import pyLDAvis.gensim_models as gensimvis
pyLDAvis.enable_notebook()
import matplotlib.pyplot as plt
%matplotlib inline<jupyter_output>/Users/tejasrikurapati/opt/anaconda3/lib/python3.8/site-packages/ipykernel/ipkernel.py:287: DeprecationWarning:

`should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.<jupyter_text>Importing data<jupyter_code>reddit = pd.DataFrame(reddit)
reddit['data'] = reddit
reddit
reddit = reddit.drop([0],axis=1)
reddit
reddit.to_csv('./reddit.csv')
#converting dataset to list
corpus=[]
a=[]
for i in range(len(reddit['data'])):
        a=reddit['data'][i]
        corpus.append(a)
        
corpus[0:10]
TEMP_FOLDER = tempfile.gettempdir()
print('Folder "{}" will be used to save temporary dictionary and corpus.'.format(TEMP_FOLDER))

logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)<jupyter_output>Folder "/<KEY>" will be used to save temporary dictionary and corpus.<jupyter_text>Data cleaning Converting upper case to lower case. Removing stop words.<jupyter_code>list1 = ['f','k','r','ist','gmo','said','even','la','el','est','qu','gt','en','se','rt','via','en','l','que','n','un','las','es','con','por','para','et','al','el','si','a','q','w','e','r','t','y','i','o','p','s','g','h','j','k','l','z','x','c','v','b','n','m',':','/','(',')','?','|',";","]","[",".",'@',"&","-","!"]
stoplist = stopwords.words('english') + list1

texts = [[word for word in str(document).lower().split() if word not in stoplist] for document in corpus]

dictionary = corpora.Dictionary(texts)
dictionary.save(os.path.join(TEMP_FOLDER, 'data.dict'))  # store the dictionary, for future reference
corpus = [dictionary.doc2bow(text) for text in texts]
corpora.MmCorpus.serialize(os.path.join(TEMP_FOLDER, 'elon.mm'), corpus)  # store to disk, for later use<jupyter_output>/Users/tejasrikurapati/opt/anaconda3/lib/python3.8/site-packages/ipykernel/ipkernel.py:287: DeprecationWarning:

`should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.

2021-12-17 11:27:07,036 : INFO : storing corpus in Matrix Market format to /var/folders/yd/dwdfnncj7g10ds7bnpksx5bc0000gn/T/elon.mm
2021-12-17 11:27:07,037 : INFO : saving sparse matrix to /var/folders/yd/dwdfnncj7g10ds7bnpksx5bc0000gn/T/elon.mm
2021-12-17 11:27:07,038 : INFO : PROGRESS: saving document #0
2021-12-17 11:27:07,074 : INFO : saved 374x8435 matrix, density=0.731% (23048/3154690)
2021-12-17 11:27:07,076 : INFO : saving MmCorpus index to /var/folders/yd/dwdfnncj7g10ds7bnpksx5bc0000gn/T/elon.mm.index<jupyter_text>Implementing Co-occurrence matrix heatmap<jupyter_code>tfidf = models.TfidfModel(corpus) # step 1 -- initialize a model
corpus_tfidf = tfidf[corpus]  # step 2 -- use the model to transform vectors
total_topics = 5
lda = models.LdaModel(corpus, id2word=dictionary, num_topics=total_topics)
corpus_lda = lda[corpus_tfidf] # create a double wrapper over the original corpus: bow->tfidf->fold-in-lsi
lda.show_topics(total_topics,10)
data_lda = {i: OrderedDict(lda.show_topic(i,25)) for i in range(total_topics)}
df_lda = pd.DataFrame(data_lda)
df_lda = df_lda.fillna(0).T
print(df_lda.shape)
g=sns.clustermap(df_lda.corr(), center=0, standard_scale=1, cmap="RdBu", metric='cosine', linewidths=.75, figsize=(15, 15))
plt.setp(g.ax_heatmap.yaxis.get_majorticklabels(), rotation=0)
plt.show()<jupyter_output>/Users/tejasrikurapati/opt/anaconda3/lib/python3.8/site-packages/ipykernel/ipkernel.py:287: DeprecationWarning:

`should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.
{'fraction_non_alphanumeric': 0.1460549645390071, 'fraction_numerical': 0.04188829787234043, 'mean_word_length': 6.781034482758621, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '435012', 'license': 'MIT', 'n_tokens_mistral': 1739, 'n_tokens_neox': 1625, 'n_words': 364, 'path': 'sensitive-topic-analysis/Co-occurence-matrix-analysis/co-occurence-matrix-reddit.ipynb', 'repo_name': 'Sapphirine/Comprehensive-Analysis-on-Microblogging-sites-Reddit-vs-Twitter', 'size': '4554'}