<jupyter_start><jupyter_text>Importing packages<jupyter_code>import os
import pickle
import pandas as pd
import warnings
warnings.filterwarnings('ignore', category=DeprecationWarning)

import pyLDAvis
import pyLDAvis.sklearn
pyLDAvis.enable_notebook()

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation

os.chdir('/Volumes/My <NAME>/Dropbox/Data Science/DRepositories/NYCODW-STEW-MAP-Project')
with open('processed_tweets.pickle', 'rb') as read_file:
    df = pickle.load(read_file)

tokens_list_of_lists = list(df.tweet.values)
tokens_list = df.tweet.sum()
tokens_list_of_strings = [",".join(x).replace(",", " ") for x in tokens_list_of_lists]
vectorizer = CountVectorizer()
tf = vectorizer.fit_transform(tokens_list)
tf_feature_names = vectorizer.get_feature_names()<jupyter_output><empty_output><jupyter_text>n_components = 3<jupyter_code>lda_3 = LatentDirichletAllocation(n_components=3, random_state=42)
lda_3.fit(tf)
def display_topics(model, feature_names, no_top_words):
    topic_dict = {}
    for topic_idx, topic in enumerate(model.components_):
        topic_dict["Topic %d words" % (topic_idx)]= ['{}'.format(feature_names[i])
                        for i in topic.argsort()[:-no_top_words - 1:-1]]
        topic_dict["Topic %d weights" % (topic_idx)]= ['{:.1f}'.format(topic[i])
                        for i in topic.argsort()[:-no_top_words - 1:-1]]
    return pd.DataFrame(topic_dict)
no_top_words = 10
display_topics(lda_3, tf_feature_names, no_top_words)
pyLDAvis.sklearn.prepare(lda_3, tf, vectorizer)<jupyter_output><empty_output><jupyter_text>n_components = 4<jupyter_code>lda_4 = LatentDirichletAllocation(n_components=4, random_state=42)
lda_4.fit(tf)
display_topics(lda_4, tf_feature_names, no_top_words)
pyLDAvis.sklearn.prepare(lda_4, tf, vectorizer)<jupyter_output>/Users/examsherpa/opt/anaconda3/envs/nlp-env/lib/python3.8/site-packages/past/builtins/misc.py:45: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
  from imp import reload
/Users/examsherpa/opt/anaconda3/envs/nlp-env/lib/python3.8/site-packages/past/builtins/misc.py:45: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
  from imp import reload
/Users/examsherpa/opt/anaconda3/envs/nlp-env/lib/python3.8/site-packages/past/builtins/misc.py:45: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
  from imp import reload
/Users/examsherpa/opt/anaconda3/envs/nlp-env/lib/python3.8/site-packages/past/builtins/misc.py:45: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
  from imp import reload
/Use[...]<jupyter_text>n_components = 5<jupyter_code>lda_5 = LatentDirichletAllocation(n_components=5, random_state=42)
lda_5.fit(tf)
display_topics(lda_5, tf_feature_names, no_top_words)
pyLDAvis.sklearn.prepare(lda_5, tf, vectorizer)<jupyter_output>/Users/examsherpa/opt/anaconda3/envs/nlp-env/lib/python3.8/site-packages/past/builtins/misc.py:45: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
  from imp import reload<jupyter_text>n_components = 6<jupyter_code>lda_6 = LatentDirichletAllocation(n_components=6, random_state=42)
lda_6.fit(tf)
display_topics(lda_6, tf_feature_names, no_top_words)
pyLDAvis.sklearn.prepare(lda_6, tf, vectorizer)<jupyter_output><empty_output><jupyter_text>n_components = 10<jupyter_code>lda_10 = LatentDirichletAllocation(n_components=10, random_state=42)
lda_10.fit(tf)
display_topics(lda_10, tf_feature_names, no_top_words)
pyLDAvis.sklearn.prepare(lda_10, tf, vectorizer)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10781532112458087, 'fraction_numerical': 0.02011864843951509, 'mean_word_length': 6.709741550695825, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '571817', 'license': 'MIT', 'n_tokens_mistral': 1454, 'n_tokens_neox': 1385, 'n_words': 245, 'path': 'Unsupervised_LDA.ipynb', 'repo_name': 'datascisteven/NYCODW-STEW-MAP-Project', 'size': '3887'}