<jupyter_start><jupyter_text>Topic Modeling of Twitter Statuses with LDA<jupyter_code>from gensim import corpora, models, similarities
from pymongo import MongoClient
import time
import os
import random
import numpy as np
import pyLDAvis.gensim
import csv
from collections import defaultdict
import matplotlib.pyplot as plt
import networkx as nx
from networkx.drawing.nx_agraph import graphviz_layout
import warnings
warnings.filterwarnings('ignore')

# Ugly hackery, but necessary: stackoverflow.com/questions/4383571/importing-files-from-different-folder
import sys
sys.path.append('../../')

from src.streaming import spark_functions

corpus_filename = './../../data/processed/tweets_stream.mm'
dict_filename = './../../data/processed/tweets_stream.dict'
lda_filename = './../../models/lda/gensim/tweets_stream.lda'

preprocess = spark_functions.preprocessor()
tokenize = spark_functions.tokenizer(remove_stopwords=True)<jupyter_output><empty_output><jupyter_text>Loading the Corpus and Dictionary<jupyter_code>corpus = corpora.MmCorpus(corpus_filename)
dictionary = corpora.Dictionary.load(dict_filename)
num_topics = 10<jupyter_output><empty_output><jupyter_text>Train the LDA model<jupyter_code>#lda = models.LdaModel.load(lda_filename)
lda = models.LdaModel(corpus, id2word=dictionary, num_topics=num_topics, alpha="auto", eta=0.01)
#lda.print_topics()
lda.save(lda_filename)<jupyter_output><empty_output><jupyter_text>Some Experiments<jupyter_code>followers_data =  pyLDAvis.gensim.prepare(lda, corpus, dictionary)
pyLDAvis.display(followers_data)
# TODO document
#brew install graphviz
#sudo pip install pygraphviz

graph = nx.Graph()

for i in range(num_topics):    
    for (term_id, term_probability) in lda.get_topic_terms(topicid=i, topn=5):
        graph.add_node(i,color="#75DDFF")
        graph.add_edge(i,dictionary[term_id],weight=term_probability*100)    

plt.figure(figsize=(16,16))

pos = graphviz_layout(graph, prog="twopi", root='1')
nodes = graph.nodes()

# nodes
nodes = graph.nodes(data=True)
colors = [node[1]['color'] if 'color' in node[1] else "#FFFFFF" for node in nodes]
nx.draw_networkx_nodes(graph,pos,node_color=colors, node_size=2000)

# edges
edges = graph.edges()
weights = [graph[u][v]['weight'] for u,v in edges]
nx.draw_networkx_edges(graph,pos,edgelist=edges,width=weights)

# labels
sizes = [16 if 'color' in node[1] else 8 for node in nodes]
nx.draw_networkx_labels(graph,pos,font_size=12,font_family='sans-serif')

plt.axis('off')
plt.savefig('lda_network_graph.pdf', format="pdf",bbox_inches='tight')
plt.show()
# Pie chart of topics in the sample dataset
db = MongoClient()['thesis-dev']
topic_frequency = defaultdict(int)
for tweet in db.tweets.find():
    
    topics = lda[dictionary.doc2bow(tokenize(preprocess(tweet['text'])))]
    for topic in topics:
        #print(topic)
        topic_frequency[topic[0]] += topic[1]
        

import matplotlib.pyplot as plt
from matplotlib import rcParams
labels,values = [],[]
for key in topic_frequency:
    values.append(topic_frequency[key])
    labels.append(", ".join(dictionary[term_id] for (term_id,_) in lda.get_topic_terms(topicid=key, topn=5)))
plt.pie(values, labels=labels, shadow=False, autopct='%1.1f%%',)
plt.axis('equal')
plt.savefig('topic_distribution.pdf', format="pdf",bbox_inches='tight')
plt.show()
percentages = [value/sum(values) for value in values]
for index,label in enumerate(labels):
    print(str(percentages[index]) + "; Topic " + str(index) + ": " + str("".join(label)))
def print_topics(text):
    doc_bow = [dictionary.doc2bow(text) for text in [tokenize(preprocess(text))]]
    topics = lda[doc_bow][0]

    print(text)
    for topic in topics:  # Maximum 3 Topics per tweet
        terms = lda.get_topic_terms(topicid=topic[0], topn=5) # Maximum 5 Terms per tweet
        print("Probability %d%%: %s" % (topic[1]*100, ", ".join(dictionary[term[0]] for term in terms)))
        
print_topics("lol i hate the stupid #iphone")
print_topics("android is pretty cool I guess, with the new ice cream sandwich update")<jupyter_output>lol i hate the stupid #iphone
Probability 1%: love, read, mersal, emoji, first
Probability 22%: damn, preaching, ever, need, look
Probability 1%: still, appears, playing, baby, day
Probability 1%: mtvhottest, lady, gaga, zara, larsson
Probability 1%: video, sex, happy, youtube, playlist
Probability 38%: tell, feel, give, win, home
Probability 1%: already, take, stop, outfit, care
Probability 1%: miss, someone, remember, today, give
Probability 1%: single, minaj, nicki, mtvhottest, made
Probability 26%: need, plan, people, away, since
android is pretty cool I guess, with the new ice cream sandwich update
Probability 32%: love, read, mersal, emoji, first
Probability 1%: damn, preaching, ever, need, look
Probability 1%: still, appears, playing, baby, day
Probability 1%: mtvhottest, lady, gaga, zara, larsson
Probability 28%: video, sex, happy, youtube, playlist
Probability 31%: tell, feel, give, win, home
Probability 1%: already, take, stop, outfit, care
Probability 1%: miss, someone, reme[...]
{'fraction_non_alphanumeric': 0.12944664031620554, 'fraction_numerical': 0.015217391304347827, 'mean_word_length': 5.94238683127572, 'pattern_counts': {'":': 1, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '439118', 'license': 'MIT', 'n_tokens_mistral': 1839, 'n_tokens_neox': 1725, 'n_words': 438, 'path': 'notebooks/topic_modeling/LDA.ipynb', 'repo_name': 'ClaasM/streamed-sentiment-topic-intent', 'size': '5060'}