<jupyter_start><jupyter_text>Topic Modelling using Biterm Model<jupyter_code>import eland as ed
from eland.conftest import *
import pandas as pd
import numpy as np
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from gensim import corpora, models
from wordcloud import WordCloud
from biterm.utility import vec_to_biterms
from biterm.btm import oBTM
import gensim
import matplotlib.pyplot as plt
import pyLDAvis.gensim
import pickle 
import pyLDAvis
from pprint import pprint
import re
import warnings
warnings.filterwarnings('ignore')
pd.set_option('display.max_columns', 500)
pd.set_option('display.max_rows', 100)<jupyter_output><empty_output><jupyter_text>Importing data from Elasticsearch<jupyter_code>ed_df = ed.DataFrame('localhost', 'twitter', columns=['full_text_processed', 'user_id', 'verified', 'name', 'location', 'entities.hashtags.text', 'entities.user_mentions.name'])

# defining the full-text query we need: Retrieving records for full_text_processed with the condition is_retweet=False and is_quote_status=False
query_unique = {
    "bool": {
        "must": {
            "term":{"is_retweet":"false"},
        },
        "filter": {
            "term":{"is_quote_status":"false"}
        },
    }
}
# using full-text search capabilities with Eland:
df_ed = ed_df.es_query(query_unique)
df_tweets = df_ed.to_pandas()
df_tweets.head()<jupyter_output><empty_output><jupyter_text>Tokenising and removing short tweets (less than 4 words)<jupyter_code>def remove_emoji(text):
    emoji_pattern = re.compile("["
                               u"\U0001F600-\U0001F64F"  # emoticons
                               u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                               u"\U0001F680-\U0001F6FF"  # transport & map symbols
                               u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                               u"\U00002500-\U00002BEF"  # chinese char
                               u"\U00002702-\U000027B0"
                               u"\U00002702-\U000027B0"
                               u"\U000024C2-\U0001F251"
                               u"\U0001f926-\U0001f937"
                               u"\U00010000-\U0010ffff"
                               u"\u2640-\u2642"
                               u"\u2600-\u2B55"
                               u"\u200d"
                               u"\u23cf"
                               u"\u23e9"
                               u"\u231a"
                               u"\ufe0f"  # dingbats
                               u"\u3030"
                               "]+", flags=re.UNICODE)
    return emoji_pattern.sub(r'', text)
df_tweets['full_text_processed'] = df_tweets['full_text_processed'].apply(lambda x: remove_emoji(x))
df_tweets['full_text_tokens'] = df_tweets['full_text_processed'].apply(lambda x: [w for w in x.split()])
df_tweets['length'] = df_tweets['full_text_tokens'].apply(lambda x: len(x))
df_tweets = df_tweets[df_tweets['length']>4]
stop_words = stopwords.words('english')
stop_words.extend(['from','not', 'would', 'say', 'could', '_', 'be', 'go', 'do', 'rather', 'seem', 'due', 'via', 'done', 'said'])

tweets = df_tweets.full_text_processed.to_list()<jupyter_output><empty_output><jupyter_text>Building the Vectorizer<jupyter_code># Create a TF-IDF vectorizer
vectorizer = TfidfVectorizer(ngram_range=(2,4), stop_words=stop_words, min_df=5, token_pattern='(?u)\\b[^\\s|\\.]+\\b')
# Create the TF-IDF matrix
tweets_tfidf = vectorizer.fit_transform(tweets)
tweets_tfidf.shape
# Tokenize the documents according to the parameters in the vectorizer
for i in range(len(tweets)):
    tweets[i] = vectorizer.build_analyzer()(tweets[i])
# Create a dictionary and corpus for Gensim to compute coherence metrics
dictionary=corpora.Dictionary(tweets)
corpus=[dictionary.doc2bow(doc) for doc in tweets]
# Adding the TF-IDF for better insight 
tfidf = gensim.models.TfidfModel(corpus)
tfidf_corpus = tfidf[corpus]<jupyter_output><empty_output><jupyter_text>Get the vocabulary and the biterms from the tweets<jupyter_code>vocab = np.array(vectorizer.get_feature_names())
biterms = vec_to_biterms(tweets_tfidf)
# create btm
btm = oBTM(num_topics=10, V=vocab)
print("\n\n Train BTM ..")
for i in range(0, len(biterms), 10000): # process chunkwise
    biterms_chunk = biterms[i:i + 10000]
    btm.fit(biterms_chunk, iterations=1)
    topics = btm.transform(biterms)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11730337078651686, 'fraction_numerical': 0.04224719101123595, 'mean_word_length': 2.9459219858156027, 'pattern_counts': {'":': 7, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '418543', 'license': 'MIT', 'n_tokens_mistral': 1652, 'n_tokens_neox': 1508, 'n_words': 288, 'path': 'notebooks/3.1-km-topic-modelling-biterm.ipynb', 'repo_name': 'joaopfonseca/solve-iwmi', 'size': '4450'}