<jupyter_start><jupyter_text>Topic Modelling and Analysis  Import Packages and Defining Helper Functions<jupyter_code># import required packages
#warnings :)
import warnings
warnings.filterwarnings('ignore')

# for df purpose
import pandas as pd
import numpy as np
from tqdm.auto import tqdm
tqdm.pandas()

# for text processing
import nltk
import re
import string
from autocorrect import Speller
import malaya

# for topic modelling
import gensim as gensim
from gensim import matutils, models, corpora
import scipy.sparse
import pyLDAvis.gensim

# for graph plotting / visualisation
import matplotlib.pyplot as plt
import matplotlib.dates
import seaborn as sns
%matplotlib inline
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
from sklearn.feature_extraction.text import CountVectorizer

#for storing intermediate results
import pickle

# for notebook function
from IPython.display import display, clear_output
import json
import time
def print_bold(text):
    text_bold = '\x1b[1;30;47m'+text+ '\x1b[0m'
    print(text_bold)
def glance(x,n=5):
    try:
        iterator = iter(x)
    except:
        print(x)
        return
    else:
        if type(x) == str or len(str(x)) <= 50:
            print(x)
            return
        if type(x) == dict:
            iterator = x.items()
        i = 0
        for ele in iterator:
            if i >= n:
                break
            glance(ele,n)
            i += 1<jupyter_output><empty_output><jupyter_text>Read Data and perform secondary cleaning<jupyter_code>df = pd.read_pickle('pickles/df_analyse')
df
# Separate df into English and Malay
df_eng = df[df['lang_group'] == 'eng']
df_malay = df[df['lang_group'] == 'malay']
print('English Posts: %s' % len(df_eng))
print('Malay Posts: %s' % len(df_malay))<jupyter_output>English Posts: 854
Malay Posts: 3327<jupyter_text>Removing Stop Words  Malay<jupyter_code>word_counts = {}
def count_words(tokens):
    for t in tokens:
        if t not in word_counts:
            word_counts[t] = 0
        word_counts[t] += 1
df_malay['stem'].progress_apply(count_words)
count_df = pd.DataFrame({'count':word_counts}).sort_values('count',ascending=False)
print(count_df.head(20)['count'])
def load_ms_stopwords():
    try:
        ms_stopwords = json.load(open('pickles/ms_stopwords.json','rb'))
        print('stopwords found: %s' % len(ms_stopwords))
        return ms_stopwords
    except:
        print('stopwords not found')
        json.dump([],open('pickles/ms_stopwords.json','a'))
        return []
def add_ms_stopwords(new_list):
    ms_stopwords = load_ms_stopwords()
    for t in new_list:
        if t not in ms_stopwords:
            ms_stopwords.append(t)
    json.dump(ms_stopwords,open('pickles/ms_stopwords.json','w'))
    return load_ms_stopwords()
def remove_stopwords(series):
    ms_stopwords = load_ms_stopwords()
    return series.apply(lambda x: [t for t in x if t not in ms_stopwords])
add_ms_stopwords(['ke','bagi'])
df_malay['stem'] = remove_stopwords(df_malay['stem'])
word_counts = {}
df_malay['stem'].progress_apply(count_words)
count_df = pd.DataFrame({'count':word_counts}).sort_values('count',ascending=False)
print(count_df.head(20)['count'])<jupyter_output>stopwords found: 16
stopwords found: 18
stopwords found: 18<jupyter_text>English<jupyter_code>stopset = set(nltk.corpus.stopwords.words('english'))
df_eng['stem'] = df_eng['stem'].apply(lambda x: [t for t in x if t not in stopset])
word_counts = {}
df_eng['stem'].progress_apply(count_words)
count_df = pd.DataFrame({'count':word_counts}).sort_values('count',ascending=False)
print(count_df.head(20)['count'])
stopset.update(['blood','donation','u','http'])
df_eng['stem'] = df_eng['stem'].apply(lambda x: [t for t in x if t not in stopset])
word_counts = {}
df_eng['stem'].progress_apply(count_words)
count_df = pd.DataFrame({'count':word_counts}).sort_values('count',ascending=False)
print(count_df.head(20)['count'])<jupyter_output><empty_output><jupyter_text>Topic Modelling  Malay Posts<jupyter_code>cv_malay = CountVectorizer()  # remove stop words from the build-in CountVectorizer
data_cv = cv_malay.fit_transform(df_malay['stem'].apply(lambda x: ' '.join(x)))
data_dtm = pd.DataFrame(data_cv.toarray(),columns=cv_malay.get_feature_names())
data_dtm.index = df_malay.index
data_dtm
tdm = data_dtm.transpose()
sparse_counts = scipy.sparse.csr_matrix(tdm)
corpus_malay = matutils.Sparse2Corpus(sparse_counts)
dictionary_malay = corpora.Dictionary(df_malay['stem'])
# base model
lda_malay = models.LdaModel(corpus=corpus_malay, id2word=dictionary_malay, num_topics=3, passes=50)
lda_malay.print_topics()
coherence_score = models.CoherenceModel(model=lda_malay, texts=df_malay['stem'], dictionary=dictionary_malay, coherence='c_v').get_coherence()
coherence_score<jupyter_output><empty_output><jupyter_text>Hyperparameter Tuning<jupyter_code>def grid_search_tuning(corpus, dictionary,topics_range=None, alpha=None, beta=None, random_state=100, texts=df_malay['stem']):
    
    # supporting function
    def compute_coherence_values(corpus, dictionary, k, a, b, random_state=100):

        lda_model = gensim.models.LdaMulticore(corpus=corpus,
                                               id2word=dictionary,
                                               num_topics=k, 
                                               random_state=random_state,
                                               chunksize=100,
                                               passes=10,
                                               alpha=a,
                                               eta=b,
                                              )

        coherence_model_lda = models.CoherenceModel(model=lda_model, texts=texts, dictionary=dictionary, coherence='c_v')

        return coherence_model_lda.get_coherence()

    # Topics range
    if topics_range is None:
        min_topics = 3
        max_topics = 5
        step_size = 1
        topics_range = range(min_topics, max_topics, step_size)
        print('Using default topics range: ',topics_range)
    else:
        print('topics_range: ',topics_range)

    # Alpha parameter
    if alpha is None:
        alpha = list(np.arange(0.1, 1, 0.1))
        alpha.append('symmetric')
        alpha.append('asymmetric')
        print('Using default alpha list: ',alpha)
    else:
        print('alpha: ',alpha)

    # Beta parameter
    if beta is None:
        beta = list(np.arange(0.1, 1, 0.1))
        beta.append('symmetric')
        print('Using default beta list: ',beta)
    else:
        print('beta: ',beta)

    model_results = {
        'Topics': [],
        'Alpha': [],
        'Beta': [],
        'Coherence': []
    }

    # Can take a long time to run
    if 1 == 1:
        pbar = tqdm(total=len(topics_range)*len(alpha)*len(beta))
        # iterate through number of topics
        for k in topics_range:
            # iterate through alpha values
            for a in alpha:
                # iterare through beta values
                for b in beta:
                    # get the coherence score for the given parameters
                    cv = compute_coherence_values(corpus=corpus, dictionary=dictionary, k=k, a=a, b=b, random_state=random_state)
                    # Save the model results
                    model_results['Topics'].append(k)
                    model_results['Alpha'].append(a)
                    model_results['Beta'].append(b)
                    model_results['Coherence'].append(cv)

                    pbar.update(1)
#         pd.DataFrame(model_results).to_csv('outputs/lda_tuning_results.csv', index=False)
        pbar.close()
    return model_results
model_results = pd.DataFrame(grid_search_tuning(corpus_malay,dictionary_malay,[3,4]))
model_results.sample(5)
top_models = model_results.sort_values('Coherence',ascending=False).head(5)
# tuning_results['Group'] = tuning_results[['Alpha','Beta']].apply(lambda x: ', '.join(x.astype(str)), axis=1)
# sns.lineplot(x='Topics', y='Coherence', data=tuning_results, hue='Group')
top_models
# Best Performing Topic
[num_topics, alpha, eta] = top_models.iloc[0][['Topics','Alpha','Beta']]

lda_malay = models.LdaModel(corpus=corpus_malay, id2word=dictionary_malay, num_topics=num_topics, passes=100, alpha=alpha, eta=eta)
lda_malay.save('models/lda_malay.gensim')
lda_malay.print_topics()
df_malay['topic'] = [max(ele, key = lambda x:x[1])[0] for ele in lda_malay[corpus_malay]]
for i in range(0,4,1):
    print('='*20)
    print('topic: ', i)
    for title in df_malay[df_malay['topic'] == i].sample(10)['title']:
        print('-'*10)
        print(title)<jupyter_output>====================
topic:  0
----------
Terima ksh kepada yg menderma hr ni... 👍🏻
----------
Kriteria untuk menderma darah. #DermaDarah #DarahAndaPenyelamatNyawa
----------
J<NAME> 1 Mac 2017 (Rabu) #jomdermadarah
----------
Penang...:)
----------
<NAME> (<NAME>) Sambutan sangat menggalakkan. Sebanyak 107 pint darah berjaya dikutip setakat ini.
----------
Salam sejahtera.. Jom derma darah minggu ini.. Jadual Kempen <NAME> di kawasan Pulau Pinang dan Seberang Perai pada tarikh 27, 28 & 30 Julai 2019 (Sabtu, Ahad & Selasa) adalah seperti berikut: #jomdermadarah #dermadarahpenang
----------
Miri : )
----------
Penang : )
----------
…Jom ,kita boleh pi Hospital untuk derma darah penuhi tabung darah. semoga pihak hospital ada darah yang cukup .Orang ramai juga dijemput datang ke Aeon Jusco Mall ALMA,BM.menyertai Kempen derma darah pada Ahad ni 29-10-2017 ,pukul 10 hingga 2 pm. Takziah kepada keluarga mangsa yang terlibat.... Yang cedera,semoga berseman[...]<jupyter_text>Understanding Topics* Topic 0 - Informal Advertisement of Blood Donation Campaign* Topic 1 - Information on Blood Supply Shortage and Existing Blood Donation Programs* Topic 2 - Educating Blood Donors on Related Information* Topic 3 - Formal Advertisement of Blood Donation Campaigns<jupyter_code>plt.rcParams["font.size"] = "20"
df_malay['topic'].value_counts().plot.pie(figsize=(10,10),autopct='%1.1f%%')
df_topic_year = pd.DataFrame()
df_topic_month = pd.DataFrame()
for topic in range(0,4,1):
    temp_df = df_malay[df_malay['topic'] == topic]
    temp_series = temp_df['topic']
    temp_series.index = temp_df['date']
    df_topic_year[topic] = temp_series.resample('Y').count()
    df_topic_month[topic] = temp_series.resample('M').count()
df_topic_year.plot(figsize=(16,5), title='Topic Yearly Count')
ax = df_topic_month.plot(figsize=(16,5), title='Topic Monthly Count')
# ax.xaxis.set_major_locator(matplotlib.dates.YearLocator())
# ax.xaxis.set_minor_locator(matplotlib.dates.MonthLocator((1,4,7,10)))
# ax.xaxis.set_major_formatter(matplotlib.dates.DateFormatter("\n%Y"))
# ax.xaxis.set_minor_formatter(matplotlib.dates.DateFormatter("%b"))<jupyter_output><empty_output><jupyter_text>English Posts<jupyter_code>cv_eng = CountVectorizer()  # remove stop words from the build-in CountVectorizer
data_cv = cv_eng.fit_transform(df_eng['stem'].apply(lambda x: ' '.join(x)))
data_dtm = pd.DataFrame(data_cv.toarray(),columns=cv_eng.get_feature_names())
data_dtm.index = df_eng.index
data_dtm
# remove Chinese Characters
df_eng['stem'] = df_eng['stem'].apply(lambda x: [t for t in x if re.match('[a-zA-Z]', t)])
cv_eng = CountVectorizer()  # remove stop words from the build-in CountVectorizer
data_cv = cv_eng.fit_transform(df_eng['stem'].apply(lambda x: ' '.join(x)))
data_dtm = pd.DataFrame(data_cv.toarray(),columns=cv_eng.get_feature_names())
data_dtm.index = df_eng.index
data_dtm
tdm = data_dtm.transpose()
sparse_counts = scipy.sparse.csr_matrix(tdm)
corpus_eng = matutils.Sparse2Corpus(sparse_counts)
dictionary_eng = corpora.Dictionary(df_eng['stem'])
# base model
lda_eng = models.LdaModel(corpus=corpus_eng, id2word=dictionary_eng, num_topics=3, passes=50)
lda_eng.print_topics()
coherence_score = models.CoherenceModel(model=lda_eng, texts=df_eng['stem'], dictionary=dictionary_eng, coherence='c_v').get_coherence()
coherence_score
model_results = pd.DataFrame(grid_search_tuning(corpus_eng,dictionary_eng,[3,4],texts=df_eng['stem']))
model_results.sample(5)
top_models = model_results.sort_values('Coherence',ascending=False).head(5)
# tuning_results['Group'] = tuning_results[['Alpha','Beta']].apply(lambda x: ', '.join(x.astype(str)), axis=1)
# sns.lineplot(x='Topics', y='Coherence', data=tuning_results, hue='Group')
top_models
# Best Performing Topic
[num_topics, alpha, eta] = top_models.iloc[0][['Topics','Alpha','Beta']]

lda_eng = models.LdaModel(corpus=corpus_eng, id2word=dictionary_eng, num_topics=num_topics, passes=100, alpha=alpha, eta=eta)
lda_eng.save('models/lda_eng.gensim')
lda_eng.print_topics()
df_eng['topic'] = [max(ele, key = lambda x:x[1])[0] for ele in lda_eng[corpus_eng]]
for i in range(0,3,1):
    print('\n')
    print('='*20)
    print_bold('topic: %s'% i)
    for title in df_eng[df_eng['topic'] == i].sample(20)['title']:
        print('-'*10)
        print(title)<jupyter_output>====================
[1;30;47mtopic: 0[0m
----------
…thalassemia patients. He has established blood donation center to provide blood for the community hospital on a regular basis since year 2007 until to-date, which has attracted more than 4,000 donors to participate Blood donation campaign. Over thousands packs of blood have been brought to the local blood bank. He, himself has already donated 108 times so far, and because of the insistency of Mr. Firdaus, many people were touched by his good deed and joined the blood donation activity. - 一个动念，血源不断 --- 一股助人热忱，推动Firdaus成为捐血常客，尤其本着想帮助地中海贫血症患者的初心，进而创立一所捐血中心，定期为社区医院提供血液，从2007年迄今已招纳逾4千名自愿者参与捐血运动。为当地的血库带来了数千包血。他本身至今也已经捐了109次，也因为Firdaus先生的坚持感动了许多人，响应他的号召参与捐血活动。
----------
HOPE worldwide Malaysia Blood Donation Campaign is coming back this year on 22 September! Email your name and contact number to <EMAIL> and register as a Blood Donor today to reserve your Sakae Sushi… https://instagram.com/p/B0zmkrfgM4T/?igshid=18pzsaty[...]<jupyter_text>Understanding Topics* Topic 0 - Raising Public Awareness and Educating Public on Blood Donation related Issues.* Topic 1 - Sharing Info of currently ongoing blood donation campaigns.* Topic 2 - Blood Donation campaign advertisement. Urge public to participate.<jupyter_code>df_eng['topic'].value_counts().plot.pie(figsize=(10,10),autopct='%1.1f%%')
df_topic_year = pd.DataFrame()
df_topic_month = pd.DataFrame()
for topic in range(0,3,1):
    temp_df = df_eng[df_eng['topic'] == topic]
    temp_series = temp_df['topic']
    temp_series.index = temp_df['date']
    df_topic_year[topic] = temp_series.resample('Y').count()
    df_topic_month[topic] = temp_series.resample('M').count()
df_topic_year.plot(figsize=(16,5), title='Topic Yearly Count')
df_topic_month.plot(figsize=(16,5), title='Topic Monthly Count')
# Convert notebook to html
!jupyter nbconvert --to html_ch --output-dir='.\html' "3. Topic Modelling and Insights.ipynb"<jupyter_output>[NbConvertApp] Converting notebook 3. Topic Modelling and Insights.ipynb to html_ch
[NbConvertApp] Writing 740409 bytes to .\html\3. Topic Modelling and Insights.html
{'fraction_non_alphanumeric': 0.13139704904958127, 'fraction_numerical': 0.015352917718995081, 'mean_word_length': 4.074873524451939, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '84918', 'license': 'MIT', 'n_tokens_mistral': 5459, 'n_tokens_neox': 5199, 'n_words': 1102, 'path': '2. Analysis/Submission/source codes/3. Topic Modelling and Insights.ipynb', 'repo_name': 'chin0813/Blood-Donation-Campaign-Analysis', 'size': '15099'}