pages/2 Topic Modeling.py

#import module
import streamlit as st
import pandas as pd
import numpy as np
import re
import nltk
nltk.download('wordnet')
from nltk.stem import WordNetLemmatizer
nltk.download('stopwords')
from nltk.corpus import stopwords
import gensim
import gensim.corpora as corpora
from gensim.corpora import Dictionary
from gensim.models.coherencemodel import CoherenceModel
from gensim.models.ldamodel import LdaModel
from pprint import pprint
import pickle
import pyLDAvis
import pyLDAvis.gensim_models as gensimvis
import matplotlib.pyplot as plt
import streamlit.components.v1 as components
from io import StringIO
from ipywidgets.embed import embed_minimal_html
from nltk.stem.snowball import SnowballStemmer
from bertopic import BERTopic
import plotly.express as px
from sklearn.cluster import KMeans
import bitermplus as btm
import tmplot as tmp
import tomotopy
import sys
import spacy
import en_core_web_sm
import pipeline


#===config===
st.set_page_config(
     page_title="Coconut",
     page_icon="🥥",
     layout="wide"
)
st.header("Topic Modeling")
st.subheader('Put your file here...')

#========unique id========
@st.cache_resource(ttl=3600)
def create_list():
    l = [1, 2, 3]
    return l

l = create_list()
first_list_value = l[0]
l[0] = first_list_value + 1
uID = str(l[0])

@st.cache_data(ttl=3600)
def get_ext(uploaded_file):
    extype = uID+uploaded_file.name
    return extype

#===clear cache===

def reset_biterm():
     try:
          biterm_map.clear()
          biterm_bar.clear()
     except NameError:
          biterm_topic.clear()

def reset_all():
     st.cache_data.clear()
        
#===clean csv===
@st.cache_data(ttl=3600, show_spinner=False)
def clean_csv(extype):
    try:
        paper = papers.dropna(subset=['Abstract'])
    except KeyError:
        st.error('Error: Please check your Abstract column.')
        sys.exit(1)
    paper = paper[~paper.Abstract.str.contains("No abstract available")]
    paper = paper[~paper.Abstract.str.contains("STRAIT")]
            
        #===mapping===
    paper['Abstract_pre'] = paper['Abstract'].map(lambda x: re.sub('[,:;\.!-?•=]', '', x))
    paper['Abstract_pre'] = paper['Abstract_pre'].map(lambda x: x.lower())
    paper['Abstract_pre'] = paper['Abstract_pre'].map(lambda x: re.sub('©.*', '', x))
          
         #===stopword removal===
    stop = stopwords.words('english')
    paper['Abstract_stop'] = paper['Abstract_pre'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))
     
        #===lemmatize===
    lemmatizer = WordNetLemmatizer()
    def lemmatize_words(text):
        words = text.split()
        words = [lemmatizer.lemmatize(word) for word in words]
        return ' '.join(words)
    paper['Abstract_lem'] = paper['Abstract_stop'].apply(lemmatize_words)
     
    topic_abs = paper.Abstract_lem.values.tolist()
    return topic_abs, paper

#===upload file===
@st.cache_data(ttl=3600)
def upload(file):
    papers = pd.read_csv(uploaded_file)
    return papers

@st.cache_data(ttl=3600)
def conv_txt(extype):
    col_dict = {'TI': 'Title',
            'SO': 'Source title',
            'DT': 'Document Type',
            'AB': 'Abstract',
            'PY': 'Year'}
    papers = pd.read_csv(uploaded_file, sep='\t', lineterminator='\r')
    papers.rename(columns=col_dict, inplace=True)
    return papers


#===Read data===
uploaded_file = st.file_uploader("Choose a file", type=['csv', 'txt'], on_change=reset_all)

if uploaded_file is not None:
    extype = get_ext(uploaded_file)

    if extype.endswith('.csv'):
         papers = upload(extype) 
    elif extype.endswith('.txt'):
         papers = conv_txt(extype)
          
    topic_abs, paper=clean_csv(extype)
    method = st.selectbox(
            'Choose method',
            ('Choose...', 'pyLDA', 'Biterm','BERTopic'), on_change=reset_all)
        
    #===topic===
    if method == 'Choose...':
        st.write('')

    elif method == 'pyLDA':
         num_topic = st.slider('Choose number of topics', min_value=2, max_value=15, step=1, on_change=reset_all)
         @st.cache_data(ttl=3600, show_spinner=False)
         def pylda(extype):
            topic_abs_LDA = [t.split(' ') for t in topic_abs]
            id2word = Dictionary(topic_abs_LDA)
            corpus = [id2word.doc2bow(text) for text in topic_abs_LDA]
            #===LDA===
            lda_model = LdaModel(corpus=corpus,
                        id2word=id2word,
                        num_topics=num_topic, 
                        random_state=0,
                        chunksize=100,
                        alpha='auto',
                        per_word_topics=True)

            pprint(lda_model.print_topics())
            doc_lda = lda_model[corpus]

            #===visualization===
            coherence_model_lda = CoherenceModel(model=lda_model, texts=topic_abs_LDA, dictionary=id2word, coherence='c_v')
            coherence_lda = coherence_model_lda.get_coherence()
            vis = pyLDAvis.gensim_models.prepare(lda_model, corpus, id2word)
            py_lda_vis_html = pyLDAvis.prepared_data_to_html(vis)
            return py_lda_vis_html, coherence_lda
         
         tab1, tab2, tab3 = st.tabs(["📈 Generate visualization & Calculate coherence", "📃 Reference", "📓 Recommended Reading"])

         with tab1:
         #===visualization===
             with st.spinner('Calculating and Creating pyLDAvis Visualization ...'):
              py_lda_vis_html, coherence_lda = pylda(extype)
              st.write('Coherence: ', (coherence_lda))
              components.html(py_lda_vis_html, width=1700, height=800)
              st.markdown('Copyright (c) 2015, Ben Mabey. https://github.com/bmabey/pyLDAvis')

         with tab2:
             st.markdown('**Sievert, C., & Shirley, K. (2014). LDAvis: A method for visualizing and interpreting topics. Proceedings of the Workshop on Interactive Language Learning, Visualization, and Interfaces.** https://doi.org/10.3115/v1/w14-3110')

         with tab3:
             st.markdown('**Chen, X., & Wang, H. (2019, January). Automated chat transcript analysis using topic modeling for library reference services. Proceedings of the Association for Information Science and Technology, 56(1), 368–371.** https://doi.org/10.1002/pra2.31')
             st.markdown('**Joo, S., Ingram, E., & Cahill, M. (2021, December 15). Exploring Topics and Genres in Storytime Books: A Text Mining Approach. Evidence Based Library and Information Practice, 16(4), 41–62.** https://doi.org/10.18438/eblip29963')
             st.markdown('**Lamba, M., & Madhusudhan, M. (2021, July 31). Topic Modeling. Text Mining for Information Professionals, 105–137.** https://doi.org/10.1007/978-3-030-85085-2_4')
             st.markdown('**Lamba, M., & Madhusudhan, M. (2019, June 7). Mapping of topics in DESIDOC Journal of Library and Information Technology, India: a study. Scientometrics, 120(2), 477–505.** https://doi.org/10.1007/s11192-019-03137-5')
     
     #===Biterm===
    elif method == 'Biterm':
        num_bitopic = st.slider('Choose number of topics', min_value=2, max_value=20, step=1, on_change=reset_all)     
        #===optimize Biterm===
        @st.cache_data(ttl=3600)
        def biterm_topic(extype):
            X, vocabulary, vocab_dict = btm.get_words_freqs(topic_abs)
            tf = np.array(X.sum(axis=0)).ravel()
            docs_vec = btm.get_vectorized_docs(topic_abs, vocabulary)
            docs_lens = list(map(len, docs_vec))
            biterms = btm.get_biterms(docs_vec)
            model = btm.BTM(
              X, vocabulary, seed=12321, T=num_bitopic, M=20, alpha=50/8, beta=0.01)
            model.fit(biterms, iterations=20)
            p_zd = model.transform(docs_vec)
            coherence = model.coherence_
            phi = tmp.get_phi(model)
            topics_coords = tmp.prepare_coords(model)
            totaltop = topics_coords.label.values.tolist()
            return topics_coords, phi, totaltop

        try:
          topics_coords, phi, totaltop = biterm_topic(extype)
          #with st.spinner('Visualizing, please wait ....'):          
          tab1, tab2, tab3 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading"])
          with tab1:
            col1, col2 = st.columns(2)
                  
            @st.cache_data(ttl=3600)
            def biterm_map(extype):
              btmvis_coords = tmp.plot_scatter_topics(topics_coords, size_col='size', label_col='label', topic=numvis)
              return btmvis_coords
                  
            @st.cache_data(ttl=3600)
            def biterm_bar(extype):
              terms_probs = tmp.calc_terms_probs_ratio(phi, topic=numvis, lambda_=1)
              btmvis_probs = tmp.plot_terms(terms_probs, font_size=12)
              return btmvis_probs
                  
            with col1:
              numvis = st.selectbox(
                'Choose topic',
                (totaltop), on_change=reset_biterm)
              btmvis_coords = biterm_map(extype)
              st.altair_chart(btmvis_coords, use_container_width=True)
            with col2:
              btmvis_probs = biterm_bar(extype)
              st.altair_chart(btmvis_probs, use_container_width=True)

          with tab2: 
            st.markdown('**Yan, X., Guo, J., Lan, Y., & Cheng, X. (2013, May 13). A biterm topic model for short texts. Proceedings of the 22nd International Conference on World Wide Web.** https://doi.org/10.1145/2488388.2488514')
          with tab3:
            st.markdown('**Cai, M., Shah, N., Li, J., Chen, W. H., Cuomo, R. E., Obradovich, N., & Mackey, T. K. (2020, August 26). Identification and characterization of tweets related to the 2015 Indiana HIV outbreak: A retrospective infoveillance study. PLOS ONE, 15(8), e0235150.** https://doi.org/10.1371/journal.pone.0235150')
            st.markdown('**Chen, Y., Dong, T., Ban, Q., & Li, Y. (2021). What Concerns Consumers about Hypertension? A Comparison between the Online Health Community and the Q&A Forum. International Journal of Computational Intelligence Systems, 14(1), 734.** https://doi.org/10.2991/ijcis.d.210203.002')
            st.markdown('**George, Crissandra J., "AMBIGUOUS APPALACHIANNESS: A LINGUISTIC AND PERCEPTUAL INVESTIGATION INTO ARC-LABELED PENNSYLVANIA COUNTIES" (2022). Theses and Dissertations-- Linguistics. 48.** https://doi.org/10.13023/etd.2022.217')
            st.markdown('**Li, J., Chen, W. H., Xu, Q., Shah, N., Kohler, J. C., & Mackey, T. K. (2020). Detection of self-reported experiences with corruption on twitter using unsupervised machine learning. Social Sciences & Humanities Open, 2(1), 100060.** https://doi.org/10.1016/j.ssaho.2020.100060')
          
        except ValueError:
          st.error('Please raise the number of topics')
    
     #===BERTopic===
    elif method == 'BERTopic':
        num_btopic = st.slider('Choose number of topics', min_value=4, max_value=20, step=1, on_change=reset_all)
        @st.cache_data(ttl=3600)
        def bertopic_vis(extype):
          topic_time = paper.Year.values.tolist()
          cluster_model = KMeans(n_clusters=num_btopic)
          nlp = en_core_web_sm.load(exclude=['tagger', 'parser', 'ner', 'attribute_ruler', 'lemmatizer'])
          topic_model = BERTopic(embedding_model=nlp, hdbscan_model=cluster_model, language="multilingual").fit(topic_abs)
          topics, probs = topic_model.fit_transform(topic_abs)
          return topic_model, topic_time, topics, probs
        
        @st.cache_data(ttl=3600)
        def Vis_Topics(extype):
          fig1 = topic_model.visualize_topics()
          return fig1
        
        @st.cache_data(ttl=3600)
        def Vis_Documents(extype):
          fig2 = topic_model.visualize_documents(topic_abs)
          return fig2

        @st.cache_data(ttl=3600)
        def Vis_Hierarchy(extype):
          fig3 = topic_model.visualize_hierarchy(top_n_topics=num_btopic)
          return fig3
    
        @st.cache_data(ttl=3600)
        def Vis_Heatmap(extype):
          global topic_model
          fig4 = topic_model.visualize_heatmap(n_clusters=num_btopic-1, width=1000, height=1000)
          return fig4

        @st.cache_data(ttl=3600)
        def Vis_Barchart(extype):
          fig5 = topic_model.visualize_barchart(top_n_topics=num_btopic, n_words=10)
          return fig5
    
        @st.cache_data(ttl=3600)
        def Vis_ToT(extype):
          topics_over_time = topic_model.topics_over_time(topic_abs, topic_time)
          fig6 = topic_model.visualize_topics_over_time(topics_over_time)
          return fig6
        
        tab1, tab2, tab3 = st.tabs(["📈 Generate visualization", "📃 Reference", "📓 Recommended Reading"])
        with tab1:
          topic_model, topic_time, topics, probs = bertopic_vis(extype)
          #===visualization===
          viz = st.selectbox(
            'Choose visualization',
            ('Visualize Topics', 'Visualize Documents', 'Visualize Document Hierarchy', 'Visualize Topic Similarity', 'Visualize Terms', 'Visualize Topics over Time'))

          if viz == 'Visualize Topics':
                 fig1 = Vis_Topics(extype)
                 st.write(fig1)

          elif viz == 'Visualize Documents':
                 fig2 = Vis_Documents(extype)
                 st.write(fig2)

          elif viz == 'Visualize Document Hierarchy':
                 fig3 = Vis_Hierarchy(extype)
                 st.write(fig3)

          elif viz == 'Visualize Topic Similarity':
                 fig4 = Vis_Heatmap(extype)
                 st.write(fig4)

          elif viz == 'Visualize Terms':
                 fig5 = Vis_Barchart(extype)
                 st.write(fig5)

          elif viz == 'Visualize Topics over Time':
                 fig6 = Vis_ToT(extype)
                 st.write(fig6)

        with tab2:
          st.markdown('**Grootendorst, M. (2022). BERTopic: Neural topic modeling with a class-based TF-IDF procedure. arXiv preprint arXiv:2203.05794.** https://doi.org/10.48550/arXiv.2203.05794')
          
        with tab3:
          st.markdown('**Jeet Rawat, A., Ghildiyal, S., & Dixit, A. K. (2022, December 1). Topic modelling of legal documents using NLP and bidirectional encoder representations from transformers. Indonesian Journal of Electrical Engineering and Computer Science, 28(3), 1749.** https://doi.org/10.11591/ijeecs.v28.i3.pp1749-1755')
          st.markdown('**Yao, L. F., Ferawati, K., Liew, K., Wakamiya, S., & Aramaki, E. (2023, April 20). Disruptions in the Cystic Fibrosis Community’s Experiences and Concerns During the COVID-19 Pandemic: Topic Modeling and Time Series Analysis of Reddit Comments. Journal of Medical Internet Research, 25, e45249.** https://doi.org/10.2196/45249')