<jupyter_start><jupyter_text>BERTopic Model AnalysisBERTopic is a topic modeling technique that leverages 🤗 transformers and c-TF-IDF to create dense clusters allowing for easily interpretable topics whilst keeping important words in the topic descriptions. This will be used for the Topic Modelling phaseIt implements DTM and multiple visualisation for the topics extracted  Setup phase- Installing third-party libraries and extensions- Import data<jupyter_code>import time
start = time.time()

import re
import string
from copy import deepcopy
import torch
import nltk
import pandas as pd
import plotly.express as px
import spacy
import swifter
import tensorflow as tf
import tensorflow_hub
from bertopic import BERTopic
from flair.embeddings import TransformerDocumentEmbeddings, SentenceTransformerDocumentEmbeddings
from hdbscan import HDBSCAN
from keras.preprocessing.text import text_to_word_sequence
from nltk import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from pymongo import MongoClient
from sentence_transformers import SentenceTransformer
from sklearn.feature_extraction.text import CountVectorizer
from tqdm.notebook import tqdm
from umap import UMAP
from thinc.api import prefer_gpu,use_tensorflow_for_gpu_memory

if prefer_gpu():
    use_tensorflow_for_gpu_memory()

nltk.download('stopwords')
nltk.download('omw-1.4')
nltk.download('punkt')
nltk.download('wordnet')<jupyter_output><empty_output><jupyter_text>Adjust parameters<jupyter_code># For loading data directly from a local file
local_data = False
dataset = 'collab_dataset_07_22__220000'

# For extracting data from a MongoDB instance
start_year = '07'
end_year = '22'
size_cutoff = 150000  # Maximum value: 740000

# Available cleaning methods are:
# - emma: The cleaning pipeline used by <NAME> in the paper:
#                     https://github.com/etattershall/burst-detection
# - nltk: A cleaning pipeline using the builtin methods
#         from NLTK library
# - custom: A cleaning pipeline built using several iterations
#           and experiments to get the best results for BERTopic
cleaning_method = 'custom'

# Available cleaning methods are:
# - sentence-transformers: https://www.sbert.net/docs/pretrained_models.html
# - flair: using flair library to utilize any hugging face model available
# - spacy: using spaCy English Transformers model
# - use: Universal Sentence Encoder from tensorflow
embedding = 'use'

# BERTopic parameter tuning

# Available hyperparameter tuning options:
# - default : uses the default BERTopic configuration
# - custom : uses separate submodules for each step of the algorithm
#           it requires extra parameters to be set (default values for each
#           parameter are the best based on the experiments ran)
# - full : uses a breakdown of BERTopic to drill further down into the separate
#          submodules for each step of the algorithm. (default values for each
#          parameter are the best based on the experiments ran)
#
# By default, the model will be selected to work on english and
# will compute probabilities for each corresponding topic

model_complexity = 'custom'

# custom parameters (also applicable for full version)
top_n_words = 10
n_gram_range = (1, 2)
nr_topics = 800

# full parameters

# UMAP parameters
low_memory = True
n_neighbors = 15
n_components = 10

# HDBSCAN parameters
min_cluster_size = 10
min_samples = 1
metric = 'manhattan'
prediction_data = True

if model_complexity == 'default':
    parameters_str = f'exp_local_{cleaning_method}_{embedding}_{model_complexity}'
elif model_complexity == 'custom':
    parameters_str = f'exp_local_{cleaning_method}_{embedding}_{model_complexity}_{top_n_words}_{n_gram_range}_{nr_topics}'
else:
    parameters_str = f'exp_local_{cleaning_method}_{embedding}_{model_complexity}_{top_n_words}_{n_gram_range}_{nr_topics}_'<jupyter_output><empty_output><jupyter_text>Loading the data<jupyter_code>if local_data:
    df = pd.read_csv(f'{dataset}.csv')
else:
    # Connect to local instance of MongoDB
    client = MongoClient('127.0.0.1', 27017)
    db = client.frtp
    collection = db.documents

    # Extract all the data available with specific restrictions
    result = collection.find({
        "year": {
            '$lt': end_year,
            '$gte': start_year
        },
        "size": {
            '$lt': size_cutoff
        }
    })
    df = pd.DataFrame(list(result))
    df['year'] = pd.to_datetime(df['year'], format='%y')

print(df.shape)

df[df['ticker'].str.contains('AAPL')].iloc[3]<jupyter_output><empty_output><jupyter_text>Cleaning the dataset<jupyter_code># cleaning method - emma (this is the preprocessing pipeline that was used in the burstiness model paper)
alphabets = "([A-Za-z])"
prefixes = "(Mr|St|Mrs|Ms|Dr)[.]"
suffixes = "(Inc|Ltd|Jr|Sr|Co)"
starters = "(Mr|Mrs|Ms|Dr|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)"
acronyms = "([A-Z][.][A-Z][.](?:[A-Z][.])?)"
websites = "[.](com|net|org|io|gov)"
htmltags = '<[^>]+>'
htmlspecial = '&#?[xX]?[a-zA-Z0-9]{2,8};'

start_delimiter = 'documentstart'
sent_delimiter = 'sentenceboundary'
end_delimiter = 'documentend'

ngram_length = 3

delimiters = [start_delimiter, sent_delimiter, end_delimiter]

# Download the lemmatisesr
wnl = WordNetLemmatizer()

# Create a tokeniser
count = CountVectorizer(strip_accents='ascii', min_df=1)
tokeniser = count.build_analyzer()


def normalise_acronymns(text):
    '''
    Remove the periods in acronyms. 
    Adapted from the method found at https://stackoverflow.com/a/40197005 
    '''
    return re.sub(r'(?<!\w)([A-Z, a-z])\.', r'\1', text)


def normalise_decimals(text):
    '''
    Remove the periods in decimal numbers and replace with POINT
    '''
    return re.sub(r'([0-9])\.([0-9])', r'\1POINT\2', text)


def split_into_sentences(text):
    '''
    Sentence splitter adapted from https://stackoverflow.com/a/31505798
    '''
    text = text.replace("\n", " ")
    text = re.sub(prefixes, "\\1<prd>", text)
    text = re.sub(websites, "<prd>\\1", text)

    # my addition
    text = re.sub(htmltags, " ", text)
    text = re.sub(htmlspecial, " ", text)

    if "Ph.D" in text:
        text = text.replace("Ph.D.", "PhD")

    text = re.sub("\s" + alphabets + "[.] ", " \\1", text)
    text = re.sub(acronyms+" "+starters, "\\1<stop> \\2", text)
    text = re.sub(alphabets + "[.]" + alphabets + "[.]" + alphabets + "[.]", "\\1\\2\\3", text)
    text = re.sub(alphabets + "[.]" + alphabets + "[.]", "\\1\\2", text)
    text = re.sub(" "+suffixes+"[.] "+starters, " \\1 \\2", text)
    text = re.sub(" "+suffixes+"[.]", " \\1", text)
    text = re.sub(" " + alphabets + "[.]", " \\1", text)

    if "”" in text:
        text = text.replace(".”", "”.")
    if "\"" in text:
        text = text.replace(".\"", "\".")
    if "!" in text:
        text = text.replace("!\"", "\"!")
    if "?" in text:
        text = text.replace("?\"", "\"?")

    text = text.replace(".", "<stop>")
    text = text.replace("?", "<stop>")
    text = text.replace("!", "<stop>")

    sentences = text.split("<stop>")
    sentences = [s.strip() for s in sentences]

    non_empty = []

    for s in sentences:
        # we require that there be two alphanumeric characters in a row
        if len(re.findall("[A-Za-z0-9][A-Za-z0-9]", s)) > 0:
            non_empty.append(s)
    return non_empty


def pad_sentences(sentences):
    '''
    Takes a list of sentences and returns a string in which:
        - The beginning of the abstract is indicated by DOCUMENTSTART
        - The end is indicated by DOCUMENTEND
        - Sentence boundaries are indicated by SENTENCEBOUNDARY

    The number of delimiters used is dependent on the ngram length
    '''
    sent_string = (' '+(sent_delimiter+' ')*(ngram_length-1)).join(sentences)

    return (start_delimiter+' ')*(ngram_length-1) + sent_string + (' '+end_delimiter)*(ngram_length-1)


def get_stopwords():
    stop = set(stopwords.words('english'))
    stop = set([s.replace("'", "") for s in stop])

    # Add years to prevent spikes
    for year in range(1900, 2020):
        stop.add(str(year))

    # Add small numbers
    for num in range(0, 100):
        if len(str(num)) < 2:
            stop.add(str(num))
            num = '0' + str(num)

        stop.add(str(num))

    # Add several specific terms considered stopwords based on
    # manual dataset inspection
    extra = [
        'use', 'using', 'uses', 'used', 'based', 'including', 'include', 'approach', 'factors', 'business', 'risk'
        'wa', 'ha', 'doe', 'item', '1a', 'factor', '1b', '1aitem', '10-k', '1AITEM', 'could', 'regarding'
    ]
    for word in extra:
        stop.add(word)
    return stop


def cleaning_pipeline(text):
    '''
    Takes a binary string and returns a list of cleaned sentences, stripped of punctuation and lemmatised
    '''

    stopwords = get_stopwords()
    text = normalise_decimals(normalise_acronymns(text))
    text = ' '.join([word for word in text.split() if word not in stopwords])
    sentences = split_into_sentences(text)

    # strip out punctuation and make lowercase
    clean_sentences = []
    for s in sentences:

        # Deal with special cases
        s = re.sub(r'[-/]', ' ', s)

        # Remove all other punctuation
        s = re.sub(r'[^\w\s]', '', s)

        clean_sentences.append(s.lower())

    # pad sentences with delimiters
    text = pad_sentences(clean_sentences)

    # Lemmatise word by word
    lemmas = []
    for word in tokeniser(text):
        lemmas.append(wnl.lemmatize(word))

    return ' '.join(lemmas)


def cleaning_pipeline_df(row):
    row['clean_text'] = cleaning_pipeline(row['text'])
    return row

# cleaning method - custom
def get_stopwords_manual():
    stop = set(stopwords.words('english'))
    stop = set([s.replace("'", "") for s in stop])

    # Add years to prevent spikes
    for year in range(1900, 2020):
        stop.add(str(year))

    # Add small numbers
    for num in range(0, 100):
        if len(str(num)) < 2:
            stop.add(str(num))
            num = '0' + str(num)

        stop.add(str(num))

    # Add several specific terms considered stopwords based on
    # manual dataset inspection
    extra = [
        'use', 'using', 'uses', 'used', 'based', 'including', 'include',
        'approach', 'factors', 'business', 'risk', 'factors16',
        'wa', 'ha', 'doe', 'item', '1a', 'factor', '1b', '1aitem', '10-k',
        'item', '1arisk', 'factors11', '1arisk', 'factors10k', 'factorsk13', 'could',
        'factorsk10', 'may', 'looking', 'forward', 'statement'
    ]

    # Add factors combined with numbers as
    # they appear often as references
    for number in range(1, 300):
        factor_string = 'factors'+str(number)
        stop.add(factor_string)

    for word in extra:
        stop.add(word)

    return stop

# cleaning method - nltk
punct = []
punct += list(string.punctuation)
punct += '’'
punct.remove("'")


def remove_punctuations(text):
    for punctuation in punct:
        text = text.replace(punctuation, ' ')
    return text


def nlp(df):
    df['token'] = df['text'].swifter.apply(lambda x: x.lower())
    df['token'] = df['token'].swifter.apply(lambda x: x.replace('\n', ' '))
    df['token'] = df['token'].str.replace('http\S+|www.\S+', '', case=False)
    df['token'] = df['token'].swifter.apply(lambda x: x.replace('&gt;', ''))
    df['token'] = df['token'].swifter.apply(remove_punctuations)
    df['token'] = df['token'].swifter.apply(lambda x: str(x).replace(" s ", " "))
    df['final_text'] = df['token'].swifter.apply(word_tokenize)
    my_stopwords = set(stopwords.words('english'))
    df['final_text'] = df['final_text'].swifter.apply(
        lambda text_list: [x for x in text_list if x not in my_stopwords])
    df['final_text'] = df['final_text'].swifter.apply(lambda list_data: [x for x in list_data if x.isalpha()])
    df['clean_text'] = df['final_text'].swifter.apply(lambda row: ' '.join([word for word in row]))
    return df

if cleaning_method == 'custom':
    df['clean_text'] = df['text'].str.lower()
    df['clean_text'] = df['clean_text'].swifter.apply(lambda x: ' '.join(
        [word for word in text_to_word_sequence(x)]))
    stop_words = get_stopwords_manual()
    df['clean_text'] = df['clean_text'].swifter.apply(lambda x: ' '.join(
        [word for word in x.split() if word not in (stop_words)]))
elif cleaning_method == 'emma':
    df['clean_text'] = ''
    df = df.swifter.apply(cleaning_pipeline_df, axis=1)
else:
    df = nlp(df)

# Extracting the text with timestamps
timestamps = df.year.to_list()
text = df.clean_text.to_list()<jupyter_output><empty_output><jupyter_text>Topic Modelling utilising BERTopic<jupyter_code>if embedding == 'sentence-transformers':
    embedding_model = SentenceTransformer("all-mpnet-base-v2") #all-roberta-large-v1 all-mpnet-base-v1
elif embedding == 'flair':
    embedding_model = TransformerDocumentEmbeddings(
        'sentence-transformers/all-MiniLM-L6-v2')
    # embedding_model = SentenceTransformerDocumentEmbeddings('stsb-distilroberta-base-v2') #  stsb-roberta-large
elif embedding == 'spacy':
    embedding_model = spacy.load("en_core_web_trf", exclude=['tagger', 'parser', 'ner',
                                                             'attribute_ruler', 'lemmatizer'])
elif embedding == 'use':
    embedding_model = tensorflow_hub.load(
        "https://tfhub.dev/google/universal-sentence-encoder-lite/2")
else:
    embedding_model = SentenceTransformer("all-MiniLM-L6-v2") # all-MiniLM-L12-v1 

if model_complexity == 'default':
    topic_model = BERTopic(verbose=True, language="english", nr_topics="auto",
                           embedding_model=embedding_model)
elif model_complexity == 'custom':
    topic_model = BERTopic(verbose=True, n_gram_range=n_gram_range,
                           language="english", top_n_words=top_n_words,
                           nr_topics=nr_topics, low_memory=low_memory,
                           embedding_model=embedding_model)
else:
    umap_model = UMAP(n_neighbors=n_neighbors, n_components=n_components, metric=metric,
                      low_memory=low_memory)
    hdbscan_model = HDBSCAN(min_cluster_size=min_cluster_size, metric=metric,
                            prediction_data=prediction_data)
    topic_model = BERTopic(verbose=True, n_gram_range=n_gram_range,
                           nr_topics=nr_topics, umap_model=umap_model,
                           hdbscan_model=hdbscan_model,
                           embedding_model=embedding_model)

topics, probs = topic_model.fit_transform(text)

# Due to a bug that I had to report on library's page: https://github.com/MaartenGr/BERTopic/issues/470
if embedding != 'use':
    topic_model.save(f"models/BERTopic_model_{parameters_str}.model")

# Explore a topic
topic_model.get_topic(21)

# Overview of the topics
topic_info = topic_model.get_topic_info()
topic_info

# topic_model.visualize_hierarchy()

topic_model.visualize_barchart()

# topic_model.visualize_heatmap()

topic_model.visualize_term_rank()
topic_model.visualize_topics()

# test_topic_model = BERTopic.load('models/BERTopic_model_local_nltk_sentence-transformers_full_10_(1, 3)_1000_.model')
# test_topic_model = BERTopic.load('models/BERTopic_model_local_nltk_flair_full_10_(1, 2)_800_.model',embedding_model='all-mpnet-base-v2')

# test_topic_model.visualize_topics()<jupyter_output><empty_output><jupyter_text>Utilising DTM to convert the topics resulted into a timeserie<jupyter_code># Running DTM on the entire dataset
topics_over_time = topic_model.topics_over_time(text, topics, timestamps,global_tuning=False)
topic_model.visualize_topics_over_time(topics_over_time)

# Running DTM on the entire dataset
# topics_over_time_global = topic_model.topics_over_time(text, topics, timestamps)
# # TODO: Decide if it is always excluded
# topics_over_time_global=topics_over_time_global[topics_over_time_global['Topic'] != -1]
# topic_model.visualize_topics_over_time(topics_over_time_global)<jupyter_output><empty_output><jupyter_text>Cleaning DTM results<jupyter_code># TODO: Decide if it is always excluded
topics_over_time=topics_over_time[topics_over_time['Topic'] != -1]
smaller_reporting_topics = topics_over_time[(topics_over_time['Words'].str.contains("smaller"))]['Topic'].unique()
topics_over_time = topics_over_time[~(topics_over_time['Topic'].isin(smaller_reporting_topics))]

spaces_topics = topics_over_time[(topics_over_time['Words'].str.contains(", , ,"))]['Topic'].unique()
topics_over_time = topics_over_time[~(topics_over_time['Topic'].isin(spaces_topics))]

# First 10 entries in the timeseries generated by DTM
topics_over_time.head(10)

topic_model.visualize_topics_over_time(topics_over_time, top_n_topics=10)

topic_model.visualize_topics_over_time(topics_over_time)

topics_over_time.to_csv(f'output/DTM/DTM_{parameters_str}.csv')

"the code you want to test stays here"
end = time.time()
print(end-start)
print(len(topics_over_time['Topic'].unique()))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13333333333333333, 'fraction_numerical': 0.011844034007622397, 'mean_word_length': 3.887106017191977, 'pattern_counts': {'":': 2, '<': 41, '<?xml version=': 0, '>': 40, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '551751', 'license': 'MIT', 'n_tokens_mistral': 5871, 'n_tokens_neox': 5619, 'n_words': 1328, 'path': 'BERTopic_Model.ipynb', 'repo_name': 'armandanusca/topic-modelling-risk-factors', 'size': '17065'}