<jupyter_start><jupyter_text><jupyter_code>!pip install bertopic
!pip install bertopic[visualization]
!pip install bert-extractive-summarizer
!pip install plotly
!pip install torch
!pip install tensorflow
!pip install flair
!pip install googletrans==3.1.0a0
!pip install gensim==4.0.0b
!pip install 'umap-learn==0.5.0'
!git clone https://github.com/zain711/BERTopic.git
!cp /content/BERTopic/bertopic/_bertopic.py /usr/local/lib/python3.7/dist-packages/bertopic/_bertopic.py
import json
import pandas as pd
import torch
from bertopic import BERTopic
import os
import nltk 
import numpy as np
nltk.download('punkt')
nltk.download('wordnet')
from nltk.stem import PorterStemmer , WordNetLemmatizer
from nltk.tokenize import word_tokenize 
from google.colab import drive
drive.mount('/content/drive')
import six
from google.cloud import translate_v2 as translate
import getpass

APIKEY = getpass.getpass()
!pip list -v | grep bertopic<jupyter_output>bertopic                      0.7.0          /usr/local/lib/python3.7/dist-packages pip<jupyter_text>Preprocessing Text<jupyter_code>#previous vaccine forums
data = pd.read_table('/content/drive/MyDrive/BrazilAnxNews.tsv')
docs_raw = list(data.loc[:, "matches.value"].values)
times = list(data.loc[:, "Date"].values)
data.head(5)
import re
wnl = WordNetLemmatizer()
ps = PorterStemmer()

#tokens = list(data.loc[:, "tokens"].values)
docs = []
for_embed = []
rem_num = []
my_tokens = []

docs_lower = [k.lower() for k in docs_raw]
#remove numbers and symbols from token list and lemmatize
for b in docs_lower:
  rem_num.append(re.sub('[0-9]+', '', b)) #remove numerical tokens

for a in rem_num:
  docs.append(re.sub(r'[^\w]', ' ', a))

#for c in new_tokens:
  #my_tokens.append(" ".join([wnl.lemmatize(a) for a in c.split()]))

#split tokens for word embeddings
for c in docs:
  for_embed.append(c.split())
#clean times for topic evolution visualization
clean_times = []
for a in times:
  #ts = pd.to_datetime(str(a)) 
  #d = a.strftime('%Y.%m')
  clean_times.append(a)
clean_times = [i.replace('"', '') for i in times]
from gensim.models.callbacks import CallbackAny2Vec
import matplotlib.pyplot as plt 

#calculate and graph loss of embeddings
class callback(CallbackAny2Vec):
    '''Callback to print loss after each epoch.'''
    def __init__(self):
        self.epoch = 0
        self.loss_to_be_subed = 0
        self.epochs = []
        self.losses = []

    def on_epoch_end(self, modeling):
        plt.axis([0, 100, 0, 3000000])
        loss = modeling.running_training_loss()
        loss_now = loss - self.loss_to_be_subed
        self.loss_to_be_subed = loss
        print('Loss after epoch {}: {}'.format(self.epoch, loss_now))
        epochs = self.epoch
        losses = loss_now
        plt.plot(self.epoch, loss_now, '.r-')
        self.epoch += 1
        if (self.epoch == 100):
           plt.show()<jupyter_output><empty_output><jupyter_text>**FINE TUNED FASTTEXT MODEL**<jupyter_code>!pip install gensim==4.0.0b
from gensim.models.fasttext import FastText, load_facebook_model, save_facebook_model
from multiprocessing import cpu_count
from typing import List
from math import log10
import numpy as np
from gensim.test.utils import datapath

model_path = "/content/drive/MyDrive/portuguese.bin"
model = load_facebook_model(datapath(model_path))
model.workers = cpu_count() - 2
model.build_vocab(corpus_iterable=for_embed, update=False, keep_raw_vocab=False)
num_docs = len(for_embed)
num_tokens = sum([len(doc) for doc in for_embed])
num_epochs = min(round(100.0 / (log10(num_docs + 1) + 1e-5)), 100)
model.train(corpus_iterable=for_embed,
            total_examples=num_docs,
            total_words = num_tokens,
            epochs=100)
# 10 topics, 100 epochs, fine tuned - 0.5557887476532941, 0.45324172191545986
# 10 topics, 100 epochs, with updated embeddings - ~0.44, ~0.38
# 20 epochs, 50 epochs, fine tuned - ~0.45, ~.51
from gensim.models.callbacks import CallbackAny2Vec
import matplotlib.pyplot as plt 

#calculate and graph loss of embeddings
class callback(CallbackAny2Vec):
    '''Callback to print loss after each epoch.'''
    def __init__(self):
        self.epoch = 0
        self.loss_to_be_subed = 0
        self.epochs = []
        self.losses = []

    def on_epoch_end(self, modeling):
        plt.axis([0, 100, 0, 3000000])
        loss = model.get_latest_training_loss()
        #loss_now = loss - self.loss_to_be_subed
        #self.loss_to_be_subed = loss
        print('Loss after epoch {}: {}'.format(self.epoch, loss))
        epochs = self.epoch
        #losses = loss_now
        plt.plot(self.epoch, loss, '.r-')
        self.epoch += 1
        if (self.epoch == 100):
           plt.show()
# save model
save_facebook_model(model, "finetuned_model2.bin")
# After training the model
# 1. Word vector extraction
def get_word_vector(word):
    return model.wv.get_vector(word)

# 2. Sentence vector extraction
def get_sentence_vector(text: List[str]):
    def div_norm(x):
        norm_value = np.linalg.norm(x)
        if norm_value > 0:
            return x * (1.0 / norm_value)
        else:
            return x
    
    token_vectors = np.array([div_norm(get_word_vector(token))
                                  for token in text if len(token) > 0])

    if len(token_vectors) == 0:
        return np.zeros(100)

    return np.mean(token_vectors, axis=0)
word_vectors = model.wv
word_vectors.save("/content/drive/My Drive/fasttext_finetune_port")
from flair.embeddings import WordEmbeddings, DocumentPoolEmbeddings

glove_embedding = WordEmbeddings("/content/drive/My Drive/fasttext_finetune_port")
document_glove_embeddings = DocumentPoolEmbeddings([glove_embedding])
topic_model = BERTopic(nr_topics=10, embedding_model=document_glove_embeddings)
topics, prob = topic_model.fit_transform(docs)
#remove stop words and change n-gram range
from sklearn.feature_extraction import text 
from sklearn.feature_extraction.text import CountVectorizer
nltk.download('stopwords')
from nltk.corpus import stopwords

stop = ['ter', 'ser','tempo','dia','porque','bem','ainda','tudo','pode','estar']
stopped = stopwords.words("portuguese")
for a in stop:
  stopped.append(a)
cv = CountVectorizer(ngram_range=(1, 1), stop_words=stopped)
topic_model.update_topics(docs, topics, vectorizer_model=cv)
topic_model.get_topic_freq()
topic_model.get_topics()
import six
from google.cloud import translate_v2 as translate
import os
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/content/drive/My Drive/rugged-alloy-311800.json"

def translate_docs(words, lang):
  translate_client = translate.Client()
  result = translate_client.translate(words, target_language='en', source_language=lang)
  return result.get('translatedText')
topic_list = get_topic_list(topic_model)
translated_list = []
for a in topic_list:
  translated_list.append([translate_docs(i,'pt') for i in a])
translated_list
topic_model.save('/content/drive/My Drive/port_ft_finettune1')
np.save('/content/drive/My Drive/topic_portfine1', topics)
topic_model = BERTopic.load('/content/drive/My Drive/port_ft_finettune1')
topics = np.load('/content/drive/My Drive/topic_portfine1.npy')
topics_over_time = topic_model.topics_over_time(docs, topics, clean_times, global_tuning=False, translate = "pt")
topics_over_time
topic_model.visualize_topics_over_time(topics_over_time, top_n=10)
topics_over_time['Words'] = topics_over_time['Words'].apply(translate_docs, lang = translate)
topics_over_time = topic_model.topics_over_time(docs, topics, clean_times, global_tuning=False, translate='pt')
topic_model.visualize_topics_over_time(topics_over_time, top_n=10, translate='pt')<jupyter_output><empty_output><jupyter_text>**Bert**<jupyter_code>from transformers import AutoTokenizer  # Or BertTokenizer

input_ids = []
tokenizer = AutoTokenizer.from_pretrained('neuralmind/bert-base-portuguese-cased', do_lower_case=False)
for a in docs:
  input_ids.append(tokenizer.encode(a, return_tensors='pt'))
from transformers import AutoModel
model = AutoModel.from_pretrained('neuralmind/bert-base-portuguese-cased')
encoded = []
with torch.no_grad():
  for b in range(len(input_ids)):
    outs = model(input_ids[b])
    encoded.append(outs[0][0, 1:-1])
import tensorflow as tf
embeddings = tf.make_ndarray(encoded)
for c in encoded:
  embeddings.append(encoded.numpy())
np.save('/content/drive/MyDrive/portuguese-bert', embeddings)
topic_model = BERTopic(nr_topics = 10)
topics, prob = topic_model.fit_transform(docs, embeddings=embeddings)
#remove stop words and change n-gram range
from sklearn.feature_extraction import text 
from sklearn.feature_extraction.text import CountVectorizer
nltk.download('stopwords')
from nltk.corpus import stopwords

stop = ['ter', 'ser','tempo','dia','porque','bem','ainda','tudo']
stoped = stopwords.words("portuguese")
for a in stop:
  stoped.append(a)
cv = CountVectorizer(ngram_range=(1, 1), stop_words=stoped)
topic_model.update_topics(docs, topics, vectorizer_model=cv)
topic_model.get_topics()<jupyter_output><empty_output><jupyter_text>**COHERENCE**<jupyter_code>def get_topic_list(topic_model):
  dictionary = list(topic_model.get_topics().values())
  diction = dictionary[1:] #remove outlier topic
  p = 0
  final_t = []
  for i in range(len(diction)):
    final_t.append([])

  for key in diction:
     for i in key:
          final_t[p].append(i[0])
     p+=1
  return final_t
from gensim.models.coherencemodel import CoherenceModel
import gensim.corpora as corpora
dictionary = corpora.Dictionary(for_embed)
corpus = [dictionary.doc2bow(text) for text in for_embed]
topics_over_time = topics_over_time[topics_over_time.Topic != -1]
y = topics_over_time.groupby(["Topic",'Timestamp']).agg({"Words": ', '.join})
local = y.values.tolist()
pd.set_option('display.max_rows', None)
y
local = local[1:] #remove outlier topic
local
b = []
p = 0
topics_local = []
for a in local:
  b.append(a[0].split())

for i in range(len(local)):
   topics_local.append([])

for c in b:
  for e in c:
    topics_local[p].append(e.replace(',', ''))
  p+=1
cm = CoherenceModel(topics=get_topic_list(topic_model), texts = for_embed, dictionary = dictionary, coherence='c_v')
coherence = cm.get_coherence()
coherence
# 10 fine tuned - 0.5557887476532941, 0.45324172191545986
# with updated embeddings ~0.437, ~0.376<jupyter_output><empty_output><jupyter_text>**EXTRA**<jupyter_code>#import six
#from google.cloud import translate_v2 as translate
#import os
#os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/content/drive/My Drive/rugged-alloy-311800.json"



 #   def translate_names(self,
  #                      words,
   ##    translate_client = translate.Client()
     #   trans_names = words.replace('_', ' ')
         #    result = translate_client.translate(trans_names, target_language='en', source_language=lang)

      ## return result.get('translatedText')       
       
       #if translate:
       #   topics_over_time['Name'] = topics_over_time['Name'].apply(self.translate_names, lang = translate)
        #  data = topics_over_time.loc[topics_over_time.Topic.isin(selected_topics), :]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12253178252637273, 'fraction_numerical': 0.022180146064376523, 'mean_word_length': 4.656297807241203, 'pattern_counts': {'":': 1, '<': 24, '<?xml version=': 0, '>': 26, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '401040', 'license': 'MIT', 'n_tokens_mistral': 4031, 'n_tokens_neox': 3809, 'n_words': 813, 'path': 'BrazilNews.ipynb', 'repo_name': 'zain711/BERTopic', 'size': '11091'}