changes for topic modelling and embeddings

app.py

@@ -1,6 +1,5 @@
 import streamlit as st
-from generation import Insights
-
+from topics import Insights
 
 import time
 
@@ -10,15 +9,16 @@ st.title("Drop the second document")
 file2 = st.file_uploader("Upload a file", type=["md", "txt"], key="second")
 
 if file1 is not None and file2 is not None:
-    st.title("Contents of the first file")
-    st.title("Contents of the second file")
-    
+       
     st.title("Generating insights")
     with st.spinner('Generating insights...'):
         insight1 = Insights(file1.read().decode("utf-8"))
         insight2 = Insights(file2.read().decode("utf-8"))
-        st.write(insight1.text)
-        st.write(insight2.text)
         st.write(insight1.generate_topics())
         st.write(insight2.generate_topics())
+        st.write(insight1.text)
+        st.write(insight2.text)
+        embed1 = insight1.generate_embeddings()
+        embed2 = insight2.generate_embeddings()
+        
         st.success('Done!')

requirements.txt

@@ -1,4 +1,9 @@
 keybert==0.7.0
+mdforest==1.5.0
+nltk==3.8.1
 openai==0.27.2
+pandas==1.5.3
+sentence_transformers==2.2.2
+spacy==3.5.2
 streamlit==1.21.0
 transformers==4.27.2

generation.py → topics.py

@@ -1,23 +1,38 @@
 import openai 
 from utils import *
 import mdforest 
+import pandas as pd
+import spacy
 
 class Insights:
     
-    EMBEDDING_MAX_TOKENS = 8191
+    EMBEDDING_MAX_TOKENS = 1023
     
     def __init__(self, text:str) -> None:
-        self.corpus = preprocess(mdforest.clean_markdown(text))
-        self.text = create_nest_sentences(self.corpus, self.EMBEDDING_MAX_TOKENS)
+        cleaned_text = mdforest.clean_markdown(text)
         self.keywords = []
+        
+        self.corpus = preprocess(cleaned_text)        
+        self.text = create_nest_sentences(self.corpus, self.EMBEDDING_MAX_TOKENS)
         self.model = load_keyword_model()
-    
+        self.embedder = load_embedder()
+        
+        
     def generate_topics(self) -> list:
-        print("We are here for generating topics")
         for sentence in self.text:
             self.keywords = self.keywords  + generate_keywords(self.model, sentence)
         return self.keywords
     
+    def generate_embeddings(self) -> list:
+        # generate embeddings for all the sentences
+        nlp = spacy.load("en_core_web_sm")
+        final_embeddings = []
+        for text in self.text:
+            print(text[0])
+            doc = nlp(text[0])
+            sentence_embeddings = [sent.vector for sent in doc.sents]    
+            final_embeddings += sentence_embeddings
+
     
     
         

utils.py

@@ -3,6 +3,13 @@ from keybert import KeyBERT
 from nltk.corpus import stopwords
 from transformers import AutoTokenizer
 import re
+import spacy
+from sentence_transformers import SentenceTransformer
+
+# @st.cache_data
+# def load_nlp():
+#   nlp = 
+
 
 @st.cache_data
 def load_autotoken():
@@ -14,7 +21,12 @@ def load_keyword_model():
   kw_model = KeyBERT()
   return kw_model
 
-def create_nest_sentences(document:str, token_max_length = 8191):
+@st.cache_data
+def load_embedder():
+  embedder = SentenceTransformer('all-MiniLM-L6-v2')
+  return embedder 
+
+def create_nest_sentences(document:str, token_max_length = 1023):
   nested = []
   sent = []
   length = 0