lexical search haystack plus spacy

requirements.txt

@@ -11,6 +11,7 @@ pdfplumber==0.6.2
 Pillow==9.1.1
 seaborn==0.11.2
 transformers==4.21.2
-rank_bm25
+st-annotated-text
+markdown
 python-docx
 streamlit_option_menu

utils/search.py

@@ -1,145 +1,68 @@
-import glob, os, sys; sys.path.append('../udfPreprocess')
-
-#import helper
-import udfPreprocess.docPreprocessing as pre
-import udfPreprocess.cleaning as clean
-
-#import needed libraries
-import seaborn as sns
-from pandas import DataFrame
-from sentence_transformers import SentenceTransformer, CrossEncoder, util
-# from keybert import KeyBERT
-from transformers import pipeline
-import matplotlib.pyplot as plt
-import numpy as np
-import streamlit as st
-import pandas as pd 
-from rank_bm25 import BM25Okapi
-from sklearn.feature_extraction import _stop_words
-import string
-from tqdm.autonotebook import tqdm
-import numpy as np
-import docx
-from docx.shared import Inches
-from docx.shared import Pt
-from docx.enum.style import WD_STYLE_TYPE 
-import logging
-logger = logging.getLogger(__name__)
-import tempfile
-import sqlite3
+from haystack.nodes import TfidfRetriever
+from haystack.document_stores import InMemoryDocumentStore
 import configparser
+import spacy
+import re
+from spacy.matcher import Matcher
+import streamlit as st
+from markdown import markdown
+from annotated_text import annotation
 
-### These are lexcial search related functions/methods#####
-
-def bm25_tokenizer(text):
-    tokenized_doc = []
-    for token in text.lower().split():
-        token = token.strip(string.punctuation)
-
-        if len(token) > 0 and token not in _stop_words.ENGLISH_STOP_WORDS:
-            tokenized_doc.append(token)
-    return tokenized_doc
-    
-def bm25TokenizeDoc(paraList):
-    tokenized_corpus = []
-    ##########Commenting this for now########### will incorporate paragrpah splitting later.
-    # for passage in tqdm(paraList):
-        # if len(passage.split()) >256:
-        #     # st.write("Splitting")
-        #     temp  = " ".join(passage.split()[:256])
-        #     tokenized_corpus.append(bm25_tokenizer(temp))
-        #     temp  = " ".join(passage.split()[256:])
-        #     tokenized_corpus.append(bm25_tokenizer(temp))
-        # else:
-        #     tokenized_corpus.append(bm25_tokenizer(passage))
-    ######################################################################################33333
-    for passage in tqdm(paraList):
-        tokenized_corpus.append(bm25_tokenizer(passage))    
-
-    return tokenized_corpus
+config = configparser.ConfigParser()
+config.read_file(open('paramconfig.py'))
 
-def lexical_search(keyword, document_bm25):
-    config = configparser.ConfigParser()
-    config.read_file(open('udfPreprocess/paramconfig.cfg'))
-    top_k = int(config.get('lexical_search','TOP_K'))
-    bm25_scores = document_bm25.get_scores(bm25_tokenizer(keyword))
-    top_n = np.argpartition(bm25_scores, -top_k)[-top_k:]
-    bm25_hits = [{'corpus_id': idx, 'score': bm25_scores[idx]} for idx in top_n]
-    bm25_hits = sorted(bm25_hits, key=lambda x: x['score'], reverse=True)
-    return bm25_hits
 
-@st.cache(allow_output_mutation=True)
-def load_sentenceTransformer(name):
-    return SentenceTransformer(name)
+def tokenize_lexical_query(query):
+    nlp = spacy.load("en_core_web_sm")    
+    token_list = [token.text.lower() for token in nlp(query) if not token.is_stop]
+    return token_list
 
+def runSpacyMatcher(token_list, document):
+    nlp = spacy.load("en_core_web_sm")
+    spacydoc = nlp(document)
+    matcher = Matcher(nlp.vocab)
+    token_pattern = [[{"LOWER":token}] for token in token_list]
+    matcher.add(",".join(token_list), token_pattern)
+    spacymatches = matcher(spacydoc)
 
-def semantic_search(keywordlist,paraList):
+    matches = []
+    for match_id, start, end in spacymatches:
+        matches = matches + [[start, end]]
     
-    ##### Sematic Search #####
-    #query = "Does document contain {} issues ?".format(keyword)
-    config = configparser.ConfigParser()
-    config.read_file(open('udfPreprocess/paramconfig.cfg'))
-    model_name = config.get('semantic_search','MODEL_NAME')
+    return matches, spacydoc
 
-    bi_encoder = load_sentenceTransformer(model_name)
-    bi_encoder.max_seq_length = int(config.get('semantic_search','MAX_SEQ_LENGTH'))     #Truncate long passages to 256 tokens
-    top_k = int(config.get('semantic_search','TOP_K'))  
-    document_embeddings = bi_encoder.encode(paraList, convert_to_tensor=True, show_progress_bar=False)
-    question_embedding = bi_encoder.encode(keywordlist, convert_to_tensor=True)
-
-    hits = util.semantic_search(question_embedding, document_embeddings, top_k=top_k)
+def runRegexMatcher(token_list, document):
+    matches = []
+    for token in token_list:
+        matches = matches + [[val.start(), val.start()+ len(token)] for val in re.finditer(token, document)]
     
-    return hits
-
-def show_results(keywordList):
-            document = docx.Document()
-            # document.add_heading('Document name:{}'.format(file_name), 2)
-            section = document.sections[0]
-
-           # Calling the footer
-            footer = section.footer
-        
-            # Calling the paragraph already present in
-        # the footer section
-            footer_para = footer.paragraphs[0]
-        
-            font_styles = document.styles
-            font_charstyle = font_styles.add_style('CommentsStyle', WD_STYLE_TYPE.CHARACTER)
-            font_object = font_charstyle.font
-            font_object.size = Pt(7)
-        # Adding the centered zoned footer
-            footer_para.add_run('''\tPowered by GIZ Data and the Sustainable Development Solution Network hosted at Hugging-Face spaces: https://huggingface.co/spaces/ppsingh/streamlit_dev''', style='CommentsStyle')
-            document.add_heading('Your Seacrhed for {}'.format(keywordList), level=1)
-            for keyword in keywordList:
-          
-              st.write("Results for Query: {}".format(keyword))
-              para = document.add_paragraph().add_run("Results for Query: {}".format(keyword))
-              para.font.size = Pt(12)
-              bm25_hits, hits = search(keyword)     
-
-              st.markdown("""
-                      We will provide with 2 kind of results. The 'lexical search' and the semantic search. 
-                      """)  
-              # In the semantic search part we provide two kind of results one with only Retriever (Bi-Encoder) and other the ReRanker (Cross Encoder)           
-              st.markdown("Top few lexical search (BM25) hits")
-              document.add_paragraph("Top few lexical search (BM25) hits")
-
-              for hit in bm25_hits[0:5]:
-                  if hit['score'] > 0.00:   
-                      st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
-                      document.add_paragraph("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
-          
-        
-        
-        #   st.table(bm25_hits[0:3])
-          
-              st.markdown("\n-------------------------\n")
-              st.markdown("Top few Bi-Encoder Retrieval hits")
-              document.add_paragraph("\n-------------------------\n")
-              document.add_paragraph("Top few Bi-Encoder Retrieval hits")
+    return matches, document
+
+def searchAnnotator(matches, document):
+    start = 0
+    annotated_text = ""
+    for match in matches:
+        start_idx = match[0]
+        end_idx = match[1]
+        annotated_text = annotated_text + document[start:start_idx] + str(annotation(body=document[start_idx:end_idx], label="ANSWER", background="#964448", color='#ffffff'))
+        start = end_idx
+    
+    st.write(
+            markdown(annotated_text),
+            unsafe_allow_html=True,
+        )
+
+def lexical_search(query,documents):
+
+    document_store = InMemoryDocumentStore()
+    document_store.write_documents(documents)
+    retriever = TfidfRetriever(document_store)
+    results = retriever.retrieve(query=query, 
+                            top_k= int(config.get('lexical_search','TOP_K')))
+    query_tokens = tokenize_lexical_query(query)
+    for result in results:
+        matches, doc = runSpacyMatcher(query_tokens,result.content)
+        searchAnnotator(matches, doc)
 
-              hits = sorted(hits, key=lambda x: x['score'], reverse=True)
-              for hit in hits[0:5]:
-                #  if hit['score'] > 0.45:
-                  st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
-                  document.add_paragraph("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+    
+    

ver0.1 scripts/search.py

@@ -0,0 +1,141 @@
+import glob, os, sys; sys.path.append('../utils')
+
+#import needed libraries
+import seaborn as sns
+from pandas import DataFrame
+from sentence_transformers import SentenceTransformer, CrossEncoder, util
+# from keybert import KeyBERT
+from transformers import pipeline
+import matplotlib.pyplot as plt
+import numpy as np
+import streamlit as st
+import pandas as pd 
+from rank_bm25 import BM25Okapi
+from sklearn.feature_extraction import _stop_words
+import string
+from tqdm.autonotebook import tqdm
+import numpy as np
+import docx
+from docx.shared import Inches
+from docx.shared import Pt
+from docx.enum.style import WD_STYLE_TYPE 
+import logging
+logger = logging.getLogger(__name__)
+import tempfile
+import sqlite3
+import configparser
+
+### These are lexcial search related functions #####
+
+def bm25_tokenizer(text):
+    tokenized_doc = []
+    for token in text.lower().split():
+        token = token.strip(string.punctuation)
+
+        if len(token) > 0 and token not in _stop_words.ENGLISH_STOP_WORDS:
+            tokenized_doc.append(token)
+    return tokenized_doc
+    
+def bm25TokenizeDoc(paraList):
+    tokenized_corpus = []
+    ##########Commenting this for now########### will incorporate paragrpah splitting later.
+    # for passage in tqdm(paraList):
+        # if len(passage.split()) >256:
+        #     # st.write("Splitting")
+        #     temp  = " ".join(passage.split()[:256])
+        #     tokenized_corpus.append(bm25_tokenizer(temp))
+        #     temp  = " ".join(passage.split()[256:])
+        #     tokenized_corpus.append(bm25_tokenizer(temp))
+        # else:
+        #     tokenized_corpus.append(bm25_tokenizer(passage))
+    ######################################################################################33333
+    for passage in tqdm(paraList):
+        tokenized_corpus.append(bm25_tokenizer(passage))    
+
+    return tokenized_corpus
+
+def lexical_search(keyword, document_bm25):
+    config = configparser.ConfigParser()
+    config.read_file(open('udfPreprocess/paramconfig.cfg'))
+    top_k = int(config.get('lexical_search','TOP_K'))
+    bm25_scores = document_bm25.get_scores(bm25_tokenizer(keyword))
+    top_n = np.argpartition(bm25_scores, -top_k)[-top_k:]
+    bm25_hits = [{'corpus_id': idx, 'score': bm25_scores[idx]} for idx in top_n]
+    bm25_hits = sorted(bm25_hits, key=lambda x: x['score'], reverse=True)
+    return bm25_hits
+
+@st.cache(allow_output_mutation=True)
+def load_sentenceTransformer(name):
+    return SentenceTransformer(name)
+
+
+def semantic_search(keywordlist,paraList):
+    
+    ##### Sematic Search #####
+    #query = "Does document contain {} issues ?".format(keyword)
+    config = configparser.ConfigParser()
+    config.read_file(open('udfPreprocess/paramconfig.cfg'))
+    model_name = config.get('semantic_search','MODEL_NAME')
+
+    bi_encoder = load_sentenceTransformer(model_name)
+    bi_encoder.max_seq_length = int(config.get('semantic_search','MAX_SEQ_LENGTH'))     #Truncate long passages to 256 tokens
+    top_k = int(config.get('semantic_search','TOP_K'))  
+    document_embeddings = bi_encoder.encode(paraList, convert_to_tensor=True, show_progress_bar=False)
+    question_embedding = bi_encoder.encode(keywordlist, convert_to_tensor=True)
+
+    hits = util.semantic_search(question_embedding, document_embeddings, top_k=top_k)
+    
+    return hits
+
+def show_results(keywordList):
+            document = docx.Document()
+            # document.add_heading('Document name:{}'.format(file_name), 2)
+            section = document.sections[0]
+
+           # Calling the footer
+            footer = section.footer
+        
+            # Calling the paragraph already present in
+        # the footer section
+            footer_para = footer.paragraphs[0]
+        
+            font_styles = document.styles
+            font_charstyle = font_styles.add_style('CommentsStyle', WD_STYLE_TYPE.CHARACTER)
+            font_object = font_charstyle.font
+            font_object.size = Pt(7)
+        # Adding the centered zoned footer
+            footer_para.add_run('''\tPowered by GIZ Data and the Sustainable Development Solution Network hosted at Hugging-Face spaces: https://huggingface.co/spaces/ppsingh/streamlit_dev''', style='CommentsStyle')
+            document.add_heading('Your Seacrhed for {}'.format(keywordList), level=1)
+            for keyword in keywordList:
+          
+              st.write("Results for Query: {}".format(keyword))
+              para = document.add_paragraph().add_run("Results for Query: {}".format(keyword))
+              para.font.size = Pt(12)
+              bm25_hits, hits = search(keyword)     
+
+              st.markdown("""
+                      We will provide with 2 kind of results. The 'lexical search' and the semantic search. 
+                      """)  
+              # In the semantic search part we provide two kind of results one with only Retriever (Bi-Encoder) and other the ReRanker (Cross Encoder)           
+              st.markdown("Top few lexical search (BM25) hits")
+              document.add_paragraph("Top few lexical search (BM25) hits")
+
+              for hit in bm25_hits[0:5]:
+                  if hit['score'] > 0.00:   
+                      st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+                      document.add_paragraph("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+          
+        
+        
+        #   st.table(bm25_hits[0:3])
+          
+              st.markdown("\n-------------------------\n")
+              st.markdown("Top few Bi-Encoder Retrieval hits")
+              document.add_paragraph("\n-------------------------\n")
+              document.add_paragraph("Top few Bi-Encoder Retrieval hits")
+
+              hits = sorted(hits, key=lambda x: x['score'], reverse=True)
+              for hit in hits[0:5]:
+                #  if hit['score'] > 0.45:
+                  st.write("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
+                  document.add_paragraph("\t Score: {:.3f}:  \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))