File size: 7,758 Bytes
22b8e0b cc5c327 22b8e0b 72e4dad cc5c327 2a8e40d 7d78a3b 43cd965 7d78a3b 43cd965 22b8e0b 72e4dad 22b8e0b 72e4dad 22b8e0b 72e4dad b26b139 948ca1f b26b139 22b8e0b 72e4dad cc5c327 72e4dad 8c4c590 72e4dad 43cd965 22b8e0b 72e4dad 22b8e0b 1984bd1 cc5c327 72e4dad 948ca1f 4e2e62f 43cd965 72e4dad f9949bb 72e4dad cc5c327 a4bf4e8 f9949bb 43cd965 948ca1f 43cd965 a4bf4e8 99ae6d0 7d78a3b 43cd965 550b85d 7d78a3b 99ae6d0 550b85d 908bb07 2ce67a7 908bb07 a4bf4e8 048a702 22b8e0b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 |
# set path
import glob, os, sys;
sys.path.append('../utils')
import streamlit as st
import json
import logging
from utils.lexical_search import runLexicalPreprocessingPipeline, lexical_search
from utils.semantic_search import runSemanticPreprocessingPipeline, semantic_search
from utils.checkconfig import getconfig
# Declare all the necessary variables
config = getconfig('paramconfig.cfg')
split_by = config.get('semantic_search','SPLIT_BY')
split_length = int(config.get('semantic_search','SPLIT_LENGTH'))
split_overlap = int(config.get('semantic_search','SPLIT_OVERLAP'))
split_respect_sentence_boundary = bool(int(config.get('semantic_search',
'RESPECT_SENTENCE_BOUNDARY')))
remove_punc = bool(int(config.get('semantic_search','REMOVE_PUNC')))
embedding_model = config.get('semantic_search','RETRIEVER')
embedding_model_format = config.get('semantic_search','RETRIEVER_FORMAT')
embedding_layer = int(config.get('semantic_search','RETRIEVER_EMB_LAYER'))
retriever_top_k = int(config.get('semantic_search','RETRIEVER_TOP_K'))
reader_model = config.get('semantic_search','READER')
reader_top_k = int(config.get('semantic_search','RETRIEVER_TOP_K'))
lexical_split_by= config.get('lexical_search','SPLIT_BY')
lexical_split_length=int(config.get('lexical_search','SPLIT_LENGTH'))
lexical_split_overlap = int(config.get('lexical_search','SPLIT_OVERLAP'))
lexical_remove_punc = bool(int(config.get('lexical_search','REMOVE_PUNC')))
lexical_top_k=int(config.get('lexical_search','TOP_K'))
def app():
with st.container():
st.markdown("<h1 style='text-align: center; \
color: black;'> Search</h1>",
unsafe_allow_html=True)
st.write(' ')
st.write(' ')
with st.expander("ℹ️ - About this app", expanded=False):
st.write(
"""
The *Keyword Search* app is an easy-to-use interface \
built in Streamlit for doing keyword search in \
policy document - developed by GIZ Data and the \
Sustainable Development Solution Network.
""")
st.write("")
st.write(""" The application allows its user to perform a keyword search\
based on two options: a lexical ([TFIDF](https://en.wikipedia.org/wiki/Tf%E2%80%93idf))\
search and semantic bi-encoder search. The difference between both \
approaches is quite straightforward; while the lexical search only \
displays paragraphs in the document with exact matching results, \
the semantic search shows paragraphs with meaningful connections \
(e.g., synonyms) based on the context as well. The semantic search \
allows for a personalized experience in using the application. Both \
methods employ a probabilistic retrieval framework in its identification\
of relevant paragraphs. By defualt the search is performed using \
'Semantic Search' to find 'Exact/Lexical Matches' please tick the \
checkbox provided, which will by pass semantic search.. Furthermore,\
the application allows the user to search for pre-defined keywords \
from different thematic buckets present in sidebar.""")
with st.sidebar:
with open('docStore/sample/keywordexample.json','r') as json_file:
keywordexample = json.load(json_file)
genre = st.radio("Select Keyword Category", list(keywordexample.keys()))
if genre:
keywordList = keywordexample[genre]
else:
keywordList = None
st.markdown("---")
with st.container():
# if keywordList is not None:
# queryList = st.text_input("You selected the {} category we \
# will look for these keywords in document".format(genre),
# value="{}".format(keywordList))
queryList = st.text_input("Please enter here your question and we \
will look for an answer in the document\
OR enter the keyword you are looking \
for and we will we will look for similar\
context in the document. You can select the \
presets of keywords from sidebar.",
value = "{}".format(keywordList))
searchtype = st.checkbox("Show only Exact Matches")
if st.button("Find them"):
if queryList == "":
st.info("🤔 No keyword provided, if you dont have any, \
please try example sets from sidebar!")
logging.warning("Terminated as no keyword provided")
else:
if 'filepath' in st.session_state:
if searchtype:
allDocuments = runLexicalPreprocessingPipeline(
file_name=st.session_state['filename'],
file_path=st.session_state['filepath'],
split_by=lexical_split_by,
split_length=lexical_split_length,
split_overlap=lexical_split_overlap,
removePunc=lexical_remove_punc)
logging.info("performing lexical search")
with st.spinner("Performing Exact matching search \
(Lexical search) for you"):
st.markdown("##### Top few lexical search (TFIDF) hits #####")
lexical_search(
query=queryList,
documents = allDocuments['documents'],
top_k = lexical_top_k )
else:
allDocuments = runSemanticPreprocessingPipeline(
file_path= st.session_state['filepath'],
file_name = st.session_state['filename'],
split_by=split_by,
split_length= split_length,
split_overlap=split_overlap,
removePunc= remove_punc,
split_respect_sentence_boundary=split_respect_sentence_boundary)
if len(allDocuments['documents']) > 100:
warning_msg = ": This might take sometime, please sit back and relax."
else:
warning_msg = ""
logging.info("starting semantic search")
with st.spinner("Performing Similar/Contextual search{}".format(warning_msg)):
semantic_search(query = queryList,
documents = allDocuments['documents'],
embedding_model=embedding_model,
embedding_layer=embedding_layer,
embedding_model_format=embedding_model_format,
reader_model=reader_model,reader_top_k=reader_top_k,
retriever_top_k=retriever_top_k)
else:
st.info("🤔 No document found, please try to upload it at the sidebar!")
logging.warning("Terminated as no document provided")
|