File size: 7,462 Bytes
cc5c327 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 |
# set path
import glob, os, sys
from udfPreprocess.search import semantic_search
sys.path.append('../udfPreprocess')
#import helper
import udfPreprocess.docPreprocessing as pre
import udfPreprocess.cleaning as clean
from udfPreprocess.search import bm25_tokenizer, bm25TokenizeDoc, lexical_search
#import needed libraries
import seaborn as sns
from pandas import DataFrame
from sentence_transformers import SentenceTransformer, CrossEncoder, util
# from keybert import KeyBERT
from transformers import pipeline
import matplotlib.pyplot as plt
import numpy as np
import streamlit as st
import pandas as pd
from rank_bm25 import BM25Okapi
from sklearn.feature_extraction import _stop_words
import string
from tqdm.autonotebook import tqdm
import numpy as np
import docx
from docx.shared import Inches
from docx.shared import Pt
from docx.enum.style import WD_STYLE_TYPE
import logging
logger = logging.getLogger(__name__)
import tempfile
import sqlite3
import json
import configparser
def app():
with st.container():
st.markdown("<h1 style='text-align: center; \
color: black;'> Search</h1>",
unsafe_allow_html=True)
st.write(' ')
st.write(' ')
with st.expander("ℹ️ - About this app", expanded=False):
st.write(
"""
The *Keyword Search* app is an easy-to-use interface \
built in Streamlit for doing keyword search in \
policy document - developed by GIZ Data and the \
Sustainable Development Solution Network.
""")
st.markdown("")
with st.sidebar:
with open('sample/keywordexample.json','r') as json_file:
keywordexample = json.load(json_file)
genre = st.radio("Select Keyword Category", list(keywordexample.keys()))
if genre == 'Food':
keywordList = keywordexample['Food']
elif genre == 'Climate':
keywordList = keywordexample['Climate']
elif genre == 'Social':
keywordList = keywordexample['Social']
elif genre == 'Nature':
keywordList = keywordexample['Nature']
elif genre == 'Implementation':
keywordList = keywordexample['Implementation']
else:
keywordList = None
searchtype = st.selectbox("Do you want to find exact macthes or similar meaning/context", ['Exact Matches', 'Similar context/meaning'])
with st.container():
if keywordList is not None:
queryList = st.text_input("You selcted the {} category we will look for these keywords in document".format(genre),
value="{}".format(keywordList))
else:
queryList = st.text_input("Please enter here your question and we will look \
for an answer in the document OR enter the keyword you \
are looking for and we will \
we will look for similar context \
in the document.",
placeholder="Enter keyword here")
if st.button("Find them"):
if queryList == "":
st.info("🤔 No keyword provided, if you dont have any, please try example sets from sidebar!")
logging.warning("Terminated as no keyword provided")
else:
if 'docs' in st.session_state:
docs = st.session_state['docs']
paraList = st.session_state['paraList']
if searchtype == 'Exact Matches':
queryList = list(queryList.split(","))
logging.info("performing lexical search")
tokenized_corpus = bm25TokenizeDoc(paraList)
# st.write(len(tokenized_corpus))
document_bm25 = BM25Okapi(tokenized_corpus)
with st.spinner("Performing Exact matching search (Lexical search) for you"):
st.markdown("##### Top few lexical search (BM25) hits #####")
for keyword in queryList:
bm25_hits = lexical_search(keyword,document_bm25)
counter = 0
for hit in bm25_hits:
if hit['score'] > 0.00:
counter += 1
if counter == 1:
st.markdown("###### Results for keyword: **{}** ######".format(keyword))
# st.write("\t Score: {:.3f}: \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
st.write("\t {}: {}\t".format(counter, paraList[hit['corpus_id']].replace("\n", " ")))
if counter == 0:
st.write("No results found for '**{}**' ".format(keyword))
st.markdown("---")
else:
logging.info("starting semantic search")
with st.spinner("Performing Similar/Contextual search"):
query = "Find {} related issues ?".format(queryList)
config = configparser.ConfigParser()
config.read_file(open('udfPreprocess/paramconfig.cfg'))
threshold = float(config.get('semantic_search','THRESHOLD'))
# st.write(query)
semantic_hits = semantic_search(query,paraList)
st.markdown("##### Few Semantic search hits for {} related topics #####".format(queryList))
for i,queryhit in enumerate(semantic_hits):
# st.markdown("###### Results for query: **{}** ######".format(queryList[i]))
counter = 0
for hit in queryhit:
counter += 1
if hit['score'] > threshold:
# st.write("\t Score: {:.3f}: \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
st.write("\t {}: \t {}".format(counter, paraList[hit['corpus_id']].replace("\n", " ")))
# document.add_paragraph("\t Score: {:.3f}: \t{}".format(hit['score'], paraList[hit['corpus_id']].replace("\n", " ")))
st.markdown("---")
# st.write(semantic_hits)
else:
st.info("🤔 No document found, please try to upload it at the sidebar!")
logging.warning("Terminated as no keyword provided")
|