Merge branch 'add_find_demo' into main

pages/6_🔎_Find_Demo.py

@@ -1,11 +1,116 @@
 import os
+from io import StringIO
+import re
+
+import pandas as pd
 
 import streamlit as st
 import streamlit_analytics
+
+import  streamlit_toggle as tog
+
 from utils import add_logo_to_sidebar, add_footer, add_email_signup_form
 
+from huggingface_hub import snapshot_download
+
+from haystack.document_stores import InMemoryDocumentStore
+from haystack.nodes import BM25Retriever, EmbeddingRetriever
+
+HF_TOKEN = os.environ.get("HF_TOKEN")
+DATA_REPO_ID = "simplexico/cuad-qa-answers"
+DATA_FILENAME = "cuad_questions_answers.json"
+EMBEDDING_MODEL = "sentence-transformers/paraphrase-MiniLM-L3-v2"
+if EMBEDDING_MODEL == "sentence-transformers/multi-qa-MiniLM-L6-cos-v1" or EMBEDDING_MODEL == "sentence-transformers/paraphrase-MiniLM-L3-v2":
+    EMBEDDING_DIM = 384
+else:
+    EMBEDDING_DIM = 768
+
+EXAMPLE_TEXT = "the governing law is the State of Texas"
+
 streamlit_analytics.start_tracking()
 
+@st.cache(allow_output_mutation=True)
+def load_dataset():
+    snapshot_download(repo_id=DATA_REPO_ID, token=HF_TOKEN, local_dir='./', repo_type='dataset')
+    df = pd.read_json(DATA_FILENAME)
+    return df
+
+@st.cache(allow_output_mutation=True)
+def generate_document_store(df):
+    """Create haystack document store using contract clause data 
+    """
+    document_dicts = []
+
+    for idx, row in df.iterrows():
+        document_dicts.append(
+            {
+                'content': row['paragraph'],
+                'meta': {'contract_title': row['contract_title']}
+            } 
+        )
+
+    document_store = InMemoryDocumentStore(use_bm25=True, embedding_dim=EMBEDDING_DIM, similarity='cosine')
+
+    document_store.write_documents(document_dicts)
+
+    return document_store
+
+def files_to_dataframe(uploaded_files, limit=10):
+    texts = []
+    titles = []
+    for uploaded_file in uploaded_files[:limit]:
+
+        stringio = StringIO(uploaded_file.getvalue().decode("utf-8"))
+
+        text = stringio.read().strip()
+        paragraphs = text.split("\n\n")
+        paragraphs = [p.strip() for p in paragraphs if len(p.split()) > 10]
+        texts.extend(paragraphs)
+        titles.extend([uploaded_file.name]*len(paragraphs))
+    
+    return pd.DataFrame({'paragraph': texts, 'contract_title': titles})
+
+@st.cache(allow_output_mutation=True)
+def generate_bm25_retriever(document_store):
+    return BM25Retriever(document_store)
+
+@st.cache(allow_output_mutation=True)
+def generate_embeddings(embedding_model, document_store):
+    embedding_retriever = EmbeddingRetriever(
+        embedding_model=embedding_model,
+        document_store=document_store,
+        model_format="sentence_transformers",
+        scale_score=True
+    )
+    document_store.update_embeddings(embedding_retriever)
+    return embedding_retriever
+
+def process_query(query, retriever):
+    """Generates dataframe with top ten results"""
+    texts = []
+    contract_titles = []
+    scores = []
+    ranking = []
+    candidate_documents = retriever.retrieve(
+        query=query,
+        top_k=10,
+    )
+        
+    for idx, document in enumerate(candidate_documents):
+        texts.append(document.content)
+        contract_titles.append(document.meta["contract_title"])
+        scores.append(str(round(document.score, 2)))
+        ranking.append(idx + 1)
+    
+    return pd.DataFrame(
+        {
+            "Ranking": ranking,
+            "Text": texts,
+            "Source Contract": contract_titles,
+            "Similarity": scores
+        }
+    )
+
 st.set_page_config(
     page_title="Find Demo",
     page_icon="🔎",
@@ -22,7 +127,79 @@ add_logo_to_sidebar()
 st.sidebar.success("👆 Select a demo above.")
 
 st.title('🔎 Find Demo')
-st.markdown("🏗 This demo is currently under construction. Please visit back soon.")
+
+st.write("""
+This demo shows how a set of clauses can be searched.
+Upload a set of contracts on the left and the paragraphs can be searched using **keywords** or using **semantic search**.
+Semantic search leverages an AI model which matches on clauses with a similar meaning to the input text. 
+""")
+st.write("**👈 Upload a set of contracts on the left** to start the demo")
+
+
+#df = load_dataset()
+
+#document_store = generate_document_store(df)
+
+#bm25_retriever = generate_bm25_retriever(document_store)
+
+#embedding_retriever = generate_embeddings(EMBEDDING_MODEL, document_store)
+col1, col2, col3, col4, col5 = st.columns(5)
+
+uploaded_files = st.sidebar.file_uploader("Select contracts to search **(upload up to 10 files)**", accept_multiple_files=True)
+
+if uploaded_files:
+    with col1:
+        st.write("Toggle between **keyword** or **semantic** search:")
+        value = tog.st_toggle_switch(
+            label="Keyword/Semantic",
+            label_after=True, 
+            inactive_color='#D3D3D3', 
+            active_color="#11567f", 
+            track_color="#29B5E8"
+        )
+        if value:
+            search_type = "semantic"
+        else:
+            search_type = "keyword"
+
+    print(value)
+
+    df = files_to_dataframe(uploaded_files)
+    document_store = generate_document_store(df)
+    bm25_retriever = generate_bm25_retriever(document_store)
+    st.write("**👇 Enter search query below** and hit the button **Find Clauses** to see the demo in action")
+    query = st.text_area(label='Enter Search Query', value=EXAMPLE_TEXT, height=250)
+    button = st.button('**Find Clauses**', type='primary', use_container_width=True)
+
+    if button:
+
+        hide_dataframe_row_index = """
+            <style>
+            .row_heading.level0 {display:none}
+            .blank {display:none}
+            </style>
+            """
+
+        st.subheader(f'Search Results ({search_type}):')
+        # Inject CSS with Markdown
+        st.markdown(hide_dataframe_row_index, unsafe_allow_html=True)
+
+        if search_type == "keyword":
+            df_bm25 = process_query(query, bm25_retriever)
+            st.table(df_bm25)
+        
+        if search_type == "semantic":
+            embedding_retriever = generate_embeddings(EMBEDDING_MODEL, document_store)
+            df_embed = process_query(query, embedding_retriever)
+            st.table(df_embed)
+            
+        # with col2:
+
+        #     st.subheader('Semantic Search Results:')
+        #     # Inject CSS with Markdown
+        #     st.markdown(hide_dataframe_row_index, unsafe_allow_html=True)
+        #     df_embed = process_query(query, embedding_retriever)
+        #     st.table(df_embed)
 
 add_email_signup_form()
 

requirements.txt

@@ -8,6 +8,7 @@ click==8.1.3
 cloudpickle==2.2.1
 decorator==5.1.1
 entrypoints==0.4
+farm-haystack==1.15.1
 filelock==3.10.0
 gitdb==4.0.10
 GitPython==3.1.31
@@ -25,7 +26,8 @@ matplotlib==3.7.1
 mdurl==0.1.2
 nltk==3.8.1
 numba==0.56.4
-numpy==1.24.2
+numpy==1.23.5
+>>>>>>> add_find_demo
 packaging==23.0
 pandas==1.5.3
 Pillow==9.4.0
@@ -53,6 +55,7 @@ https://huggingface.co/spacy/en_core_web_md/resolve/main/en_core_web_md-any-py3-
 smmap==5.0.0
 streamlit==1.20.0
 streamlit-analytics==0.4.1
+streamlit-toggle-switch==1.0.2
 threadpoolctl==3.1.0
 toml==0.10.2
 toolz==0.12.0