updating organise demo to take pdfs and some design changes

pages/1_🏷_Label_Clause_Demo.py

@@ -43,8 +43,6 @@ st.write("""
 This demo shows how AI can be used to label text.
 We've trained an AI model to label a clause by its clause type.
 """)
-st.write("**👈 Enter a clause on the left** and hit the button **Label Clause** to see the demo in action")
-
 
 @st.cache(allow_output_mutation=True)
 def load_model():
@@ -61,8 +59,9 @@ def get_prediction_prob(text):
     return y_pred, y_probs
 
 
-text = st.sidebar.text_area(label='Enter Clause Text', value=EXAMPLE_TEXT, height=250)
-button = st.sidebar.button('**Label Clause**', type='primary', use_container_width=True)
+st.markdown('### 🖊 Enter clause text')
+text = st.text_area(label='**Enter Clause Text**', label_visibility='collapsed', value=EXAMPLE_TEXT, height=100)
+button = st.button('**Label Clause**', type='primary', use_container_width=True)
 
 with st.spinner('⚙️ Loading model...'):
     model = load_model()

pages/2_🏷_Label_Contract_Demo.py

@@ -67,8 +67,6 @@ st.write("""
 This demo shows how AI can be used to label text.
 We've trained an AI model to label a contract by its contract type.
 """)
-st.write("**👈 Enter a contract on the left** and hit the button **Label Contract** to see the demo in action")
-
 
 @st.cache(allow_output_mutation=True)
 def load_model():
@@ -90,8 +88,9 @@ with st.spinner('⚙️ Loading model...'):
 
 classes = [s.title() for s in model.classes_]
 
-text = st.sidebar.text_area('Enter Contract Text', value=EXAMPLE_TEXT, height=250)
-button = st.sidebar.button('Label Contract', type='primary', use_container_width=True)
+st.markdown("### 🖊 Enter contract text")
+text = st.text_area('Enter Contract Text', label_visibility='collapsed', value=EXAMPLE_TEXT, height=250)
+button = st.button('Label Contract', type='primary', use_container_width=True)
 
 if button:
     text = text[:250]

pages/3_⛏_Extract_Demo.py

@@ -48,8 +48,6 @@ st.write("""
 This demo shows how AI can be used to extract information from text.
 We've trained an AI model to extract key pieces of information from a contract recital.
 """)
-st.write("**👈 Enter a contract recital on the left** and hit the button **Extract Data** to see the demo in action")
-
 
 @st.cache(allow_output_mutation=True)
 def load_model():
@@ -57,9 +55,9 @@ def load_model():
     nlp = spacy.load('model-best')
     return nlp
 
-
-text = st.sidebar.text_area('Enter Clause Text', value=EXAMPLE_TEXT, height=250)
-button = st.sidebar.button('Extract Data', type='primary', use_container_width=True)
+st.markdown('### 🖊 Enter a contract recital')
+text = st.text_area('Enter Clause Text', label_visibility='collapsed', value=EXAMPLE_TEXT, height=100)
+button = st.button('Extract Data', type='primary', use_container_width=True)
 
 with st.spinner('⚙️ Loading model...'):
     nlp = load_model()

pages/4_🔗_Compare_Demo.py

@@ -31,7 +31,6 @@ add_logo_to_sidebar()
 
 st.title('🔗 Compare Demo')
 st.write("""This demo shows how AI can be used to compare passages of text.""")
-st.write("**👈 Enter two passages of text on the left** and hit the button **Compare** to see the demo in action")
 
 with st.spinner('⚙️ Loading model...'):
     nlp = load_model()
@@ -41,10 +40,15 @@ State of Delaware without regard to its conflicts of law provisions."""
 
 EXAMPLE_TEXT_2 = """This agreement will be governed by and must be construed in accordance with the laws of the State of Israel."""
 
-text_1 = st.sidebar.text_area('Enter a passage of text', value=EXAMPLE_TEXT_1, height=150, key='input1')
-text_2 = st.sidebar.text_area('Enter a second passage of text', value=EXAMPLE_TEXT_2, height=150, key='input2')
+col1, col2 = st.columns(2)
+with col1:
+    st.markdown('### 🖊 Enter a passage of text')
+    text_1 = st.text_area('Enter a passage of text', label_visibility='collapsed', value=EXAMPLE_TEXT_1, height=100, key='input1')
+with col2:
+    st.markdown('### 🖊 Enter a second passage of text')
+    text_2 = st.text_area('Enter a second passage of text', label_visibility='collapsed', value=EXAMPLE_TEXT_2, height=100, key='input2')
 
-button = st.sidebar.button('Compare', type='primary', use_container_width=True)
+button = st.button('Compare', type='primary', use_container_width=True)
 
 
 def get_tokens(doc):

pages/5_🗂_Organise_Demo.py

@@ -1,8 +1,10 @@
 import os
+from io import StringIO
+
 import joblib
 
 from copy import deepcopy
-
+from pypdf import PdfReader
 import pandas as pd
 import plotly.express as px
 
@@ -33,18 +35,19 @@ st.set_page_config(
 )
 
 add_logo_to_sidebar()
-st.sidebar.success("👆 Select a demo above.")
 
 st.title('🗂 Organise Demo')
 st.write("""
-This demo shows how AI can be used to organise contracts.
-We've trained a model to group contracts into similar types.
+This demo shows how AI can be used to organise a collection of texts.
+We've trained a model to group documents into similar types.
 The plot below shows a sample set of contracts that have been automatically grouped together.
 Each point in the plot represents how the model interprets a contract, the closer together a pair of points are, the more similar they appear to the model.
 Similar documents are grouped by color.
 \n**TIP:** Hover over each point to see the filename of the contract. Groups can be added or removed by clicking on the symbol in the plot legend.
 """)
-st.write("**👈 Upload your own contracts on the left (as .txt files)** and hit the button **Organise Data** to see how your own contracts can be grouped together")
+
+st.info("👈 Upload your own documents on the left (as .txt or .pdf files) to see how your own documents can be organised using AI.")
+
 
 @st.cache(allow_output_mutation=True)
 def load_model():
@@ -53,24 +56,27 @@ def load_model():
     )
     return model
 
+
 @st.cache(allow_output_mutation=True)
 def load_dataset():
     snapshot_download(repo_id=DATA_REPO_ID, token=HF_TOKEN, local_dir='./', repo_type='dataset')
     df = pd.read_json(DATA_FILENAME)
     return df
 
+
 def get_transform_and_predictions(model, X):
     y = model.predict(X)
     X_transform = model[:2].transform(X)
     return X_transform, y
 
+
 def generate_plot(X, y, filenames):
     fig = px.scatter_3d(
-        x=X[:,0],
-        y=X[:,1],
-        z=X[:,2],
+        x=X[:, 0],
+        y=X[:, 1],
+        z=X[:, 2],
         color=[str(y_i) for y_i in y], hover_name=filenames)
-    
+
     fig.update_traces(
         marker_size=8,
         marker_line=dict(width=2),
@@ -91,6 +97,7 @@ def generate_plot(X, y, filenames):
 
     return fig
 
+
 @st.cache(allow_output_mutation=True)
 def prepare_figure(model, df):
     X = [text[:500] for text in df['text'].to_list()]
@@ -102,6 +109,7 @@ def prepare_figure(model, df):
 
     return fig
 
+
 @st.cache()
 def prepare_page():
     model = load_model()
@@ -117,31 +125,43 @@ def prepare_page():
     return fig, model
 
 
-uploaded_files = st.sidebar.file_uploader("Select contracts to organise ", accept_multiple_files=True)
+uploaded_files = st.sidebar.file_uploader("Upload your documents", accept_multiple_files=True,
+                                          type=['pdf', 'txt'],
+                                          help="Upload your own documents. Don't worry we don't store any data.")
 
-button = st.sidebar.button('Organise Contracts', type='primary', use_container_width=True)
+# button = st.sidebar.button('Organise Contracts', type='primary', use_container_width=True)
 
 with st.spinner('⚙️ Loading model...'):
     fig, cuad_tfidf_umap_kmeans = prepare_page()
     figure = st.plotly_chart(fig, use_container_width=True)
 
-if button:
-    figure.empty()
 
-    with st.spinner('⚙️ Training model...'):
+if uploaded_files:
+    figure.empty()
+    filenames = []
+    X_train = []
+    if len(uploaded_files) < 5:
+        st.error('### 💔 Please upload more than 4 files.')
+    else:
+        with st.spinner('⚙️ Training model...'):
+            for uploaded_file in uploaded_files:
+                print(uploaded_file.name)
+                if '.pdf' in uploaded_file.name.lower():
+                    reader = PdfReader(uploaded_file)
+                    page_texts = [page.extract_text() for page in reader.pages]
+                    text = "\n".join(page_texts)
+
+                if '.txt' in uploaded_file.name.lower():
+                    stringio = StringIO(uploaded_file.getvalue().decode("utf-8"))
+                    text = stringio.read()
+
+                X_train.append(text[:500])
+                filenames.append(uploaded_file.name)
 
-        if not uploaded_files or not len(uploaded_files) > 2:
-            st.write(
-                "**Please add at least three contracts**"
-            )
-        else:
             if len(uploaded_files) < 10:
                 n_clusters = 3
             else:
                 n_clusters = 8
-        
-            X_train = [uploaded_file.read()[:500] for uploaded_file in uploaded_files]
-            filenames = [uploaded_file.name for uploaded_file in uploaded_files]
 
             tfidf_umap_kmeans = deepcopy(cuad_tfidf_umap_kmeans)
             tfidf_umap_kmeans.set_params(kmeans__n_clusters=n_clusters)
@@ -149,14 +169,15 @@ if button:
 
             X_transform, y = get_transform_and_predictions(cuad_tfidf_umap_kmeans, X_train)
 
-            fig = generate_plot(X_transform, y, filenames)
+        fig = generate_plot(X_transform, y, filenames)
+
+        st.markdown("## 🗂 Your Organised Documents")
+
+        st.plotly_chart(fig, use_container_width=True)
 
-            st.write("**Your organised contracts:**")
 
-            st.plotly_chart(fig, use_container_width=True)
-            
 add_email_signup_form()
 
 add_footer()
 
-streamlit_analytics.stop_tracking(unsafe_password=os.environ["ANALYTICS_PASSWORD"])
+streamlit_analytics.stop_tracking(unsafe_password=os.environ["ANALYTICS_PASSWORD"])

requirements.txt

@@ -35,6 +35,7 @@ pyarrow==11.0.0
 pydeck==0.8.0
 Pygments==2.14.0
 Pympler==1.0.1
+pypdf==3.7.1
 pyrsistent==0.19.3
 python-dateutil==2.8.2
 pytz==2022.7.1

utils.py

@@ -52,12 +52,14 @@ def add_share_to_twitter_button():
 
 
 def add_footer():
-    st.markdown("""
+    st.info("""
     ### 🙋‍♂️ Interested in building out your own tailored Legal AI solutions?
     - 🌐 Check out our [website](https://simplexico.ai)
     - 📞 Book a call with [us](https://calendly.com/uwais-iqbal/discovery-call)
     - ✉️ Send us an [email](mailto:[email protected])
-    
+    """)
+
+    st.success("""
     #### 🙌 Follow Us on Social Media - [🐥 Twitter](https://twitter.com/_simplexico) | [💼 LinkedIn](https://www.linkedin.com/company/simplexico/?viewAsMember=true)
     """)
 

🏡_Home.py

@@ -23,7 +23,7 @@ add_logo_to_sidebar()
 
 st.title("👋 Welcome - Legal AI Demos from simplexico!")
 
-st.sidebar.success("👆 Select a demo above.")
+# st.sidebar.success("👆 Select a demo above.")
 
 st.markdown(
     """
@@ -40,13 +40,13 @@ st.markdown(
     - 🔎 **Find** - Using AI to **find** relevant information from a collection of texts
     - ✍️ **Draft** - Using AI to **draft** text
     - 📝 **Summarise** - Using AI to **summarise** text
-    
-    📢 FYI - These demos are to help you understand AI better. The AI models have not been optimised for prediction performance.
-
-    ### 👈 Select a demo from the sidebar to see some examples of what Legal AI can do!
-    
     """)
 
+st.warning(" 📢 These demos are to help you understand AI better. The AI models have not been optimised for prediction performance. "
+           "Read more about what went into making these demos in our [blog post](https://www.simplexico.ai/blog/legal-ai-demos-intro).")
+
+st.info("#### 👈 Select a demo from the sidebar to see some examples of what Legal AI can do!")
+
 add_email_signup_form()
 
 st.markdown(