Adds URL indexing support

interface/components.py

@@ -1,5 +1,5 @@
 import streamlit as st
-from interface.utils import get_pipelines
+from interface.utils import get_pipelines, extract_text_from_url
 from interface.draw_pipelines import get_pipeline_graph
 
 
@@ -64,3 +64,23 @@ def component_text_input(container):
             {"text": doc["text"], "id": doc_id} for doc_id, doc in enumerate(texts)
         ]
         return corpus
+
+
+def component_article_url(container):
+    """Draw the Article URL widget"""
+    with container:
+        urls = []
+        doc_id = 1
+        with st.expander("Enter URLs"):
+            while True:
+                url = st.text_input(f"URL {doc_id}", key=doc_id)
+                if url != "":
+                    urls.append({"text": extract_text_from_url(url)})
+                    doc_id += 1
+                    st.markdown("---")
+                else:
+                    break
+        corpus = [
+            {"text": doc["text"], "id": doc_id} for doc_id, doc in enumerate(urls)
+        ]
+        return corpus

interface/pages.py

@@ -5,6 +5,7 @@ from interface.components import (
     component_show_pipeline,
     component_show_search_result,
     component_text_input,
+    component_article_url,
 )
 
 
@@ -57,6 +58,7 @@ def page_index(container):
 
         input_funcs = {
             "Raw Text": (component_text_input, "card-text"),
+            "URL": (component_article_url, "card-link"),
         }
         selected_input = option_menu(
             "Input Text",

interface/utils.py

@@ -1,5 +1,7 @@
 import core.pipelines as pipelines_functions
 from inspect import getmembers, isfunction
+from newspaper import Article
+import streamlit as st
 
 
 def get_pipelines():
@@ -10,3 +12,12 @@ def get_pipelines():
         " ".join([n.capitalize() for n in name.split("_")]) for name in pipeline_names
     ]
     return pipeline_names, pipeline_funcs
+
+
+@st.experimental_memo
+def extract_text_from_url(url: str):
+    article = Article(url)
+    article.download()
+    article.parse()
+
+    return article.text

requirements.txt

@@ -1,5 +1,6 @@
-streamlit
-streamlit_option_menu
-farm-haystack
-black
-plotly
+streamlit==1.12.2
+streamlit_option_menu==0.3.2
+farm-haystack==1.8.0
+black==22.8.0
+plotly==5.10.0
+newspaper3k==0.2.8