add url support for summarization

app.py

@@ -1,13 +1,13 @@
 import torch
+import validators
 import streamlit as st
-from transformers import T5Tokenizer, T5ForConditionalGeneration
+from transformers import pipeline, T5Tokenizer, T5ForConditionalGeneration
 
 # local modules
 from extractive_summarizer.model_processors import Summarizer
-from src.utils import clean_text
+from src.utils import clean_text, fetch_article_text
 from src.abstractive_summarizer import abstractive_summarizer
 
-
 # abstractive summarizer model
 @st.cache()
 def load_abs_model():
@@ -25,9 +25,14 @@ if __name__ == "__main__":
         "Summarization type", options=["Extractive", "Abstractive"]
     )
 
-    inp_text = st.text_input("Enter the text here")
+    inp_text = st.text_input("Enter text or a url here")
 
-    inp_text = clean_text(inp_text)
+    is_url = validators.url(inp_text)
+    if is_url:
+        # complete text, chunks to summarize (list of sentences for long docs)
+        text, text_to_summarize = fetch_article_text(url=inp_text)
+    else:
+        text_to_summarize = clean_text(inp_text)
 
     # view summarized text (expander)
     with st.expander("View input text"):
@@ -44,7 +49,7 @@ if __name__ == "__main__":
                 text="Creating extractive summary. This might take a few seconds ..."
             ):
                 ext_model = Summarizer()
-                summarized_text = ext_model(inp_text, num_sentences=6)
+                summarized_text = ext_model(text_to_summarize, num_sentences=6)
 
         elif summarize_type == "Abstractive":
             with st.spinner(
@@ -52,8 +57,14 @@ if __name__ == "__main__":
             ):
                 abs_tokenizer, abs_model = load_abs_model()
                 summarized_text = abstractive_summarizer(
-                    abs_tokenizer, abs_model, inp_text
+                    abs_tokenizer, abs_model, text_to_summarize
                 )
+        elif summarize_type == "Abstractive" and is_url:
+            abs_url_summarizer = pipeline("summarization")
+            tmp_sum = abs_url_summarizer(
+                text_to_summarize, max_length=120, min_length=30, do_sample=False
+            )
+            summarized_text = " ".join([summ["summary_text"] for summ in tmp_sum])
 
         # final summarized output
         st.subheader("Summarized text")

src/utils.py

@@ -1,4 +1,6 @@
 import re
+import requests
+from bs4 import BeautifulSoup
 
 emoji_pattern = re.compile(
     "["
@@ -27,3 +29,30 @@ def clean_text(x):
     x = re.sub("[^A-Za-z0-9]+", " ", x)  # special charachters
 
     return x
+
+
+def fetch_article_text(url: str):
+
+    r = requests.get(url)
+    soup = BeautifulSoup(r.text, "html.parser")
+    results = soup.find_all(["h1", "p"])
+    text = [result.text for result in results]
+    ARTICLE = " ".join(text)
+    sentences = ARTICLE.split("<eos>")
+    current_chunk = 0
+    chunks = []
+    for sentence in sentences:
+        if len(chunks) == current_chunk + 1:
+            if len(chunks[current_chunk]) + len(sentence.split(" ")) <= 500:
+                chunks[current_chunk].extend(sentence.split(" "))
+            else:
+                current_chunk += 1
+                chunks.append(sentence.split(" "))
+        else:
+            print(current_chunk)
+            chunks.append(sentence.split(" "))
+
+    for chunk_id in range(len(chunks)):
+        chunks[chunk_id] = " ".join(chunks[chunk_id])
+
+    return ARTICLE, chunks