Update pages/1_📈_predict.py

pages/1_📈_predict.py

@@ -1,37 +1,36 @@
 import streamlit as st
 import pandas as pd
+from transformers import BertTokenizer, TFBertForSequenceClassification
+from transformers import TextClassificationPipeline
 from transformers import pipeline
 from stqdm import stqdm
 from simplet5 import SimpleT5
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 from transformers import BertTokenizer, TFBertForSequenceClassification
-from datetime import datetime
 import logging
-from transformers import TextClassificationPipeline
-import gc
 from datasets import load_dataset
-from utils.openllmapi.api import ChatBot
-from utils.openllmapi.exceptions import *
-import time
+import gc
 from typing import List
 from collections import OrderedDict
+from datetime import datetime
 
+tokenizer_kwargs = dict(max_length=128, truncation=True, padding=True)
 
-tokenizer_kwargs = dict(
-    max_length=128,
-    truncation=True,
-    padding=True,
-)
+
+flan_t5_kwargs = dict(repetition_penalty=1.2)
 SLEEP = 2
 
 
-def cleanMemory(obj: TextClassificationPipeline):
+date = datetime.now().strftime(r"%Y-%m-%d")
+
+
+def clean_memory(obj: TextClassificationPipeline):
     del obj
     gc.collect()
 
 
 @st.cache_data
-def getAllCats():
+def get_all_cats():
     data = load_dataset("ashhadahsan/amazon_theme")
     data = data["train"].to_pandas()
     labels = [x for x in list(set(data.iloc[:, 1].values.tolist())) if x != "Unknown"]
@@ -40,108 +39,97 @@ def getAllCats():
 
 
 @st.cache_data
-def getAllSubCats():
-    data = load_dataset("ashhadahsan/amazon_theme")
+def get_all_subcats():
+    data = load_dataset("ashhadahsan/amazon_subtheme")
     data = data["train"].to_pandas()
     labels = [x for x in list(set(data.iloc[:, 1].values.tolist())) if x != "Unknown"]
     del data
     return labels
 
 
-def assignHF(bot, what: str, to: str, old: List):
-    try:
-        old = ", ".join(old)
-        message_content = bot.chat(
-            f"""'Assign a one-line {what} to this summary of the text of a review
-        {to}
-        already assigned themes are , {old}
-    theme""",
-        )
-        try:
-            return message_content.split(":")[1].strip()
-        except:
-            return message_content.strip()
-    except ChatError:
-        return ""
-
-
-def assignOpen(bot, what: str, to: str, old: List):
-    old = ", ".join(old)
-    template = """'Assign a one-line {what} to this summary of the text of a review
-        {to}
-        already assigned themes are , {old}
-    theme"""
-    prompt = PromptTemplate(template=template, input_variables=["what", "to", "old"])
-    llm_chain = LLMChain(prompt=prompt, llm=bot)
-    generated = llm_chain.run(what=what, to=summary, old=old)
-    return generated
-
-
-@st.cache_resource
-def loadZeroShotClassification():
-    classifierzero = pipeline(
-        "zero-shot-classification", model="facebook/bart-large-mnli"
-    )
-    return classifierzero
-
-
 @st.cache_resource
-def loadopenModel():
-    llm = OpenLLM(
-        model_name="dolly-v2",
-        model_id="databricks/dolly-v2-3b",
-        temperature=0.94,
-        repetition_penalty=1.2,
+def load_zero_shot_classification_large():
+    classifier_zero = pipeline(
+        "zero-shot-classification",
+        model="facebook/bart-large-mnli",
     )
-    return llm
+    return classifier_zero
 
 
-def assignZeroShot(zero, to: str, old: List):
+def assign_label_zeroshot(zero, to: str, old: List):
     assigned = zero(to, old)
-    assigneddict = dict(zip(assigned["labels"], assigned["scores"]))
-    od = OrderedDict(sorted(assigneddict.items(), key=lambda x: x[1], reverse=True))
+    assigned_dict = dict(zip(assigned["labels"], assigned["scores"]))
+    od = OrderedDict(sorted(assigned_dict.items(), key=lambda x: x[1], reverse=True))
     print(list(od.keys())[0])
     print(type(list(od.keys())[0]))
 
     return list(od.keys())[0]
 
 
-date = datetime.now().strftime(r"%Y-%m-%d")
+def assign_labels_flant5(pipe, what: str, to: str, old: List):
+    old = ", ".join(old)
+
+    return pipe(
+        f"""'Generate a new one word {what} to this summary of the text of a review
+        {to} for context
+        already assigned {what} are , {themes}
+    theme:"""
+    )[0]["generated_text"]
 
 
 @st.cache_resource
 def load_t5() -> (AutoModelForSeq2SeqLM, AutoTokenizer):
-    model = AutoModelForSeq2SeqLM.from_pretrained("t5-base")
+    model = AutoModelForSeq2SeqLM.from_pretrained(
+        "t5-base",
+    )
 
-    tokenizer = AutoTokenizer.from_pretrained("t5-base")
+    tokenizer = AutoTokenizer.from_pretrained(
+        pretrained_model_name_or_path="t5-base",
+    )
     return model, tokenizer
 
 
+@st.cache_resource
+def load_flan_t5_large():
+    return pipeline(
+        task="text2text-generation",
+        model="google/flan-t5-large",
+        model_kwargs=flan_t5_kwargs,
+    )
+
+
 @st.cache_resource
 def summarizationModel():
-    return pipeline("summarization", model="my_awesome_sum/")
+    return pipeline(
+        task="summarization",
+        model="my_awesome_sum/",
+    )
 
 
 @st.cache_resource
 def convert_df(df: pd.DataFrame):
-    # IMPORTANT: Cache the conversion to prevent computation on every rerun
     return df.to_csv(index=False).encode("utf-8")
 
 
 def load_one_line_summarizer(model):
-    return model.load_model("t5", "snrspeaks/t5-one-line-summary")
+    return model.load_model(
+        "t5",
+        "snrspeaks/t5-one-line-summary",
+    )
 
 
 @st.cache_resource
 def classify_theme() -> TextClassificationPipeline:
     tokenizer = BertTokenizer.from_pretrained(
-        "ashhadahsan/amazon-theme-bert-base-finetuned"
+        "ashhadahsan/amazon-theme-bert-base-finetuned",
     )
     model = TFBertForSequenceClassification.from_pretrained(
-        "ashhadahsan/amazon-theme-bert-base-finetuned"
+        "ashhadahsan/amazon-theme-bert-base-finetuned",
     )
     pipeline = TextClassificationPipeline(
-        model=model, tokenizer=tokenizer, top_k=1, **tokenizer_kwargs
+        model=model,
+        tokenizer=tokenizer,
+        **tokenizer_kwargs,
     )
     return pipeline
 
@@ -149,46 +137,37 @@ def classify_theme() -> TextClassificationPipeline:
 @st.cache_resource
 def classify_sub_theme() -> TextClassificationPipeline:
     tokenizer = BertTokenizer.from_pretrained(
-        "ashhadahsan/amazon-subtheme-bert-base-finetuned"
+        "ashhadahsan/amazon-subtheme-bert-base-finetuned",
     )
     model = TFBertForSequenceClassification.from_pretrained(
-        "ashhadahsan/amazon-subtheme-bert-base-finetuned"
+        "ashhadahsan/amazon-subtheme-bert-base-finetuned",
     )
     pipeline = TextClassificationPipeline(
-        model=model, tokenizer=tokenizer, top_k=1, **tokenizer_kwargs
+        model=model, tokenizer=tokenizer, **tokenizer_kwargs
     )
     return pipeline
 
 
 st.set_page_config(layout="wide", page_title="Amazon Review | Summarizer")
-st.title("Amazon Review Summarizer")
+st.title(body="Amazon Review Summarizer")
 
-uploaded_file = st.file_uploader("Choose a file", type=["xlsx", "xls", "csv"])
+uploaded_file = st.file_uploader(label="Choose a file", type=["xlsx", "xls", "csv"])
 
-# try:
-#     bot = ChatBot(
-#         cookies={
-#             "hf-chat": st.secrets["hf-chat"],
-#             "token": st.secrets["token"],
-#         }
-#     )
-# except ChatBotInitError as e:
-#     print(e)
 
 summarizer_option = st.selectbox(
-    "Select Summarizer",
-    ("Custom trained on the dataset", "t5-base", "t5-one-line-summary"),
+    label="Select Summarizer",
+    options=("Custom trained on the dataset", "t5-base", "t5-one-line-summary"),
 )
-col1, col2, col3 = st.columns([1, 1, 1])
+col1, col2, col3 = st.columns(spec=[1, 1, 1])
 
 with col1:
-    summary_yes = st.checkbox("Summrization", value=False)
+    summary_yes = st.checkbox(label="Summrization", value=False)
 
 with col2:
-    classification = st.checkbox("Classify Category", value=True)
+    classification = st.checkbox(label="Classify Category", value=True)
 
 with col3:
-    sub_theme = st.checkbox("Sub theme classification", value=True)
+    sub_theme = st.checkbox(label="Sub theme classification", value=True)
 
 treshold = st.slider(
     label="Model Confidence value",
@@ -202,23 +181,22 @@ treshold = st.slider(
 ps = st.empty()
 
 if st.button("Process", type="primary"):
-    themes = getAllCats()
-    subthemes = getAllSubCats()
-    # st.write(themes)
+    themes = get_all_cats()
+    subthemes = get_all_subcats()
 
     oneline = SimpleT5()
     load_one_line_summarizer(model=oneline)
-    zeroline = loadZeroShotClassification()
-    bot = loadopenModel()
+    zeroline = load_zero_shot_classification_large()
+    bot = load_flan_t5_large()
 
     cancel_button = st.empty()
     cancel_button2 = st.empty()
     cancel_button3 = st.empty()
     if uploaded_file is not None:
         if uploaded_file.name.split(".")[-1] in ["xls", "xlsx"]:
-            df = pd.read_excel(uploaded_file, engine="openpyxl")
+            df = pd.read_excel(io=uploaded_file, engine="openpyxl")
         if uploaded_file.name.split(".")[-1] in [".csv"]:
-            df = pd.read_csv(uploaded_file)
+            df = pd.read_csv(filepath_or_buffer=uploaded_file)
         columns = df.columns.values.tolist()
         columns = [x.lower() for x in columns]
         df.columns = columns
@@ -234,7 +212,7 @@ if st.button("Process", type="primary"):
                     progress_text = "Summarization in progress. Please wait."
                     summary = []
 
-                    for x in stqdm(range(len(text))):
+                    for x in stqdm(iterable=range(len(text))):
                         if cancel_button.button("Cancel", key=x):
                             del model
                             break
@@ -256,28 +234,28 @@ if st.button("Process", type="primary"):
                     classesUnlabel = []
                     classesUnlabelZero = []
                     for x in stqdm(
-                        text,
+                        iterable=text,
                         desc="Assigning Themes ...",
                         total=len(text),
                         colour="#BF1A1A",
                     ):
-                        output = themePipe(x)[0][0]["label"]
+                        output = themePipe(x)[0]["label"]
                         classes.append(output)
-                        score = round(themePipe(x)[0][0]["score"], 2)
+                        score = round(number=themePipe(x)[0]["score"], ndigits=2)
                         if score <= treshold:
-                            onelineoutput = oneline.predict(x)[0]
-                            time.sleep(SLEEP)
+                            onelineoutput = oneline.predict(source_text=x)[0]
+
                             print("hit")
                             classesUnlabel.append(
-                                assignOpen(
-                                    bot=bot,
+                                assign_labels_flant5(
+                                    bot,
                                     what="theme",
                                     to=onelineoutput,
                                     old=themes,
                                 )
                             )
                             classesUnlabelZero.append(
-                                assignZeroShot(
+                                assign_label_zeroshot(
                                     zero=zeroline, to=onelineoutput, old=themes
                                 )
                             )
@@ -289,37 +267,35 @@ if st.button("Process", type="primary"):
                     outputdf["Review Theme"] = classes
                     outputdf["Review Theme-issue-new"] = classesUnlabel
                     outputdf["Review SubTheme-issue-zero"] = classesUnlabelZero
-                    cleanMemory(themePipe)
+                    clean_memory(themePipe)
                 if sub_theme:
                     subThemePipe = classify_sub_theme()
                     classes = []
                     classesUnlabel = []
                     classesUnlabelZero = []
                     for x in stqdm(
-                        text,
+                        iterable=text,
                         desc="Assigning Subthemes ...",
                         total=len(text),
                         colour="green",
                     ):
-                        output = subThemePipe(x)[0][0]["label"]
+                        output = subThemePipe(x)[0]["label"]
                         classes.append(output)
-                        score = round(subThemePipe(x)[0][0]["score"], 2)
+                        score = round(subThemePipe(x)[0]["score"], 2)
                         if score <= treshold:
                             onelineoutput = oneline.predict(x)[0]
 
-                            time.sleep(SLEEP)
-
                             print("hit")
                             classesUnlabel.append(
-                                assignOpen(
-                                    bot=bot,
+                                assign_labels_flant5(
+                                    bot,
                                     what="subtheme",
                                     to=onelineoutput,
                                     old=subthemes,
                                 )
                             )
                             classesUnlabelZero.append(
-                                assignZeroShot(
+                                assign_label_zeroshot(
                                     zero=zeroline,
                                     to=onelineoutput,
                                     old=subthemes,
@@ -334,7 +310,7 @@ if st.button("Process", type="primary"):
                     outputdf["Review SubTheme-issue-new"] = classesUnlabel
                     outputdf["Review SubTheme-issue-zero"] = classesUnlabelZero
 
-                    cleanMemory(subThemePipe)
+                    clean_memory(subThemePipe)
 
                 csv = convert_df(outputdf)
                 st.download_button(
@@ -380,25 +356,24 @@ if st.button("Process", type="primary"):
                     for x in stqdm(
                         text, desc="Assigning Themes ...", total=len(text), colour="red"
                     ):
-                        output = themePipe(x)[0][0]["label"]
+                        output = themePipe(x)[0]["label"]
                         classes.append(output)
-                        score = round(themePipe(x)[0][0]["score"], 2)
+                        score = round(themePipe(x)[0]["score"], 2)
                         if score <= treshold:
                             onelineoutput = oneline.predict(x)[0]
 
                             print("hit")
-                            time.sleep(SLEEP)
 
                             classesUnlabel.append(
-                                assignOpen(
-                                    bot=bot,
+                                assign_labels_flant5(
+                                    bot,
                                     what="theme",
                                     to=onelineoutput,
                                     old=themes,
                                 )
                             )
                             classesUnlabelZero.append(
-                                assignZeroShot(
+                                assign_label_zeroshot(
                                     zero=zeroline, to=onelineoutput, old=themes
                                 )
                             )
@@ -409,7 +384,7 @@ if st.button("Process", type="primary"):
                     outputdf["Review Theme"] = classes
                     outputdf["Review Theme-issue-new"] = classesUnlabel
                     outputdf["Review SubTheme-issue-zero"] = classesUnlabelZero
-                    cleanMemory(themePipe)
+                    clean_memory(themePipe)
 
                 if sub_theme:
                     subThemePipe = classify_sub_theme()
@@ -422,24 +397,23 @@ if st.button("Process", type="primary"):
                         total=len(text),
                         colour="green",
                     ):
-                        output = subThemePipe(x)[0][0]["label"]
+                        output = subThemePipe(x)[0]["label"]
                         classes.append(output)
-                        score = round(subThemePipe(x)[0][0]["score"], 2)
+                        score = round(subThemePipe(x)[0]["score"], 2)
                         if score <= treshold:
                             onelineoutput = oneline.predict(x)[0]
 
-                            time.sleep(SLEEP)
                             print("hit")
                             classesUnlabel.append(
-                                assignOpen(
-                                    bot=bot,
+                                assign_labels_flant5(
+                                    bot,
                                     what="subtheme",
                                     to=onelineoutput,
                                     old=subthemes,
                                 )
                             )
                             classesUnlabelZero.append(
-                                assignZeroShot(
+                                assign_label_zeroshot(
                                     zero=zeroline,
                                     to=onelineoutput,
                                     old=subthemes,
@@ -454,7 +428,7 @@ if st.button("Process", type="primary"):
                     outputdf["Review SubTheme-issue-new"] = classesUnlabel
                     outputdf["Review SubTheme-issue-zero"] = classesUnlabelZero
 
-                    cleanMemory(subThemePipe)
+                    clean_memory(subThemePipe)
 
                 csv = convert_df(outputdf)
                 st.download_button(
@@ -471,12 +445,12 @@ if st.button("Process", type="primary"):
                     load_one_line_summarizer(model=model)
 
                     summary = []
-                    for x in stqdm(range(len(text))):
-                        if cancel_button3.button("Cancel", key=x):
+                    for x in stqdm(iterable=range(len(text))):
+                        if cancel_button3.button(label="Cancel", key=x):
                             del model
                             break
                         try:
-                            summary.append(model.predict(text[x])[0])
+                            summary.append(model.predict(source_text=text[x])[0])
                         except:
                             pass
                     outputdf["summary"] = summary
@@ -488,27 +462,28 @@ if st.button("Process", type="primary"):
                     classesUnlabel = []
                     classesUnlabelZero = []
                     for x in stqdm(
-                        text, desc="Assigning Themes ...", total=len(text), colour="red"
+                        iterable=text,
+                        desc="Assigning Themes ...",
+                        total=len(text),
+                        colour="red",
                     ):
-                        output = themePipe(x)[0][0]["label"]
+                        output = themePipe(x)[0]["label"]
                         classes.append(output)
-                        score = round(themePipe(x)[0][0]["score"], 2)
+                        score = round(number=themePipe(x)[0]["score"], ndigits=2)
                         if score <= treshold:
                             onelineoutput = oneline.predict(x)[0]
 
-                            time.sleep(SLEEP)
-
                             print("hit")
                             classesUnlabel.append(
-                                assignOpen(
-                                    bot=bot,
+                                assign_labels_flant5(
+                                    bot,
                                     what="theme",
                                     to=onelineoutput,
                                     old=themes,
                                 )
                             )
                             classesUnlabelZero.append(
-                                assignZeroShot(
+                                assign_label_zeroshot(
                                     zero=zeroline, to=onelineoutput, old=themes
                                 )
                             )
@@ -526,29 +501,28 @@ if st.button("Process", type="primary"):
                     classesUnlabelZero = []
 
                     for x in stqdm(
-                        text,
+                        iterable=text,
                         desc="Assigning Subthemes ...",
                         total=len(text),
                         colour="green",
                     ):
-                        output = subThemePipe(x)[0][0]["label"]
+                        output = subThemePipe(x)[0]["label"]
                         classes.append(output)
-                        score = round(subThemePipe(x)[0][0]["score"], 2)
+                        score = round(subThemePipe(x)[0]["score"], 2)
                         if score <= treshold:
                             print("hit")
-                            onelineoutput = oneline.predict(x)[0]
+                            onelineoutput = oneline.predict(source_text=x)[0]
 
-                            time.sleep(SLEEP)
                             classesUnlabel.append(
-                                assignOpen(
-                                    bot=bot,
+                                assign_labels_flant5(
+                                    bot,
                                     what="subtheme",
                                     to=onelineoutput,
                                     old=subthemes,
                                 )
                             )
                             classesUnlabelZero.append(
-                                assignZeroShot(
+                                assign_label_zeroshot(
                                     zero=zeroline,
                                     to=onelineoutput,
                                     old=subthemes,
@@ -563,7 +537,7 @@ if st.button("Process", type="primary"):
                     outputdf["Review SubTheme-issue-new"] = classesUnlabel
                     outputdf["Review SubTheme-issue-zero"] = classesUnlabelZero
 
-                    cleanMemory(subThemePipe)
+                    clean_memory(subThemePipe)
 
                 csv = convert_df(outputdf)
                 st.download_button(
@@ -576,11 +550,11 @@ if st.button("Process", type="primary"):
 
         except KeyError as e:
             st.error(
-                "Please Make sure that your data must have a column named text",
+                body="Please Make sure that your data must have a column named text",
                 icon="🚨",
             )
-            st.info("Text column must have amazon reviews", icon="ℹ️")
-            # st.exception(e)
+            st.info(body="Text column must have amazon reviews", icon="ℹ️")
+            st.exception(e)
 
         except BaseException as e:
-            logging.exception("An exception was occurred")
+            logging.exception(msg="An exception was occurred")