rouge score

app.py

@@ -6,6 +6,7 @@ import nltk
 import validators
 import streamlit as st
 from summarizer import summarizer_init, summarizer_summarize
+from rouge_evaluate import get_rouge_scores
 from config import MODELS
 from warnings import filterwarnings
 
@@ -28,6 +29,11 @@ def initialize_app():
         "model_name": "Boardpac summarizer v1",
         "summarizer_type": "Map Reduce",
         "is_parameters_changed":False,
+        "is_evaluate_pressed":False,
+        "is_evaluate_pressed":False,
+        "reference_summary":'',
+        "generated_summary":'',
+        "summary_time":'',
         # "user_question":'',
         'openai_api_key':'',
     }
@@ -155,6 +161,7 @@ def load_app():
     # inp_text = st.text_input(
     #     "Enter text or a url here"
     # )
+    # with st.form("Text Summarizer"):
     inp_text = st.text_area(
         "Enter text here"
     )
@@ -177,7 +184,7 @@ def load_app():
         cleaned_txt = clean_text(cleaned_txt)
     else:
         logger.info("Text Input Type: INPUT TEXT")
-        cleaned_txt = clean_text(inp_text)
+        cleaned_txt = inp_text # clean_text(inp_text)
 
     # view summarized text (expander)
     with st.expander("View input text"):
@@ -186,7 +193,14 @@ def load_app():
         else:
             st.write(cleaned_txt)
 
+    st.subheader('Optional - Evaluate summary against a reference')
+    # with st.form('Evaluate summary against a reference'):
+    reference_summary = st.text_area(
+        "Enter reference summary here"
+    )
+
     submitted = st.button("Summarize")
+    # submitted = st.form_submit_button("Summarize")
 
     if submitted:
         if is_url:
@@ -194,11 +208,61 @@ def load_app():
         else:
             text_to_summarize = cleaned_txt
 
-        submit_text_to_summarize(text_to_summarize)
+        summarized_text=submit_text_to_summarize(text_to_summarize)
+        # summarized_text=submit_text_to_summarize2(text_to_summarize, reference_summary)
+        #   "reference_summary":'',
+        # st.session_state.generated_summary=summarized_text
+        # evaluate_block(summarized_text)
+        # st.subheader('Evaluate summary against a reference')
+        # with st.form('Evaluate summary against a reference'):
+        # reference_summary = st.text_area(
+        #     "Enter reference summary here"
+        # )
+        # Evaluate = st.button(
+        #     "Evaluate",
+        #     # on_click=update_evaluate_button_change,
+        #     on_click=testtttt,
+        #     # args=[summarized_text, reference_summary]
+        # )
+
+        # if Evaluate :
+        if reference_summary.strip() != '':
+            summarized_text=st.session_state.generated_summary
+            rouge_result = get_rouge_scores(summarized_text, clean_text(reference_summary))
+            # st.text(f"evaluate scores-----: {scores}")
+            # st.info("evaluate scores-----:")
+            # st.info("evaluated scores-----:")
+            # with st.expander(f"evaluated scores: rouge1: {scores['rouge1']:.2f}%"):
+            # st.table(scores)
+            st.text("Evaluated scores:")
+            col1, col2, col3 = st.columns(3)
+            col1.metric('Rogue1', "{:.2f}".format(rouge_result['rouge1']))
+            col2.metric('rouge2', "{:.2f}".format(rouge_result['rouge2']))
+            col3.metric('rougeL', "{:.2f}".format(rouge_result['rougeL']))
+            # col4.metric('rougeLsum', "{:.2f}".format(rouge_result['rougeLsum']))
+
+
 
 def submit_text_to_summarize(text_to_summarize):
     summarized_text, time = get_summary(text_to_summarize)
+    st.session_state.generated_summary=summarized_text
     display_output(summarized_text,time)
+    # evaluate_block(summarized_text)
+    # return summarized_text
+
+def submit_text_to_summarize2(text_to_summarize, reference_summary):
+    summarized_text, time = get_summary(text_to_summarize)
+    # display_output(summarized_text,time)
+    logger.info(f"SUMMARY: {summarized_text}")
+    logger.info(f"Summary took {time}s")
+    st.subheader("Summarized text")
+    st.info(f"{summarized_text}")
+    # st.info(f"Time: {time}s")
+    st.text(f"Time taken: {time}s")
+
+    # scores = get_rouge_scores(summarized_text, reference_summary)
+    # st.markdown(f"evaluate scores: {scores}")
+
 
 
 def get_summary(text_to_summarize):
@@ -221,21 +285,57 @@ def get_summary(text_to_summarize):
             summarized_text, time = summarizer_summarize(model_type,tokenizer, base_summarizer, text_to_summarize ,summarizer_type = "map_reduce")
             return summarized_text, time
 
+def evaluate_block(summarized_text):
+    st.subheader('Evaluate summary against a reference')
+    # with st.form('Evaluate summary against a reference'):
+    reference_summary = st.text_area(
+        "Enter reference summary here"
+    )
+    Evaluate = st.button(
+        "Evaluate",
+        on_click=testtttt,
+        args=[summarized_text,reference_summary]
+    )
+
+    # Evaluate = st.form_submit_button(
+    #             "Evaluate key",
+    #             # on_click=update_parameters_change
+    #             on_click=testtttt,
+    #             args=[summarized_text,reference_summary]
+    # )
+        # if Evaluate or st.session_state.is_evaluate_pressed:
+        # if reference_summary:
+        #     # if 
+        #     scores = get_rouge_scores(summarized_text, reference_summary)
+        #     st.text(f"evaluate scores: {scores}")
+        #     st.session_state.is_evaluate_pressed = False
+
+def testtttt(summarized_text, reference_summary):
+    print(summarized_text, reference_summary)
+    scores = get_rouge_scores(summarized_text, reference_summary)
+    st.text(f"evaluate scores-----: {scores}")
+
+
+def update_evaluate_button_change():
+    st.session_state.is_evaluate_pressed = True
+
+# def evaluate(summarized_text, reference_summary):
+#         return get_rouge_scores(summarized_text, reference_summary)
+    
 
 def display_output(summarized_text,time):
     logger.info(f"SUMMARY: {summarized_text}")
     logger.info(f"Summary took {time}s")
     st.subheader("Summarized text")
     st.info(f"{summarized_text}")
-    st.markdown(f"Time: {time}s")
-
+    # st.info(f"Time: {time}s")
+    st.text(f"Time taken: {time}s")
 
 def main():
    
     initialize_app()
     side_bar()
     load_app()
-    # chat_body()
 
 
 if __name__ == "__main__":

rouge_evaluate.py

@@ -0,0 +1,8 @@
+import evaluate
+rouge_score = evaluate.load("rouge")
+
+def get_rouge_scores(genererated_summary:str,reference_summary:str)->dict:
+
+    rouge_result = rouge_score.compute(predictions=[genererated_summary], references=[reference_summary])
+
+    return rouge_result