optimized app

.gitignore

@@ -1,3 +1,6 @@
 venv/
 sent_clf_onnx/
-sentiment_model_dir/
+sentiment_model_dir/
+zs_model_dir/
+zeroshot_onnx_dir/
+sent_clf_onnx_dir/

app.py

@@ -2,21 +2,19 @@ import numpy as np
 import pandas as pd
 import streamlit as st
 from streamlit_text_rating.st_text_rater import st_text_rater
-from sentiment import classify_sentiment
-from sentiment_onnx_classify import classify_sentiment_onnx, classify_sentiment_onnx_quant,create_onnx_model
-from zeroshot_clf import zero_shot_classification
 from transformers import AutoTokenizer,AutoModelForSequenceClassification
-from onnxruntime.quantization import quantize_dynamic,QuantType
-import transformers.convert_graph_to_onnx as onnx_convert
-from pathlib import Path
+import onnxruntime as ort
 import os
 import time
 import plotly.express as px
 import plotly.graph_objects as go
-import onnxruntime as ort
 global _plotly_config
 _plotly_config={'displayModeBar': False}
 
+from sentiment_clf_helper import classify_sentiment,create_onnx_model_sentiment,classify_sentiment_onnx
+from zeroshot_clf_helper import zero_shot_classification,create_onnx_model_zs,zero_shot_classification_onnx
+
+
 st.set_page_config(  # Alternate names: setup_page, page, layout
     layout="wide",  # Can be "centered" or "wide". In the future also "dashboard", etc.
     initial_sidebar_state="auto",  # Can be "auto", "expanded", "collapsed"
@@ -73,7 +71,7 @@ st.markdown(hide_streamlit_style, unsafe_allow_html=True)
 
 @st.cache(allow_output_mutation=True, suppress_st_warning=True, max_entries=None, ttl=None)
 def create_model_dir(chkpt, model_dir):
-    if not os.path.exists(chkpt):
+    if not os.path.exists(model_dir):
         try:
             os.mkdir(path=model_dir)
         except:
@@ -101,30 +99,57 @@ if select_task=='README':
 
 sent_chkpt = "distilbert-base-uncased-finetuned-sst-2-english"
 sent_model_dir="sentiment_model_dir"
-#create model/token dir
+#create model/token dir for sentiment classification
 create_model_dir(chkpt=sent_chkpt, model_dir=sent_model_dir)
+#create onnx model for sentiment classification
+model_sentiment=AutoModelForSequenceClassification.from_pretrained(sent_model_dir)
+tokenizer_sentiment=AutoTokenizer.from_pretrained(sent_model_dir)
+create_onnx_model_sentiment(_model=model_sentiment, _tokenizer=tokenizer_sentiment)
 
 @st.cache(allow_output_mutation=True, suppress_st_warning=True, max_entries=None, ttl=None)
-def task_selected(task,sent_model_dir=sent_model_dir):
+def sentiment_task_selected(task,sent_model_dir=sent_model_dir):
     model_sentiment=AutoModelForSequenceClassification.from_pretrained(sent_model_dir)
     tokenizer_sentiment=AutoTokenizer.from_pretrained(sent_model_dir)
+    # create_onnx_model_sentiment(_model=model_sentiment, _tokenizer=tokenizer_sentiment)
+    #create inference session
+    sentiment_session = ort.InferenceSession("sent_clf_onnx_dir/sentiment_classifier_onnx.onnx")
+    sentiment_session_quant = ort.InferenceSession("sent_clf_onnx_dir/sentiment_classifier_onnx_quant.onnx")
 
-    create_onnx_model(_model=model_sentiment, _tokenizer=tokenizer_sentiment)
+    return model_sentiment,tokenizer_sentiment,sentiment_session,sentiment_session_quant
 
-    #create inference session
-    sentiment_session = ort.InferenceSession("sent_clf_onnx/sentiment_classifier_onnx.onnx")
-    sentiment_session_int8 = ort.InferenceSession("sent_clf_onnx/sentiment_classifier_onnx_int8.onnx")
+############## Pre-Download & instantiate objects for sentiment analysis ********************* END **********************************
 
-    return model_sentiment,tokenizer_sentiment,sentiment_session,sentiment_session_int8
 
+############### Pre-Download & instantiate objects for Zero shot clf *********************** START **********************
+
+zs_chkpt = "valhalla/distilbart-mnli-12-1"
+zs_model_dir = "zs_model_dir"
+# create model/token dir for zeroshot clf
+create_model_dir(chkpt=zs_chkpt, model_dir=zs_model_dir)
+#ceate onnx model for zeroshot
+create_onnx_model_zs()
+
+@st.cache(allow_output_mutation=True, suppress_st_warning=True, max_entries=None, ttl=None)
+def zs_task_selected(task, zs_model_dir=zs_model_dir,onnx_dir='zeroshot_onnx_dir'):
+
+    #model & tokenizer initialization for normal ZS classification
+    model_zs=AutoModelForSequenceClassification.from_pretrained(zs_model_dir)
+    tokenizer_zs=AutoTokenizer.from_pretrained(zs_model_dir)
+
+    #create inference session from onnx model
+    zs_session = ort.InferenceSession(f"{onnx_dir}/model.onnx")
+    zs_session_quant = ort.InferenceSession(f"{onnx_dir}/model_quant.onnx")
+
+    return model_zs,tokenizer_zs,zs_session,zs_session_quant
+
+############## Pre-Download & instantiate objects for Zero shot analysis ********************* END **********************************
 
-############## Pre-Download & instantiate objects for sentiment analysis ********************* END **********************************
 
 if select_task == 'Detect Sentiment':
 
     t1=time.time()
     model_sentiment,tokenizer_sentiment,\
-    sentiment_session,sentiment_session_int8 = task_selected(task=select_task)
+    sentiment_session,sentiment_session_quant = sentiment_task_selected(task=select_task)
     t2 = time.time()
     st.write(f"Total time to load Model is {(t2-t1)*1000:.1f} ms")
 
@@ -159,8 +184,8 @@ if select_task == 'Detect Sentiment':
             st.write(f"Time taken for computation {(end - start) * 1000:.1f} ms")
         elif response3:
             start = time.time()
-            sentiments=classify_sentiment_onnx_quant(input_texts,
-                                                     _session=sentiment_session_int8,
+            sentiments=classify_sentiment_onnx(input_texts,
+                                                     _session=sentiment_session_quant,
                                                      _tokenizer=tokenizer_sentiment)
             end = time.time()
             st.write(f"Time taken for computation {(end - start) * 1000:.1f} ms")
@@ -190,8 +215,8 @@ if select_task == 'Detect Sentiment':
             onnx_runtime_quant=[]
             for i in range(100):
                 start=time.time()
-                sentiments = classify_sentiment_onnx_quant(input_texts,
-                                               _session=sentiment_session,
+                sentiments = classify_sentiment_onnx(input_texts,
+                                               _session=sentiment_session_quant,
                                                _tokenizer=tokenizer_sentiment)
                 end=time.time()
 
@@ -227,6 +252,12 @@ if select_task == 'Detect Sentiment':
                                          color_background='rgb(233, 116, 81)',key=t)
 
 if select_task=='Zero Shot Classification':
+
+    t1=time.time()
+    model_zs,tokenizer_zs,zs_session,zs_session_quant = zs_task_selected(task=select_task)
+    t2 = time.time()
+    st.write(f"Total time to load Model is {(t2-t1)*1000:.1f} ms")
+
     st.header("You are now performing Zero Shot Classification")
     input_texts = st.text_input(label="Input text to classify into topics")
     input_lables = st.text_input(label="Enter labels separated by commas")
@@ -240,14 +271,97 @@ if select_task=='Zero Shot Classification':
     with c3:
         response3=st.button("ONNX runtime with Quantization")
     with c4:
-        response4 = st.button("Simulate 100 runs each runtime")
+        response4 = st.button("Simulate 10 runs each runtime")
 
     if any([response1,response2,response3,response4]):
         if response1:
             start=time.time()
-            output = zero_shot_classification(input_texts, input_lables)
+            df_output = zero_shot_classification(input_texts, input_lables,model=model_zs,tokenizer=tokenizer_zs)
             end=time.time()
             st.write("")
             st.write(f"Time taken for computation {(end-start)*1000:.1f} ms")
-            st.plotly_chart(output, config=_plotly_config)
+            fig = px.bar(x='Probability',
+                         y='labels',
+                         text='Probability',
+                         data_frame=df_output,
+                         title='Zero Shot Normalized Probabilities')
+
+            st.plotly_chart(fig, config=_plotly_config)
+        elif response2:
+            start = time.time()
+            df_output=zero_shot_classification_onnx(premise=input_texts,labels=input_lables,_session=zs_session,_tokenizer=tokenizer_zs)
+            end=time.time()
+            st.write("")
+            st.write(f"Time taken for computation {(end-start)*1000:.1f} ms")
+
+            fig = px.bar(x='Probability',
+                         y='labels',
+                         text='Probability',
+                         data_frame=df_output,
+                         title='Zero Shot Normalized Probabilities')
+
+            st.plotly_chart(fig,config=_plotly_config)
+        elif response3:
+            start = time.time()
+            df_output = zero_shot_classification_onnx(premise=input_texts, labels=input_lables, _session=zs_session_quant,
+                                                   _tokenizer=tokenizer_zs)
+            end = time.time()
+            st.write("")
+            st.write(f"Time taken for computation {(end-start)*1000:.1f} ms")
+            fig = px.bar(x='Probability',
+                         y='labels',
+                         text='Probability',
+                         data_frame=df_output,
+                         title='Zero Shot Normalized Probabilities')
+
+            st.plotly_chart(fig, config=_plotly_config)
+        elif response4:
+            normal_runtime = []
+            for i in range(100):
+                start = time.time()
+                _ = zero_shot_classification(input_texts, input_lables,model=model_zs,tokenizer=tokenizer_zs)
+                end = time.time()
+                t = (end - start) * 1000
+                normal_runtime.append(t)
+            normal_runtime = np.clip(normal_runtime, 50, 400)
+
+            onnx_runtime = []
+            for i in range(100):
+                start = time.time()
+                _ = zero_shot_classification_onnx(premise=input_texts, labels=input_lables, _session=zs_session,
+                                                   _tokenizer=tokenizer_zs)
+                end = time.time()
+                t = (end - start) * 1000
+                onnx_runtime.append(t)
+            onnx_runtime = np.clip(onnx_runtime, 50, 200)
+
+            onnx_runtime_quant = []
+            for i in range(100):
+                start = time.time()
+                _ = zero_shot_classification_onnx(premise=input_texts, labels=input_lables, _session=zs_session_quant,
+                                                   _tokenizer=tokenizer_zs)
+                end = time.time()
+
+                t = (end - start) * 1000
+                onnx_runtime_quant.append(t)
+            onnx_runtime_quant = np.clip(onnx_runtime_quant, 50, 200)
+
+            temp_df = pd.DataFrame({'Normal Runtime (ms)': normal_runtime,
+                                    'ONNX Runtime (ms)': onnx_runtime,
+                                    'ONNX Quant Runtime (ms)': onnx_runtime_quant})
+
+            from plotly.subplots import make_subplots
+
+            fig = make_subplots(rows=1, cols=3, start_cell="bottom-left",
+                                subplot_titles=['Normal Runtime', 'ONNX Runtime', 'ONNX Runtime with Quantization'])
+
+            fig.add_trace(go.Histogram(x=temp_df['Normal Runtime (ms)']), row=1, col=1)
+            fig.add_trace(go.Histogram(x=temp_df['ONNX Runtime (ms)']), row=1, col=2)
+            fig.add_trace(go.Histogram(x=temp_df['ONNX Quant Runtime (ms)']), row=1, col=3)
+            fig.update_layout(height=400, width=1000,
+                              title_text="10 Simulations of different Runtimes",
+                              showlegend=False)
+            st.plotly_chart(fig, config=_plotly_config)
+        else:
+            pass
 

sentiment.py

@@ -1,23 +0,0 @@
-import torch
-from transformers import AutoModelForSequenceClassification,AutoTokenizer
-
-chkpt='distilbert-base-uncased-finetuned-sst-2-english'
-model=AutoModelForSequenceClassification.from_pretrained(chkpt)
-tokenizer=AutoTokenizer.from_pretrained(chkpt)
-# tokenizer=AutoTokenizer.from_pretrained('sentiment_classifier/')
-
-def classify_sentiment(texts,model,tokenizer):
-    """
-        user will pass texts separated by comma
-    """
-    try:
-        texts=texts.split(',')
-    except:
-        pass
-
-    input = tokenizer(texts, padding=True, truncation=True,
-                      return_tensors="pt")
-    logits = model(**input)['logits'].softmax(dim=1)
-    logits = torch.argmax(logits, dim=1)
-    output = ['Positive' if i == 1 else 'Negative' for i in logits]
-    return output

sentiment_onnx_classify.py → sentiment_clf_helper.py

@@ -1,6 +1,3 @@
-import onnxruntime as ort
-import torch
-from transformers import AutoTokenizer,AutoModelForSequenceClassification
 import numpy as np
 import transformers
 from onnxruntime.quantization import quantize_dynamic,QuantType
@@ -8,11 +5,32 @@ import transformers.convert_graph_to_onnx as onnx_convert
 from pathlib import Path
 import os
 
-# chkpt='distilbert-base-uncased-finetuned-sst-2-english'
-# model= AutoModelForSequenceClassification.from_pretrained(chkpt)
-# tokenizer= AutoTokenizer.from_pretrained(chkpt)
+import torch
+from transformers import AutoModelForSequenceClassification,AutoTokenizer
+
+chkpt='distilbert-base-uncased-finetuned-sst-2-english'
+model=AutoModelForSequenceClassification.from_pretrained(chkpt)
+tokenizer=AutoTokenizer.from_pretrained(chkpt)
+# tokenizer=AutoTokenizer.from_pretrained('sentiment_classifier/')
+
+def classify_sentiment(texts,model,tokenizer):
+    """
+        user will pass texts separated by comma
+    """
+    try:
+        texts=texts.split(',')
+    except:
+        pass
+
+    input = tokenizer(texts, padding=True, truncation=True,
+                      return_tensors="pt")
+    logits = model(**input)['logits'].softmax(dim=1)
+    logits = torch.argmax(logits, dim=1)
+    output = ['Positive' if i == 1 else 'Negative' for i in logits]
+    return output
+
 
-def create_onnx_model(_model, _tokenizer):
+def create_onnx_model_sentiment(_model, _tokenizer):
     """
 
     Args:
@@ -23,44 +41,25 @@ def create_onnx_model(_model, _tokenizer):
         Creates a simple ONNX model & int8 Quantized Model in the directory "sent_clf_onnx/" if directory not present
 
     """
-    if not os.path.exists('sent_clf_onnx'):
+    if not os.path.exists('sent_clf_onnx_dir'):
         try:
-            os.mkdir('sent_clf_onnx')
+            os.mkdir('sent_clf_onnx_dir')
         except:
             pass
-        """ 
-            Making ONNX model object
-        """
         pipeline=transformers.pipeline("text-classification", model=_model, tokenizer=_tokenizer)
 
-        """ 
-            convert pipeline to onnx object
-        """
         onnx_convert.convert_pytorch(pipeline,
                                      opset=11,
-                                     output=Path("sent_clf_onnx/sentiment_classifier_onnx.onnx"),
+                                     output=Path("sent_clf_onnx_dir/sentiment_classifier_onnx.onnx"),
                                      use_external_format=False
                                      )
 
-        """ 
-            convert onnx object to another onnx object with int8 quantization 
-        """
-        quantize_dynamic("sent_clf_onnx/sentiment_classifier_onnx.onnx","sent_clf_onnx/sentiment_classifier_onnx_int8.onnx",
+        quantize_dynamic("sent_clf_onnx_dir/sentiment_classifier_onnx.onnx","sent_clf_onnx_dir/sentiment_classifier_onnx_quant.onnx",
                          weight_type=QuantType.QUInt8)
     else:
         pass
 
 
-
-
-# #create onnx & onnx_int_8 sessions
-# session = ort.InferenceSession("sent_clf_onnx/sentiment_classifier_onnx.onnx")
-# session_int8 = ort.InferenceSession("sent_clf_onnx/sentiment_classifier_onnx_int8.onnx")
-
-# options=ort.SessionOptions()
-# options.inter_op_num_threads=1
-# options.intra_op_num_threads=1
-
 def classify_sentiment_onnx(texts, _session, _tokenizer):
     """
 
@@ -92,41 +91,3 @@ def classify_sentiment_onnx(texts, _session, _tokenizer):
     output = ['Positive' if i == 1 else 'Negative' for i in output]
     return output
 
-def classify_sentiment_onnx_quant(texts, _session, _tokenizer):
-    """
-    Args:
-        texts: input texts from user
-        _session: pass ONNX runtime session
-        _tokenizer: Relevant Tokenizer e.g. AutoTokenizer.from_pretrained("same checkpoint as the model")
-
-    Returns:
-        list of Positve and Negative texts
-
-    """
-    try:
-        texts=texts.split(',')
-    except:
-        pass
-
-    _inputs = _tokenizer(texts, padding=True, truncation=True,
-                      return_tensors="np")
-
-
-    input_feed={
-        "input_ids":np.array(_inputs['input_ids']),
-        "attention_mask":np.array((_inputs['attention_mask']))
-    }
-
-    output = _session.run(input_feed=input_feed, output_names=['output_0'])[0]
-
-    output=np.argmax(output,axis=1)
-    output = ['Positive' if i == 1 else 'Negative' for i in output]
-
-    return output
-
-
-
-
-
-
-

sentiment_onnx.py

@@ -1,41 +0,0 @@
-from transformers import AutoTokenizer,AutoModelForSequenceClassification
-import transformers.convert_graph_to_onnx as onnx_convert
-from pathlib import Path
-import transformers
-from onnxruntime.quantization import quantize_dynamic,QuantType
-import onnx
-import onnxruntime as ort
-
-"""
-type in cmd  to create onnx model of hugging face chkpt
-python3 -m transformers.onnx --model= distilbert-base-uncased-finetuned-sst-2-english sentiment_onnx/
-"""
-chkpt='distilbert-base-uncased-finetuned-sst-2-english'
-model= AutoModelForSequenceClassification.from_pretrained(chkpt)
-tokenizer= AutoTokenizer.from_pretrained(chkpt)
-
-""" 
-or download the model directly from hub --
-chkpt='distilbert-base-uncased-finetuned-sst-2-english'
-model= AutoModelForSequenceClassification.from_pretrained(chkpt)
-tokenizer= AutoTokenizer.from_pretrained(chkpt)
-"""
-
-
-pipeline=transformers.pipeline("text-classification",model=model,tokenizer=tokenizer)
-
-""" convert pipeline to onnx object"""
-onnx_convert.convert_pytorch(pipeline,
-                             opset=11,
-                             output=Path("sent_clf_onnx/sentiment_classifier_onnx.onnx"),
-                             use_external_format=False
-                             )
-
-""" convert onnx object to another onnx object with int8 quantization """
-quantize_dynamic("sent_clf_onnx/sentiment_classifier_onnx.onnx","sent_clf_onnx/sentiment_classifier_onnx_int8.onnx",
-                 weight_type=QuantType.QUInt8)
-
-print(ort.__version__)
-print(onnx.__version__)
-
-

test.py

@@ -0,0 +1,34 @@
+from transformers import AutoTokenizer
+from onnxruntime import InferenceSession
+import numpy as np
+import subprocess
+import os
+
+
+#create onnx model using
+if not os.path.exists("zs_model_onnx"):
+    try:
+        subprocess.run(['python3','-m','transformers.onnx',
+                        '--model=facebook/bart-large-mnli',
+                        '--feature=sequence-classification',
+                        'zs_model_onnx/'])
+    except:
+        pass
+
+#create session of saved onnx model
+session = InferenceSession("zs_model_onnx/model.onnx")
+
+#tokenizer for the chkpt
+tokenizer=AutoTokenizer.from_pretrained('zs_model_dir')
+
+# ONNX Runtime expects NumPy arrays as input
+inputs = tokenizer("Using DistilBERT with ONNX Runtime!","you know how", return_tensors="np")
+input_feed = {
+    "input_ids": np.array(inputs['input_ids']),
+    "attention_mask": np.array((inputs['attention_mask']))
+}
+
+#output
+outputs = session.run(output_names=["logits"], input_feed=dict(input_feed))
+
+print(outputs)

zeroshot_clf.py → zeroshot_clf_helper.py

@@ -1,13 +1,10 @@
-import pandas as pd
-import streamlit
 import torch
-from transformers import AutoModelForSequenceClassification,AutoTokenizer
+from onnxruntime.quantization import quantize_dynamic,QuantType
+import os
+import subprocess
 import numpy as np
-import plotly.express as px
+import pandas as pd
 
-# chkpt='valhalla/distilbart-mnli-12-1'
-# model=AutoModelForSequenceClassification.from_pretrained(chkpt)
-# tokenizer=AutoTokenizer.from_pretrained('zero_shot_clf/')
 
 def zero_shot_classification(premise: str, labels: str, model, tokenizer):
     try:
@@ -35,14 +32,66 @@ def zero_shot_classification(premise: str, labels: str, model, tokenizer):
 
     df=pd.DataFrame({'labels':labels,
                      'Probability':labels_prob_norm})
-    fig=px.bar(x='Probability',
-               y='labels',
-               text='Probability',
-               data_frame=df,
-               title='Zero Shot Normalized Probabilities')
-    return fig
 
+    return df
+
+##example
 # zero_shot_classification(premise='Tiny worms and breath analyzers could screen for \disease while it’s early and treatable',
 #                          labels='science, sports, museum')
 
 
+def create_onnx_model_zs(art_path='zeroshot_onnx_dir'):
+
+    # create onnx model using
+    if not os.path.exists(art_path):
+        try:
+            subprocess.run(['python3', '-m', 'transformers.onnx',
+                            '--model=facebook/bart-large-mnli',
+                            '--feature=sequence-classification',
+                            art_path])
+        except:
+            pass
+
+        #create quanitzed model from vanila onnx
+        quantize_dynamic(f"{art_path}/model.onnx",f"{art_path}/model_quant.onnx",weight_type=QuantType.QUInt8)
+    else:
+        pass
+
+def zero_shot_classification_onnx(premise,labels,_session,_tokenizer):
+    try:
+        labels=labels.split(',')
+        labels=[l.lower() for l in labels]
+    except:
+        raise Exception("please pass atleast 2 labels to classify")
+
+    premise=premise.lower()
+
+    labels_prob=[]
+
+    for l in labels:
+
+        hypothesis= f'this is an example of {l}'
+
+        inputs = _tokenizer(premise,hypothesis,
+                             return_tensors='pt',
+                                 truncation_strategy='only_first')
+
+        input_feed = {
+            "input_ids": np.array(inputs['input_ids']),
+            "attention_mask": np.array((inputs['attention_mask']))
+        }
+
+        output = _session.run(output_names=["logits"],input_feed=dict(input_feed))[0] #returns logits as array
+        output=torch.from_numpy(output)
+        entail_contra_prob = output[:,[0,2]].softmax(dim=1)[:,1].item() #only normalizing entail & contradict probabilties
+        labels_prob.append(entail_contra_prob)
+
+    labels_prob_norm=[np.round(100*c/np.sum(labels_prob),1) for c in labels_prob]
+
+    df=pd.DataFrame({'labels':labels,
+                     'Probability':labels_prob_norm})
+
+    return df
+
+
+