first commit

app.py

@@ -0,0 +1,72 @@
+import pandas as pd
+import streamlit as st
+import numpy as np
+import torch
+from transformers import AlbertTokenizer
+import io
+import time
+
+@st.cache(show_spinner=True,allow_output_mutation=True)
+def load_model(model_name):
+    if model_name.startswith('albert'):
+        tokenizer = AlbertTokenizer.from_pretrained(model_name)
+    return tokenizer
+
+
+if __name__=='__main__':
+
+    # Config
+    max_width = 1500
+    padding_top = 0
+    padding_right = 2
+    padding_bottom = 0
+    padding_left = 2
+
+    define_margins = f"""
+    <style>
+        .appview-container .main .block-container{{
+            max-width: {max_width}px;
+            padding-top: {padding_top}rem;
+            padding-right: {padding_right}rem;
+            padding-left: {padding_left}rem;
+            padding-bottom: {padding_bottom}rem;
+        }}
+    </style>
+    """
+    hide_table_row_index = """
+                <style>
+                tbody th {display:none}
+                .blank {display:none}
+                </style>
+                """
+    st.markdown(define_margins, unsafe_allow_html=True)
+    st.markdown(hide_table_row_index, unsafe_allow_html=True)
+    input_type = st.sidebar.radio(
+        label='1. Choose the input type',
+        on_change=clear_df,
+        options=('Use one of the example sentences','Use your own initial sentence')
+    )
+
+    # Title
+    st.header("Tokenizer Demo")
+
+    tokenizer = load_model('albert-xxlarge-v2')
+    sent_cols = st.columns(2)
+    num_tokens = {}
+    for sent_id, sent_col in enumerate(sent_cols):
+        with sent_col:
+            sentence = st.text_input(f'Sentence {sent_id+1}')
+            input_sent = tokenizer(sentence)['input_ids']
+            decoded_sent = [tokenizer.decode([token]) for token in input_sent[1:-1]]
+            num_tokens[f'sent_{sent_id}'] = len(decoded_sent)
+
+            char_nums = [len(word)+2 for word in decoded_sent]
+            word_cols = st.columns(char_nums)
+            for word_col,word in zip(word_cols,decoded_sent):
+                with word_col:
+                    st.write(word)
+            st.write(f'{num_tokens[f'sent_{sent_id}']} tokens')
+    if num_tokens[f'sent_1']==num_tokens[f'sent_2']:
+        st.write('Matched!')
+    else:
+        st.write('Not Matched...')

packages.txt

@@ -0,0 +1 @@
+libgl1-mesa-dev

requirements.txt

@@ -0,0 +1,7 @@
+torch
+transformers
+sentence_transformers
+opencv-python
+seaborn
+sklearn
+protobuf~=3.19.0