Spaces:
Running
Running
taka-yamakoshi
commited on
Commit
•
6751661
1
Parent(s):
ef0b5c6
fix a minor bug
Browse files
app.py
CHANGED
@@ -95,7 +95,7 @@ if __name__=='__main__':
|
|
95 |
|
96 |
# Title
|
97 |
st.markdown(generate_markdown('WordPiece Explorer',size=32), unsafe_allow_html=True)
|
98 |
-
st.markdown(generate_markdown('quick and easy way to explore how tokenizers work',size=24), unsafe_allow_html=True)
|
99 |
|
100 |
# Select and load the tokenizer
|
101 |
st.sidebar.write('1. Choose the tokenizer from below')
|
@@ -139,7 +139,7 @@ if __name__=='__main__':
|
|
139 |
default_tokens = tokenizer('Tokenizers decompose bigger words into smaller tokens')['input_ids']
|
140 |
else:
|
141 |
default_tokens = tokenizer('Tokenizers decompose bigger words into smaller tokens')['input_ids'][1:-1]
|
142 |
-
sentence = st.text_input(f'Tokenized IDs',value=' '.join(default_tokens))
|
143 |
num_tokens = DeTokenizeText(sentence)
|
144 |
else:
|
145 |
sentence = st.text_input(f'Text',value='Tokenizers decompose bigger words into smaller tokens')
|
|
|
95 |
|
96 |
# Title
|
97 |
st.markdown(generate_markdown('WordPiece Explorer',size=32), unsafe_allow_html=True)
|
98 |
+
st.markdown(generate_markdown('- quick and easy way to explore how tokenizers work -',size=24), unsafe_allow_html=True)
|
99 |
|
100 |
# Select and load the tokenizer
|
101 |
st.sidebar.write('1. Choose the tokenizer from below')
|
|
|
139 |
default_tokens = tokenizer('Tokenizers decompose bigger words into smaller tokens')['input_ids']
|
140 |
else:
|
141 |
default_tokens = tokenizer('Tokenizers decompose bigger words into smaller tokens')['input_ids'][1:-1]
|
142 |
+
sentence = st.text_input(f'Tokenized IDs',value=' '.join([str(token) for token in default_tokens]))
|
143 |
num_tokens = DeTokenizeText(sentence)
|
144 |
else:
|
145 |
sentence = st.text_input(f'Text',value='Tokenizers decompose bigger words into smaller tokens')
|