"""Gradio app to showcase the LLM tokenization."""
import os
import gradio as gr
from huggingface_hub import login
from playground_app import demo as playground_tab
from compression_app import demo as compression_tab
from character_app import demo as character_tab
auth_token = os.environ.get('HF_TOKEN', None)
if auth_token:
login(token=auth_token)
title = '
Tokenizer Arena ⚔️
'
interface_list = [playground_tab, compression_tab, character_tab]
tab_names = [" ⚔️ Playground", "🏆 Compression Leaderboard", "📊 Character Statistics"]
with gr.Blocks(css="css/style.css", js="js/onload.js") as demo:
gr.HTML(
f"{title}
"
)
with gr.Tabs():
for interface, tab_name in zip(interface_list, tab_names):
with gr.Tab(label=tab_name):
interface.render()
model_name = gr.Textbox(
placeholder="🔍 Add tokenizer from Hugging Face (e.g. Xenova/gpt-4o) and press ENTER...",
show_label=False,
)
model_name.submit()
# demo.load(js=open("js/onload.js", "r", encoding="utf-8").read())
if __name__ == "__main__":
demo.launch()
# demo.queue(max_size=1024, default_concurrency_limit=80).launch()