app.TabbedInterface.py

"""Gradio app to showcase the LLM tokenization.



"""

import os
from playground_app import demo as playground_tab
from compression_app import demo as compression_tab
from character_app import demo as character_tab
from patcher.gr_interface import TabbedInterface
from huggingface_hub import login

auth_token = os.environ.get('HF_TOKEN', None)
if auth_token:
    login(token=auth_token)

# More Tab: encoding efficiency; decoding efficiency;
demo = TabbedInterface(
    [playground_tab, compression_tab, character_tab],
    [" ⚔️ Playground", "🏆 Compression Leaderboard", "📊 Character Statistics"],
    title='<div align="center">Tokenizer Arena ⚔️</div>',
    css="css/style.css"
)

demo.load(js=open("js/onload.js", "r", encoding="utf-8").read())

if __name__ == "__main__":
    demo.queue(max_size=1024, default_concurrency_limit=80).launch()