Create app.py

app.py

@@ -0,0 +1,27 @@
+import gradio as gr
+
+# Assuming we have a function to load a tokenizer by name (you would need to replace this with actual implementation)
+def load_tokenizer(tokenizer_name):
+    if tokenizer_name == "aranizer_sp32k":
+        tokenizer = aranizer_sp32k.get_tokenizer()
+    # Add conditions for other tokenizers
+
+    return tokenizer
+
+def tokenize_and_encode(text, tokenizer_choice):
+    tokenizer = load_tokenizer(tokenizer_choice)
+    tokens = tokenizer.tokenize(text)
+    encoded_output = tokenizer.encode(text, add_special_tokens=True)
+    decoded_text = tokenizer.decode(encoded_output)
+    return tokens, encoded_output, decoded_text
+
+iface = gr.Interface(
+    fn=tokenize_and_encode,
+    inputs=[gr.inputs.Textbox(lines=5, label="النص العربي"), gr.inputs.Dropdown(choices=["aranizer_bpe32k", "aranizer_bpe50k", "aranizer_bpe64k", "aranizer_bpe86k", "aranizer_sp32k", "aranizer_sp50k", "aranizer_sp64k", "aranizer_sp86k"], label="اختر المحلل اللفظي")],
+    outputs=[gr.outputs.Textbox(label="Tokens"), gr.outputs.Textbox(label="Encoded Output"), gr.outputs.Textbox(label="Decoded Text")],
+    title="مقارنة المحللات اللفظية للنص العربي",
+    description="حدد نوع المحلل اللفظي وأدخل نصًا لرؤية النتائج.",
+    language="ar",
+)
+
+iface.launch()