update

app.py

@@ -4,13 +4,13 @@
 
 """
 ## TODO:
-- http get方式获取参数，
+- http get方式获取参数，(高优先级)
+- i18 国际化  https://blog.csdn.net/qq_26212731/article/details/78457198   request.header中也有language
 - iter_vocab 的 warmup
 - add_special_token 开关
 - theme 开关 light/dark
 - token_id/tokens/bytes 开关
 - 通过 javascript 添加 hover_text
-- i18
 - 给方法 + 缓存，避免重复调用
 - 英文 utf-8编码
 - 词典支持下载
@@ -41,15 +41,22 @@ from vocab import all_tokenizers
 from util import *
 
 # llama chatglm_6b gpt_nexo_20b baichuan  baichuan_7b
-examples = [
+examples_zh = [
     ["空格测试：  2个空格        8个空格", "llama", "chatglm_6b"],  # chatglm 有blank_n,
     ["标点测试：，。！？；", "baichuan_7b", "llama"],
     ["符号测试：🦙❤❥웃유♋☮✊☏☢☚✔☑♚▢♪✈✞÷↑↓▤▥⊙■□▣▽¿─│♥❣▬▫☿Ⓐ ✋✉☣☤", "baichuan_7b", "llama"],
     ["数字测试：(10086 + 98) = 100184", "baichuan_7b", "llama"],
     ["中文简体：宽带，繁体：樂來", "baichuan_7b", "llama"],
+]
 
+examples = [
+    ["spaces:  2spaces        8spaces", "llama", "chatglm_6b"],  # chatglm 有blank_n,
+    ["punctuations: ,./?\"，。！？；", "baichuan_7b", "llama"],
+    ["symbols: 🦙❤❥웃유♋☮✊☏☢☚✔☑♚▢♪✈✞÷↑↓▤▥⊙■□▣▽¿─│♥❣▬▫☿Ⓐ ✋✉☣☤", "baichuan_7b", "llama"],
+    ["digits: (10086 + 98) = 100184", "baichuan_7b", "llama"],
 ]
 
+
 # jieba.enable_parallel()  # flask中没办法parallel