add more tokenizers

stats/character_stats.json

@@ -1803,5 +1803,119 @@
     "len(ja-kana)": "1,4,16",
     "num(ko)": 384,
     "len(ko)": "1,1,1"
+  },
+  "01-ai/Yi-1.5-34B": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/01-ai/Yi-1.5-34B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Yi-1.5-34B</a>",
+    "organization": "Yi",
+    "vocab_size": 63992,
+    "num(digit)": 195,
+    "len(digit)": "1,13,13",
+    "num(space)": 43,
+    "len(space)": "1,2,15",
+    "num(ar)": 18,
+    "len(ar)": "1,1,4",
+    "num(zh)": 21350,
+    "len(zh)": "1,2,12",
+    "num(ja)": 21401,
+    "len(ja)": "1,2,12",
+    "num(ja-kana)": 51,
+    "len(ja-kana)": "1,1,2",
+    "num(ko)": 28,
+    "len(ko)": "1,1,2"
+  },
+  "Qwen/Qwen2-72B": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-72B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-72B</a>",
+    "organization": "Alibaba",
+    "vocab_size": 151646,
+    "num(digit)": 10,
+    "len(digit)": "1,1,1",
+    "num(space)": 55883,
+    "len(space)": "1,6,128",
+    "num(ar)": 4018,
+    "len(ar)": "1,3,12",
+    "num(zh)": 25557,
+    "len(zh)": "1,2,7",
+    "num(ja)": 27206,
+    "len(ja)": "1,2,11",
+    "num(ja-kana)": 2089,
+    "len(ja-kana)": "1,3,11",
+    "num(ko)": 3495,
+    "len(ko)": "1,1,5"
+  },
+  "apple/DCLM-7B": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/apple/DCLM-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">DCLM-7B</a>",
+    "organization": "Apple",
+    "vocab_size": 50277,
+    "num(digit)": 2036,
+    "len(digit)": "1,3,35",
+    "num(space)": 28996,
+    "len(space)": "1,7,512",
+    "num(ar)": 94,
+    "len(ar)": "1,2,4",
+    "num(zh)": 313,
+    "len(zh)": "1,1,2",
+    "num(ja)": 480,
+    "len(ja)": "1,1,4",
+    "num(ja-kana)": 167,
+    "len(ja-kana)": "1,1,4",
+    "num(ko)": 25,
+    "len(ko)": "1,1,2"
+  },
+  "google/gemma-2-9b": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-9b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-9b</a>",
+    "organization": "Google",
+    "vocab_size": 256000,
+    "num(digit)": 134,
+    "len(digit)": "1,10,12",
+    "num(space)": 125662,
+    "len(space)": "1,7,31",
+    "num(ar)": 6274,
+    "len(ar)": "1,4,15",
+    "num(zh)": 23767,
+    "len(zh)": "1,2,12",
+    "num(ja)": 28852,
+    "len(ja)": "1,2,12",
+    "num(ja-kana)": 7061,
+    "len(ja-kana)": "1,3,12",
+    "num(ko)": 2295,
+    "len(ko)": "1,1,5"
+  },
+  "meta-llama/Meta-Llama-3.1-405B": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-405B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3.1</a>",
+    "organization": "Meta",
+    "vocab_size": 128256,
+    "num(digit)": 1110,
+    "len(digit)": "1,3,3",
+    "num(space)": 60860,
+    "len(space)": "1,6,128",
+    "num(ar)": 3810,
+    "len(ar)": "1,4,11",
+    "num(zh)": 4424,
+    "len(zh)": "1,1,7",
+    "num(ja)": 5387,
+    "len(ja)": "1,2,8",
+    "num(ja-kana)": 1086,
+    "len(ja-kana)": "1,2,8",
+    "num(ko)": 2281,
+    "len(ko)": "1,2,6"
+  },
+  "mistralai/Mistral-Large-Instruct-2407": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mistral-Large-Instruct-2407\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mistral-Large-Instruct-2407</a>",
+    "organization": "Mistral",
+    "vocab_size": 32768,
+    "num(digit)": 775,
+    "len(digit)": "1,13,18",
+    "num(space)": 15823,
+    "len(space)": "1,6,16",
+    "num(ar)": 71,
+    "len(ar)": "1,1,3",
+    "num(zh)": 1459,
+    "len(zh)": "1,1,2",
+    "num(ja)": 1593,
+    "len(ja)": "1,1,2",
+    "num(ja-kana)": 134,
+    "len(ja-kana)": "1,1,1",
+    "num(ko)": 346,
+    "len(ko)": "1,1,1"
   }
 }

stats/compression_rate.json

@@ -9610,5 +9610,653 @@
     "oov_ratio": 0.7118713655580824,
     "_oov_charset": "[\"윈\", \"드\", \"싣\", \"譯\", \"칵\", \"섯\", \"🥁\", \"미\", \"을\", \"펌\", \"싶\", \"퀘\", \"뭇\", \"묘\", \"폭\", \"룡\", \"^\", \"훔\", \"츈\", \"😗\", \"ㅙ\", \"Ⅲ\", \"잣\", \"른\", \"밀\", \"및\", \"근\", \"붉\", \"훑\", \"액\", \"껍\", \"參\", \"샴\", \"린\", \"업\", \"맙\", \"빵\", \"힐\", \"쫀\", \"푹\", \"꺽\", \"얻\", \"└\", \"Ⅹ\", \"믐\", \"쁨\", \"궤\", \"남\", \"햇\", \"숲\", \"밋\", \"ㅁ\", \"유\", \"늘\", \"뵙\", \"돼\", \"네\", \"쥔\", \"은\", \"넨\", \"듭\", \"륜\", \"특\", \"룹\", \"댈\", \"ㅕ\", \"️\", \"힘\", \"핏\", \"억\", \"겁\", \"딜\", \"榮\", \"🍞\", \"깥\", \"ᴄ\", \"앨\", \"위\", \"푼\", \"툼\", \"콩\", \"왼\", \"뗀\", \"음\", \"결\", \"專\", \"낮\", \"ㄱ\", \"구\", \"⁴\", \"작\", \"눔\", \"♣\", \"金\", \"禮\", \"◦\", \"팀\", \"­\", \"넓\", \"☞\", \"쬐\", \"ㅇ\", \"경\", \"득\", \"덥\", \"쩌\", \"㉰\", \"벚\", \"쪄\", \"얍\", \"켓\", \"닫\", \"송\", \"💸\", \"톤\", \"혀\", \"卽\", \"폼\", \"듯\", \"팥\", \"멜\", \"포\", \"읽\", \"맵\", \"굉\", \"껏\", \"괄\", \"셀\", \"속\", \"잎\", \"촨\", \"다\", \"례\", \"킵\", \"듣\", \"펀\", \"⋅\", \"컴\", \"？\", \"친\", \"풀\", \"ã\", \"챗\", \"짱\", \"목\", \"겟\", \"넣\", \"랬\", \"🤕\", \"란\", \"온\", \"튿\", \"ㅓ\", \"쌍\", \"같\", \"본\", \"Ⅰ\", \"벨\", \"이\", \"승\", \"죤\", \"앰\", \"밧\", \"칫\", \"쓰\", \"먼\", \"뿔\", \"픈\", \"투\", \"련\", \"로\", \"즙\", \"객\", \"뻔\", \"잇\", \"놓\", \"콰\", \"캄\", \"륭\", \"팡\", \"맴\", \"긁\", \"싱\", \"좋\", \"👍\", \"북\", \"쉘\", \"틋\", \"갖\", \"증\", \"쌀\", \"뒷\", \"캔\", \"펩\", \"뵐\", \"풋\", \"➎\", \"밍\", \"렘\", \"👋\", \"표\", \"럼\", \"쐐\", \"손\", \"덴\", \"묻\", \"훠\", \"꺼\", \"밌\", \"뛰\", \"마\", \"협\", \"맨\", \"잊\", \"휘\", \"격\", \"②\", \"➊\", \"눕\", \"낍\", \"벳\", \"ㅈ\", \"냄\", \"햄\", \"펑\", \"♧\", \"갭\", \"멤\", \"디\", \"캣\", \"춘\", \"뜯\", \"팟\", \"낱\", \"각\", \"◆\", \"뽐\", \"뭘\", \"깽\", \"뉘\", \"빕\", \"텔\", \"용\", \"준\", \"빔\", \"쾅\", \"ㅜ\", \"에\", \"밝\", \"뜹\", \"며\", \"록\", \"눌\", \"왜\", \"ㅎ\", \"ㅍ\", \"념\", \"뭐\", \"그\", \"러\", \"텍\", \"옷\", \"곳\", \"테\", \"엇\", \"믿\", \"룩\", \"뭔\", \"💰\", \"만\", \"燭\", \"즘\", \"끄\", \"섬\", \"염\", \"럭\", \"명\", \"왁\", \"엡\", \"섣\", \"↓\", \"슨\", \"랄\", \"ㅠ\", \"눅\", \"즉\", \"빛\", \"치\", \"호\", \"찰\", \"별\", \"성\", \"핸\", \"씌\", \"식\", \"촘\", \"♀\", \"떤\", \"딱\", \"완\", \"⑥\", \"왕\", \"샷\", \"꾸\", \"시\", \"섰\", \"설\", \"씨\", \"얇\", \"냐\", \"뽀\", \"♥\", \"힉\", \"슝\", \"듬\", \"최\", \"🦄\", \"향\", \"숫\", \"잴\", \"케\", \"떄\", \"팎\", \"합\", \"몸\", \"ㅊ\", \"옥\", \"애\", \"춤\", \"쾨\", \"찜\", \"맞\", \"현\", \"수\", \"카\", \"땐\", \"암\", \"갱\", \"꺄\", \"힌\", \"볕\", \"戌\", \"뜩\", \"낌\", \"짙\", \"삿\", \"짖\", \"💡\", \"베\", \"져\", \"뜬\", \"삘\", \"∮\", \"씹\", \"웨\", \"횡\", \"팅\", \"막\", \"떳\", \"끊\", \"웁\", \"딛\", \"보\", \"챌\", \"롯\", \"♡\", \"릿\", \"눴\", \"난\", \"력\", \"곶\", \"패\", \"많\", \"바\", \"센\", \"캉\", \"게\", \"춧\", \"창\", \"탰\", \"｣\", \"병\", \"므\", \"없\", \"깜\", \"黃\", \"쏟\", \"쳇\", \"맡\", \"홉\", \"안\", \"당\", \"언\", \"겐\", \"찔\", \"궜\", \"좆\", \"꽂\", \"홈\", \"돌\", \"쉴\", \"둔\", \"광\", \"렐\", \"緞\", \"몽\", \"죄\", \"습\", \"루\", \"좁\", \"상\", \"濟\", \"솜\", \"틈\", \"€\", \"워\", \"닐\", \"퀵\", \"댄\", \"낳\", \"밟\", \"⑸\", \"쥰\", \"플\", \"튬\", \"뷰\", \"겸\", \"괜\", \"굿\", \"줬\", \"휙\", \"역\", \"»\", \"≤\", \"∙\", \"끌\", \"吳\", \"텅\", \"채\", \"렴\", \"겨\", \"곽\", \"파\", \"첫\", \"공\", \"겹\", \"야\", \"藥\", \"對\", \"짚\", \"기\", \"뎀\", \"屠\", \"끼\", \"짠\", \"슬\", \"쏭\", \"깁\", \"또\", \"콕\", \"티\", \"꿨\", \"션\", \"첼\", \"싹\", \"🦅\", \"◼\", \"김\", \"뉜\", \"얕\", \"뒀\", \"롱\", \"둬\", \"휠\", \"ㅞ\", \"삽\", \"①\", \"렸\", \"�\", \"\", \"④\", \"함\", \"✔\", \"썸\", \"곱\", \"께\", \"똥\", \"늦\", \"ᴀ\", \"비\", \"중\", \"류\", \"웰\", \"將\", \"융\", \"환\", \"췄\", \"해\", \"콘\", \"女\", \"새\", \"💌\", \"쥐\", \"뻑\", \"섞\", \"홋\", \"ω\", \"쎄\", \"굶\", \"껑\", \"튜\", \"큰\", \"㎥\", \"잦\", \"폿\", \"멕\", \"샌\", \"듀\", \"척\", \"🗺\", \"쿵\", \"받\", \"밤\", \"▶\", \"걍\", \"썹\", \"빼\", \"았\", \"백\", \"런\", \"敎\", \"품\", \"길\", \"름\", \"원\", \"킨\", \"빤\", \"까\", \"랩\", \"ᴛ\", \"⑴\", \"걷\", \"섭\", \"릎\", \"짝\", \"꿰\", \"③\", \"⑨\", \"틱\", \"혐\", \"했\", \"는\", \"툴\", \"劍\", \"짭\", \"씩\", \"덮\", \"교\", \"문\", \"한\", \"딤\", \"끝\", \"취\", \"활\", \"琪\", \"큼\", \"탈\", \"빚\", \"옐\", \"☀\", \"ㄷ\", \"림\", \"둡\", \"군\", \"능\", \"롤\", \"빅\", \"불\", \"…\", \"촬\", \"전\", \"Ⅳ\", \"높\", \"셈\", \"샹\", \"淨\", \"흉\", \"뜰\", \"딴\", \"💕\", \"앱\", \"觸\", \"흩\", \"퇴\", \"켐\", \"들\", \"냥\", \"꿍\", \"➋\", \"족\", \"롬\", \"둥\", \"←\", \"끽\", \"고\", \"끙\", \"꼴\", \"갚\", \"ㅒ\", \"찮\", \"쇼\", \"항\", \"ㅔ\", \"양\", \"쎈\", \"燦\", \"무\", \"롸\", \"행\", \"말\", \"□\", \"ㅚ\", \"브\", \"둠\", \"때\", \"꿔\", \"뤄\", \"쿠\", \"셋\", \"익\", \"릴\", \"깨\", \"빴\", \"經\", \"꿈\", \"궁\", \"쫓\", \"槪\", \"대\", \"묶\", \"렬\", \"든\", \"낀\", \"셨\", \"內\", \"🔹\", \"램\", \"쩔\", \"레\", \"밸\", \"앎\", \"윌\", \"평\", \"뀔\", \"곧\", \"앤\", \"씀\", \"텐\", \"좀\", \"셔\", \"쉼\", \"략\", \"범\", \"횟\", \"낭\", \"쟤\", \"옌\", \"납\", \"려\", \"퍙\", \"쁠\", \"악\", \"웖\", \"​\", \"숭\", \"〮\", \"썰\", \"탭\", \"트\", \"졸\", \"띈\", \"긋\", \"릇\", \"아\", \"랭\", \"됐\", \"예\", \"갤\", \"찬\", \"초\", \"假\", \"켰\", \"민\", \"쨌\", \"홍\", \"않\", \"갯\", \"갑\", \"켠\", \"ㅑ\", \"悧\", \"🍕\", \"리\", \"거\", \"렜\", \"뺐\", \"귈\", \"혼\", \"숱\", \"첨\", \"➍\", \"간\", \"녜\", \"덩\", \"얘\", \"허\", \"딪\", \"뮐\", \"캠\", \"꼭\", \"賣\", \"🤔\", \"똑\", \"🏫\", \"래\", \"💥\", \"훗\", \"㉿\", \"곡\", \"킁\", \"돠\", \"👨\", \"뻗\", \"낡\", \"탓\", \"關\", \"회\", \"됩\", \"댔\", \"젠\", \"열\", \"훌\", \"價\", \"혔\", \"섹\", \"펼\", \"핵\", \"숍\", \"법\", \"빌\", \"릅\", \"임\", \"꾼\", \"탑\", \"휩\", \"데\", \"理\", \"된\", \"헨\", \"옹\", \"ㅖ\", \"앵\", \"짬\", \"쁜\", \"뿍\", \"쩐\", \"옛\", \"㉠\", \"쯤\", \"좌\", \"ㅟ\", \"ㅆ\", \"땅\", \"❏\", \"녕\", \"흑\", \"⬇\", \"어\", \"씬\", \"관\", \"솥\", \"發\", \"닙\", \"돈\", \"팝\", \"累\", \"퓰\", \"봤\", \"홀\", \"냉\", \"움\", \"좇\", \"욱\", \"쫑\", \"컷\", \"캘\", \"탐\", \"ｓ\", \"굽\", \"응\", \"앉\", \"휴\", \"큽\", \"싫\", \"줘\", \"톨\", \"識\", \"님\", \"퀄\", \"라\", \"헝\", \"짧\", \"참\", \"흠\", \"ᴇ\", \"생\", \"뚜\", \"컵\", \"꿕\", \"ᴍ\", \"맑\", \"｢\", \"췌\", \"희\", \"왘\", \"戮\", \"닷\", \"넉\", \"渴\", \"감\", \"²\", \"킥\", \"눈\", \"댁\", \"힙\", \"꺾\", \"겠\", \"🤗\", \"클\", \"폄\", \"쫒\", \"꽁\", \"첸\", \"빠\", \"박\", \"뤘\", \"켜\", \"跏\", \"샬\", \"곁\", \"릉\", \"동\", \"번\", \"랍\", \"측\", \"뮬\", \"띠\", \"‪\", \"저\", \"빈\", \"뾱\", \"궈\", \"훼\", \"😊\", \"ⅲ\", \"얏\", \"퓨\", \"℃\", \"꿀\", \"컨\", \"듐\", \"배\", \"갈\", \"났\", \"몇\", \"뛴\", \"처\", \"폰\", \"＊\", \"렌\", \"\", \"땡\", \"잘\", \"턱\", \"픔\", \"꽉\", \"샤\", \"⑵\", \"🙆\", \"균\", \"獻\", \"낄\", \"뱃\", \"즐\", \"닭\", \"청\", \"숙\", \"悖\", \"붐\", \"연\", \"쁘\", \"➌\", \"옆\", \"락\", \"낚\", \"켈\", \"크\", \"삭\", \"▼\", \"죽\", \"쓸\", \"제\", \"택\", \"븐\", \"질\", \"팔\", \"확\", \"🏻\", \"요\", \"流\", \"직\", \"턴\", \"뺑\", \"깝\", \"울\", \"핑\", \"`\", \"앗\", \"곰\", \"껴\", \"적\", \"놨\", \"여\", \"◎\", \"엠\", \"맷\", \"歲\", \"퍼\", \"％\", \"피\", \"ɪ\", \"커\", \"｜\", \"땠\", \"끓\", \"꿇\", \"💅\", \"헉\", \"깐\", \"웅\", \"◀\", \"헤\", \"땀\", \"앓\", \"맹\", \"띄\", \"닮\", \"춥\", \"ㅢ\", \"깊\", \"세\", \"일\", \"뿌\", \"됬\", \"璣\", \"룬\", \"실\", \"귀\", \"鎭\", \"툭\", \"면\", \"촉\", \"층\", \"ᴡ\", \"믄\", \"샀\", \"찼\", \"폐\", \"못\", \"낫\", \"쵸\", \"봬\", \"닳\", \"였\", \"멍\", \"헬\", \"팁\", \"퀸\", \"딩\", \"멈\", \"낸\", \"◐\", \"톰\", \"봄\", \"쌩\", \"더\", \"💦\", \"몫\", \"윙\", \"ᴏ\", \"튼\", \"쭉\", \"넷\", \"獨\", \"뿜\", \"튀\", \"쭈\", \"Ⅵ\", \"맘\", \"벅\", \"읍\", \"맥\", \"•\", \"탁\", \"얽\", \"➏\", \"소\", \"떻\", \"두\", \"렉\", \"봇\", \"잤\", \"韜\", \"탕\", \"홑\", \"히\", \"정\", \"늑\", \"걱\", \"쿤\", \"를\", \"닦\", \"썅\", \"쿨\", \"페\", \"헛\", \"살\", \"녘\", \"헐\", \"즈\", \"삼\", \"느\", \"톱\", \"됨\", \"릭\", \"괌\", \"💣\", \"주\", \"뢰\", \"뇌\", \"블\", \"㏊\", \"검\", \"돔\", \"률\", \"달\", \"흥\", \"ㆍ\", \"칭\", \"멘\", \"🤟\", \"건\", \"ㅅ\", \"🌵\", \"칼\", \"황\", \"ⓔ\", \"긍\", \"끗\", \"버\", \"쏙\", \"쐬\", \"앙\", \"댓\", \"흐\", \"끔\", \"樂\", \"ㅣ\", \"☎\", \"뀌\", \"쉽\", \"룸\", \"짢\", \"재\", \"웬\", \"德\", \"륨\", \"럿\", \"蟄\", \"등\", \"샵\", \"팩\", \"ㅘ\", \"변\", \"넬\", \"탄\", \"발\", \"🍟\", \"촌\", \"벤\", \"있\", \"썼\", \"흘\", \"랜\", \"諒\", \"줏\", \"딥\", \"😭\", \"조\", \"잔\", \"볍\", \"❤\", \"엔\", \"흰\", \"👏\", \"뎅\", \"：\", \"육\", \"혹\", \"흡\", \"뤼\", \"꽤\", \"쌈\", \"룰\", \"㎡\", \"숏\", \"셸\", \"캐\", \"춰\", \"쭙\", \"꽝\", \"녹\", \"쏜\", \"◾\", \"후\", \"삣\", \"몬\", \"튄\", \"텝\", \"깅\", \"징\", \"널\", \"틴\", \"쑥\", \"혜\", \"것\", \"옮\", \"킹\", \"싼\", \"§\", \"권\", \"틀\", \"인\", \"슴\", \"렛\", \"툰\", \"뼈\", \"펜\", \"믹\", \"샐\", \"찍\", \"앞\", \"靑\", \"✈\", \"순\", \"🙇\", \"할\", \"®\", \"뛸\", \"쉰\", \"챔\", \"렀\", \"젖\", \"雙\", \"딘\", \"깡\", \"킬\", \"榜\", \"尤\", \"벙\", \"으\", \"옴\", \"찾\", \"월\", \"싸\", \"쪽\", \"픕\", \"몹\", \"날\", \"키\", \"立\", \"엄\", \"엌\", \"⑧\", \"탱\", \"뮌\", \"계\", \"칸\", \"닥\", \"따\", \"렵\", \"삶\", \"챨\", \"슈\", \"峴\", \"냠\", \"욜\", \"뇨\", \"⑦\", \"밖\", \"델\", \"흙\", \"떨\", \"뺏\", \"점\", \"의\", \"반\", \"통\", \"윤\", \"變\", \"터\", \"臺\", \"녀\", \"㎍\", \"쌓\", \"약\", \"↕\", \"넵\", \"젊\", \"오\", \"줄\", \"필\", \"‧\", \"잉\", \"도\", \"총\", \"쩍\", \"陸\", \"가\", \"졌\", \"형\", \"욕\", \"뚝\", \"ⅰ\", \"꽃\", \"돕\", \"쥬\", \"과\", \"넹\", \"착\", \"▣\", \"년\", \"梁\", \"🙌\", \"탠\", \"윗\", \"‬\", \"팹\", \"논\", \"르\", \"숯\", \"닿\", \"쉬\", \"ń\", \"훈\", \"방\", \"잃\", \"접\", \"Ⅷ\", \"혁\", \"쌉\", \"캇\", \"태\", \"론\", \"벵\", \"맺\", \"갔\", \"럴\", \"값\", \"옵\", \"편\", \"뼘\", \"ㅐ\", \"진\", \"흔\", \"켄\", \"겼\", \"효\", \"침\", \"쾌\", \"뮈\", \"쏘\", \"젝\", \"잰\", \"뮤\", \"🍔\", \"똘\", \"갛\", \"텨\", \"엿\", \"쫗\", \"템\", \"＠\", \"힜\", \"핫\", \"솔\", \"젓\", \"엘\", \"‘\", \"떠\", \"↑\", \"절\", \"츄\", \"농\", \"딸\", \"옳\", \"팰\", \"볼\", \"너\", \"흄\", \"귤\", \"놔\", \"Ⅶ\", \"낯\", \"🌿\", \"던\", \"캡\", \"🤭\", \"부\", \"출\", \"얼\", \"럽\", \"픽\", \"팜\", \"칠\", \"딧\", \"웠\", \"멋\", \"▷\", \"켤\", \"넘\", \"⚀\", \"핀\", \"펄\", \"찌\", \"쳤\", \"ⓒ\", \"氣\", \"數\", \"뷔\", \"팠\", \"랴\", \"심\", \"올\", \"쨍\", \"쓕\", \"⇒\", \"솨\", \"종\", \"붓\", \"알\", \"핥\", \"ㅡ\", \"메\", \"극\", \"걔\", \"료\", \"飮\", \"壞\", \"먹\", \"돋\", \"타\", \"렷\", \"골\", \"짜\", \"쏠\", \"랑\", \"넥\", \"곤\", \"뭥\", \"떴\", \"독\", \"◑\", \"▒\", \"≫\", \"뜨\", \"선\", \"듈\", \"璿\", \"쟁\", \"蚩\", \"될\", \"樂\", \"굵\", \"닝\", \"띕\", \"붕\", \"틔\", \"걀\", \"誡\", \"엉\", \"曆\", \"🙋\", \"不\", \"큐\", \"떼\", \"❍\", \"ㅗ\", \"철\", \"맬\", \"늙\", \"싯\", \"국\", \"렁\", \"⑹\", \"밥\", \"망\", \"석\", \"뉴\", \"율\", \"贖\", \"둘\", \"나\", \"봉\", \"링\", \"산\", \"Ⅱ\", \"쳐\", \"趺\", \"쟈\", \"량\", \"텁\", \"엽\", \"㎞\", \"규\", \"🚿\", \"사\", \"깃\", \"밭\", \"靈\", \"코\", \"弛\", \"쫄\", \"차\", \"웹\", \"분\", \"뚫\", \"매\", \"쿄\", \"댐\", \"개\", \"욥\", \"줌\", \"닛\", \"🍎\", \"Ⅸ\", \"킌\", \"줍\", \"◇\", \"峙\", \"🍰\", \"노\", \"와\", \"龍\", \"쓴\", \"렇\", \"즌\", \"팬\", \"🚨\", \"셉\", \"돗\", \"술\", \"내\", \"🍭\", \"ｖ\", \"↔\", \"었\", \"죠\", \"짤\", \"끈\", \"얹\", \"장\", \"스\", \"롭\", \"촛\", \"😂\", \"털\", \"풍\", \"칙\", \"◈\", \"쩡\", \"녔\", \"£\", \"넌\", \"훨\", \"꼬\", \"답\", \"벽\", \"ⅱ\", \"멀\", \"쪼\", \"몰\", \"모\", \"놈\", \"폈\", \"㉣\", \"천\", \"갓\", \"섦\", \"물\", \"찢\", \"잠\", \"콜\", \"험\", \"령\", \"륙\", \"셌\", \"‍\", \"겉\", \"눠\", \"빙\", \"셜\", \"십\", \"킷\", \"떡\", \"추\", \"츠\", \"외\", \"푸\", \"⊙\", \"›\", \"갉\", \"하\", \"덕\", \"폴\", \"띤\", \"누\", \"😣\", \"컬\", \"금\", \" \", \"낼\", \"덧\", \"괴\", \"밉\", \"뽑\", \"©\", \"헌\", \"뱀\", \"깔\", \"색\", \"짐\", \"잭\", \"걸\", \"쇠\", \"쿡\", \"숨\", \"俠\", \"읊\", \"낙\", \"說\", \"슐\", \"綃\", \"🌳\", \"토\", \"엣\", \"압\", \"첩\", \"꼽\", \"짊\", \"體\", \"擒\", \"벌\", \"ʏ\", \"ㅏ\", \"펙\", \"닌\", \"뒤\", \"충\", \"뜻\", \"︎\", \"퍽\", \"Ⅴ\", \"강\", \"쒀\", \"쿰\", \"늄\", \"지\", \"봐\", \"솟\", \"굴\", \"굳\", \"뻤\", \"⑤\", \"콤\", \"급\", \"뱉\", \"콥\", \"왠\", \"쇄\", \"≪\", \"봅\", \"단\", \"랐\", \"텀\", \"펠\", \"쯔\", \"화\", \"벼\", \"축\", \"良\", \"륵\", \"혈\", \"째\", \"핍\", \"폍\", \"뻐\", \"∼\", \"\", \"뜸\", \"뫼\", \"벗\", \"겔\", \"뿐\", \"훅\", \"ⅳ\", \"꼈\", \"람\", \"판\", \"견\", \"칩\", \"😱\", \"프\", \"퀀\", \"臀\", \"복\", \"겪\", \"잖\", \"운\", \"․\", \"궐\", \"집\", \"젤\", \"놀\", \"‥\", \"잡\", \"　\", \"뺀\", \"존\", \"랙\", \"냈\", \"묵\", \"얀\", \"빳\", \"숟\", \"흗\", \"슷\", \"니\", \"웃\", \"립\", \"뀐\", \"샘\", \"녁\", \"엑\", \"컸\", \"긴\", \"왔\", \"획\", \"밴\", \"㈜\", \"‎\", \"¹\", \"빗\", \"튠\", \"붙\", \"뱅\", \"멸\", \"ㄹ\", \"劫\", \"덜\", \"⑶\", \"써\", \"삐\", \"잼\", \"덟\", \"뭉\", \"팍\", \"자\", \"탬\", \"♤\", \"ㅛ\", \"닉\", \"체\", \"틸\", \"서\", \"슛\", \"킴\", \"우\", \"글\", \"되\", \"짓\", \"맛\", \"담\", \"웍\", \"셰\", \"꾀\", \"썬\", \"학\", \"씽\", \"랫\", \"빡\", \"ʟ\", \"냅\", \"갇\", \"덤\", \"펴\", \"둑\", \"엮\", \"입\", \"퀴\", \"팽\", \"챙\", \"밑\", \"왓\", \"씻\", \"ㅋ\", \"拜\", \"넛\", \"퉁\", \"썩\", \"쑤\", \"🏷\", \"책\", \"엎\", \"ㄴ\", \"늬\", \"신\", \"볶\", \"머\", \"껀\", \"ㅝ\", \"쿼\", \"영\", \"빨\", \"얄\", \"톡\"]",
     "lossless": false
+  },
+  "01-ai/Yi-1.5-34B @ cc100/ar": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/01-ai/Yi-1.5-34B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Yi-1.5-34B</a>",
+    "organization": "Yi",
+    "vocab_size": 63992,
+    "_n_bytes": 2813283,
+    "_n_tokens": 1795801,
+    "_n_chars": 1560987,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "01-ai/Yi-1.5-34B @ cc100/de": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/01-ai/Yi-1.5-34B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Yi-1.5-34B</a>",
+    "organization": "Yi",
+    "vocab_size": 63992,
+    "_n_bytes": 1814876,
+    "_n_tokens": 698366,
+    "_n_chars": 1784021,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "01-ai/Yi-1.5-34B @ cc100/en": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/01-ai/Yi-1.5-34B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Yi-1.5-34B</a>",
+    "organization": "Yi",
+    "vocab_size": 63992,
+    "_n_bytes": 1124813,
+    "_n_tokens": 270399,
+    "_n_chars": 1121360,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "01-ai/Yi-1.5-34B @ cc100/es": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/01-ai/Yi-1.5-34B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Yi-1.5-34B</a>",
+    "organization": "Yi",
+    "vocab_size": 63992,
+    "_n_bytes": 1664455,
+    "_n_tokens": 577018,
+    "_n_chars": 1630297,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "01-ai/Yi-1.5-34B @ cc100/fa": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/01-ai/Yi-1.5-34B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Yi-1.5-34B</a>",
+    "organization": "Yi",
+    "vocab_size": 63992,
+    "_n_bytes": 2054052,
+    "_n_tokens": 1337264,
+    "_n_chars": 1145876,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "01-ai/Yi-1.5-34B @ cc100/fr": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/01-ai/Yi-1.5-34B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Yi-1.5-34B</a>",
+    "organization": "Yi",
+    "vocab_size": 63992,
+    "_n_bytes": 1540504,
+    "_n_tokens": 533106,
+    "_n_chars": 1484970,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "01-ai/Yi-1.5-34B @ cc100/ja": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/01-ai/Yi-1.5-34B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Yi-1.5-34B</a>",
+    "organization": "Yi",
+    "vocab_size": 63992,
+    "_n_bytes": 1774770,
+    "_n_tokens": 740791,
+    "_n_chars": 603065,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "01-ai/Yi-1.5-34B @ cc100/ko": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/01-ai/Yi-1.5-34B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Yi-1.5-34B</a>",
+    "organization": "Yi",
+    "vocab_size": 63992,
+    "_n_bytes": 1524839,
+    "_n_tokens": 1203134,
+    "_n_chars": 655190,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "01-ai/Yi-1.5-34B @ cc100/zh-Hans": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/01-ai/Yi-1.5-34B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Yi-1.5-34B</a>",
+    "organization": "Yi",
+    "vocab_size": 63992,
+    "_n_bytes": 2633047,
+    "_n_tokens": 588729,
+    "_n_chars": 927311,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "Qwen/Qwen2-72B @ cc100/ar": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-72B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-72B</a>",
+    "organization": "Alibaba",
+    "vocab_size": 151646,
+    "_n_bytes": 2813283,
+    "_n_tokens": 614959,
+    "_n_chars": 1560987,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "Qwen/Qwen2-72B @ cc100/de": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-72B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-72B</a>",
+    "organization": "Alibaba",
+    "vocab_size": 151646,
+    "_n_bytes": 1814876,
+    "_n_tokens": 503561,
+    "_n_chars": 1784021,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "Qwen/Qwen2-72B @ cc100/en": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-72B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-72B</a>",
+    "organization": "Alibaba",
+    "vocab_size": 151646,
+    "_n_bytes": 1124813,
+    "_n_tokens": 257983,
+    "_n_chars": 1121360,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "Qwen/Qwen2-72B @ cc100/es": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-72B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-72B</a>",
+    "organization": "Alibaba",
+    "vocab_size": 151646,
+    "_n_bytes": 1664455,
+    "_n_tokens": 434264,
+    "_n_chars": 1630297,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "Qwen/Qwen2-72B @ cc100/fa": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-72B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-72B</a>",
+    "organization": "Alibaba",
+    "vocab_size": 151646,
+    "_n_bytes": 2054052,
+    "_n_tokens": 643421,
+    "_n_chars": 1145876,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "Qwen/Qwen2-72B @ cc100/fr": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-72B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-72B</a>",
+    "organization": "Alibaba",
+    "vocab_size": 151646,
+    "_n_bytes": 1540504,
+    "_n_tokens": 413637,
+    "_n_chars": 1484970,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "Qwen/Qwen2-72B @ cc100/ja": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-72B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-72B</a>",
+    "organization": "Alibaba",
+    "vocab_size": 151646,
+    "_n_bytes": 1774770,
+    "_n_tokens": 377144,
+    "_n_chars": 603065,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "Qwen/Qwen2-72B @ cc100/ko": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-72B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-72B</a>",
+    "organization": "Alibaba",
+    "vocab_size": 151646,
+    "_n_bytes": 1524839,
+    "_n_tokens": 457492,
+    "_n_chars": 655190,
+    "_n_oov_chars": 25,
+    "oov_ratio": 3.815687052610693e-05,
+    "_oov_charset": "[\"立\", \"樂\", \"陸\", \"累\", \"龍\", \"梁\", \"金\", \"識\", \"女\", \"樂\", \"流\", \"良\", \"不\", \"靈\", \"理\"]",
+    "lossless": false
+  },
+  "Qwen/Qwen2-72B @ cc100/zh-Hans": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/Qwen/Qwen2-72B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Qwen2-72B</a>",
+    "organization": "Alibaba",
+    "vocab_size": 151646,
+    "_n_bytes": 2633047,
+    "_n_tokens": 589211,
+    "_n_chars": 927311,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "apple/DCLM-7B @ cc100/ar": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/apple/DCLM-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">DCLM-7B</a>",
+    "organization": "Apple",
+    "vocab_size": 50277,
+    "_n_bytes": 2813283,
+    "_n_tokens": 1106277,
+    "_n_chars": 1560987,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "apple/DCLM-7B @ cc100/de": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/apple/DCLM-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">DCLM-7B</a>",
+    "organization": "Apple",
+    "vocab_size": 50277,
+    "_n_bytes": 1814876,
+    "_n_tokens": 583628,
+    "_n_chars": 1784021,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "apple/DCLM-7B @ cc100/en": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/apple/DCLM-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">DCLM-7B</a>",
+    "organization": "Apple",
+    "vocab_size": 50277,
+    "_n_bytes": 1124813,
+    "_n_tokens": 259357,
+    "_n_chars": 1121360,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "apple/DCLM-7B @ cc100/es": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/apple/DCLM-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">DCLM-7B</a>",
+    "organization": "Apple",
+    "vocab_size": 50277,
+    "_n_bytes": 1664455,
+    "_n_tokens": 494577,
+    "_n_chars": 1630297,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "apple/DCLM-7B @ cc100/fa": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/apple/DCLM-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">DCLM-7B</a>",
+    "organization": "Apple",
+    "vocab_size": 50277,
+    "_n_bytes": 2054052,
+    "_n_tokens": 866434,
+    "_n_chars": 1145876,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "apple/DCLM-7B @ cc100/fr": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/apple/DCLM-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">DCLM-7B</a>",
+    "organization": "Apple",
+    "vocab_size": 50277,
+    "_n_bytes": 1540504,
+    "_n_tokens": 458961,
+    "_n_chars": 1484970,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "apple/DCLM-7B @ cc100/ja": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/apple/DCLM-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">DCLM-7B</a>",
+    "organization": "Apple",
+    "vocab_size": 50277,
+    "_n_bytes": 1774770,
+    "_n_tokens": 605168,
+    "_n_chars": 603065,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "apple/DCLM-7B @ cc100/ko": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/apple/DCLM-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">DCLM-7B</a>",
+    "organization": "Apple",
+    "vocab_size": 50277,
+    "_n_bytes": 1524839,
+    "_n_tokens": 973288,
+    "_n_chars": 655190,
+    "_n_oov_chars": 25,
+    "oov_ratio": 3.815687052610693e-05,
+    "_oov_charset": "[\"立\", \"樂\", \"陸\", \"累\", \"龍\", \"梁\", \"金\", \"識\", \"女\", \"樂\", \"流\", \"良\", \"不\", \"靈\", \"理\"]",
+    "lossless": false
+  },
+  "apple/DCLM-7B @ cc100/zh-Hans": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/apple/DCLM-7B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">DCLM-7B</a>",
+    "organization": "Apple",
+    "vocab_size": 50277,
+    "_n_bytes": 2633047,
+    "_n_tokens": 1220529,
+    "_n_chars": 927311,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "google/gemma-2-9b @ cc100/ar": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-9b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-9b</a>",
+    "organization": "Google",
+    "vocab_size": 256000,
+    "_n_bytes": 2813283,
+    "_n_tokens": 563788,
+    "_n_chars": 1560987,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "google/gemma-2-9b @ cc100/de": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-9b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-9b</a>",
+    "organization": "Google",
+    "vocab_size": 256000,
+    "_n_bytes": 1814876,
+    "_n_tokens": 406876,
+    "_n_chars": 1784021,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "google/gemma-2-9b @ cc100/en": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-9b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-9b</a>",
+    "organization": "Google",
+    "vocab_size": 256000,
+    "_n_bytes": 1124813,
+    "_n_tokens": 258010,
+    "_n_chars": 1121360,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "google/gemma-2-9b @ cc100/es": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-9b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-9b</a>",
+    "organization": "Google",
+    "vocab_size": 256000,
+    "_n_bytes": 1664455,
+    "_n_tokens": 361321,
+    "_n_chars": 1630297,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "google/gemma-2-9b @ cc100/fa": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-9b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-9b</a>",
+    "organization": "Google",
+    "vocab_size": 256000,
+    "_n_bytes": 2054052,
+    "_n_tokens": 363762,
+    "_n_chars": 1145876,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "google/gemma-2-9b @ cc100/fr": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-9b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-9b</a>",
+    "organization": "Google",
+    "vocab_size": 256000,
+    "_n_bytes": 1540504,
+    "_n_tokens": 364551,
+    "_n_chars": 1484970,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "google/gemma-2-9b @ cc100/ja": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-9b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-9b</a>",
+    "organization": "Google",
+    "vocab_size": 256000,
+    "_n_bytes": 1774770,
+    "_n_tokens": 307873,
+    "_n_chars": 603065,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "google/gemma-2-9b @ cc100/ko": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-9b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-9b</a>",
+    "organization": "Google",
+    "vocab_size": 256000,
+    "_n_bytes": 1524839,
+    "_n_tokens": 454410,
+    "_n_chars": 655190,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "google/gemma-2-9b @ cc100/zh-Hans": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/google/gemma-2-9b\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">gemma-2-9b</a>",
+    "organization": "Google",
+    "vocab_size": 256000,
+    "_n_bytes": 2633047,
+    "_n_tokens": 631795,
+    "_n_chars": 927311,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "meta-llama/Meta-Llama-3.1-405B @ cc100/ar": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-405B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3.1</a>",
+    "organization": "Meta",
+    "vocab_size": 128256,
+    "_n_bytes": 2813283,
+    "_n_tokens": 614186,
+    "_n_chars": 1560987,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "meta-llama/Meta-Llama-3.1-405B @ cc100/de": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-405B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3.1</a>",
+    "organization": "Meta",
+    "vocab_size": 128256,
+    "_n_bytes": 1814876,
+    "_n_tokens": 499766,
+    "_n_chars": 1784021,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "meta-llama/Meta-Llama-3.1-405B @ cc100/en": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-405B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3.1</a>",
+    "organization": "Meta",
+    "vocab_size": 128256,
+    "_n_bytes": 1124813,
+    "_n_tokens": 254944,
+    "_n_chars": 1121360,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "meta-llama/Meta-Llama-3.1-405B @ cc100/es": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-405B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3.1</a>",
+    "organization": "Meta",
+    "vocab_size": 128256,
+    "_n_bytes": 1664455,
+    "_n_tokens": 433286,
+    "_n_chars": 1630297,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "meta-llama/Meta-Llama-3.1-405B @ cc100/fa": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-405B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3.1</a>",
+    "organization": "Meta",
+    "vocab_size": 128256,
+    "_n_bytes": 2054052,
+    "_n_tokens": 384591,
+    "_n_chars": 1145876,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "meta-llama/Meta-Llama-3.1-405B @ cc100/fr": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-405B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3.1</a>",
+    "organization": "Meta",
+    "vocab_size": 128256,
+    "_n_bytes": 1540504,
+    "_n_tokens": 412141,
+    "_n_chars": 1484970,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "meta-llama/Meta-Llama-3.1-405B @ cc100/ja": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-405B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3.1</a>",
+    "organization": "Meta",
+    "vocab_size": 128256,
+    "_n_bytes": 1774770,
+    "_n_tokens": 414715,
+    "_n_chars": 603065,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "meta-llama/Meta-Llama-3.1-405B @ cc100/ko": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-405B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3.1</a>",
+    "organization": "Meta",
+    "vocab_size": 128256,
+    "_n_bytes": 1524839,
+    "_n_tokens": 412433,
+    "_n_chars": 655190,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "meta-llama/Meta-Llama-3.1-405B @ cc100/zh-Hans": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/meta-llama/Meta-Llama-3.1-405B\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">llama3.1</a>",
+    "organization": "Meta",
+    "vocab_size": 128256,
+    "_n_bytes": 2633047,
+    "_n_tokens": 747405,
+    "_n_chars": 927311,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": false
+  },
+  "mistralai/Mistral-Large-Instruct-2407 @ cc100/ar": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mistral-Large-Instruct-2407\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mistral-Large-Instruct-2407</a>",
+    "organization": "Mistral",
+    "vocab_size": 32768,
+    "_n_bytes": 2813283,
+    "_n_tokens": 1388980,
+    "_n_chars": 1560987,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "mistralai/Mistral-Large-Instruct-2407 @ cc100/de": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mistral-Large-Instruct-2407\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mistral-Large-Instruct-2407</a>",
+    "organization": "Mistral",
+    "vocab_size": 32768,
+    "_n_bytes": 1814876,
+    "_n_tokens": 570172,
+    "_n_chars": 1784021,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "mistralai/Mistral-Large-Instruct-2407 @ cc100/en": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mistral-Large-Instruct-2407\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mistral-Large-Instruct-2407</a>",
+    "organization": "Mistral",
+    "vocab_size": 32768,
+    "_n_bytes": 1124813,
+    "_n_tokens": 276803,
+    "_n_chars": 1121360,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "mistralai/Mistral-Large-Instruct-2407 @ cc100/es": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mistral-Large-Instruct-2407\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mistral-Large-Instruct-2407</a>",
+    "organization": "Mistral",
+    "vocab_size": 32768,
+    "_n_bytes": 1664455,
+    "_n_tokens": 504711,
+    "_n_chars": 1630297,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "mistralai/Mistral-Large-Instruct-2407 @ cc100/fa": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mistral-Large-Instruct-2407\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mistral-Large-Instruct-2407</a>",
+    "organization": "Mistral",
+    "vocab_size": 32768,
+    "_n_bytes": 2054052,
+    "_n_tokens": 1114580,
+    "_n_chars": 1145876,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "mistralai/Mistral-Large-Instruct-2407 @ cc100/fr": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mistral-Large-Instruct-2407\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mistral-Large-Instruct-2407</a>",
+    "organization": "Mistral",
+    "vocab_size": 32768,
+    "_n_bytes": 1540504,
+    "_n_tokens": 467708,
+    "_n_chars": 1484970,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "mistralai/Mistral-Large-Instruct-2407 @ cc100/ja": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mistral-Large-Instruct-2407\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mistral-Large-Instruct-2407</a>",
+    "organization": "Mistral",
+    "vocab_size": 32768,
+    "_n_bytes": 1774770,
+    "_n_tokens": 665746,
+    "_n_chars": 603065,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "mistralai/Mistral-Large-Instruct-2407 @ cc100/ko": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mistral-Large-Instruct-2407\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mistral-Large-Instruct-2407</a>",
+    "organization": "Mistral",
+    "vocab_size": 32768,
+    "_n_bytes": 1524839,
+    "_n_tokens": 711304,
+    "_n_chars": 655190,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
+  },
+  "mistralai/Mistral-Large-Instruct-2407 @ cc100/zh-Hans": {
+    "tokenizer": "<a target=\"_blank\" href=\"https://huggingface.co/mistralai/Mistral-Large-Instruct-2407\" style=\"color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;\">Mistral-Large-Instruct-2407</a>",
+    "organization": "Mistral",
+    "vocab_size": 32768,
+    "_n_bytes": 2633047,
+    "_n_tokens": 1022751,
+    "_n_chars": 927311,
+    "_n_oov_chars": 0,
+    "oov_ratio": 0.0,
+    "_oov_charset": "[]",
+    "lossless": true
   }
 }

vocab.py

@@ -238,6 +238,7 @@ _all_tokenizer_config = [
     TokenizerConfig("Qwen/Qwen1.5-14B", impl=TokenizerImpl.SentencePiece, org="Alibaba"),  # 15万，速度有点慢
     TokenizerConfig("Qwen/Qwen1.5-110B", impl=TokenizerImpl.SentencePiece, org="Alibaba"),
     TokenizerConfig("Qwen/Qwen1.5-1.8B", impl=TokenizerImpl.SentencePiece, org="Alibaba"),
+    TokenizerConfig("Qwen/Qwen2-72B", impl=TokenizerImpl.SentencePiece, org="Alibaba"),
     TokenizerConfig("HuggingFaceH4/starchat-alpha", impl=TokenizerImpl.SentencePiece, org="-"),
 
     ####### google/sentencepiece tokenizer:
@@ -254,14 +255,19 @@ _all_tokenizer_config = [
 
     TokenizerConfig("ClueAI/ChatYuan-large-v2", impl=TokenizerImpl.SentencePiece, org="CLUE"),
     TokenizerConfig("ClueAI/PromptCLUE-base", impl=TokenizerImpl.SentencePiece, org="CLUE"),
+
+    # byte-level BPE
+    # '中文单字': 700, '中文多字': 0  meta-llama/Meta-Llama-3.1-405B
+    TokenizerConfig("meta-llama/Meta-Llama-3.1-405B", name_display="Meta/llama3.1", impl=TokenizerImpl.SentencePiece,
+                    org="Meta"),
     TokenizerConfig("gradientai/Llama-3-8B-Instruct-Gradient-1048k", name_display="Meta/llama3",
                     impl=TokenizerImpl.SentencePiece, org="Meta",
                     desc="llama split all numbers into individual digits, and fallback to bytes to decompose unknown UTF-8 characters"),
-    # byte-level BPE
-    # '中文单字': 700, '中文多字': 0
     TokenizerConfig("NousResearch/Llama-2-7b-chat-hf", name_display="Meta/llama2", impl=TokenizerImpl.SentencePiece,
                     org="Meta"),
     TokenizerConfig("huggyllama/llama-7b", name_display="Meta/llama", impl=TokenizerImpl.SentencePiece, org="Meta"),
+
+
     TokenizerConfig("hpcai-tech/grok-1", name_display="xai-org/grok-1", impl=TokenizerImpl.SentencePiece, org="xAI"),
     # 由.model文件转化为了
     TokenizerConfig("hfl/chinese-llama-lora-7b", impl=TokenizerImpl.SentencePiece, org="-",
@@ -328,14 +334,17 @@ _all_tokenizer_config = [
     # 未分类
     # ("amber", ""),
     TokenizerConfig("LLM360/CrystalCoder", org="MBZUAI"),
+    TokenizerConfig("apple/DCLM-7B", org="Apple"),
     TokenizerConfig("mistralai/Mistral-7B-v0.1", org="Mistral"),
     TokenizerConfig("mistralai/Mixtral-8x7B-v0.1", org="Mistral"),
+    TokenizerConfig("mistralai/Mistral-Large-Instruct-2407", org="Mistral"),
 
     TokenizerConfig("paust/pko-t5-large", org="PAUST"),
 
     TokenizerConfig("01-ai/Yi-6B", org="Yi"),
     TokenizerConfig("01-ai/Yi-34B", org="Yi"),
     TokenizerConfig("01-ai/Yi-VL-34B", org="Yi"),
+    TokenizerConfig("01-ai/Yi-1.5-34B", org="Yi"),
     TokenizerConfig("OrionStarAI/Orion-14B-Chat", org="OrionStar"),
     TokenizerConfig("microsoft/phi-1", org="Microsoft"),
     TokenizerConfig("microsoft/phi-2", org="Microsoft"),
@@ -356,6 +365,7 @@ _all_tokenizer_config = [
     TokenizerConfig("deepseek-ai/deepseek-llm-7b-base", org="DeepSeek"),
     TokenizerConfig("deepseek-ai/DeepSeek-V2", org="DeepSeek"),
     TokenizerConfig("google/gemma-7b", org="Google"),
+    TokenizerConfig("google/gemma-2-9b", org="Google"),
     TokenizerConfig("allenai/OLMo-7B", org="Allen AI"),
     TokenizerConfig("HuggingFaceH4/zephyr-7b-beta", org="HuggingFace"),
     TokenizerConfig("ai21labs/Jamba-v0.1", org="AI21"),