add xlm-roberta

vocab/__init__.py

@@ -116,6 +116,7 @@ all_tokenizers = [
     ("falcon_180b", "", "sentencepiece"),
     ("skywork_13b_base",),
     ("skywork_13b_math",),
+    ("xlm_roberta", ),
     # "goat",
 
     # ##### glm系列
@@ -140,6 +141,7 @@ all_tokenizers = [
     ("mistral_7b",),
     ("mixtral_8_7b",),
 
+
     ("flan_t5_base",),
     ("fastchat_t5_3b",),
     ("pko_t5_large",),

vocab/xlm_roberta/__init__.py

@@ -0,0 +1,3 @@
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-base", trust_remote_code=True)