Update config and tokenizer

README.md

@@ -1,14 +1,14 @@
 ---
 language: ti
 widget:
-- text: "ዓቕሚ ደቀንስትዮ [MASK] ብግብሪ ተራእዩ"
+- text: "ዓቕሚ መንእሰይ ኤርትራ <mask> ተራእዩ"
 ---
 
-# RoBERTa Pretrained for Tigrinya Language
+# TiRoBERTa: RoBERTa Pretrained for the Tigrinya Language
 
 We pretrain a RoBERTa base model for Tigrinya on a dataset of 40 million tokens trained for 40 epochs.
 
-Contained in this repo are the original pretrained Flax model that was trained on a TPU v3.8 and it's correponding PyTorch version.
+Contained in this repo is the original pretrained Flax model that was trained on a TPU v3.8 and it's corresponding PyTorch version.
 
 
 ## Hyperparameters
@@ -17,6 +17,27 @@ The hyperparameters corresponding to model sizes mentioned above are as follows:
 
 | Model Size | L  | AH | HS  | FFN  | P    | Seq  |
 |------------|----|----|-----|------|------|------|
-| BASE       | 12 | 12 | 768 | 3072 | 125M | 128  |
+| BASE       | 12 | 12 | 768 | 3072 | 125M | 512  |
 
 (L = number of layers; AH = number of attention heads; HS = hidden size; FFN = feedforward network dimension; P = number of parameters; Seq = maximum sequence length.)
+
+### Framework versions
+
+- Transformers 4.12.0.dev0
+- Pytorch 1.9.0+cu111
+- Datasets 1.13.3
+- Tokenizers 0.10.3
+
+
+## Citation
+
+If you use this model in your product or research, please cite as follows:
+
+```
+@article{Fitsum2021TiPLMs,
+  author={Fitsum Gaim and Wonsuk Yang and Jong C. Park},
+  title={Monolingual Pre-trained Language Models for Tigrinya},
+  year=2021,
+  publisher={WiNLP 2021 at EMNLP 2021}
+}
+```

config.json

@@ -5,6 +5,7 @@
   ],
   "attention_probs_dropout_prob": 0.1,
   "bos_token_id": 0,
+  "classifier_dropout": null,
   "eos_token_id": 2,
   "gradient_checkpointing": false,
   "hidden_act": "gelu",
@@ -20,7 +21,7 @@
   "pad_token_id": 1,
   "position_embedding_type": "absolute",
   "torch_dtype": "float32",
-  "transformers_version": "4.9.0.dev0",
+  "transformers_version": "4.12.0.dev0",
   "type_vocab_size": 1,
   "use_cache": true,
   "vocab_size": 50265

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "cls_token": "<s>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}}
+{"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "sep_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "cls_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true}}

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "add_prefix_space": false, "errors": "replace", "sep_token": "</s>", "cls_token": "<s>", "pad_token": "<pad>", "mask_token": "<mask>", "special_tokens_map_file": null, "name_or_path": "roberta-base-ti", "tokenizer_class": "RobertaTokenizer"}
+{"errors": "replace", "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "sep_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "cls_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "special_tokens_map_file": null, "name_or_path": "./", "tokenizer_class": "RobertaTokenizer"}