config and tokenizer

config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "RobertaForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.9.0.dev0",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 50265
+}

create_config.py

@@ -0,0 +1,7 @@
+from transformers import RobertaConfig
+
+model_dir = "./"
+
+config = RobertaConfig.from_pretrained("roberta-base")
+config.save_pretrained(model_dir)
+

train_tokenizer.py

@@ -0,0 +1,49 @@
+from datasets import load_dataset, concatenate_datasets
+from tokenizers import ByteLevelBPETokenizer
+from pathlib import Path
+
+dataset_language = "su"
+validation_split_percentage = 10
+
+# load dataset
+# only the train subset for tokenizing purposes
+oscar = load_dataset(
+    "oscar", f"unshuffled_deduplicated_{dataset_language}", split="train",
+)
+
+cc100 = load_dataset("cc100", lang=dataset_language, split="train")
+
+mc4 = load_dataset("mc4", dataset_language, split="train")
+
+wiki_files = [str(x) for x in Path("../docs").glob("*.txt")]
+wiki = load_dataset("text", data_files=wiki_files)
+
+# want: text column only!
+oscar = oscar.remove_columns("id")
+mc4 = mc4.remove_columns(["url", "timestamp"])
+cc100 = cc100.remove_columns("id")
+
+dataset = concatenate_datasets([oscar, mc4, cc100, wiki["train"]])
+dataset = dataset.train_test_split(test_size=validation_split_percentage / 100, seed=42)
+
+# Instantiate tokenizer
+tokenizer = ByteLevelBPETokenizer()
+
+
+def batch_iterator(batch_size=10000):
+    for i in range(0, len(dataset), batch_size):
+        yield dataset["train"][i : i + batch_size]["text"]
+
+
+# Customized training
+tokenizer.train_from_iterator(
+    batch_iterator(),
+    vocab_size=50265,
+    min_frequency=2,
+    special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>",],
+)
+
+# Save files to disk
+model_dir = "."
+tokenizer.save(f"{model_dir}/tokenizer.json")
+