Spaces:
Runtime error
Runtime error
nroggendorff
commited on
Commit
•
88a2b73
1
Parent(s):
b04b412
Update app.py
Browse files
app.py
CHANGED
@@ -26,9 +26,9 @@ tokenizer.train_from_iterator(
|
|
26 |
special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>", "<|user|>", "<|bot|>", "<|end|>"]
|
27 |
)
|
28 |
|
29 |
-
tokenizer.save("
|
30 |
|
31 |
-
tokenizer = PreTrainedTokenizerFast(tokenizer_file="
|
32 |
|
33 |
tokenizer.bos_token = "<s>"
|
34 |
tokenizer.eos_token = "</s>"
|
|
|
26 |
special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>", "<|user|>", "<|bot|>", "<|end|>"]
|
27 |
)
|
28 |
|
29 |
+
tokenizer.save("custom_tokenizer.json")
|
30 |
|
31 |
+
tokenizer = PreTrainedTokenizerFast(tokenizer_file="custom_tokenizer.json")
|
32 |
|
33 |
tokenizer.bos_token = "<s>"
|
34 |
tokenizer.eos_token = "</s>"
|