nroggendorff commited on
Commit
88a2b73
1 Parent(s): b04b412

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -26,9 +26,9 @@ tokenizer.train_from_iterator(
26
  special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>", "<|user|>", "<|bot|>", "<|end|>"]
27
  )
28
 
29
- tokenizer.save("./custom_tokenizer.json")
30
 
31
- tokenizer = PreTrainedTokenizerFast(tokenizer_file="./custom_tokenizer.json")
32
 
33
  tokenizer.bos_token = "<s>"
34
  tokenizer.eos_token = "</s>"
 
26
  special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>", "<|user|>", "<|bot|>", "<|end|>"]
27
  )
28
 
29
+ tokenizer.save("custom_tokenizer.json")
30
 
31
+ tokenizer = PreTrainedTokenizerFast(tokenizer_file="custom_tokenizer.json")
32
 
33
  tokenizer.bos_token = "<s>"
34
  tokenizer.eos_token = "</s>"