nroggendorff commited on
Commit
47875c3
1 Parent(s): 1f2ceb7

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -52,9 +52,9 @@ tokenizer.add_special_tokens({
52
  tokenizer.user_token_id = tokenizer.convert_tokens_to_ids("<|user|>")
53
  tokenizer.assistant_token_id = tokenizer.convert_tokens_to_ids("<|bot|>")
54
 
55
- tokenizer.save_pretrained("llama-tokenizer")
56
 
57
- tokenizer = AutoTokenizer.from_pretrained("llama-tokenizer")
58
  print(tokenizer.apply_chat_template([{"role": "user", "content": "Why is the sky blue?"}, {"role": "assistant", "content": "Due to rayleigh scattering."}, {"role": "user", "content": "That's cool."}, {"role": "assistant", "content": "Yeah, I agree."}], tokenize=False))
59
 
60
  config = LlamaConfig(
 
52
  tokenizer.user_token_id = tokenizer.convert_tokens_to_ids("<|user|>")
53
  tokenizer.assistant_token_id = tokenizer.convert_tokens_to_ids("<|bot|>")
54
 
55
+ tokenizer.save_pretrained("/tmp/llama-tokenizer")
56
 
57
+ tokenizer = AutoTokenizer.from_pretrained("/tmp/llama-tokenizer")
58
  print(tokenizer.apply_chat_template([{"role": "user", "content": "Why is the sky blue?"}, {"role": "assistant", "content": "Due to rayleigh scattering."}, {"role": "user", "content": "That's cool."}, {"role": "assistant", "content": "Yeah, I agree."}], tokenize=False))
59
 
60
  config = LlamaConfig(