Spaces:
Runtime error
Runtime error
nroggendorff
commited on
Commit
•
47875c3
1
Parent(s):
1f2ceb7
Update app.py
Browse files
app.py
CHANGED
@@ -52,9 +52,9 @@ tokenizer.add_special_tokens({
|
|
52 |
tokenizer.user_token_id = tokenizer.convert_tokens_to_ids("<|user|>")
|
53 |
tokenizer.assistant_token_id = tokenizer.convert_tokens_to_ids("<|bot|>")
|
54 |
|
55 |
-
tokenizer.save_pretrained("llama-tokenizer")
|
56 |
|
57 |
-
tokenizer = AutoTokenizer.from_pretrained("llama-tokenizer")
|
58 |
print(tokenizer.apply_chat_template([{"role": "user", "content": "Why is the sky blue?"}, {"role": "assistant", "content": "Due to rayleigh scattering."}, {"role": "user", "content": "That's cool."}, {"role": "assistant", "content": "Yeah, I agree."}], tokenize=False))
|
59 |
|
60 |
config = LlamaConfig(
|
|
|
52 |
tokenizer.user_token_id = tokenizer.convert_tokens_to_ids("<|user|>")
|
53 |
tokenizer.assistant_token_id = tokenizer.convert_tokens_to_ids("<|bot|>")
|
54 |
|
55 |
+
tokenizer.save_pretrained("/tmp/llama-tokenizer")
|
56 |
|
57 |
+
tokenizer = AutoTokenizer.from_pretrained("/tmp/llama-tokenizer")
|
58 |
print(tokenizer.apply_chat_template([{"role": "user", "content": "Why is the sky blue?"}, {"role": "assistant", "content": "Due to rayleigh scattering."}, {"role": "user", "content": "That's cool."}, {"role": "assistant", "content": "Yeah, I agree."}], tokenize=False))
|
59 |
|
60 |
config = LlamaConfig(
|