ProdeusUnity commited on
Commit
141ba70
1 Parent(s): 94b4655

Upload 2 files

Browse files
Files changed (2) hide show
  1. tokenizer.json +2 -2
  2. tokenizer_config.json +1 -2
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4b4c8fcd33487a449c07f423d47adb035bba8347ccf13eb074b4d1fef8acf919
3
- size 17078288
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ba3c07c153070f7da72cf185508260ca07d9a037a703a44a42da9c790924271
3
+ size 9264441
tokenizer_config.json CHANGED
@@ -8008,9 +8008,8 @@
8008
  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
8009
  "clean_up_tokenization_spaces": true,
8010
  "eos_token": "</s>",
8011
- "model_max_length": 1024000,
8012
  "pad_token": "<pad>",
8013
- "padding_side": "left",
8014
  "tokenizer_class": "GPT2Tokenizer",
8015
  "unk_token": "<unk>"
8016
  }
 
8008
  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
8009
  "clean_up_tokenization_spaces": true,
8010
  "eos_token": "</s>",
8011
+ "model_max_length": 1000000000000000019884624838656,
8012
  "pad_token": "<pad>",
 
8013
  "tokenizer_class": "GPT2Tokenizer",
8014
  "unk_token": "<unk>"
8015
  }