File size: 393 Bytes
c790826
 
 
05bceed
 
c790826
 
 
 
 
 
 
05bceed
c790826
05bceed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
{
  "auto_map": {
    "AutoTokenizer": [
      "tokenization_internlm2.InternLM2Tokenizer",
      "tokenization_internlm2_fast.InternLM2TokenizerFast"
    ]
  },
  "bos_token": "<s>",
  "clean_up_tokenization_spaces": false,
  "eos_token": "</s>",
  "model_max_length": 1000000000000000019884624838656,
  "pad_token": "</s>",
  "tokenizer_class": "InternLM2Tokenizer",
  "unk_token": "<unk>"
}