eduagarcia commited on
Commit
0bd41c9
1 Parent(s): db8227e

internlm/internlm2* models changed it's tokenizer name to tokenization_internlm2.InternLM2Tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -29,7 +29,7 @@
29
  },
30
  "auto_map": {
31
  "AutoTokenizer": [
32
- "internlm/internlm2-base-7b--tokenization_internlm.InternLMTokenizer",
33
  null
34
  ]
35
  },
 
29
  },
30
  "auto_map": {
31
  "AutoTokenizer": [
32
+ "internlm/internlm2-base-7b--tokenization_internlm2.InternLM2Tokenizer",
33
  null
34
  ]
35
  },