Redwood0 commited on
Commit
71a067b
1 Parent(s): 52db581

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +0 -6
tokenizer_config.json CHANGED
@@ -27,12 +27,6 @@
27
  "special": true
28
  }
29
  },
30
- "auto_map": {
31
- "AutoTokenizer": [
32
- "tokenization_yi.YiTokenizer",
33
- null
34
- ]
35
- },
36
  "bos_token": "<|startoftext|>",
37
  "clean_up_tokenization_spaces": false,
38
  "eos_token": "<|endoftext|>",
 
27
  "special": true
28
  }
29
  },
 
 
 
 
 
 
30
  "bos_token": "<|startoftext|>",
31
  "clean_up_tokenization_spaces": false,
32
  "eos_token": "<|endoftext|>",