JustinLin610 commited on
Commit
2d040d3
1 Parent(s): 57f417d

Upload folder using huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer.json +6 -10
tokenizer.json CHANGED
@@ -12556,21 +12556,17 @@
12556
  }
12557
  ]
12558
  },
12559
- "pre_tokenizer": {
12560
- "type": "Metaspace",
12561
- "replacement": "▁",
12562
- "add_prefix_space": false,
12563
- "prepend_scheme": "first"
12564
- },
12565
  "post_processor": null,
12566
  "decoder": {
12567
  "type": "Sequence",
12568
  "decoders": [
12569
  {
12570
- "type": "Metaspace",
12571
- "replacement": "▁",
12572
- "add_prefix_space": false,
12573
- "prepend_scheme": "first"
 
12574
  },
12575
  {
12576
  "type": "ByteFallback"
 
12556
  }
12557
  ]
12558
  },
12559
+ "pre_tokenizer": null,
 
 
 
 
 
12560
  "post_processor": null,
12561
  "decoder": {
12562
  "type": "Sequence",
12563
  "decoders": [
12564
  {
12565
+ "type": "Replace",
12566
+ "pattern": {
12567
+ "String": "▁"
12568
+ },
12569
+ "content": " "
12570
  },
12571
  {
12572
  "type": "ByteFallback"