Hugging Face's logo

rinabuoy
/
khmer-tokenizer

Transformers
Inference Endpoints
Model card Files Community
rinabuoy commited on
Commit
bd39485
•
1 Parent(s): 39cd040

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +2 -2
  2. tokenizer_config.json +2 -2
tokenizer.json CHANGED
@@ -50,7 +50,7 @@
50
  },
51
  {
52
  "id": 2894,
53
- "content": "<sep>",
54
  "single_word": false,
55
  "lstrip": false,
56
  "rstrip": false,
@@ -59,7 +59,7 @@
59
  },
60
  {
61
  "id": 2895,
62
- "content": "<cls>",
63
  "single_word": false,
64
  "lstrip": false,
65
  "rstrip": false,
 
50
  },
51
  {
52
  "id": 2894,
53
+ "content": "<cls>",
54
  "single_word": false,
55
  "lstrip": false,
56
  "rstrip": false,
 
59
  },
60
  {
61
  "id": 2895,
62
+ "content": "<sep>",
63
  "single_word": false,
64
  "lstrip": false,
65
  "rstrip": false,
tokenizer_config.json CHANGED
@@ -41,7 +41,7 @@
41
  "special": true
42
  },
43
  "2894": {
44
- "content": "<sep>",
45
  "lstrip": false,
46
  "normalized": false,
47
  "rstrip": false,
@@ -49,7 +49,7 @@
49
  "special": true
50
  },
51
  "2895": {
52
- "content": "<cls>",
53
  "lstrip": false,
54
  "normalized": false,
55
  "rstrip": false,
 
41
  "special": true
42
  },
43
  "2894": {
44
+ "content": "<cls>",
45
  "lstrip": false,
46
  "normalized": false,
47
  "rstrip": false,
 
49
  "special": true
50
  },
51
  "2895": {
52
+ "content": "<sep>",
53
  "lstrip": false,
54
  "normalized": false,
55
  "rstrip": false,