Upload tokenizer

merges.txt

@@ -2056,8 +2056,8 @@ d o
 Ġchan ged
 Ġcontro l
 Ġs ense
-it ch
 as ure
+it ch
 Ġfe bruary
 Ġ3 0
 Ġdad dy
@@ -2569,9 +2569,9 @@ in c
 Ġm ir
 Ġf em
 Ġbec om
-Ġ q
 em pt
 a f
+Ġ q
 ri age
 Ġbr own
 od e
@@ -3068,10 +3068,10 @@ ur ity
 Ġb ath
 Ġl ength
 Ġtr ade
-bo ard
 Ġkn ock
 ast ic
 Ġjo ined
+bo ard
 b a
 ter day
 Ġco ffee
@@ -4739,7 +4739,6 @@ u ly
 Ġcom ment
 Ġtr ave
 Ġdis play
-Ġsw itch
 Ġass um
 Ġadv ice
 Ġstep s
@@ -4749,6 +4748,7 @@ u ly
 Ġk im
 im um
 Ġrec om
+Ġsw itch
 Ġsign al
 Ġill ust
 ipp ing

tokenizer.json

@@ -1,19 +1,7 @@
 {
   "version": "1.0",
-  "truncation": {
-    "direction": "Right",
-    "max_length": 128,
-    "strategy": "LongestFirst",
-    "stride": 0
-  },
-  "padding": {
-    "strategy": "BatchLongest",
-    "direction": "Right",
-    "pad_to_multiple_of": null,
-    "pad_id": 1,
-    "pad_type_id": 0,
-    "pad_token": "<pad>"
-  },
+  "truncation": null,
+  "padding": null,
   "added_tokens": [
     {
       "id": 0,
@@ -2415,8 +2403,8 @@
       "Ġchanged": 2315,
       "Ġcontrol": 2316,
       "Ġsense": 2317,
-      "itch": 2318,
-      "asure": 2319,
+      "asure": 2318,
+      "itch": 2319,
       "Ġfebruary": 2320,
       "Ġ30": 2321,
       "Ġdaddy": 2322,
@@ -2928,9 +2916,9 @@
       "Ġmir": 2828,
       "Ġfem": 2829,
       "Ġbecom": 2830,
-      "Ġq": 2831,
-      "empt": 2832,
-      "af": 2833,
+      "empt": 2831,
+      "af": 2832,
+      "Ġq": 2833,
       "riage": 2834,
       "Ġbrown": 2835,
       "ode": 2836,
@@ -3427,10 +3415,10 @@
       "Ġbath": 3327,
       "Ġlength": 3328,
       "Ġtrade": 3329,
-      "board": 3330,
-      "Ġknock": 3331,
-      "astic": 3332,
-      "Ġjoined": 3333,
+      "Ġknock": 3330,
+      "astic": 3331,
+      "Ġjoined": 3332,
+      "board": 3333,
       "ba": 3334,
       "terday": 3335,
       "Ġcoffee": 3336,
@@ -5098,16 +5086,16 @@
       "Ġcomment": 4998,
       "Ġtrave": 4999,
       "Ġdisplay": 5000,
-      "Ġswitch": 5001,
-      "Ġassum": 5002,
-      "Ġadvice": 5003,
-      "Ġsteps": 5004,
-      "Ġdefeated": 5005,
-      "Ġresources": 5006,
-      "Ġrick": 5007,
-      "Ġkim": 5008,
-      "imum": 5009,
-      "Ġrecom": 5010,
+      "Ġassum": 5001,
+      "Ġadvice": 5002,
+      "Ġsteps": 5003,
+      "Ġdefeated": 5004,
+      "Ġresources": 5005,
+      "Ġrick": 5006,
+      "Ġkim": 5007,
+      "imum": 5008,
+      "Ġrecom": 5009,
+      "Ġswitch": 5010,
       "Ġsignal": 5011,
       "Ġillust": 5012,
       "ipping": 5013,
@@ -18540,8 +18528,8 @@
       "Ġchan ged",
       "Ġcontro l",
       "Ġs ense",
-      "it ch",
       "as ure",
+      "it ch",
       "Ġfe bruary",
       "Ġ3 0",
       "Ġdad dy",
@@ -19053,9 +19041,9 @@
       "Ġm ir",
       "Ġf em",
       "Ġbec om",
-      "Ġ q",
       "em pt",
       "a f",
+      "Ġ q",
       "ri age",
       "Ġbr own",
       "od e",
@@ -19552,10 +19540,10 @@
       "Ġb ath",
       "Ġl ength",
       "Ġtr ade",
-      "bo ard",
       "Ġkn ock",
       "ast ic",
       "Ġjo ined",
+      "bo ard",
       "b a",
       "ter day",
       "Ġco ffee",
@@ -21223,7 +21211,6 @@
       "Ġcom ment",
       "Ġtr ave",
       "Ġdis play",
-      "Ġsw itch",
       "Ġass um",
       "Ġadv ice",
       "Ġstep s",
@@ -21233,6 +21220,7 @@
       "Ġk im",
       "im um",
       "Ġrec om",
+      "Ġsw itch",
       "Ġsign al",
       "Ġill ust",
       "ipp ing",

tokenizer_config.json

@@ -13,7 +13,7 @@
     "single_word": false
   },
   "model_max_length": 1000000000000000019884624838656,
-  "name_or_path": "CamBabyTrainers/BabyBERTa-3-8192-tokenizer",
+  "name_or_path": "CamBabyTrainers/CamBabyTokenizer-8192",
   "pad_token": "<pad>",
   "sep_token": "</s>",
   "special_tokens_map_file": null,