Upload tokenizer

Browse files

Files changed (5) hide show

added_tokens.json +130 -0
special_tokens_map.json +135 -0
spiece.model +3 -0
tokenizer.json +0 -0
tokenizer_config.json +112 -0

added_tokens.json ADDED Viewed

	@@ -0,0 +1,130 @@

+{
+  "<DR_0.15>": 32167,
+  "<DR_0.25>": 32145,
+  "<DR_0.2>": 32214,
+  "<DR_0.35>": 32100,
+  "<DR_0.3>": 32138,
+  "<DR_0.45>": 32210,
+  "<DR_0.4>": 32106,
+  "<DR_0.55>": 32101,
+  "<DR_0.5>": 32200,
+  "<DR_0.65>": 32114,
+  "<DR_0.6>": 32111,
+  "<DR_0.75>": 32181,
+  "<DR_0.7>": 32201,
+  "<DR_0.85>": 32136,
+  "<DR_0.8>": 32141,
+  "<DR_0.95>": 32205,
+  "<DR_0.9>": 32168,
+  "<DR_1.05>": 32193,
+  "<DR_1.0>": 32216,
+  "<DR_1.15>": 32128,
+  "<DR_1.1>": 32113,
+  "<DR_1.25>": 32219,
+  "<DR_1.2>": 32151,
+  "<DR_1.35>": 32129,
+  "<DR_1.3>": 32186,
+  "<DR_1.45>": 32226,
+  "<DR_1.4>": 32218,
+  "<DR_1.55>": 32171,
+  "<DR_1.5>": 32177,
+  "<DR_1.65>": 32132,
+  "<DR_1.6>": 32194,
+  "<DR_1.75>": 32166,
+  "<DR_1.7>": 32215,
+  "<DR_1.85>": 32104,
+  "<DR_1.8>": 32103,
+  "<DR_1.95>": 32191,
+  "<DR_1.9>": 32150,
+  "<DR_2.0>": 32134,
+  "<LS_0.25>": 32160,
+  "<LS_0.2>": 32146,
+  "<LS_0.35>": 32208,
+  "<LS_0.3>": 32212,
+  "<LS_0.45>": 32165,
+  "<LS_0.4>": 32133,
+  "<LS_0.55>": 32155,
+  "<LS_0.5>": 32206,
+  "<LS_0.65>": 32123,
+  "<LS_0.6>": 32180,
+  "<LS_0.75>": 32164,
+  "<LS_0.7>": 32188,
+  "<LS_0.85>": 32195,
+  "<LS_0.8>": 32190,
+  "<LS_0.95>": 32115,
+  "<LS_0.9>": 32158,
+  "<LS_1.0>": 32221,
+  "<NC_0.15>": 32225,
+  "<NC_0.1>": 32137,
+  "<NC_0.25>": 32185,
+  "<NC_0.2>": 32144,
+  "<NC_0.35>": 32118,
+  "<NC_0.3>": 32105,
+  "<NC_0.45>": 32125,
+  "<NC_0.4>": 32140,
+  "<NC_0.55>": 32170,
+  "<NC_0.5>": 32107,
+  "<NC_0.65>": 32213,
+  "<NC_0.6>": 32179,
+  "<NC_0.75>": 32217,
+  "<NC_0.7>": 32197,
+  "<NC_0.85>": 32135,
+  "<NC_0.8>": 32227,
+  "<NC_0.95>": 32109,
+  "<NC_0.9>": 32209,
+  "<NC_1.05>": 32127,
+  "<NC_1.0>": 32211,
+  "<NC_1.15>": 32223,
+  "<NC_1.1>": 32156,
+  "<NC_1.25>": 32178,
+  "<NC_1.2>": 32207,
+  "<NC_1.35>": 32176,
+  "<NC_1.3>": 32199,
+  "<NC_1.45>": 32117,
+  "<NC_1.4>": 32184,
+  "<NC_1.55>": 32162,
+  "<NC_1.5>": 32169,
+  "<NC_1.65>": 32139,
+  "<NC_1.6>": 32163,
+  "<NC_1.75>": 32183,
+  "<NC_1.7>": 32121,
+  "<NC_1.85>": 32174,
+  "<NC_1.8>": 32147,
+  "<NC_1.95>": 32182,
+  "<NC_1.9>": 32157,
+  "<NC_2.0>": 32204,
+  "<WR_0.35>": 32222,
+  "<WR_0.45>": 32110,
+  "<WR_0.4>": 32153,
+  "<WR_0.55>": 32143,
+  "<WR_0.5>": 32192,
+  "<WR_0.65>": 32187,
+  "<WR_0.6>": 32198,
+  "<WR_0.75>": 32202,
+  "<WR_0.7>": 32108,
+  "<WR_0.85>": 32142,
+  "<WR_0.8>": 32189,
+  "<WR_0.95>": 32149,
+  "<WR_0.9>": 32196,
+  "<WR_1.05>": 32120,
+  "<WR_1.0>": 32112,
+  "<WR_1.15>": 32130,
+  "<WR_1.1>": 32173,
+  "<WR_1.25>": 32172,
+  "<WR_1.2>": 32119,
+  "<WR_1.35>": 32152,
+  "<WR_1.3>": 32122,
+  "<WR_1.45>": 32224,
+  "<WR_1.4>": 32102,
+  "<WR_1.55>": 32131,
+  "<WR_1.5>": 32154,
+  "<WR_1.65>": 32126,
+  "<WR_1.6>": 32116,
+  "<WR_1.75>": 32148,
+  "<WR_1.7>": 32175,
+  "<WR_1.85>": 32159,
+  "<WR_1.8>": 32161,
+  "<WR_1.95>": 32220,
+  "<WR_1.9>": 32203,
+  "<WR_2.0>": 32124
+}

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,135 @@

+{
+  "additional_special_tokens": [
+    "<DR_0.35>",
+    "<DR_0.55>",
+    "<WR_1.4>",
+    "<DR_1.8>",
+    "<DR_1.85>",
+    "<NC_0.3>",
+    "<DR_0.4>",
+    "<NC_0.5>",
+    "<WR_0.7>",
+    "<NC_0.95>",
+    "<WR_0.45>",
+    "<DR_0.6>",
+    "<WR_1.0>",
+    "<DR_1.1>",
+    "<DR_0.65>",
+    "<LS_0.95>",
+    "<WR_1.6>",
+    "<NC_1.45>",
+    "<NC_0.35>",
+    "<WR_1.2>",
+    "<WR_1.05>",
+    "<NC_1.7>",
+    "<WR_1.3>",
+    "<LS_0.65>",
+    "<WR_2.0>",
+    "<NC_0.45>",
+    "<WR_1.65>",
+    "<NC_1.05>",
+    "<DR_1.15>",
+    "<DR_1.35>",
+    "<WR_1.15>",
+    "<WR_1.55>",
+    "<DR_1.65>",
+    "<LS_0.4>",
+    "<DR_2.0>",
+    "<NC_0.85>",
+    "<DR_0.85>",
+    "<NC_0.1>",
+    "<DR_0.3>",
+    "<NC_1.65>",
+    "<NC_0.4>",
+    "<DR_0.8>",
+    "<WR_0.85>",
+    "<WR_0.55>",
+    "<NC_0.2>",
+    "<DR_0.25>",
+    "<LS_0.2>",
+    "<NC_1.8>",
+    "<WR_1.75>",
+    "<WR_0.95>",
+    "<DR_1.9>",
+    "<DR_1.2>",
+    "<WR_1.35>",
+    "<WR_0.4>",
+    "<WR_1.5>",
+    "<LS_0.55>",
+    "<NC_1.1>",
+    "<NC_1.9>",
+    "<LS_0.9>",
+    "<WR_1.85>",
+    "<LS_0.25>",
+    "<WR_1.8>",
+    "<NC_1.55>",
+    "<NC_1.6>",
+    "<LS_0.75>",
+    "<LS_0.45>",
+    "<DR_1.75>",
+    "<DR_0.15>",
+    "<DR_0.9>",
+    "<NC_1.5>",
+    "<NC_0.55>",
+    "<DR_1.55>",
+    "<WR_1.25>",
+    "<WR_1.1>",
+    "<NC_1.85>",
+    "<WR_1.7>",
+    "<NC_1.35>",
+    "<DR_1.5>",
+    "<NC_1.25>",
+    "<NC_0.6>",
+    "<LS_0.6>",
+    "<DR_0.75>",
+    "<NC_1.95>",
+    "<NC_1.75>",
+    "<NC_1.4>",
+    "<NC_0.25>",
+    "<DR_1.3>",
+    "<WR_0.65>",
+    "<LS_0.7>",
+    "<WR_0.8>",
+    "<LS_0.8>",
+    "<DR_1.95>",
+    "<WR_0.5>",
+    "<DR_1.05>",
+    "<DR_1.6>",
+    "<LS_0.85>",
+    "<WR_0.9>",
+    "<NC_0.7>",
+    "<WR_0.6>",
+    "<NC_1.3>",
+    "<DR_0.5>",
+    "<DR_0.7>",
+    "<WR_0.75>",
+    "<WR_1.9>",
+    "<NC_2.0>",
+    "<DR_0.95>",
+    "<LS_0.5>",
+    "<NC_1.2>",
+    "<LS_0.35>",
+    "<NC_0.9>",
+    "<DR_0.45>",
+    "<NC_1.0>",
+    "<LS_0.3>",
+    "<NC_0.65>",
+    "<DR_0.2>",
+    "<DR_1.7>",
+    "<DR_1.0>",
+    "<NC_0.75>",
+    "<DR_1.4>",
+    "<DR_1.25>",
+    "<WR_1.95>",
+    "<LS_1.0>",
+    "<WR_0.35>",
+    "<NC_1.15>",
+    "<WR_1.45>",
+    "<NC_0.15>",
+    "<DR_1.45>",
+    "<NC_0.8>"
+  ],
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

spiece.model ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:d60acb128cf7b7f2536e8f38a5b18a05535c9e14c7a355904270e15b0945ea86
+size 791656

tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,112 @@

+{
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "extra_ids": 100,
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>"
+}