Upload 17 files
Browse files- model_index.json +32 -0
- scheduler/scheduler_config.json +11 -0
- text_encoder/config.json +23 -0
- text_encoder/model.safetensors +3 -0
- text_encoder_2/config.json +32 -0
- text_encoder_2/model.safetensors +3 -0
- tokenizer/merges.txt +647 -0
- tokenizer/special_tokens_map.json +30 -0
- tokenizer/tokenizer_config.json +30 -0
- tokenizer/vocab.json +1002 -0
- tokenizer_2/special_tokens_map.json +125 -0
- tokenizer_2/tokenizer.json +0 -0
- tokenizer_2/tokenizer_config.json +937 -0
- transformer/config.json +18 -0
- transformer/diffusion_pytorch_model.safetensors +3 -0
- vae/config.json +28 -0
- vae/diffusion_pytorch_model.safetensors +3 -0
model_index.json
ADDED
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_class_name": "FluxPipeline",
|
3 |
+
"_diffusers_version": "0.30.0",
|
4 |
+
"scheduler": [
|
5 |
+
"diffusers",
|
6 |
+
"FlowMatchEulerDiscreteScheduler"
|
7 |
+
],
|
8 |
+
"text_encoder": [
|
9 |
+
"transformers",
|
10 |
+
"CLIPTextModel"
|
11 |
+
],
|
12 |
+
"text_encoder_2": [
|
13 |
+
"transformers",
|
14 |
+
"T5EncoderModel"
|
15 |
+
],
|
16 |
+
"tokenizer": [
|
17 |
+
"transformers",
|
18 |
+
"CLIPTokenizer"
|
19 |
+
],
|
20 |
+
"tokenizer_2": [
|
21 |
+
"transformers",
|
22 |
+
"T5TokenizerFast"
|
23 |
+
],
|
24 |
+
"transformer": [
|
25 |
+
"diffusers",
|
26 |
+
"FluxTransformer2DModel"
|
27 |
+
],
|
28 |
+
"vae": [
|
29 |
+
"diffusers",
|
30 |
+
"AutoencoderKL"
|
31 |
+
]
|
32 |
+
}
|
scheduler/scheduler_config.json
ADDED
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_class_name": "FlowMatchEulerDiscreteScheduler",
|
3 |
+
"_diffusers_version": "0.30.0",
|
4 |
+
"base_image_seq_len": 256,
|
5 |
+
"base_shift": 0.5,
|
6 |
+
"max_image_seq_len": 4096,
|
7 |
+
"max_shift": 1.15,
|
8 |
+
"num_train_timesteps": 1000,
|
9 |
+
"shift": 1.0,
|
10 |
+
"use_dynamic_shifting": false
|
11 |
+
}
|
text_encoder/config.json
ADDED
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"architectures": [
|
3 |
+
"CLIPTextModel"
|
4 |
+
],
|
5 |
+
"attention_dropout": 0.0,
|
6 |
+
"bos_token_id": 0,
|
7 |
+
"eos_token_id": 2,
|
8 |
+
"hidden_act": "gelu",
|
9 |
+
"hidden_size": 32,
|
10 |
+
"initializer_factor": 1.0,
|
11 |
+
"initializer_range": 0.02,
|
12 |
+
"intermediate_size": 37,
|
13 |
+
"layer_norm_eps": 1e-05,
|
14 |
+
"max_position_embeddings": 77,
|
15 |
+
"model_type": "clip_text_model",
|
16 |
+
"num_attention_heads": 4,
|
17 |
+
"num_hidden_layers": 5,
|
18 |
+
"pad_token_id": 1,
|
19 |
+
"projection_dim": 32,
|
20 |
+
"torch_dtype": "float32",
|
21 |
+
"transformers_version": "4.43.3",
|
22 |
+
"vocab_size": 1000
|
23 |
+
}
|
text_encoder/model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:103860291b96610a23d9dda96f6a3c4e6c6dd67e5984a0d5e7c8e62769ac6412
|
3 |
+
size 283212
|
text_encoder_2/config.json
ADDED
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_name_or_path": "hf-internal-testing/tiny-random-t5",
|
3 |
+
"architectures": [
|
4 |
+
"T5EncoderModel"
|
5 |
+
],
|
6 |
+
"bos_token_id": 0,
|
7 |
+
"classifier_dropout": 0.0,
|
8 |
+
"d_ff": 37,
|
9 |
+
"d_kv": 8,
|
10 |
+
"d_model": 32,
|
11 |
+
"decoder_start_token_id": 0,
|
12 |
+
"dense_act_fn": "relu",
|
13 |
+
"dropout_rate": 0.1,
|
14 |
+
"eos_token_id": 1,
|
15 |
+
"feed_forward_proj": "relu",
|
16 |
+
"gradient_checkpointing": false,
|
17 |
+
"initializer_factor": 0.002,
|
18 |
+
"is_encoder_decoder": true,
|
19 |
+
"is_gated_act": false,
|
20 |
+
"layer_norm_epsilon": 1e-06,
|
21 |
+
"model_type": "t5",
|
22 |
+
"num_decoder_layers": 5,
|
23 |
+
"num_heads": 4,
|
24 |
+
"num_layers": 5,
|
25 |
+
"pad_token_id": 0,
|
26 |
+
"relative_attention_max_distance": 128,
|
27 |
+
"relative_attention_num_buckets": 8,
|
28 |
+
"torch_dtype": "float32",
|
29 |
+
"transformers_version": "4.43.3",
|
30 |
+
"use_cache": true,
|
31 |
+
"vocab_size": 1103
|
32 |
+
}
|
text_encoder_2/model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:a3ff1d3d80459481c9847d35de9a618835224139a4de455b40fcfe6f8544193b
|
3 |
+
size 276696
|
tokenizer/merges.txt
ADDED
@@ -0,0 +1,647 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#version: 0.2
|
2 |
+
Ġ t
|
3 |
+
Ġt h
|
4 |
+
Ġ a
|
5 |
+
Ġth e</w>
|
6 |
+
i n
|
7 |
+
Ġ o
|
8 |
+
Ġ ,</w>
|
9 |
+
Ġ s
|
10 |
+
e d</w>
|
11 |
+
Ġ w
|
12 |
+
e r
|
13 |
+
Ġ .</w>
|
14 |
+
Ġ i
|
15 |
+
r e
|
16 |
+
Ġ c
|
17 |
+
n d</w>
|
18 |
+
Ġ f
|
19 |
+
Ġ b
|
20 |
+
a t
|
21 |
+
Ġo f</w>
|
22 |
+
e r</w>
|
23 |
+
e n
|
24 |
+
a r
|
25 |
+
o r
|
26 |
+
i t
|
27 |
+
Ġ p
|
28 |
+
Ġ h
|
29 |
+
Ġa nd</w>
|
30 |
+
o n
|
31 |
+
in g</w>
|
32 |
+
a n
|
33 |
+
r o
|
34 |
+
Ġ m
|
35 |
+
Ġ d
|
36 |
+
e s</w>
|
37 |
+
Ġi n</w>
|
38 |
+
o n</w>
|
39 |
+
Ġt o</w>
|
40 |
+
o u
|
41 |
+
i s
|
42 |
+
Ġ a</w>
|
43 |
+
i c
|
44 |
+
Ġ T
|
45 |
+
a l
|
46 |
+
Ġ l
|
47 |
+
Ġ =</w>
|
48 |
+
Ġ re
|
49 |
+
Ġ "</w>
|
50 |
+
e s
|
51 |
+
Ġ S
|
52 |
+
a s</w>
|
53 |
+
a l</w>
|
54 |
+
i l
|
55 |
+
e l
|
56 |
+
i on</w>
|
57 |
+
Ġ A
|
58 |
+
Ġ C
|
59 |
+
Ġ 1
|
60 |
+
Ġ Ċ</w>
|
61 |
+
u r
|
62 |
+
ĠT h
|
63 |
+
Ġ n
|
64 |
+
a s
|
65 |
+
Ġ @
|
66 |
+
e c
|
67 |
+
o m
|
68 |
+
a c
|
69 |
+
Ġ e
|
70 |
+
Ġw as</w>
|
71 |
+
Ġ M
|
72 |
+
o r</w>
|
73 |
+
a n</w>
|
74 |
+
a m
|
75 |
+
e n</w>
|
76 |
+
o l
|
77 |
+
Ġ in
|
78 |
+
Ġ g
|
79 |
+
Ġ '</w>
|
80 |
+
Ġ B
|
81 |
+
l y</w>
|
82 |
+
a t</w>
|
83 |
+
i v
|
84 |
+
t s</w>
|
85 |
+
ĠTh e</w>
|
86 |
+
u s
|
87 |
+
- @</w>
|
88 |
+
Ġ@ -@</w>
|
89 |
+
i s</w>
|
90 |
+
Ġ I
|
91 |
+
Ġw h
|
92 |
+
i g
|
93 |
+
Ġ H
|
94 |
+
Ġs t
|
95 |
+
o s
|
96 |
+
u n
|
97 |
+
t h
|
98 |
+
Ġ P
|
99 |
+
Ġw it
|
100 |
+
Ġth at</w>
|
101 |
+
i r
|
102 |
+
Ġa s</w>
|
103 |
+
e m
|
104 |
+
Ġo n</w>
|
105 |
+
r a
|
106 |
+
Ġf or</w>
|
107 |
+
Ġ R
|
108 |
+
e t
|
109 |
+
o w
|
110 |
+
Ġ 2
|
111 |
+
i d
|
112 |
+
Ġ D
|
113 |
+
l e</w>
|
114 |
+
Ġwit h</w>
|
115 |
+
l a
|
116 |
+
en t</w>
|
117 |
+
i m
|
118 |
+
Ġ F
|
119 |
+
e a
|
120 |
+
i on
|
121 |
+
Ġb y</w>
|
122 |
+
Ġ )</w>
|
123 |
+
Ġ (</w>
|
124 |
+
Ġa l
|
125 |
+
Ġc on
|
126 |
+
en t
|
127 |
+
Ġ W
|
128 |
+
Ġi s</w>
|
129 |
+
er e</w>
|
130 |
+
Ġ G
|
131 |
+
Ġ N
|
132 |
+
Ġ L
|
133 |
+
Ġh a
|
134 |
+
er s</w>
|
135 |
+
r i
|
136 |
+
t h</w>
|
137 |
+
t ed</w>
|
138 |
+
u c
|
139 |
+
Ġ J
|
140 |
+
Ġ1 9
|
141 |
+
e v
|
142 |
+
u l
|
143 |
+
Ġ v
|
144 |
+
c e</w>
|
145 |
+
at ion</w>
|
146 |
+
ro m</w>
|
147 |
+
Ġb e
|
148 |
+
Ġ E
|
149 |
+
i n</w>
|
150 |
+
Ġth e
|
151 |
+
Ġf rom</w>
|
152 |
+
Ġ O
|
153 |
+
t er</w>
|
154 |
+
Ġp ro
|
155 |
+
Ġa r
|
156 |
+
a d
|
157 |
+
Ġc om
|
158 |
+
i c</w>
|
159 |
+
a g
|
160 |
+
Ġh is</w>
|
161 |
+
Ġs h
|
162 |
+
Ġa t</w>
|
163 |
+
o v
|
164 |
+
i es</w>
|
165 |
+
o o
|
166 |
+
p p
|
167 |
+
s t
|
168 |
+
c h
|
169 |
+
Ġ r
|
170 |
+
Ġ2 0
|
171 |
+
a y</w>
|
172 |
+
i f
|
173 |
+
Ġw ere</w>
|
174 |
+
Ġc h
|
175 |
+
u t</w>
|
176 |
+
s t</w>
|
177 |
+
u t
|
178 |
+
d s</w>
|
179 |
+
o p
|
180 |
+
u m
|
181 |
+
Ġi t</w>
|
182 |
+
o c
|
183 |
+
t er
|
184 |
+
l e
|
185 |
+
ig h
|
186 |
+
u d
|
187 |
+
Ġe x
|
188 |
+
ion s</w>
|
189 |
+
at e</w>
|
190 |
+
it y</w>
|
191 |
+
at ed</w>
|
192 |
+
Ġ un
|
193 |
+
e p
|
194 |
+
q u
|
195 |
+
Ġn o
|
196 |
+
Ġ K
|
197 |
+
iv e</w>
|
198 |
+
is t
|
199 |
+
Ġo n
|
200 |
+
am e</w>
|
201 |
+
ou n
|
202 |
+
i r</w>
|
203 |
+
a b
|
204 |
+
Ġ â
|
205 |
+
in g
|
206 |
+
Ġh e</w>
|
207 |
+
l d</w>
|
208 |
+
u g
|
209 |
+
ic h</w>
|
210 |
+
Ġa n</w>
|
211 |
+
e d
|
212 |
+
Ġ k
|
213 |
+
Ġâ Ģ
|
214 |
+
Ġha d</w>
|
215 |
+
v e</w>
|
216 |
+
a in
|
217 |
+
Ġs e
|
218 |
+
t ion</w>
|
219 |
+
or e</w>
|
220 |
+
re s
|
221 |
+
Ġwh ich</w>
|
222 |
+
ĠI n</w>
|
223 |
+
o d
|
224 |
+
th er</w>
|
225 |
+
a k
|
226 |
+
Ġs p
|
227 |
+
a r</w>
|
228 |
+
Ġ y
|
229 |
+
ĠC h
|
230 |
+
on g</w>
|
231 |
+
Ġa c
|
232 |
+
es t</w>
|
233 |
+
Ġ U
|
234 |
+
a p
|
235 |
+
f f
|
236 |
+
al ly</w>
|
237 |
+
r it
|
238 |
+
ĠS t
|
239 |
+
u b
|
240 |
+
g e</w>
|
241 |
+
b er</w>
|
242 |
+
e t</w>
|
243 |
+
Ġb e</w>
|
244 |
+
e ar
|
245 |
+
Ġre c
|
246 |
+
er s
|
247 |
+
Ġf ir
|
248 |
+
o t
|
249 |
+
Ġar e</w>
|
250 |
+
Ġa n
|
251 |
+
c h</w>
|
252 |
+
o g
|
253 |
+
i a</w>
|
254 |
+
es t
|
255 |
+
in e</w>
|
256 |
+
il l
|
257 |
+
an d
|
258 |
+
e l</w>
|
259 |
+
ar y</w>
|
260 |
+
e w</w>
|
261 |
+
i d</w>
|
262 |
+
Ġf or
|
263 |
+
Ġ ;</w>
|
264 |
+
Ġcom p
|
265 |
+
Ġ V
|
266 |
+
Ġin c
|
267 |
+
t r
|
268 |
+
Ġ20 0
|
269 |
+
Ġthe ir</w>
|
270 |
+
u s</w>
|
271 |
+
Ġb ut</w>
|
272 |
+
r an
|
273 |
+
ic al</w>
|
274 |
+
Ġfir st</w>
|
275 |
+
Ġd e
|
276 |
+
Ġin t
|
277 |
+
Ġ ro
|
278 |
+
s o</w>
|
279 |
+
ĠâĢ ĵ</w>
|
280 |
+
Ġno t</w>
|
281 |
+
d ing</w>
|
282 |
+
f ter</w>
|
283 |
+
ur e</w>
|
284 |
+
Ġp ar
|
285 |
+
Ġ :</w>
|
286 |
+
i an</w>
|
287 |
+
Ġt w
|
288 |
+
ou ld</w>
|
289 |
+
Ġal so</w>
|
290 |
+
Ġi ts</w>
|
291 |
+
Ġw or
|
292 |
+
u m</w>
|
293 |
+
Ġo r</w>
|
294 |
+
os t</w>
|
295 |
+
0 0</w>
|
296 |
+
ou r
|
297 |
+
ar d</w>
|
298 |
+
Ġre s
|
299 |
+
m p
|
300 |
+
u e</w>
|
301 |
+
Ġa b
|
302 |
+
is h</w>
|
303 |
+
Ġcon t
|
304 |
+
Ġa d
|
305 |
+
ow n</w>
|
306 |
+
al l</w>
|
307 |
+
ou g
|
308 |
+
Ġh er</w>
|
309 |
+
as t</w>
|
310 |
+
Ġ en
|
311 |
+
om e</w>
|
312 |
+
al l
|
313 |
+
d ed</w>
|
314 |
+
o w</w>
|
315 |
+
Ġha ve</w>
|
316 |
+
Ġ us
|
317 |
+
ea r</w>
|
318 |
+
ac k</w>
|
319 |
+
d uc
|
320 |
+
i al</w>
|
321 |
+
s s
|
322 |
+
en ts</w>
|
323 |
+
a in</w>
|
324 |
+
t ing</w>
|
325 |
+
Ġon e</w>
|
326 |
+
es s
|
327 |
+
Ġh as</w>
|
328 |
+
igh t</w>
|
329 |
+
a v
|
330 |
+
Ġe v
|
331 |
+
ou t</w>
|
332 |
+
a y
|
333 |
+
en ce</w>
|
334 |
+
Ġbe en</w>
|
335 |
+
e w
|
336 |
+
Ġtw o</w>
|
337 |
+
Ġc l
|
338 |
+
d er</w>
|
339 |
+
im e</w>
|
340 |
+
k s</w>
|
341 |
+
es s</w>
|
342 |
+
is h
|
343 |
+
. @</w>
|
344 |
+
Ġ@ .@</w>
|
345 |
+
Ġp la
|
346 |
+
Ġp l
|
347 |
+
Ġo r
|
348 |
+
u p</w>
|
349 |
+
m ent</w>
|
350 |
+
ur ing</w>
|
351 |
+
ol l
|
352 |
+
ĠI n
|
353 |
+
Ġth is</w>
|
354 |
+
Ġb ec
|
355 |
+
Ġcom m
|
356 |
+
Ġd is
|
357 |
+
at er</w>
|
358 |
+
ag e</w>
|
359 |
+
Ġa pp
|
360 |
+
ou s</w>
|
361 |
+
e y</w>
|
362 |
+
i l</w>
|
363 |
+
p er
|
364 |
+
ĠA l
|
365 |
+
ion al</w>
|
366 |
+
l ud
|
367 |
+
el y</w>
|
368 |
+
t t
|
369 |
+
il e</w>
|
370 |
+
i z
|
371 |
+
Ġ j
|
372 |
+
Ġwh o</w>
|
373 |
+
Ġa g
|
374 |
+
i b
|
375 |
+
Ġthe y</w>
|
376 |
+
f or
|
377 |
+
Ġo v
|
378 |
+
at h
|
379 |
+
e g
|
380 |
+
Ġs c
|
381 |
+
i p
|
382 |
+
Ġ20 1
|
383 |
+
Ġ 3
|
384 |
+
Ġp er
|
385 |
+
or y</w>
|
386 |
+
Ġd es
|
387 |
+
id e</w>
|
388 |
+
Ġs er
|
389 |
+
s e</w>
|
390 |
+
ĠH e</w>
|
391 |
+
la nd</w>
|
392 |
+
at ions</w>
|
393 |
+
r ic
|
394 |
+
i t</w>
|
395 |
+
re s</w>
|
396 |
+
er ed</w>
|
397 |
+
Ġp re
|
398 |
+
ĠS h
|
399 |
+
an ce</w>
|
400 |
+
or t</w>
|
401 |
+
an t</w>
|
402 |
+
, @</w>
|
403 |
+
Ġ@ ,@</w>
|
404 |
+
el l</w>
|
405 |
+
Ġ Y
|
406 |
+
n ed</w>
|
407 |
+
el l
|
408 |
+
it e</w>
|
409 |
+
Ġinc lud
|
410 |
+
Ġre p
|
411 |
+
Ġa fter</w>
|
412 |
+
Ġs uc
|
413 |
+
re e</w>
|
414 |
+
an y</w>
|
415 |
+
i m</w>
|
416 |
+
or t
|
417 |
+
Ġ1 8
|
418 |
+
Ġs u
|
419 |
+
ad e</w>
|
420 |
+
ou r</w>
|
421 |
+
ĠU n
|
422 |
+
ĠI t</w>
|
423 |
+
i k
|
424 |
+
ĠM ar
|
425 |
+
em ber</w>
|
426 |
+
Ġ 1</w>
|
427 |
+
e en</w>
|
428 |
+
a nd</w>
|
429 |
+
Ġs ec
|
430 |
+
ic e</w>
|
431 |
+
Ġt ime</w>
|
432 |
+
ĠA n
|
433 |
+
Ġint o</w>
|
434 |
+
Ġf in
|
435 |
+
Ġo ther</w>
|
436 |
+
Ġa tt
|
437 |
+
il l</w>
|
438 |
+
re n
|
439 |
+
ac h
|
440 |
+
as s
|
441 |
+
er al</w>
|
442 |
+
es e</w>
|
443 |
+
s h
|
444 |
+
al s</w>
|
445 |
+
it ion</w>
|
446 |
+
oug h</w>
|
447 |
+
l es</w>
|
448 |
+
am p
|
449 |
+
Ġw ould</w>
|
450 |
+
Ġm ore</w>
|
451 |
+
ro ug
|
452 |
+
ri b
|
453 |
+
er y</w>
|
454 |
+
ac e</w>
|
455 |
+
Ġ A</w>
|
456 |
+
Ġpla y
|
457 |
+
it ed</w>
|
458 |
+
k ed</w>
|
459 |
+
is t</w>
|
460 |
+
i ed</w>
|
461 |
+
Ġ 2</w>
|
462 |
+
as ed</w>
|
463 |
+
ing s</w>
|
464 |
+
an g
|
465 |
+
a m</w>
|
466 |
+
i p</w>
|
467 |
+
Ġb o
|
468 |
+
ab le</w>
|
469 |
+
t y</w>
|
470 |
+
Ġch ar
|
471 |
+
Ġc ent
|
472 |
+
et w
|
473 |
+
at es</w>
|
474 |
+
ro p
|
475 |
+
Ġ I</w>
|
476 |
+
u nd</w>
|
477 |
+
ĠA m
|
478 |
+
c es</w>
|
479 |
+
o in
|
480 |
+
Ġin ter
|
481 |
+
u p
|
482 |
+
c t
|
483 |
+
on e</w>
|
484 |
+
Ġt ra
|
485 |
+
an t
|
486 |
+
ec t
|
487 |
+
Ġal l</w>
|
488 |
+
e f
|
489 |
+
Ġcon s
|
490 |
+
ub l
|
491 |
+
n ing</w>
|
492 |
+
an s</w>
|
493 |
+
Ġf e
|
494 |
+
us t</w>
|
495 |
+
Ġ 0
|
496 |
+
Ġre m
|
497 |
+
as e</w>
|
498 |
+
on g
|
499 |
+
Ġwh en</w>
|
500 |
+
e b
|
501 |
+
ĠW h
|
502 |
+
Ġe ar
|
503 |
+
ev er</w>
|
504 |
+
Ġov er</w>
|
505 |
+
Ġk n
|
506 |
+
a us
|
507 |
+
Ġp os
|
508 |
+
a d</w>
|
509 |
+
er m
|
510 |
+
Ġsh e</w>
|
511 |
+
Ġ ra
|
512 |
+
Ġd uring</w>
|
513 |
+
as on</w>
|
514 |
+
v i
|
515 |
+
Ġex p
|
516 |
+
Ġl ea
|
517 |
+
Ġ el
|
518 |
+
Ġ 4
|
519 |
+
Ġon ly</w>
|
520 |
+
o nd</w>
|
521 |
+
Ġd ec
|
522 |
+
Ġac c
|
523 |
+
Ġo ff
|
524 |
+
is s
|
525 |
+
Ġf l
|
526 |
+
ĠE n
|
527 |
+
o t</w>
|
528 |
+
en s
|
529 |
+
os e</w>
|
530 |
+
ak e</w>
|
531 |
+
o m</w>
|
532 |
+
Ġs ev
|
533 |
+
ac h</w>
|
534 |
+
etw een</w>
|
535 |
+
er n
|
536 |
+
Ġ 3</w>
|
537 |
+
Ġp r
|
538 |
+
Ġg ro
|
539 |
+
r uc
|
540 |
+
Ġd i
|
541 |
+
Ġ19 9
|
542 |
+
ĠA r
|
543 |
+
Ġg ame</w>
|
544 |
+
Ġh im</w>
|
545 |
+
oo k</w>
|
546 |
+
Ġ up</w>
|
547 |
+
Ġab out</w>
|
548 |
+
Ġre l
|
549 |
+
for m
|
550 |
+
Ġth ree</w>
|
551 |
+
at t
|
552 |
+
ĠC om
|
553 |
+
Ġs a
|
554 |
+
ear s</w>
|
555 |
+
Ġ 5
|
556 |
+
r y</w>
|
557 |
+
Ġi mp
|
558 |
+
Ġm ost</w>
|
559 |
+
f er
|
560 |
+
Ġp res
|
561 |
+
Ġf il
|
562 |
+
Ġb etween</w>
|
563 |
+
Ġbe g
|
564 |
+
p h
|
565 |
+
or s</w>
|
566 |
+
Ġth an</w>
|
567 |
+
Ġrec or
|
568 |
+
o b
|
569 |
+
er ic
|
570 |
+
at ing</w>
|
571 |
+
Ġth roug
|
572 |
+
k ing</w>
|
573 |
+
Ġo ut</w>
|
574 |
+
Ġn um
|
575 |
+
oo d</w>
|
576 |
+
oll ow
|
577 |
+
ac t
|
578 |
+
u il
|
579 |
+
Ġc re
|
580 |
+
ol og
|
581 |
+
at ional</w>
|
582 |
+
Ġpro duc
|
583 |
+
Ġwh ile</w>
|
584 |
+
Ġl ater</w>
|
585 |
+
Ġw rit
|
586 |
+
e x
|
587 |
+
Ġst ar
|
588 |
+
Ġsp ec
|
589 |
+
e e
|
590 |
+
ish ed</w>
|
591 |
+
Ġre g
|
592 |
+
is ion</w>
|
593 |
+
ou th</w>
|
594 |
+
Ġre le
|
595 |
+
Ġa ss
|
596 |
+
Ġse ason</w>
|
597 |
+
Ġm ade</w>
|
598 |
+
il y</w>
|
599 |
+
r u
|
600 |
+
o y
|
601 |
+
t ur
|
602 |
+
t e</w>
|
603 |
+
Ġ qu
|
604 |
+
Ġm ov
|
605 |
+
ur y</w>
|
606 |
+
ĠAm eric
|
607 |
+
em ent</w>
|
608 |
+
c c
|
609 |
+
ou nd</w>
|
610 |
+
Ġl ar
|
611 |
+
Ġfor m
|
612 |
+
ec t</w>
|
613 |
+
Ġde f
|
614 |
+
Ġm us
|
615 |
+
ĠP ar
|
616 |
+
Ġm e
|
617 |
+
Ġs ub
|
618 |
+
w ay</w>
|
619 |
+
o p</w>
|
620 |
+
o h
|
621 |
+
el d</w>
|
622 |
+
i e</w>
|
623 |
+
em p
|
624 |
+
am es</w>
|
625 |
+
er n</w>
|
626 |
+
Ġn or
|
627 |
+
iv ed</w>
|
628 |
+
ev el
|
629 |
+
Ġsuc h</w>
|
630 |
+
ar ds</w>
|
631 |
+
Ġin d
|
632 |
+
ik e</w>
|
633 |
+
Ġg en
|
634 |
+
er t
|
635 |
+
Ġy ear</w>
|
636 |
+
Ġus ed</w>
|
637 |
+
Ġn ew</w>
|
638 |
+
Ġ 5</w>
|
639 |
+
Ġal b
|
640 |
+
s p
|
641 |
+
y p
|
642 |
+
Ġwit h
|
643 |
+
Ġwh ere</w>
|
644 |
+
ic s</w>
|
645 |
+
ĠTh is</w>
|
646 |
+
Ġthe m</w>
|
647 |
+
w n</w>
|
tokenizer/special_tokens_map.json
ADDED
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"bos_token": {
|
3 |
+
"content": "<|startoftext|>",
|
4 |
+
"lstrip": false,
|
5 |
+
"normalized": true,
|
6 |
+
"rstrip": false,
|
7 |
+
"single_word": false
|
8 |
+
},
|
9 |
+
"eos_token": {
|
10 |
+
"content": "<|endoftext|>",
|
11 |
+
"lstrip": false,
|
12 |
+
"normalized": false,
|
13 |
+
"rstrip": false,
|
14 |
+
"single_word": false
|
15 |
+
},
|
16 |
+
"pad_token": {
|
17 |
+
"content": "<|endoftext|>",
|
18 |
+
"lstrip": false,
|
19 |
+
"normalized": false,
|
20 |
+
"rstrip": false,
|
21 |
+
"single_word": false
|
22 |
+
},
|
23 |
+
"unk_token": {
|
24 |
+
"content": "<|endoftext|>",
|
25 |
+
"lstrip": false,
|
26 |
+
"normalized": false,
|
27 |
+
"rstrip": false,
|
28 |
+
"single_word": false
|
29 |
+
}
|
30 |
+
}
|
tokenizer/tokenizer_config.json
ADDED
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"add_prefix_space": false,
|
3 |
+
"added_tokens_decoder": {
|
4 |
+
"0": {
|
5 |
+
"content": "<|startoftext|>",
|
6 |
+
"lstrip": false,
|
7 |
+
"normalized": true,
|
8 |
+
"rstrip": false,
|
9 |
+
"single_word": false,
|
10 |
+
"special": true
|
11 |
+
},
|
12 |
+
"1": {
|
13 |
+
"content": "<|endoftext|>",
|
14 |
+
"lstrip": false,
|
15 |
+
"normalized": false,
|
16 |
+
"rstrip": false,
|
17 |
+
"single_word": false,
|
18 |
+
"special": true
|
19 |
+
}
|
20 |
+
},
|
21 |
+
"bos_token": "<|startoftext|>",
|
22 |
+
"clean_up_tokenization_spaces": true,
|
23 |
+
"do_lower_case": true,
|
24 |
+
"eos_token": "<|endoftext|>",
|
25 |
+
"errors": "replace",
|
26 |
+
"model_max_length": 77,
|
27 |
+
"pad_token": "<|endoftext|>",
|
28 |
+
"tokenizer_class": "CLIPTokenizer",
|
29 |
+
"unk_token": "<|endoftext|>"
|
30 |
+
}
|
tokenizer/vocab.json
ADDED
@@ -0,0 +1,1002 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"!": 2,
|
3 |
+
"!</w>": 345,
|
4 |
+
"\"": 3,
|
5 |
+
"\"</w>": 344,
|
6 |
+
"#": 4,
|
7 |
+
"#</w>": 325,
|
8 |
+
"$": 5,
|
9 |
+
"$</w>": 348,
|
10 |
+
"%": 6,
|
11 |
+
"%</w>": 351,
|
12 |
+
"&": 7,
|
13 |
+
"&</w>": 352,
|
14 |
+
"'": 8,
|
15 |
+
"'</w>": 296,
|
16 |
+
"(": 9,
|
17 |
+
"(</w>": 318,
|
18 |
+
")": 10,
|
19 |
+
")</w>": 330,
|
20 |
+
"*": 11,
|
21 |
+
"*</w>": 327,
|
22 |
+
"+": 12,
|
23 |
+
"+</w>": 341,
|
24 |
+
",": 13,
|
25 |
+
",</w>": 279,
|
26 |
+
",@</w>": 754,
|
27 |
+
"-": 14,
|
28 |
+
"-</w>": 276,
|
29 |
+
"-@</w>": 439,
|
30 |
+
".": 15,
|
31 |
+
".</w>": 253,
|
32 |
+
".@</w>": 695,
|
33 |
+
"/": 16,
|
34 |
+
"/</w>": 350,
|
35 |
+
"0": 17,
|
36 |
+
"00</w>": 647,
|
37 |
+
"0</w>": 216,
|
38 |
+
"1": 18,
|
39 |
+
"1</w>": 222,
|
40 |
+
"2": 19,
|
41 |
+
"2</w>": 231,
|
42 |
+
"3": 20,
|
43 |
+
"3</w>": 243,
|
44 |
+
"4": 21,
|
45 |
+
"4</w>": 233,
|
46 |
+
"5": 22,
|
47 |
+
"5</w>": 240,
|
48 |
+
"6": 23,
|
49 |
+
"6</w>": 226,
|
50 |
+
"7": 24,
|
51 |
+
"7</w>": 215,
|
52 |
+
"8": 25,
|
53 |
+
"8</w>": 236,
|
54 |
+
"9": 26,
|
55 |
+
"9</w>": 242,
|
56 |
+
":": 27,
|
57 |
+
":</w>": 353,
|
58 |
+
";": 28,
|
59 |
+
";</w>": 317,
|
60 |
+
"<": 29,
|
61 |
+
"<</w>": 340,
|
62 |
+
"<|endoftext|>": 1,
|
63 |
+
"<|startoftext|>": 0,
|
64 |
+
"=": 30,
|
65 |
+
"=</w>": 342,
|
66 |
+
">": 31,
|
67 |
+
"></w>": 300,
|
68 |
+
"?": 32,
|
69 |
+
"?</w>": 346,
|
70 |
+
"@": 33,
|
71 |
+
"@</w>": 320,
|
72 |
+
"A": 34,
|
73 |
+
"A</w>": 227,
|
74 |
+
"B": 35,
|
75 |
+
"B</w>": 258,
|
76 |
+
"C": 36,
|
77 |
+
"C</w>": 239,
|
78 |
+
"D": 37,
|
79 |
+
"D</w>": 255,
|
80 |
+
"E": 38,
|
81 |
+
"E</w>": 246,
|
82 |
+
"F": 39,
|
83 |
+
"F</w>": 213,
|
84 |
+
"G": 40,
|
85 |
+
"G</w>": 283,
|
86 |
+
"H": 41,
|
87 |
+
"H</w>": 219,
|
88 |
+
"I": 42,
|
89 |
+
"I</w>": 237,
|
90 |
+
"J": 43,
|
91 |
+
"J</w>": 251,
|
92 |
+
"K": 44,
|
93 |
+
"K</w>": 254,
|
94 |
+
"L": 45,
|
95 |
+
"L</w>": 218,
|
96 |
+
"M": 46,
|
97 |
+
"M</w>": 234,
|
98 |
+
"N": 47,
|
99 |
+
"N</w>": 238,
|
100 |
+
"O": 48,
|
101 |
+
"O</w>": 265,
|
102 |
+
"P": 49,
|
103 |
+
"P</w>": 245,
|
104 |
+
"Q": 50,
|
105 |
+
"Q</w>": 309,
|
106 |
+
"R": 51,
|
107 |
+
"R</w>": 264,
|
108 |
+
"S": 52,
|
109 |
+
"S</w>": 230,
|
110 |
+
"T": 53,
|
111 |
+
"T</w>": 235,
|
112 |
+
"U": 54,
|
113 |
+
"U</w>": 268,
|
114 |
+
"V": 55,
|
115 |
+
"V</w>": 248,
|
116 |
+
"W": 56,
|
117 |
+
"W</w>": 274,
|
118 |
+
"X": 57,
|
119 |
+
"X</w>": 263,
|
120 |
+
"Y": 58,
|
121 |
+
"Y</w>": 310,
|
122 |
+
"Z": 59,
|
123 |
+
"Z</w>": 207,
|
124 |
+
"[": 60,
|
125 |
+
"[</w>": 270,
|
126 |
+
"\\": 61,
|
127 |
+
"\\</w>": 338,
|
128 |
+
"]": 62,
|
129 |
+
"]</w>": 289,
|
130 |
+
"^": 63,
|
131 |
+
"^</w>": 331,
|
132 |
+
"_": 64,
|
133 |
+
"_</w>": 334,
|
134 |
+
"`": 65,
|
135 |
+
"`</w>": 347,
|
136 |
+
"a": 66,
|
137 |
+
"a</w>": 197,
|
138 |
+
"ab": 555,
|
139 |
+
"able</w>": 820,
|
140 |
+
"ac": 420,
|
141 |
+
"ace</w>": 806,
|
142 |
+
"ach": 791,
|
143 |
+
"ach</w>": 885,
|
144 |
+
"ack</w>": 670,
|
145 |
+
"act": 929,
|
146 |
+
"ad": 508,
|
147 |
+
"ad</w>": 860,
|
148 |
+
"ade</w>": 771,
|
149 |
+
"ag": 511,
|
150 |
+
"age</w>": 710,
|
151 |
+
"ain": 568,
|
152 |
+
"ain</w>": 675,
|
153 |
+
"ak": 577,
|
154 |
+
"ake</w>": 882,
|
155 |
+
"al": 397,
|
156 |
+
"al</w>": 405,
|
157 |
+
"all": 664,
|
158 |
+
"all</w>": 658,
|
159 |
+
"ally</w>": 588,
|
160 |
+
"als</w>": 796,
|
161 |
+
"am": 426,
|
162 |
+
"am</w>": 817,
|
163 |
+
"ame</w>": 552,
|
164 |
+
"ames</w>": 976,
|
165 |
+
"amp": 800,
|
166 |
+
"an": 384,
|
167 |
+
"an</w>": 425,
|
168 |
+
"ance</w>": 751,
|
169 |
+
"and": 609,
|
170 |
+
"and</w>": 780,
|
171 |
+
"ang": 816,
|
172 |
+
"ans</w>": 844,
|
173 |
+
"ant": 837,
|
174 |
+
"ant</w>": 753,
|
175 |
+
"any</w>": 766,
|
176 |
+
"ap": 586,
|
177 |
+
"ar": 376,
|
178 |
+
"ar</w>": 579,
|
179 |
+
"ard</w>": 649,
|
180 |
+
"ards</w>": 982,
|
181 |
+
"ary</w>": 611,
|
182 |
+
"as": 416,
|
183 |
+
"as</w>": 404,
|
184 |
+
"ase</w>": 849,
|
185 |
+
"ased</w>": 814,
|
186 |
+
"ason</w>": 865,
|
187 |
+
"ass": 792,
|
188 |
+
"ast</w>": 661,
|
189 |
+
"at": 372,
|
190 |
+
"at</w>": 434,
|
191 |
+
"ate</w>": 541,
|
192 |
+
"ated</w>": 543,
|
193 |
+
"ater</w>": 709,
|
194 |
+
"ates</w>": 825,
|
195 |
+
"ath": 730,
|
196 |
+
"ating</w>": 922,
|
197 |
+
"ation</w>": 497,
|
198 |
+
"ational</w>": 933,
|
199 |
+
"ations</w>": 744,
|
200 |
+
"att": 903,
|
201 |
+
"aus": 858,
|
202 |
+
"av": 681,
|
203 |
+
"ay": 684,
|
204 |
+
"ay</w>": 523,
|
205 |
+
"b": 67,
|
206 |
+
"b</w>": 212,
|
207 |
+
"ber</w>": 593,
|
208 |
+
"c": 68,
|
209 |
+
"c</w>": 224,
|
210 |
+
"cc": 960,
|
211 |
+
"ce</w>": 496,
|
212 |
+
"ces</w>": 830,
|
213 |
+
"ch": 520,
|
214 |
+
"ch</w>": 603,
|
215 |
+
"ct": 834,
|
216 |
+
"d": 69,
|
217 |
+
"d</w>": 196,
|
218 |
+
"ded</w>": 665,
|
219 |
+
"der</w>": 690,
|
220 |
+
"ding</w>": 633,
|
221 |
+
"ds</w>": 530,
|
222 |
+
"duc": 671,
|
223 |
+
"e": 70,
|
224 |
+
"e</w>": 195,
|
225 |
+
"ea": 471,
|
226 |
+
"ear": 596,
|
227 |
+
"ear</w>": 669,
|
228 |
+
"ears</w>": 906,
|
229 |
+
"eb": 852,
|
230 |
+
"ec": 418,
|
231 |
+
"ect": 838,
|
232 |
+
"ect</w>": 964,
|
233 |
+
"ed": 563,
|
234 |
+
"ed</w>": 362,
|
235 |
+
"ee": 941,
|
236 |
+
"een</w>": 779,
|
237 |
+
"ef": 840,
|
238 |
+
"eg": 731,
|
239 |
+
"el": 407,
|
240 |
+
"el</w>": 610,
|
241 |
+
"eld</w>": 973,
|
242 |
+
"ell": 759,
|
243 |
+
"ell</w>": 756,
|
244 |
+
"ely</w>": 719,
|
245 |
+
"em": 455,
|
246 |
+
"ember</w>": 777,
|
247 |
+
"ement</w>": 959,
|
248 |
+
"emp": 975,
|
249 |
+
"en": 375,
|
250 |
+
"en</w>": 427,
|
251 |
+
"ence</w>": 685,
|
252 |
+
"ens": 880,
|
253 |
+
"ent": 478,
|
254 |
+
"ent</w>": 468,
|
255 |
+
"ents</w>": 674,
|
256 |
+
"ep": 545,
|
257 |
+
"er": 364,
|
258 |
+
"er</w>": 374,
|
259 |
+
"eral</w>": 793,
|
260 |
+
"ere</w>": 481,
|
261 |
+
"ered</w>": 748,
|
262 |
+
"eric": 921,
|
263 |
+
"erm": 861,
|
264 |
+
"ern": 887,
|
265 |
+
"ern</w>": 977,
|
266 |
+
"ers": 598,
|
267 |
+
"ers</w>": 486,
|
268 |
+
"ert": 986,
|
269 |
+
"ery</w>": 805,
|
270 |
+
"es": 402,
|
271 |
+
"es</w>": 388,
|
272 |
+
"ese</w>": 794,
|
273 |
+
"ess": 678,
|
274 |
+
"ess</w>": 693,
|
275 |
+
"est": 606,
|
276 |
+
"est</w>": 584,
|
277 |
+
"et": 460,
|
278 |
+
"et</w>": 594,
|
279 |
+
"etw": 824,
|
280 |
+
"etween</w>": 886,
|
281 |
+
"ev": 493,
|
282 |
+
"evel": 980,
|
283 |
+
"ever</w>": 855,
|
284 |
+
"ew": 687,
|
285 |
+
"ew</w>": 612,
|
286 |
+
"ex": 938,
|
287 |
+
"ey</w>": 713,
|
288 |
+
"f": 71,
|
289 |
+
"f</w>": 209,
|
290 |
+
"fer": 911,
|
291 |
+
"ff": 587,
|
292 |
+
"for": 728,
|
293 |
+
"form": 901,
|
294 |
+
"fter</w>": 634,
|
295 |
+
"g": 72,
|
296 |
+
"g</w>": 214,
|
297 |
+
"ge</w>": 592,
|
298 |
+
"h": 73,
|
299 |
+
"h</w>": 203,
|
300 |
+
"i": 74,
|
301 |
+
"i</w>": 205,
|
302 |
+
"ia</w>": 605,
|
303 |
+
"ial</w>": 672,
|
304 |
+
"ian</w>": 638,
|
305 |
+
"ib": 726,
|
306 |
+
"ic": 395,
|
307 |
+
"ic</w>": 510,
|
308 |
+
"ical</w>": 625,
|
309 |
+
"ice</w>": 782,
|
310 |
+
"ich</w>": 561,
|
311 |
+
"ics</w>": 996,
|
312 |
+
"id": 463,
|
313 |
+
"id</w>": 613,
|
314 |
+
"ide</w>": 739,
|
315 |
+
"ie</w>": 974,
|
316 |
+
"ied</w>": 812,
|
317 |
+
"ies</w>": 516,
|
318 |
+
"if": 524,
|
319 |
+
"ig": 444,
|
320 |
+
"igh": 537,
|
321 |
+
"ight</w>": 680,
|
322 |
+
"ik": 775,
|
323 |
+
"ike</w>": 984,
|
324 |
+
"il": 406,
|
325 |
+
"il</w>": 714,
|
326 |
+
"ile</w>": 721,
|
327 |
+
"ill": 608,
|
328 |
+
"ill</w>": 789,
|
329 |
+
"ily</w>": 950,
|
330 |
+
"im": 469,
|
331 |
+
"im</w>": 767,
|
332 |
+
"ime</w>": 691,
|
333 |
+
"in": 358,
|
334 |
+
"in</w>": 501,
|
335 |
+
"ine</w>": 607,
|
336 |
+
"ing": 557,
|
337 |
+
"ing</w>": 383,
|
338 |
+
"ings</w>": 815,
|
339 |
+
"ion": 472,
|
340 |
+
"ion</w>": 408,
|
341 |
+
"ional</w>": 717,
|
342 |
+
"ions</w>": 540,
|
343 |
+
"ip": 733,
|
344 |
+
"ip</w>": 818,
|
345 |
+
"ir": 453,
|
346 |
+
"ir</w>": 554,
|
347 |
+
"is": 393,
|
348 |
+
"is</w>": 441,
|
349 |
+
"ish": 694,
|
350 |
+
"ish</w>": 654,
|
351 |
+
"ished</w>": 942,
|
352 |
+
"ision</w>": 944,
|
353 |
+
"iss": 876,
|
354 |
+
"ist": 550,
|
355 |
+
"ist</w>": 811,
|
356 |
+
"it": 378,
|
357 |
+
"it</w>": 746,
|
358 |
+
"ite</w>": 760,
|
359 |
+
"ited</w>": 809,
|
360 |
+
"ition</w>": 797,
|
361 |
+
"ity</w>": 542,
|
362 |
+
"iv": 435,
|
363 |
+
"ive</w>": 549,
|
364 |
+
"ived</w>": 979,
|
365 |
+
"iz": 722,
|
366 |
+
"j": 75,
|
367 |
+
"j</w>": 288,
|
368 |
+
"k": 76,
|
369 |
+
"k</w>": 210,
|
370 |
+
"ked</w>": 810,
|
371 |
+
"king</w>": 924,
|
372 |
+
"ks</w>": 692,
|
373 |
+
"l": 77,
|
374 |
+
"l</w>": 201,
|
375 |
+
"la": 467,
|
376 |
+
"land</w>": 743,
|
377 |
+
"ld</w>": 559,
|
378 |
+
"le": 536,
|
379 |
+
"le</w>": 465,
|
380 |
+
"les</w>": 799,
|
381 |
+
"lud": 718,
|
382 |
+
"ly</w>": 433,
|
383 |
+
"m": 78,
|
384 |
+
"m</w>": 202,
|
385 |
+
"ment</w>": 701,
|
386 |
+
"mp": 651,
|
387 |
+
"n": 79,
|
388 |
+
"n</w>": 199,
|
389 |
+
"nd</w>": 369,
|
390 |
+
"ned</w>": 758,
|
391 |
+
"ning</w>": 843,
|
392 |
+
"o": 80,
|
393 |
+
"o</w>": 198,
|
394 |
+
"ob": 920,
|
395 |
+
"oc": 534,
|
396 |
+
"od": 575,
|
397 |
+
"og": 604,
|
398 |
+
"oh": 972,
|
399 |
+
"oin": 831,
|
400 |
+
"ol": 428,
|
401 |
+
"oll": 703,
|
402 |
+
"ollow": 928,
|
403 |
+
"olog": 932,
|
404 |
+
"om": 419,
|
405 |
+
"om</w>": 883,
|
406 |
+
"ome</w>": 663,
|
407 |
+
"on": 382,
|
408 |
+
"on</w>": 390,
|
409 |
+
"ond</w>": 872,
|
410 |
+
"one</w>": 835,
|
411 |
+
"ong": 850,
|
412 |
+
"ong</w>": 582,
|
413 |
+
"oo": 517,
|
414 |
+
"ood</w>": 927,
|
415 |
+
"ook</w>": 897,
|
416 |
+
"op": 531,
|
417 |
+
"op</w>": 971,
|
418 |
+
"or": 377,
|
419 |
+
"or</w>": 424,
|
420 |
+
"ore</w>": 571,
|
421 |
+
"ors</w>": 917,
|
422 |
+
"ort": 768,
|
423 |
+
"ort</w>": 752,
|
424 |
+
"ory</w>": 737,
|
425 |
+
"os": 447,
|
426 |
+
"ose</w>": 881,
|
427 |
+
"ost</w>": 646,
|
428 |
+
"ot": 600,
|
429 |
+
"ot</w>": 879,
|
430 |
+
"ou": 392,
|
431 |
+
"oug": 659,
|
432 |
+
"ough</w>": 798,
|
433 |
+
"ould</w>": 640,
|
434 |
+
"oun": 553,
|
435 |
+
"ound</w>": 961,
|
436 |
+
"our": 648,
|
437 |
+
"our</w>": 772,
|
438 |
+
"ous</w>": 712,
|
439 |
+
"out</w>": 683,
|
440 |
+
"outh</w>": 945,
|
441 |
+
"ov": 515,
|
442 |
+
"ow": 461,
|
443 |
+
"ow</w>": 666,
|
444 |
+
"own</w>": 657,
|
445 |
+
"oy": 952,
|
446 |
+
"p": 81,
|
447 |
+
"p</w>": 217,
|
448 |
+
"per": 715,
|
449 |
+
"ph": 916,
|
450 |
+
"pp": 518,
|
451 |
+
"q": 82,
|
452 |
+
"q</w>": 280,
|
453 |
+
"qu": 546,
|
454 |
+
"r": 83,
|
455 |
+
"r</w>": 204,
|
456 |
+
"ra": 457,
|
457 |
+
"ran": 624,
|
458 |
+
"re": 367,
|
459 |
+
"ree</w>": 765,
|
460 |
+
"ren": 790,
|
461 |
+
"res": 572,
|
462 |
+
"res</w>": 747,
|
463 |
+
"ri": 487,
|
464 |
+
"rib": 804,
|
465 |
+
"ric": 745,
|
466 |
+
"rit": 589,
|
467 |
+
"ro": 385,
|
468 |
+
"rom</w>": 498,
|
469 |
+
"rop": 826,
|
470 |
+
"roug": 803,
|
471 |
+
"ru": 951,
|
472 |
+
"ruc": 891,
|
473 |
+
"ry</w>": 908,
|
474 |
+
"s": 84,
|
475 |
+
"s</w>": 206,
|
476 |
+
"se</w>": 741,
|
477 |
+
"sh": 795,
|
478 |
+
"so</w>": 630,
|
479 |
+
"sp": 992,
|
480 |
+
"ss": 673,
|
481 |
+
"st": 519,
|
482 |
+
"st</w>": 528,
|
483 |
+
"t": 85,
|
484 |
+
"t</w>": 208,
|
485 |
+
"te</w>": 954,
|
486 |
+
"ted</w>": 489,
|
487 |
+
"ter": 535,
|
488 |
+
"ter</w>": 505,
|
489 |
+
"th": 449,
|
490 |
+
"th</w>": 488,
|
491 |
+
"ther</w>": 576,
|
492 |
+
"ting</w>": 676,
|
493 |
+
"tion</w>": 570,
|
494 |
+
"tr": 619,
|
495 |
+
"ts</w>": 436,
|
496 |
+
"tt": 720,
|
497 |
+
"tur": 953,
|
498 |
+
"ty</w>": 821,
|
499 |
+
"u": 86,
|
500 |
+
"u</w>": 229,
|
501 |
+
"ub": 591,
|
502 |
+
"ubl": 842,
|
503 |
+
"uc": 490,
|
504 |
+
"ud": 538,
|
505 |
+
"ue</w>": 652,
|
506 |
+
"ug": 560,
|
507 |
+
"uil": 930,
|
508 |
+
"ul": 494,
|
509 |
+
"um": 532,
|
510 |
+
"um</w>": 644,
|
511 |
+
"un": 448,
|
512 |
+
"und</w>": 828,
|
513 |
+
"up": 833,
|
514 |
+
"up</w>": 700,
|
515 |
+
"ur": 413,
|
516 |
+
"ure</w>": 635,
|
517 |
+
"uring</w>": 702,
|
518 |
+
"ury</w>": 957,
|
519 |
+
"us": 438,
|
520 |
+
"us</w>": 622,
|
521 |
+
"ust</w>": 846,
|
522 |
+
"ut": 529,
|
523 |
+
"ut</w>": 527,
|
524 |
+
"v": 87,
|
525 |
+
"v</w>": 232,
|
526 |
+
"ve</w>": 567,
|
527 |
+
"vi": 866,
|
528 |
+
"w": 88,
|
529 |
+
"w</w>": 250,
|
530 |
+
"way</w>": 970,
|
531 |
+
"wn</w>": 999,
|
532 |
+
"x": 89,
|
533 |
+
"x</w>": 269,
|
534 |
+
"y": 90,
|
535 |
+
"y</w>": 211,
|
536 |
+
"yp": 993,
|
537 |
+
"z": 91,
|
538 |
+
"z</w>": 228,
|
539 |
+
"|": 92,
|
540 |
+
"|</w>": 304,
|
541 |
+
"}": 93,
|
542 |
+
"}</w>": 336,
|
543 |
+
"~": 94,
|
544 |
+
"~</w>": 343,
|
545 |
+
"¡": 95,
|
546 |
+
"¡</w>": 220,
|
547 |
+
"¢": 96,
|
548 |
+
"¢</w>": 306,
|
549 |
+
"£": 97,
|
550 |
+
"£</w>": 323,
|
551 |
+
"¤": 98,
|
552 |
+
"¤</w>": 292,
|
553 |
+
"¥": 99,
|
554 |
+
"¥</w>": 339,
|
555 |
+
"¦": 100,
|
556 |
+
"¦</w>": 303,
|
557 |
+
"§": 101,
|
558 |
+
"§</w>": 275,
|
559 |
+
"¨": 102,
|
560 |
+
"¨</w>": 282,
|
561 |
+
"©": 103,
|
562 |
+
"©</w>": 259,
|
563 |
+
"ª": 104,
|
564 |
+
"ª</w>": 286,
|
565 |
+
"«": 105,
|
566 |
+
"«</w>": 266,
|
567 |
+
"¬": 106,
|
568 |
+
"¬</w>": 319,
|
569 |
+
"®": 107,
|
570 |
+
"®</w>": 329,
|
571 |
+
"¯": 108,
|
572 |
+
"¯</w>": 287,
|
573 |
+
"°": 109,
|
574 |
+
"°</w>": 298,
|
575 |
+
"±": 110,
|
576 |
+
"±</w>": 200,
|
577 |
+
"²": 111,
|
578 |
+
"²</w>": 284,
|
579 |
+
"³": 112,
|
580 |
+
"³</w>": 272,
|
581 |
+
"´": 113,
|
582 |
+
"´</w>": 307,
|
583 |
+
"µ": 114,
|
584 |
+
"µ</w>": 261,
|
585 |
+
"¶": 115,
|
586 |
+
"¶</w>": 301,
|
587 |
+
"·": 116,
|
588 |
+
"·</w>": 326,
|
589 |
+
"¸": 117,
|
590 |
+
"¸</w>": 257,
|
591 |
+
"¹": 118,
|
592 |
+
"¹</w>": 241,
|
593 |
+
"º": 119,
|
594 |
+
"º</w>": 260,
|
595 |
+
"»": 120,
|
596 |
+
"»</w>": 247,
|
597 |
+
"¼": 121,
|
598 |
+
"¼</w>": 305,
|
599 |
+
"½": 122,
|
600 |
+
"½</w>": 294,
|
601 |
+
"¾": 123,
|
602 |
+
"¾</w>": 316,
|
603 |
+
"¿": 124,
|
604 |
+
"¿</w>": 271,
|
605 |
+
"Â": 125,
|
606 |
+
"Ã": 126,
|
607 |
+
"Ä": 127,
|
608 |
+
"Å": 128,
|
609 |
+
"Æ": 129,
|
610 |
+
"Ç": 130,
|
611 |
+
"È": 131,
|
612 |
+
"É": 132,
|
613 |
+
"Ê": 133,
|
614 |
+
"Ë": 134,
|
615 |
+
"Ì": 135,
|
616 |
+
"Í": 136,
|
617 |
+
"Î": 137,
|
618 |
+
"Ï": 138,
|
619 |
+
"Ð": 139,
|
620 |
+
"Ñ": 140,
|
621 |
+
"Ö": 141,
|
622 |
+
"×": 142,
|
623 |
+
"Ø": 143,
|
624 |
+
"Ù": 144,
|
625 |
+
"Ü": 145,
|
626 |
+
"à": 146,
|
627 |
+
"á": 147,
|
628 |
+
"â": 148,
|
629 |
+
"ã": 149,
|
630 |
+
"ä": 150,
|
631 |
+
"å": 151,
|
632 |
+
"æ": 152,
|
633 |
+
"ç": 153,
|
634 |
+
"è": 154,
|
635 |
+
"é": 155,
|
636 |
+
"ë": 156,
|
637 |
+
"ì": 157,
|
638 |
+
"ï": 158,
|
639 |
+
"Ċ": 159,
|
640 |
+
"Ċ</w>": 349,
|
641 |
+
"Ġ": 160,
|
642 |
+
"Ġ\"</w>": 401,
|
643 |
+
"Ġ'</w>": 431,
|
644 |
+
"Ġ(</w>": 475,
|
645 |
+
"Ġ)</w>": 474,
|
646 |
+
"Ġ,</w>": 360,
|
647 |
+
"Ġ.</w>": 365,
|
648 |
+
"Ġ0": 847,
|
649 |
+
"Ġ1": 411,
|
650 |
+
"Ġ18": 769,
|
651 |
+
"Ġ19": 492,
|
652 |
+
"Ġ199": 893,
|
653 |
+
"Ġ1</w>": 778,
|
654 |
+
"Ġ2": 462,
|
655 |
+
"Ġ20": 522,
|
656 |
+
"Ġ200": 620,
|
657 |
+
"Ġ201": 734,
|
658 |
+
"Ġ2</w>": 813,
|
659 |
+
"Ġ3": 735,
|
660 |
+
"Ġ3</w>": 888,
|
661 |
+
"Ġ4": 870,
|
662 |
+
"Ġ5": 907,
|
663 |
+
"Ġ5</w>": 990,
|
664 |
+
"Ġ:</w>": 637,
|
665 |
+
"Ġ;</w>": 615,
|
666 |
+
"Ġ</w>": 333,
|
667 |
+
"Ġ=</w>": 399,
|
668 |
+
"Ġ@": 417,
|
669 |
+
"Ġ@,@</w>": 755,
|
670 |
+
"Ġ@-@</w>": 440,
|
671 |
+
"Ġ@.@</w>": 696,
|
672 |
+
"ĠA": 409,
|
673 |
+
"ĠA</w>": 807,
|
674 |
+
"ĠAl": 716,
|
675 |
+
"ĠAm": 829,
|
676 |
+
"ĠAmeric": 958,
|
677 |
+
"ĠAn": 784,
|
678 |
+
"ĠAr": 894,
|
679 |
+
"ĠB": 432,
|
680 |
+
"ĠC": 410,
|
681 |
+
"ĠCh": 581,
|
682 |
+
"ĠCom": 904,
|
683 |
+
"ĠD": 464,
|
684 |
+
"ĠE": 500,
|
685 |
+
"ĠEn": 878,
|
686 |
+
"ĠF": 470,
|
687 |
+
"ĠG": 482,
|
688 |
+
"ĠH": 445,
|
689 |
+
"ĠHe</w>": 742,
|
690 |
+
"ĠI": 442,
|
691 |
+
"ĠI</w>": 827,
|
692 |
+
"ĠIn": 704,
|
693 |
+
"ĠIn</w>": 574,
|
694 |
+
"ĠIt</w>": 774,
|
695 |
+
"ĠJ": 491,
|
696 |
+
"ĠK": 548,
|
697 |
+
"ĠL": 484,
|
698 |
+
"ĠM": 423,
|
699 |
+
"ĠMar": 776,
|
700 |
+
"ĠN": 483,
|
701 |
+
"ĠO": 504,
|
702 |
+
"ĠP": 450,
|
703 |
+
"ĠPar": 967,
|
704 |
+
"ĠR": 459,
|
705 |
+
"ĠS": 403,
|
706 |
+
"ĠSh": 750,
|
707 |
+
"ĠSt": 590,
|
708 |
+
"ĠT": 396,
|
709 |
+
"ĠTh": 414,
|
710 |
+
"ĠThe</w>": 437,
|
711 |
+
"ĠThis</w>": 997,
|
712 |
+
"ĠU": 585,
|
713 |
+
"ĠUn": 773,
|
714 |
+
"ĠV": 617,
|
715 |
+
"ĠW": 479,
|
716 |
+
"ĠWh": 853,
|
717 |
+
"ĠY": 757,
|
718 |
+
"Ġa": 356,
|
719 |
+
"Ġa</w>": 394,
|
720 |
+
"Ġab": 653,
|
721 |
+
"Ġabout</w>": 899,
|
722 |
+
"Ġac": 583,
|
723 |
+
"Ġacc": 874,
|
724 |
+
"Ġad": 656,
|
725 |
+
"Ġafter</w>": 763,
|
726 |
+
"Ġag": 725,
|
727 |
+
"Ġal": 476,
|
728 |
+
"Ġalb": 991,
|
729 |
+
"Ġall</w>": 839,
|
730 |
+
"Ġalso</w>": 641,
|
731 |
+
"Ġan": 602,
|
732 |
+
"Ġan</w>": 562,
|
733 |
+
"Ġand</w>": 381,
|
734 |
+
"Ġapp": 711,
|
735 |
+
"Ġar": 507,
|
736 |
+
"Ġare</w>": 601,
|
737 |
+
"Ġas</w>": 454,
|
738 |
+
"Ġass": 947,
|
739 |
+
"Ġat</w>": 514,
|
740 |
+
"Ġatt": 788,
|
741 |
+
"Ġb": 371,
|
742 |
+
"Ġbe": 499,
|
743 |
+
"Ġbe</w>": 595,
|
744 |
+
"Ġbec": 706,
|
745 |
+
"Ġbeen</w>": 686,
|
746 |
+
"Ġbeg": 915,
|
747 |
+
"Ġbetween</w>": 914,
|
748 |
+
"Ġbo": 819,
|
749 |
+
"Ġbut</w>": 623,
|
750 |
+
"Ġby</w>": 473,
|
751 |
+
"Ġc": 368,
|
752 |
+
"Ġcent": 823,
|
753 |
+
"Ġch": 526,
|
754 |
+
"Ġchar": 822,
|
755 |
+
"Ġcl": 689,
|
756 |
+
"Ġcom": 509,
|
757 |
+
"Ġcomm": 707,
|
758 |
+
"Ġcomp": 616,
|
759 |
+
"Ġcon": 477,
|
760 |
+
"Ġcons": 841,
|
761 |
+
"Ġcont": 655,
|
762 |
+
"Ġcre": 931,
|
763 |
+
"Ġd": 387,
|
764 |
+
"Ġde": 627,
|
765 |
+
"Ġdec": 873,
|
766 |
+
"Ġdef": 965,
|
767 |
+
"Ġdes": 738,
|
768 |
+
"Ġdi": 892,
|
769 |
+
"Ġdis": 708,
|
770 |
+
"Ġduring</w>": 864,
|
771 |
+
"Ġe": 421,
|
772 |
+
"Ġear": 854,
|
773 |
+
"Ġel": 869,
|
774 |
+
"Ġen": 662,
|
775 |
+
"Ġev": 682,
|
776 |
+
"Ġex": 539,
|
777 |
+
"Ġexp": 867,
|
778 |
+
"Ġf": 370,
|
779 |
+
"Ġfe": 845,
|
780 |
+
"Ġfil": 913,
|
781 |
+
"Ġfin": 786,
|
782 |
+
"Ġfir": 599,
|
783 |
+
"Ġfirst</w>": 626,
|
784 |
+
"Ġfl": 877,
|
785 |
+
"Ġfor": 614,
|
786 |
+
"Ġfor</w>": 458,
|
787 |
+
"Ġform": 963,
|
788 |
+
"Ġfrom</w>": 503,
|
789 |
+
"Ġg": 430,
|
790 |
+
"Ġgame</w>": 895,
|
791 |
+
"Ġgen": 985,
|
792 |
+
"Ġgro": 890,
|
793 |
+
"Ġh": 380,
|
794 |
+
"Ġha": 485,
|
795 |
+
"Ġhad</w>": 566,
|
796 |
+
"Ġhas</w>": 679,
|
797 |
+
"Ġhave</w>": 667,
|
798 |
+
"Ġhe</w>": 558,
|
799 |
+
"Ġher</w>": 660,
|
800 |
+
"Ġhim</w>": 896,
|
801 |
+
"Ġhis</w>": 512,
|
802 |
+
"Ġi": 366,
|
803 |
+
"Ġimp": 909,
|
804 |
+
"Ġin": 429,
|
805 |
+
"Ġin</w>": 389,
|
806 |
+
"Ġinc": 618,
|
807 |
+
"Ġinclud": 761,
|
808 |
+
"Ġind": 983,
|
809 |
+
"Ġint": 628,
|
810 |
+
"Ġinter": 832,
|
811 |
+
"Ġinto</w>": 785,
|
812 |
+
"Ġis</w>": 480,
|
813 |
+
"Ġit</w>": 533,
|
814 |
+
"Ġits</w>": 642,
|
815 |
+
"Ġj": 723,
|
816 |
+
"Ġk": 564,
|
817 |
+
"Ġkn": 857,
|
818 |
+
"Ġl": 398,
|
819 |
+
"Ġlar": 962,
|
820 |
+
"Ġlater</w>": 936,
|
821 |
+
"Ġlea": 868,
|
822 |
+
"Ġm": 386,
|
823 |
+
"Ġmade</w>": 949,
|
824 |
+
"Ġme": 968,
|
825 |
+
"Ġmore</w>": 802,
|
826 |
+
"Ġmost</w>": 910,
|
827 |
+
"Ġmov": 956,
|
828 |
+
"Ġmus": 966,
|
829 |
+
"Ġn": 415,
|
830 |
+
"Ġnew</w>": 989,
|
831 |
+
"Ġno": 547,
|
832 |
+
"Ġnor": 978,
|
833 |
+
"Ġnot</w>": 632,
|
834 |
+
"Ġnum": 926,
|
835 |
+
"Ġo": 359,
|
836 |
+
"Ġof</w>": 373,
|
837 |
+
"Ġoff": 875,
|
838 |
+
"Ġon": 551,
|
839 |
+
"Ġon</w>": 456,
|
840 |
+
"Ġone</w>": 677,
|
841 |
+
"Ġonly</w>": 871,
|
842 |
+
"Ġor": 699,
|
843 |
+
"Ġor</w>": 645,
|
844 |
+
"Ġother</w>": 787,
|
845 |
+
"Ġout</w>": 925,
|
846 |
+
"Ġov": 729,
|
847 |
+
"Ġover</w>": 856,
|
848 |
+
"Ġp": 379,
|
849 |
+
"Ġpar": 636,
|
850 |
+
"Ġper": 736,
|
851 |
+
"Ġpl": 698,
|
852 |
+
"Ġpla": 697,
|
853 |
+
"Ġplay": 808,
|
854 |
+
"Ġpos": 859,
|
855 |
+
"Ġpr": 889,
|
856 |
+
"Ġpre": 749,
|
857 |
+
"Ġpres": 912,
|
858 |
+
"Ġpro": 506,
|
859 |
+
"Ġproduc": 934,
|
860 |
+
"Ġqu": 955,
|
861 |
+
"Ġr": 521,
|
862 |
+
"Ġra": 863,
|
863 |
+
"Ġre": 400,
|
864 |
+
"Ġrec": 597,
|
865 |
+
"Ġrecor": 919,
|
866 |
+
"Ġreg": 943,
|
867 |
+
"Ġrel": 900,
|
868 |
+
"Ġrele": 946,
|
869 |
+
"Ġrem": 848,
|
870 |
+
"Ġrep": 762,
|
871 |
+
"Ġres": 650,
|
872 |
+
"Ġro": 629,
|
873 |
+
"Ġs": 361,
|
874 |
+
"Ġsa": 905,
|
875 |
+
"Ġsc": 732,
|
876 |
+
"Ġse": 569,
|
877 |
+
"Ġseason</w>": 948,
|
878 |
+
"Ġsec": 781,
|
879 |
+
"Ġser": 740,
|
880 |
+
"Ġsev": 884,
|
881 |
+
"Ġsh": 513,
|
882 |
+
"Ġshe</w>": 862,
|
883 |
+
"Ġsp": 578,
|
884 |
+
"Ġspec": 940,
|
885 |
+
"Ġst": 446,
|
886 |
+
"Ġstar": 939,
|
887 |
+
"Ġsu": 770,
|
888 |
+
"Ġsub": 969,
|
889 |
+
"Ġsuc": 764,
|
890 |
+
"Ġsuch</w>": 981,
|
891 |
+
"Ġt": 354,
|
892 |
+
"Ġth": 355,
|
893 |
+
"Ġthan</w>": 918,
|
894 |
+
"Ġthat</w>": 452,
|
895 |
+
"Ġthe": 502,
|
896 |
+
"Ġthe</w>": 357,
|
897 |
+
"Ġtheir</w>": 621,
|
898 |
+
"Ġthem</w>": 998,
|
899 |
+
"Ġthey</w>": 727,
|
900 |
+
"Ġthis</w>": 705,
|
901 |
+
"Ġthree</w>": 902,
|
902 |
+
"Ġthroug": 923,
|
903 |
+
"Ġtime</w>": 783,
|
904 |
+
"Ġto</w>": 391,
|
905 |
+
"Ġtra": 836,
|
906 |
+
"Ġtw": 639,
|
907 |
+
"Ġtwo</w>": 688,
|
908 |
+
"Ġun": 544,
|
909 |
+
"Ġup</w>": 898,
|
910 |
+
"Ġus": 668,
|
911 |
+
"Ġused</w>": 988,
|
912 |
+
"Ġv": 495,
|
913 |
+
"Ġw": 363,
|
914 |
+
"Ġwas</w>": 422,
|
915 |
+
"Ġwere</w>": 525,
|
916 |
+
"Ġwh": 443,
|
917 |
+
"Ġwhen</w>": 851,
|
918 |
+
"Ġwhere</w>": 995,
|
919 |
+
"Ġwhich</w>": 573,
|
920 |
+
"Ġwhile</w>": 935,
|
921 |
+
"Ġwho</w>": 724,
|
922 |
+
"Ġwit": 451,
|
923 |
+
"Ġwith": 994,
|
924 |
+
"Ġwith</w>": 466,
|
925 |
+
"Ġwor": 643,
|
926 |
+
"Ġwould</w>": 801,
|
927 |
+
"Ġwrit": 937,
|
928 |
+
"Ġy": 580,
|
929 |
+
"Ġyear</w>": 987,
|
930 |
+
"Ġâ": 556,
|
931 |
+
"ĠâĢ": 565,
|
932 |
+
"ĠâĢĵ</w>": 631,
|
933 |
+
"ĠĊ</w>": 412,
|
934 |
+
"Ģ": 161,
|
935 |
+
"Ģ</w>": 223,
|
936 |
+
"ģ": 162,
|
937 |
+
"ģ</w>": 273,
|
938 |
+
"Ĥ": 163,
|
939 |
+
"Ĥ</w>": 262,
|
940 |
+
"ĥ": 164,
|
941 |
+
"ĥ</w>": 337,
|
942 |
+
"Ħ": 165,
|
943 |
+
"Ħ</w>": 278,
|
944 |
+
"ħ": 166,
|
945 |
+
"ħ</w>": 281,
|
946 |
+
"Ĩ": 167,
|
947 |
+
"Ĩ</w>": 308,
|
948 |
+
"ĩ": 168,
|
949 |
+
"ĩ</w>": 225,
|
950 |
+
"Ī": 169,
|
951 |
+
"Ī</w>": 221,
|
952 |
+
"ī": 170,
|
953 |
+
"ī</w>": 244,
|
954 |
+
"Ĭ": 171,
|
955 |
+
"Ĭ</w>": 315,
|
956 |
+
"ĭ": 172,
|
957 |
+
"ĭ</w>": 321,
|
958 |
+
"Į": 173,
|
959 |
+
"Į</w>": 324,
|
960 |
+
"į": 174,
|
961 |
+
"į</w>": 302,
|
962 |
+
"İ": 175,
|
963 |
+
"İ</w>": 249,
|
964 |
+
"ı": 176,
|
965 |
+
"ı</w>": 332,
|
966 |
+
"IJ": 177,
|
967 |
+
"IJ</w>": 295,
|
968 |
+
"ij": 178,
|
969 |
+
"ij</w>": 313,
|
970 |
+
"Ĵ": 179,
|
971 |
+
"Ĵ</w>": 328,
|
972 |
+
"ĵ": 180,
|
973 |
+
"ĵ</w>": 312,
|
974 |
+
"Ķ": 181,
|
975 |
+
"Ķ</w>": 256,
|
976 |
+
"ķ": 182,
|
977 |
+
"ķ</w>": 314,
|
978 |
+
"ĸ": 183,
|
979 |
+
"ĸ</w>": 277,
|
980 |
+
"Ĺ": 184,
|
981 |
+
"Ĺ</w>": 322,
|
982 |
+
"ĺ": 185,
|
983 |
+
"ĺ</w>": 285,
|
984 |
+
"Ļ": 186,
|
985 |
+
"Ļ</w>": 267,
|
986 |
+
"ļ": 187,
|
987 |
+
"ļ</w>": 290,
|
988 |
+
"Ľ": 188,
|
989 |
+
"Ľ</w>": 311,
|
990 |
+
"ľ": 189,
|
991 |
+
"ľ</w>": 299,
|
992 |
+
"Ŀ": 190,
|
993 |
+
"Ŀ</w>": 291,
|
994 |
+
"ŀ": 191,
|
995 |
+
"ŀ</w>": 293,
|
996 |
+
"Ł": 192,
|
997 |
+
"Ł</w>": 335,
|
998 |
+
"ł": 193,
|
999 |
+
"ł</w>": 252,
|
1000 |
+
"Ń": 194,
|
1001 |
+
"Ń</w>": 297
|
1002 |
+
}
|
tokenizer_2/special_tokens_map.json
ADDED
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"additional_special_tokens": [
|
3 |
+
"<extra_id_0>",
|
4 |
+
"<extra_id_1>",
|
5 |
+
"<extra_id_2>",
|
6 |
+
"<extra_id_3>",
|
7 |
+
"<extra_id_4>",
|
8 |
+
"<extra_id_5>",
|
9 |
+
"<extra_id_6>",
|
10 |
+
"<extra_id_7>",
|
11 |
+
"<extra_id_8>",
|
12 |
+
"<extra_id_9>",
|
13 |
+
"<extra_id_10>",
|
14 |
+
"<extra_id_11>",
|
15 |
+
"<extra_id_12>",
|
16 |
+
"<extra_id_13>",
|
17 |
+
"<extra_id_14>",
|
18 |
+
"<extra_id_15>",
|
19 |
+
"<extra_id_16>",
|
20 |
+
"<extra_id_17>",
|
21 |
+
"<extra_id_18>",
|
22 |
+
"<extra_id_19>",
|
23 |
+
"<extra_id_20>",
|
24 |
+
"<extra_id_21>",
|
25 |
+
"<extra_id_22>",
|
26 |
+
"<extra_id_23>",
|
27 |
+
"<extra_id_24>",
|
28 |
+
"<extra_id_25>",
|
29 |
+
"<extra_id_26>",
|
30 |
+
"<extra_id_27>",
|
31 |
+
"<extra_id_28>",
|
32 |
+
"<extra_id_29>",
|
33 |
+
"<extra_id_30>",
|
34 |
+
"<extra_id_31>",
|
35 |
+
"<extra_id_32>",
|
36 |
+
"<extra_id_33>",
|
37 |
+
"<extra_id_34>",
|
38 |
+
"<extra_id_35>",
|
39 |
+
"<extra_id_36>",
|
40 |
+
"<extra_id_37>",
|
41 |
+
"<extra_id_38>",
|
42 |
+
"<extra_id_39>",
|
43 |
+
"<extra_id_40>",
|
44 |
+
"<extra_id_41>",
|
45 |
+
"<extra_id_42>",
|
46 |
+
"<extra_id_43>",
|
47 |
+
"<extra_id_44>",
|
48 |
+
"<extra_id_45>",
|
49 |
+
"<extra_id_46>",
|
50 |
+
"<extra_id_47>",
|
51 |
+
"<extra_id_48>",
|
52 |
+
"<extra_id_49>",
|
53 |
+
"<extra_id_50>",
|
54 |
+
"<extra_id_51>",
|
55 |
+
"<extra_id_52>",
|
56 |
+
"<extra_id_53>",
|
57 |
+
"<extra_id_54>",
|
58 |
+
"<extra_id_55>",
|
59 |
+
"<extra_id_56>",
|
60 |
+
"<extra_id_57>",
|
61 |
+
"<extra_id_58>",
|
62 |
+
"<extra_id_59>",
|
63 |
+
"<extra_id_60>",
|
64 |
+
"<extra_id_61>",
|
65 |
+
"<extra_id_62>",
|
66 |
+
"<extra_id_63>",
|
67 |
+
"<extra_id_64>",
|
68 |
+
"<extra_id_65>",
|
69 |
+
"<extra_id_66>",
|
70 |
+
"<extra_id_67>",
|
71 |
+
"<extra_id_68>",
|
72 |
+
"<extra_id_69>",
|
73 |
+
"<extra_id_70>",
|
74 |
+
"<extra_id_71>",
|
75 |
+
"<extra_id_72>",
|
76 |
+
"<extra_id_73>",
|
77 |
+
"<extra_id_74>",
|
78 |
+
"<extra_id_75>",
|
79 |
+
"<extra_id_76>",
|
80 |
+
"<extra_id_77>",
|
81 |
+
"<extra_id_78>",
|
82 |
+
"<extra_id_79>",
|
83 |
+
"<extra_id_80>",
|
84 |
+
"<extra_id_81>",
|
85 |
+
"<extra_id_82>",
|
86 |
+
"<extra_id_83>",
|
87 |
+
"<extra_id_84>",
|
88 |
+
"<extra_id_85>",
|
89 |
+
"<extra_id_86>",
|
90 |
+
"<extra_id_87>",
|
91 |
+
"<extra_id_88>",
|
92 |
+
"<extra_id_89>",
|
93 |
+
"<extra_id_90>",
|
94 |
+
"<extra_id_91>",
|
95 |
+
"<extra_id_92>",
|
96 |
+
"<extra_id_93>",
|
97 |
+
"<extra_id_94>",
|
98 |
+
"<extra_id_95>",
|
99 |
+
"<extra_id_96>",
|
100 |
+
"<extra_id_97>",
|
101 |
+
"<extra_id_98>",
|
102 |
+
"<extra_id_99>"
|
103 |
+
],
|
104 |
+
"eos_token": {
|
105 |
+
"content": "</s>",
|
106 |
+
"lstrip": false,
|
107 |
+
"normalized": false,
|
108 |
+
"rstrip": false,
|
109 |
+
"single_word": false
|
110 |
+
},
|
111 |
+
"pad_token": {
|
112 |
+
"content": "<pad>",
|
113 |
+
"lstrip": false,
|
114 |
+
"normalized": false,
|
115 |
+
"rstrip": false,
|
116 |
+
"single_word": false
|
117 |
+
},
|
118 |
+
"unk_token": {
|
119 |
+
"content": "<unk>",
|
120 |
+
"lstrip": false,
|
121 |
+
"normalized": false,
|
122 |
+
"rstrip": false,
|
123 |
+
"single_word": false
|
124 |
+
}
|
125 |
+
}
|
tokenizer_2/tokenizer.json
ADDED
The diff for this file is too large to render.
See raw diff
|
|
tokenizer_2/tokenizer_config.json
ADDED
@@ -0,0 +1,937 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"added_tokens_decoder": {
|
3 |
+
"0": {
|
4 |
+
"content": "<pad>",
|
5 |
+
"lstrip": false,
|
6 |
+
"normalized": false,
|
7 |
+
"rstrip": false,
|
8 |
+
"single_word": false,
|
9 |
+
"special": true
|
10 |
+
},
|
11 |
+
"1": {
|
12 |
+
"content": "</s>",
|
13 |
+
"lstrip": false,
|
14 |
+
"normalized": false,
|
15 |
+
"rstrip": false,
|
16 |
+
"single_word": false,
|
17 |
+
"special": true
|
18 |
+
},
|
19 |
+
"2": {
|
20 |
+
"content": "<unk>",
|
21 |
+
"lstrip": false,
|
22 |
+
"normalized": false,
|
23 |
+
"rstrip": false,
|
24 |
+
"single_word": false,
|
25 |
+
"special": true
|
26 |
+
},
|
27 |
+
"3": {
|
28 |
+
"content": "<extra_id_99>",
|
29 |
+
"lstrip": false,
|
30 |
+
"normalized": false,
|
31 |
+
"rstrip": false,
|
32 |
+
"single_word": false,
|
33 |
+
"special": true
|
34 |
+
},
|
35 |
+
"4": {
|
36 |
+
"content": "<extra_id_98>",
|
37 |
+
"lstrip": false,
|
38 |
+
"normalized": false,
|
39 |
+
"rstrip": false,
|
40 |
+
"single_word": false,
|
41 |
+
"special": true
|
42 |
+
},
|
43 |
+
"5": {
|
44 |
+
"content": "<extra_id_97>",
|
45 |
+
"lstrip": false,
|
46 |
+
"normalized": false,
|
47 |
+
"rstrip": false,
|
48 |
+
"single_word": false,
|
49 |
+
"special": true
|
50 |
+
},
|
51 |
+
"6": {
|
52 |
+
"content": "<extra_id_96>",
|
53 |
+
"lstrip": false,
|
54 |
+
"normalized": false,
|
55 |
+
"rstrip": false,
|
56 |
+
"single_word": false,
|
57 |
+
"special": true
|
58 |
+
},
|
59 |
+
"7": {
|
60 |
+
"content": "<extra_id_95>",
|
61 |
+
"lstrip": false,
|
62 |
+
"normalized": false,
|
63 |
+
"rstrip": false,
|
64 |
+
"single_word": false,
|
65 |
+
"special": true
|
66 |
+
},
|
67 |
+
"8": {
|
68 |
+
"content": "<extra_id_94>",
|
69 |
+
"lstrip": false,
|
70 |
+
"normalized": false,
|
71 |
+
"rstrip": false,
|
72 |
+
"single_word": false,
|
73 |
+
"special": true
|
74 |
+
},
|
75 |
+
"9": {
|
76 |
+
"content": "<extra_id_93>",
|
77 |
+
"lstrip": false,
|
78 |
+
"normalized": false,
|
79 |
+
"rstrip": false,
|
80 |
+
"single_word": false,
|
81 |
+
"special": true
|
82 |
+
},
|
83 |
+
"10": {
|
84 |
+
"content": "<extra_id_92>",
|
85 |
+
"lstrip": false,
|
86 |
+
"normalized": false,
|
87 |
+
"rstrip": false,
|
88 |
+
"single_word": false,
|
89 |
+
"special": true
|
90 |
+
},
|
91 |
+
"11": {
|
92 |
+
"content": "<extra_id_91>",
|
93 |
+
"lstrip": false,
|
94 |
+
"normalized": false,
|
95 |
+
"rstrip": false,
|
96 |
+
"single_word": false,
|
97 |
+
"special": true
|
98 |
+
},
|
99 |
+
"12": {
|
100 |
+
"content": "<extra_id_90>",
|
101 |
+
"lstrip": false,
|
102 |
+
"normalized": false,
|
103 |
+
"rstrip": false,
|
104 |
+
"single_word": false,
|
105 |
+
"special": true
|
106 |
+
},
|
107 |
+
"13": {
|
108 |
+
"content": "<extra_id_89>",
|
109 |
+
"lstrip": false,
|
110 |
+
"normalized": false,
|
111 |
+
"rstrip": false,
|
112 |
+
"single_word": false,
|
113 |
+
"special": true
|
114 |
+
},
|
115 |
+
"14": {
|
116 |
+
"content": "<extra_id_88>",
|
117 |
+
"lstrip": false,
|
118 |
+
"normalized": false,
|
119 |
+
"rstrip": false,
|
120 |
+
"single_word": false,
|
121 |
+
"special": true
|
122 |
+
},
|
123 |
+
"15": {
|
124 |
+
"content": "<extra_id_87>",
|
125 |
+
"lstrip": false,
|
126 |
+
"normalized": false,
|
127 |
+
"rstrip": false,
|
128 |
+
"single_word": false,
|
129 |
+
"special": true
|
130 |
+
},
|
131 |
+
"16": {
|
132 |
+
"content": "<extra_id_86>",
|
133 |
+
"lstrip": false,
|
134 |
+
"normalized": false,
|
135 |
+
"rstrip": false,
|
136 |
+
"single_word": false,
|
137 |
+
"special": true
|
138 |
+
},
|
139 |
+
"17": {
|
140 |
+
"content": "<extra_id_85>",
|
141 |
+
"lstrip": false,
|
142 |
+
"normalized": false,
|
143 |
+
"rstrip": false,
|
144 |
+
"single_word": false,
|
145 |
+
"special": true
|
146 |
+
},
|
147 |
+
"18": {
|
148 |
+
"content": "<extra_id_84>",
|
149 |
+
"lstrip": false,
|
150 |
+
"normalized": false,
|
151 |
+
"rstrip": false,
|
152 |
+
"single_word": false,
|
153 |
+
"special": true
|
154 |
+
},
|
155 |
+
"19": {
|
156 |
+
"content": "<extra_id_83>",
|
157 |
+
"lstrip": false,
|
158 |
+
"normalized": false,
|
159 |
+
"rstrip": false,
|
160 |
+
"single_word": false,
|
161 |
+
"special": true
|
162 |
+
},
|
163 |
+
"20": {
|
164 |
+
"content": "<extra_id_82>",
|
165 |
+
"lstrip": false,
|
166 |
+
"normalized": false,
|
167 |
+
"rstrip": false,
|
168 |
+
"single_word": false,
|
169 |
+
"special": true
|
170 |
+
},
|
171 |
+
"21": {
|
172 |
+
"content": "<extra_id_81>",
|
173 |
+
"lstrip": false,
|
174 |
+
"normalized": false,
|
175 |
+
"rstrip": false,
|
176 |
+
"single_word": false,
|
177 |
+
"special": true
|
178 |
+
},
|
179 |
+
"22": {
|
180 |
+
"content": "<extra_id_80>",
|
181 |
+
"lstrip": false,
|
182 |
+
"normalized": false,
|
183 |
+
"rstrip": false,
|
184 |
+
"single_word": false,
|
185 |
+
"special": true
|
186 |
+
},
|
187 |
+
"23": {
|
188 |
+
"content": "<extra_id_79>",
|
189 |
+
"lstrip": false,
|
190 |
+
"normalized": false,
|
191 |
+
"rstrip": false,
|
192 |
+
"single_word": false,
|
193 |
+
"special": true
|
194 |
+
},
|
195 |
+
"24": {
|
196 |
+
"content": "<extra_id_78>",
|
197 |
+
"lstrip": false,
|
198 |
+
"normalized": false,
|
199 |
+
"rstrip": false,
|
200 |
+
"single_word": false,
|
201 |
+
"special": true
|
202 |
+
},
|
203 |
+
"25": {
|
204 |
+
"content": "<extra_id_77>",
|
205 |
+
"lstrip": false,
|
206 |
+
"normalized": false,
|
207 |
+
"rstrip": false,
|
208 |
+
"single_word": false,
|
209 |
+
"special": true
|
210 |
+
},
|
211 |
+
"26": {
|
212 |
+
"content": "<extra_id_76>",
|
213 |
+
"lstrip": false,
|
214 |
+
"normalized": false,
|
215 |
+
"rstrip": false,
|
216 |
+
"single_word": false,
|
217 |
+
"special": true
|
218 |
+
},
|
219 |
+
"27": {
|
220 |
+
"content": "<extra_id_75>",
|
221 |
+
"lstrip": false,
|
222 |
+
"normalized": false,
|
223 |
+
"rstrip": false,
|
224 |
+
"single_word": false,
|
225 |
+
"special": true
|
226 |
+
},
|
227 |
+
"28": {
|
228 |
+
"content": "<extra_id_74>",
|
229 |
+
"lstrip": false,
|
230 |
+
"normalized": false,
|
231 |
+
"rstrip": false,
|
232 |
+
"single_word": false,
|
233 |
+
"special": true
|
234 |
+
},
|
235 |
+
"29": {
|
236 |
+
"content": "<extra_id_73>",
|
237 |
+
"lstrip": false,
|
238 |
+
"normalized": false,
|
239 |
+
"rstrip": false,
|
240 |
+
"single_word": false,
|
241 |
+
"special": true
|
242 |
+
},
|
243 |
+
"30": {
|
244 |
+
"content": "<extra_id_72>",
|
245 |
+
"lstrip": false,
|
246 |
+
"normalized": false,
|
247 |
+
"rstrip": false,
|
248 |
+
"single_word": false,
|
249 |
+
"special": true
|
250 |
+
},
|
251 |
+
"31": {
|
252 |
+
"content": "<extra_id_71>",
|
253 |
+
"lstrip": false,
|
254 |
+
"normalized": false,
|
255 |
+
"rstrip": false,
|
256 |
+
"single_word": false,
|
257 |
+
"special": true
|
258 |
+
},
|
259 |
+
"32": {
|
260 |
+
"content": "<extra_id_70>",
|
261 |
+
"lstrip": false,
|
262 |
+
"normalized": false,
|
263 |
+
"rstrip": false,
|
264 |
+
"single_word": false,
|
265 |
+
"special": true
|
266 |
+
},
|
267 |
+
"33": {
|
268 |
+
"content": "<extra_id_69>",
|
269 |
+
"lstrip": false,
|
270 |
+
"normalized": false,
|
271 |
+
"rstrip": false,
|
272 |
+
"single_word": false,
|
273 |
+
"special": true
|
274 |
+
},
|
275 |
+
"34": {
|
276 |
+
"content": "<extra_id_68>",
|
277 |
+
"lstrip": false,
|
278 |
+
"normalized": false,
|
279 |
+
"rstrip": false,
|
280 |
+
"single_word": false,
|
281 |
+
"special": true
|
282 |
+
},
|
283 |
+
"35": {
|
284 |
+
"content": "<extra_id_67>",
|
285 |
+
"lstrip": false,
|
286 |
+
"normalized": false,
|
287 |
+
"rstrip": false,
|
288 |
+
"single_word": false,
|
289 |
+
"special": true
|
290 |
+
},
|
291 |
+
"36": {
|
292 |
+
"content": "<extra_id_66>",
|
293 |
+
"lstrip": false,
|
294 |
+
"normalized": false,
|
295 |
+
"rstrip": false,
|
296 |
+
"single_word": false,
|
297 |
+
"special": true
|
298 |
+
},
|
299 |
+
"37": {
|
300 |
+
"content": "<extra_id_65>",
|
301 |
+
"lstrip": false,
|
302 |
+
"normalized": false,
|
303 |
+
"rstrip": false,
|
304 |
+
"single_word": false,
|
305 |
+
"special": true
|
306 |
+
},
|
307 |
+
"38": {
|
308 |
+
"content": "<extra_id_64>",
|
309 |
+
"lstrip": false,
|
310 |
+
"normalized": false,
|
311 |
+
"rstrip": false,
|
312 |
+
"single_word": false,
|
313 |
+
"special": true
|
314 |
+
},
|
315 |
+
"39": {
|
316 |
+
"content": "<extra_id_63>",
|
317 |
+
"lstrip": false,
|
318 |
+
"normalized": false,
|
319 |
+
"rstrip": false,
|
320 |
+
"single_word": false,
|
321 |
+
"special": true
|
322 |
+
},
|
323 |
+
"40": {
|
324 |
+
"content": "<extra_id_62>",
|
325 |
+
"lstrip": false,
|
326 |
+
"normalized": false,
|
327 |
+
"rstrip": false,
|
328 |
+
"single_word": false,
|
329 |
+
"special": true
|
330 |
+
},
|
331 |
+
"41": {
|
332 |
+
"content": "<extra_id_61>",
|
333 |
+
"lstrip": false,
|
334 |
+
"normalized": false,
|
335 |
+
"rstrip": false,
|
336 |
+
"single_word": false,
|
337 |
+
"special": true
|
338 |
+
},
|
339 |
+
"42": {
|
340 |
+
"content": "<extra_id_60>",
|
341 |
+
"lstrip": false,
|
342 |
+
"normalized": false,
|
343 |
+
"rstrip": false,
|
344 |
+
"single_word": false,
|
345 |
+
"special": true
|
346 |
+
},
|
347 |
+
"43": {
|
348 |
+
"content": "<extra_id_59>",
|
349 |
+
"lstrip": false,
|
350 |
+
"normalized": false,
|
351 |
+
"rstrip": false,
|
352 |
+
"single_word": false,
|
353 |
+
"special": true
|
354 |
+
},
|
355 |
+
"44": {
|
356 |
+
"content": "<extra_id_58>",
|
357 |
+
"lstrip": false,
|
358 |
+
"normalized": false,
|
359 |
+
"rstrip": false,
|
360 |
+
"single_word": false,
|
361 |
+
"special": true
|
362 |
+
},
|
363 |
+
"45": {
|
364 |
+
"content": "<extra_id_57>",
|
365 |
+
"lstrip": false,
|
366 |
+
"normalized": false,
|
367 |
+
"rstrip": false,
|
368 |
+
"single_word": false,
|
369 |
+
"special": true
|
370 |
+
},
|
371 |
+
"46": {
|
372 |
+
"content": "<extra_id_56>",
|
373 |
+
"lstrip": false,
|
374 |
+
"normalized": false,
|
375 |
+
"rstrip": false,
|
376 |
+
"single_word": false,
|
377 |
+
"special": true
|
378 |
+
},
|
379 |
+
"47": {
|
380 |
+
"content": "<extra_id_55>",
|
381 |
+
"lstrip": false,
|
382 |
+
"normalized": false,
|
383 |
+
"rstrip": false,
|
384 |
+
"single_word": false,
|
385 |
+
"special": true
|
386 |
+
},
|
387 |
+
"48": {
|
388 |
+
"content": "<extra_id_54>",
|
389 |
+
"lstrip": false,
|
390 |
+
"normalized": false,
|
391 |
+
"rstrip": false,
|
392 |
+
"single_word": false,
|
393 |
+
"special": true
|
394 |
+
},
|
395 |
+
"49": {
|
396 |
+
"content": "<extra_id_53>",
|
397 |
+
"lstrip": false,
|
398 |
+
"normalized": false,
|
399 |
+
"rstrip": false,
|
400 |
+
"single_word": false,
|
401 |
+
"special": true
|
402 |
+
},
|
403 |
+
"50": {
|
404 |
+
"content": "<extra_id_52>",
|
405 |
+
"lstrip": false,
|
406 |
+
"normalized": false,
|
407 |
+
"rstrip": false,
|
408 |
+
"single_word": false,
|
409 |
+
"special": true
|
410 |
+
},
|
411 |
+
"51": {
|
412 |
+
"content": "<extra_id_51>",
|
413 |
+
"lstrip": false,
|
414 |
+
"normalized": false,
|
415 |
+
"rstrip": false,
|
416 |
+
"single_word": false,
|
417 |
+
"special": true
|
418 |
+
},
|
419 |
+
"52": {
|
420 |
+
"content": "<extra_id_50>",
|
421 |
+
"lstrip": false,
|
422 |
+
"normalized": false,
|
423 |
+
"rstrip": false,
|
424 |
+
"single_word": false,
|
425 |
+
"special": true
|
426 |
+
},
|
427 |
+
"53": {
|
428 |
+
"content": "<extra_id_49>",
|
429 |
+
"lstrip": false,
|
430 |
+
"normalized": false,
|
431 |
+
"rstrip": false,
|
432 |
+
"single_word": false,
|
433 |
+
"special": true
|
434 |
+
},
|
435 |
+
"54": {
|
436 |
+
"content": "<extra_id_48>",
|
437 |
+
"lstrip": false,
|
438 |
+
"normalized": false,
|
439 |
+
"rstrip": false,
|
440 |
+
"single_word": false,
|
441 |
+
"special": true
|
442 |
+
},
|
443 |
+
"55": {
|
444 |
+
"content": "<extra_id_47>",
|
445 |
+
"lstrip": false,
|
446 |
+
"normalized": false,
|
447 |
+
"rstrip": false,
|
448 |
+
"single_word": false,
|
449 |
+
"special": true
|
450 |
+
},
|
451 |
+
"56": {
|
452 |
+
"content": "<extra_id_46>",
|
453 |
+
"lstrip": false,
|
454 |
+
"normalized": false,
|
455 |
+
"rstrip": false,
|
456 |
+
"single_word": false,
|
457 |
+
"special": true
|
458 |
+
},
|
459 |
+
"57": {
|
460 |
+
"content": "<extra_id_45>",
|
461 |
+
"lstrip": false,
|
462 |
+
"normalized": false,
|
463 |
+
"rstrip": false,
|
464 |
+
"single_word": false,
|
465 |
+
"special": true
|
466 |
+
},
|
467 |
+
"58": {
|
468 |
+
"content": "<extra_id_44>",
|
469 |
+
"lstrip": false,
|
470 |
+
"normalized": false,
|
471 |
+
"rstrip": false,
|
472 |
+
"single_word": false,
|
473 |
+
"special": true
|
474 |
+
},
|
475 |
+
"59": {
|
476 |
+
"content": "<extra_id_43>",
|
477 |
+
"lstrip": false,
|
478 |
+
"normalized": false,
|
479 |
+
"rstrip": false,
|
480 |
+
"single_word": false,
|
481 |
+
"special": true
|
482 |
+
},
|
483 |
+
"60": {
|
484 |
+
"content": "<extra_id_42>",
|
485 |
+
"lstrip": false,
|
486 |
+
"normalized": false,
|
487 |
+
"rstrip": false,
|
488 |
+
"single_word": false,
|
489 |
+
"special": true
|
490 |
+
},
|
491 |
+
"61": {
|
492 |
+
"content": "<extra_id_41>",
|
493 |
+
"lstrip": false,
|
494 |
+
"normalized": false,
|
495 |
+
"rstrip": false,
|
496 |
+
"single_word": false,
|
497 |
+
"special": true
|
498 |
+
},
|
499 |
+
"62": {
|
500 |
+
"content": "<extra_id_40>",
|
501 |
+
"lstrip": false,
|
502 |
+
"normalized": false,
|
503 |
+
"rstrip": false,
|
504 |
+
"single_word": false,
|
505 |
+
"special": true
|
506 |
+
},
|
507 |
+
"63": {
|
508 |
+
"content": "<extra_id_39>",
|
509 |
+
"lstrip": false,
|
510 |
+
"normalized": false,
|
511 |
+
"rstrip": false,
|
512 |
+
"single_word": false,
|
513 |
+
"special": true
|
514 |
+
},
|
515 |
+
"64": {
|
516 |
+
"content": "<extra_id_38>",
|
517 |
+
"lstrip": false,
|
518 |
+
"normalized": false,
|
519 |
+
"rstrip": false,
|
520 |
+
"single_word": false,
|
521 |
+
"special": true
|
522 |
+
},
|
523 |
+
"65": {
|
524 |
+
"content": "<extra_id_37>",
|
525 |
+
"lstrip": false,
|
526 |
+
"normalized": false,
|
527 |
+
"rstrip": false,
|
528 |
+
"single_word": false,
|
529 |
+
"special": true
|
530 |
+
},
|
531 |
+
"66": {
|
532 |
+
"content": "<extra_id_36>",
|
533 |
+
"lstrip": false,
|
534 |
+
"normalized": false,
|
535 |
+
"rstrip": false,
|
536 |
+
"single_word": false,
|
537 |
+
"special": true
|
538 |
+
},
|
539 |
+
"67": {
|
540 |
+
"content": "<extra_id_35>",
|
541 |
+
"lstrip": false,
|
542 |
+
"normalized": false,
|
543 |
+
"rstrip": false,
|
544 |
+
"single_word": false,
|
545 |
+
"special": true
|
546 |
+
},
|
547 |
+
"68": {
|
548 |
+
"content": "<extra_id_34>",
|
549 |
+
"lstrip": false,
|
550 |
+
"normalized": false,
|
551 |
+
"rstrip": false,
|
552 |
+
"single_word": false,
|
553 |
+
"special": true
|
554 |
+
},
|
555 |
+
"69": {
|
556 |
+
"content": "<extra_id_33>",
|
557 |
+
"lstrip": false,
|
558 |
+
"normalized": false,
|
559 |
+
"rstrip": false,
|
560 |
+
"single_word": false,
|
561 |
+
"special": true
|
562 |
+
},
|
563 |
+
"70": {
|
564 |
+
"content": "<extra_id_32>",
|
565 |
+
"lstrip": false,
|
566 |
+
"normalized": false,
|
567 |
+
"rstrip": false,
|
568 |
+
"single_word": false,
|
569 |
+
"special": true
|
570 |
+
},
|
571 |
+
"71": {
|
572 |
+
"content": "<extra_id_31>",
|
573 |
+
"lstrip": false,
|
574 |
+
"normalized": false,
|
575 |
+
"rstrip": false,
|
576 |
+
"single_word": false,
|
577 |
+
"special": true
|
578 |
+
},
|
579 |
+
"72": {
|
580 |
+
"content": "<extra_id_30>",
|
581 |
+
"lstrip": false,
|
582 |
+
"normalized": false,
|
583 |
+
"rstrip": false,
|
584 |
+
"single_word": false,
|
585 |
+
"special": true
|
586 |
+
},
|
587 |
+
"73": {
|
588 |
+
"content": "<extra_id_29>",
|
589 |
+
"lstrip": false,
|
590 |
+
"normalized": false,
|
591 |
+
"rstrip": false,
|
592 |
+
"single_word": false,
|
593 |
+
"special": true
|
594 |
+
},
|
595 |
+
"74": {
|
596 |
+
"content": "<extra_id_28>",
|
597 |
+
"lstrip": false,
|
598 |
+
"normalized": false,
|
599 |
+
"rstrip": false,
|
600 |
+
"single_word": false,
|
601 |
+
"special": true
|
602 |
+
},
|
603 |
+
"75": {
|
604 |
+
"content": "<extra_id_27>",
|
605 |
+
"lstrip": false,
|
606 |
+
"normalized": false,
|
607 |
+
"rstrip": false,
|
608 |
+
"single_word": false,
|
609 |
+
"special": true
|
610 |
+
},
|
611 |
+
"76": {
|
612 |
+
"content": "<extra_id_26>",
|
613 |
+
"lstrip": false,
|
614 |
+
"normalized": false,
|
615 |
+
"rstrip": false,
|
616 |
+
"single_word": false,
|
617 |
+
"special": true
|
618 |
+
},
|
619 |
+
"77": {
|
620 |
+
"content": "<extra_id_25>",
|
621 |
+
"lstrip": false,
|
622 |
+
"normalized": false,
|
623 |
+
"rstrip": false,
|
624 |
+
"single_word": false,
|
625 |
+
"special": true
|
626 |
+
},
|
627 |
+
"78": {
|
628 |
+
"content": "<extra_id_24>",
|
629 |
+
"lstrip": false,
|
630 |
+
"normalized": false,
|
631 |
+
"rstrip": false,
|
632 |
+
"single_word": false,
|
633 |
+
"special": true
|
634 |
+
},
|
635 |
+
"79": {
|
636 |
+
"content": "<extra_id_23>",
|
637 |
+
"lstrip": false,
|
638 |
+
"normalized": false,
|
639 |
+
"rstrip": false,
|
640 |
+
"single_word": false,
|
641 |
+
"special": true
|
642 |
+
},
|
643 |
+
"80": {
|
644 |
+
"content": "<extra_id_22>",
|
645 |
+
"lstrip": false,
|
646 |
+
"normalized": false,
|
647 |
+
"rstrip": false,
|
648 |
+
"single_word": false,
|
649 |
+
"special": true
|
650 |
+
},
|
651 |
+
"81": {
|
652 |
+
"content": "<extra_id_21>",
|
653 |
+
"lstrip": false,
|
654 |
+
"normalized": false,
|
655 |
+
"rstrip": false,
|
656 |
+
"single_word": false,
|
657 |
+
"special": true
|
658 |
+
},
|
659 |
+
"82": {
|
660 |
+
"content": "<extra_id_20>",
|
661 |
+
"lstrip": false,
|
662 |
+
"normalized": false,
|
663 |
+
"rstrip": false,
|
664 |
+
"single_word": false,
|
665 |
+
"special": true
|
666 |
+
},
|
667 |
+
"83": {
|
668 |
+
"content": "<extra_id_19>",
|
669 |
+
"lstrip": false,
|
670 |
+
"normalized": false,
|
671 |
+
"rstrip": false,
|
672 |
+
"single_word": false,
|
673 |
+
"special": true
|
674 |
+
},
|
675 |
+
"84": {
|
676 |
+
"content": "<extra_id_18>",
|
677 |
+
"lstrip": false,
|
678 |
+
"normalized": false,
|
679 |
+
"rstrip": false,
|
680 |
+
"single_word": false,
|
681 |
+
"special": true
|
682 |
+
},
|
683 |
+
"85": {
|
684 |
+
"content": "<extra_id_17>",
|
685 |
+
"lstrip": false,
|
686 |
+
"normalized": false,
|
687 |
+
"rstrip": false,
|
688 |
+
"single_word": false,
|
689 |
+
"special": true
|
690 |
+
},
|
691 |
+
"86": {
|
692 |
+
"content": "<extra_id_16>",
|
693 |
+
"lstrip": false,
|
694 |
+
"normalized": false,
|
695 |
+
"rstrip": false,
|
696 |
+
"single_word": false,
|
697 |
+
"special": true
|
698 |
+
},
|
699 |
+
"87": {
|
700 |
+
"content": "<extra_id_15>",
|
701 |
+
"lstrip": false,
|
702 |
+
"normalized": false,
|
703 |
+
"rstrip": false,
|
704 |
+
"single_word": false,
|
705 |
+
"special": true
|
706 |
+
},
|
707 |
+
"88": {
|
708 |
+
"content": "<extra_id_14>",
|
709 |
+
"lstrip": false,
|
710 |
+
"normalized": false,
|
711 |
+
"rstrip": false,
|
712 |
+
"single_word": false,
|
713 |
+
"special": true
|
714 |
+
},
|
715 |
+
"89": {
|
716 |
+
"content": "<extra_id_13>",
|
717 |
+
"lstrip": false,
|
718 |
+
"normalized": false,
|
719 |
+
"rstrip": false,
|
720 |
+
"single_word": false,
|
721 |
+
"special": true
|
722 |
+
},
|
723 |
+
"90": {
|
724 |
+
"content": "<extra_id_12>",
|
725 |
+
"lstrip": false,
|
726 |
+
"normalized": false,
|
727 |
+
"rstrip": false,
|
728 |
+
"single_word": false,
|
729 |
+
"special": true
|
730 |
+
},
|
731 |
+
"91": {
|
732 |
+
"content": "<extra_id_11>",
|
733 |
+
"lstrip": false,
|
734 |
+
"normalized": false,
|
735 |
+
"rstrip": false,
|
736 |
+
"single_word": false,
|
737 |
+
"special": true
|
738 |
+
},
|
739 |
+
"92": {
|
740 |
+
"content": "<extra_id_10>",
|
741 |
+
"lstrip": false,
|
742 |
+
"normalized": false,
|
743 |
+
"rstrip": false,
|
744 |
+
"single_word": false,
|
745 |
+
"special": true
|
746 |
+
},
|
747 |
+
"93": {
|
748 |
+
"content": "<extra_id_9>",
|
749 |
+
"lstrip": false,
|
750 |
+
"normalized": false,
|
751 |
+
"rstrip": false,
|
752 |
+
"single_word": false,
|
753 |
+
"special": true
|
754 |
+
},
|
755 |
+
"94": {
|
756 |
+
"content": "<extra_id_8>",
|
757 |
+
"lstrip": false,
|
758 |
+
"normalized": false,
|
759 |
+
"rstrip": false,
|
760 |
+
"single_word": false,
|
761 |
+
"special": true
|
762 |
+
},
|
763 |
+
"95": {
|
764 |
+
"content": "<extra_id_7>",
|
765 |
+
"lstrip": false,
|
766 |
+
"normalized": false,
|
767 |
+
"rstrip": false,
|
768 |
+
"single_word": false,
|
769 |
+
"special": true
|
770 |
+
},
|
771 |
+
"96": {
|
772 |
+
"content": "<extra_id_6>",
|
773 |
+
"lstrip": false,
|
774 |
+
"normalized": false,
|
775 |
+
"rstrip": false,
|
776 |
+
"single_word": false,
|
777 |
+
"special": true
|
778 |
+
},
|
779 |
+
"97": {
|
780 |
+
"content": "<extra_id_5>",
|
781 |
+
"lstrip": false,
|
782 |
+
"normalized": false,
|
783 |
+
"rstrip": false,
|
784 |
+
"single_word": false,
|
785 |
+
"special": true
|
786 |
+
},
|
787 |
+
"98": {
|
788 |
+
"content": "<extra_id_4>",
|
789 |
+
"lstrip": false,
|
790 |
+
"normalized": false,
|
791 |
+
"rstrip": false,
|
792 |
+
"single_word": false,
|
793 |
+
"special": true
|
794 |
+
},
|
795 |
+
"99": {
|
796 |
+
"content": "<extra_id_3>",
|
797 |
+
"lstrip": false,
|
798 |
+
"normalized": false,
|
799 |
+
"rstrip": false,
|
800 |
+
"single_word": false,
|
801 |
+
"special": true
|
802 |
+
},
|
803 |
+
"100": {
|
804 |
+
"content": "<extra_id_2>",
|
805 |
+
"lstrip": false,
|
806 |
+
"normalized": false,
|
807 |
+
"rstrip": false,
|
808 |
+
"single_word": false,
|
809 |
+
"special": true
|
810 |
+
},
|
811 |
+
"101": {
|
812 |
+
"content": "<extra_id_1>",
|
813 |
+
"lstrip": false,
|
814 |
+
"normalized": false,
|
815 |
+
"rstrip": false,
|
816 |
+
"single_word": false,
|
817 |
+
"special": true
|
818 |
+
},
|
819 |
+
"102": {
|
820 |
+
"content": "<extra_id_0>",
|
821 |
+
"lstrip": false,
|
822 |
+
"normalized": false,
|
823 |
+
"rstrip": false,
|
824 |
+
"single_word": false,
|
825 |
+
"special": true
|
826 |
+
}
|
827 |
+
},
|
828 |
+
"additional_special_tokens": [
|
829 |
+
"<extra_id_0>",
|
830 |
+
"<extra_id_1>",
|
831 |
+
"<extra_id_2>",
|
832 |
+
"<extra_id_3>",
|
833 |
+
"<extra_id_4>",
|
834 |
+
"<extra_id_5>",
|
835 |
+
"<extra_id_6>",
|
836 |
+
"<extra_id_7>",
|
837 |
+
"<extra_id_8>",
|
838 |
+
"<extra_id_9>",
|
839 |
+
"<extra_id_10>",
|
840 |
+
"<extra_id_11>",
|
841 |
+
"<extra_id_12>",
|
842 |
+
"<extra_id_13>",
|
843 |
+
"<extra_id_14>",
|
844 |
+
"<extra_id_15>",
|
845 |
+
"<extra_id_16>",
|
846 |
+
"<extra_id_17>",
|
847 |
+
"<extra_id_18>",
|
848 |
+
"<extra_id_19>",
|
849 |
+
"<extra_id_20>",
|
850 |
+
"<extra_id_21>",
|
851 |
+
"<extra_id_22>",
|
852 |
+
"<extra_id_23>",
|
853 |
+
"<extra_id_24>",
|
854 |
+
"<extra_id_25>",
|
855 |
+
"<extra_id_26>",
|
856 |
+
"<extra_id_27>",
|
857 |
+
"<extra_id_28>",
|
858 |
+
"<extra_id_29>",
|
859 |
+
"<extra_id_30>",
|
860 |
+
"<extra_id_31>",
|
861 |
+
"<extra_id_32>",
|
862 |
+
"<extra_id_33>",
|
863 |
+
"<extra_id_34>",
|
864 |
+
"<extra_id_35>",
|
865 |
+
"<extra_id_36>",
|
866 |
+
"<extra_id_37>",
|
867 |
+
"<extra_id_38>",
|
868 |
+
"<extra_id_39>",
|
869 |
+
"<extra_id_40>",
|
870 |
+
"<extra_id_41>",
|
871 |
+
"<extra_id_42>",
|
872 |
+
"<extra_id_43>",
|
873 |
+
"<extra_id_44>",
|
874 |
+
"<extra_id_45>",
|
875 |
+
"<extra_id_46>",
|
876 |
+
"<extra_id_47>",
|
877 |
+
"<extra_id_48>",
|
878 |
+
"<extra_id_49>",
|
879 |
+
"<extra_id_50>",
|
880 |
+
"<extra_id_51>",
|
881 |
+
"<extra_id_52>",
|
882 |
+
"<extra_id_53>",
|
883 |
+
"<extra_id_54>",
|
884 |
+
"<extra_id_55>",
|
885 |
+
"<extra_id_56>",
|
886 |
+
"<extra_id_57>",
|
887 |
+
"<extra_id_58>",
|
888 |
+
"<extra_id_59>",
|
889 |
+
"<extra_id_60>",
|
890 |
+
"<extra_id_61>",
|
891 |
+
"<extra_id_62>",
|
892 |
+
"<extra_id_63>",
|
893 |
+
"<extra_id_64>",
|
894 |
+
"<extra_id_65>",
|
895 |
+
"<extra_id_66>",
|
896 |
+
"<extra_id_67>",
|
897 |
+
"<extra_id_68>",
|
898 |
+
"<extra_id_69>",
|
899 |
+
"<extra_id_70>",
|
900 |
+
"<extra_id_71>",
|
901 |
+
"<extra_id_72>",
|
902 |
+
"<extra_id_73>",
|
903 |
+
"<extra_id_74>",
|
904 |
+
"<extra_id_75>",
|
905 |
+
"<extra_id_76>",
|
906 |
+
"<extra_id_77>",
|
907 |
+
"<extra_id_78>",
|
908 |
+
"<extra_id_79>",
|
909 |
+
"<extra_id_80>",
|
910 |
+
"<extra_id_81>",
|
911 |
+
"<extra_id_82>",
|
912 |
+
"<extra_id_83>",
|
913 |
+
"<extra_id_84>",
|
914 |
+
"<extra_id_85>",
|
915 |
+
"<extra_id_86>",
|
916 |
+
"<extra_id_87>",
|
917 |
+
"<extra_id_88>",
|
918 |
+
"<extra_id_89>",
|
919 |
+
"<extra_id_90>",
|
920 |
+
"<extra_id_91>",
|
921 |
+
"<extra_id_92>",
|
922 |
+
"<extra_id_93>",
|
923 |
+
"<extra_id_94>",
|
924 |
+
"<extra_id_95>",
|
925 |
+
"<extra_id_96>",
|
926 |
+
"<extra_id_97>",
|
927 |
+
"<extra_id_98>",
|
928 |
+
"<extra_id_99>"
|
929 |
+
],
|
930 |
+
"clean_up_tokenization_spaces": true,
|
931 |
+
"eos_token": "</s>",
|
932 |
+
"extra_ids": 100,
|
933 |
+
"model_max_length": 1024,
|
934 |
+
"pad_token": "<pad>",
|
935 |
+
"tokenizer_class": "T5Tokenizer",
|
936 |
+
"unk_token": "<unk>"
|
937 |
+
}
|
transformer/config.json
ADDED
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_class_name": "FluxTransformer2DModel",
|
3 |
+
"_diffusers_version": "0.30.0",
|
4 |
+
"attention_head_dim": 16,
|
5 |
+
"axes_dims_rope": [
|
6 |
+
4,
|
7 |
+
4,
|
8 |
+
8
|
9 |
+
],
|
10 |
+
"guidance_embeds": false,
|
11 |
+
"in_channels": 4,
|
12 |
+
"joint_attention_dim": 32,
|
13 |
+
"num_attention_heads": 2,
|
14 |
+
"num_layers": 1,
|
15 |
+
"num_single_layers": 1,
|
16 |
+
"patch_size": 1,
|
17 |
+
"pooled_projection_dim": 32
|
18 |
+
}
|
transformer/diffusion_pytorch_model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:036c2fa14e67535577781e8c6a3a06c6b08eec68cd345f803a6f6a3490fad17b
|
3 |
+
size 280344
|
vae/config.json
ADDED
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_class_name": "AutoencoderKL",
|
3 |
+
"_diffusers_version": "0.30.0",
|
4 |
+
"act_fn": "silu",
|
5 |
+
"block_out_channels": [
|
6 |
+
4
|
7 |
+
],
|
8 |
+
"down_block_types": [
|
9 |
+
"DownEncoderBlock2D"
|
10 |
+
],
|
11 |
+
"force_upcast": true,
|
12 |
+
"in_channels": 3,
|
13 |
+
"latent_channels": 1,
|
14 |
+
"latents_mean": null,
|
15 |
+
"latents_std": null,
|
16 |
+
"layers_per_block": 1,
|
17 |
+
"mid_block_add_attention": true,
|
18 |
+
"norm_num_groups": 1,
|
19 |
+
"out_channels": 3,
|
20 |
+
"sample_size": 32,
|
21 |
+
"scaling_factor": 1.5035,
|
22 |
+
"shift_factor": 0.0609,
|
23 |
+
"up_block_types": [
|
24 |
+
"UpDecoderBlock2D"
|
25 |
+
],
|
26 |
+
"use_post_quant_conv": false,
|
27 |
+
"use_quant_conv": false
|
28 |
+
}
|
vae/diffusion_pytorch_model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:89cdd1f3bc35f3137b689abf8317887a7345517f55e0835c7c76a73dfcfba030
|
3 |
+
size 19484
|