Update tokenization_qwen.py
#1
by
pbaylies
- opened
- tokenization_qwen.py +1 -1
tokenization_qwen.py
CHANGED
|
@@ -111,7 +111,6 @@ class QWenTokenizer(PreTrainedTokenizer):
|
|
| 111 |
quad_end_tag='</quad>',
|
| 112 |
**kwargs,
|
| 113 |
):
|
| 114 |
-
super().__init__(**kwargs)
|
| 115 |
self.image_start_tag = image_start_tag
|
| 116 |
self.image_end_tag = image_end_tag
|
| 117 |
self.image_pad_tag = image_pad_tag
|
|
@@ -128,6 +127,7 @@ class QWenTokenizer(PreTrainedTokenizer):
|
|
| 128 |
image_start_tag, image_end_tag,
|
| 129 |
image_pad_tag
|
| 130 |
)
|
|
|
|
| 131 |
|
| 132 |
self.errors = errors # how to handle errors in decoding
|
| 133 |
|
|
|
|
| 111 |
quad_end_tag='</quad>',
|
| 112 |
**kwargs,
|
| 113 |
):
|
|
|
|
| 114 |
self.image_start_tag = image_start_tag
|
| 115 |
self.image_end_tag = image_end_tag
|
| 116 |
self.image_pad_tag = image_pad_tag
|
|
|
|
| 127 |
image_start_tag, image_end_tag,
|
| 128 |
image_pad_tag
|
| 129 |
)
|
| 130 |
+
super().__init__(**kwargs)
|
| 131 |
|
| 132 |
self.errors = errors # how to handle errors in decoding
|
| 133 |
|