Gong Baitao
commited on
Commit
•
31ce9ce
1
Parent(s):
1625156
fix normal token with <>
Browse files- tokenization_cpmbee.py +1 -0
tokenization_cpmbee.py
CHANGED
@@ -236,6 +236,7 @@ class CpmBeeTokenizer(PreTrainedTokenizer):
|
|
236 |
tail = sentence_split.pop(-1)
|
237 |
sentence_split[-1] += tail
|
238 |
sentence_split.append(c)
|
|
|
239 |
elif c == ">":
|
240 |
# end of special token
|
241 |
sentence_split[-1] += c
|
|
|
236 |
tail = sentence_split.pop(-1)
|
237 |
sentence_split[-1] += tail
|
238 |
sentence_split.append(c)
|
239 |
+
is_special_token = False
|
240 |
elif c == ">":
|
241 |
# end of special token
|
242 |
sentence_split[-1] += c
|