xu-song commited on
Commit
ef8594d
1 Parent(s): 7cb27ea
vocab/qwen/Qwen-7B-Chat/qwen.tiktoken ADDED
The diff for this file is too large to render. See raw diff
 
vocab/qwen/Qwen-7B-Chat/tokenization_qwen.py ADDED
@@ -0,0 +1,246 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba Cloud.
2
+ #
3
+ # This source code is licensed under the license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ """Tokenization classes for QWen."""
7
+
8
+ import base64
9
+ import logging
10
+ import os
11
+ import unicodedata
12
+ from typing import Collection, Dict, List, Set, Tuple, Union
13
+
14
+ import tiktoken
15
+ from transformers import PreTrainedTokenizer, AddedToken
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+
20
+ VOCAB_FILES_NAMES = {"vocab_file": "qwen.tiktoken"}
21
+
22
+ PAT_STR = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
23
+ ENDOFTEXT = "<|endoftext|>"
24
+ IMSTART = "<|im_start|>"
25
+ IMEND = "<|im_end|>"
26
+ # as the default behavior is changed to allow special tokens in
27
+ # regular texts, the surface forms of special tokens need to be
28
+ # as different as possible to minimize the impact
29
+ EXTRAS = tuple((f"<|extra_{i}|>" for i in range(205)))
30
+ SPECIAL_TOKENS = (
31
+ ENDOFTEXT,
32
+ IMSTART,
33
+ IMEND,
34
+ ) + EXTRAS
35
+
36
+
37
+ def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
38
+ with open(tiktoken_bpe_file, "rb") as f:
39
+ contents = f.read()
40
+ return {
41
+ base64.b64decode(token): int(rank)
42
+ for token, rank in (line.split() for line in contents.splitlines() if line)
43
+ }
44
+
45
+ class QWenTokenizer(PreTrainedTokenizer):
46
+ """QWen tokenizer."""
47
+
48
+ vocab_files_names = VOCAB_FILES_NAMES
49
+
50
+ def __init__(
51
+ self,
52
+ vocab_file,
53
+ errors="replace",
54
+ **kwargs,
55
+ ):
56
+ super().__init__(**kwargs)
57
+
58
+ self.errors = errors # how to handle errors in decoding
59
+
60
+ self.mergeable_ranks = _load_tiktoken_bpe(vocab_file) # type: dict[bytes, int]
61
+ self.special_tokens = {
62
+ token: index
63
+ for index, token in enumerate(
64
+ SPECIAL_TOKENS, start=len(self.mergeable_ranks)
65
+ )
66
+ }
67
+
68
+ enc = tiktoken.Encoding(
69
+ "Qwen",
70
+ pat_str=PAT_STR,
71
+ mergeable_ranks=self.mergeable_ranks,
72
+ special_tokens=self.special_tokens,
73
+ )
74
+ assert (
75
+ len(self.mergeable_ranks) + len(self.special_tokens) == enc.n_vocab
76
+ ), f"{len(self.mergeable_ranks) + len(self.special_tokens)} != {enc.n_vocab} in encoding"
77
+
78
+ self.decoder = {
79
+ v: k for k, v in self.mergeable_ranks.items()
80
+ } # type: dict[int, bytes|str]
81
+ self.decoder.update({v: k for k, v in self.special_tokens.items()})
82
+
83
+ self.tokenizer = enc # type: tiktoken.Encoding
84
+
85
+ self.eod_id = self.tokenizer.eot_token
86
+ self.im_start_id = self.special_tokens[IMSTART]
87
+ self.im_end_id = self.special_tokens[IMEND]
88
+
89
+ def __getstate__(self):
90
+ # for pickle lovers
91
+ state = self.__dict__.copy()
92
+ del state['tokenizer']
93
+ return state
94
+
95
+ def __setstate__(self, state):
96
+ # tokenizer is not python native; don't pass it; rebuild it
97
+ self.__dict__.update(state)
98
+ enc = tiktoken.Encoding(
99
+ "Qwen",
100
+ pat_str=PAT_STR,
101
+ mergeable_ranks=self.mergeable_ranks,
102
+ special_tokens=self.special_tokens,
103
+ )
104
+ self.tokenizer = enc
105
+
106
+
107
+ def __len__(self) -> int:
108
+ return self.tokenizer.n_vocab
109
+
110
+ def get_vocab(self) -> Dict[bytes, int]:
111
+ return self.mergeable_ranks
112
+
113
+ def convert_tokens_to_ids(
114
+ self, tokens: Union[bytes, str, List[Union[bytes, str]]]
115
+ ) -> List[int]:
116
+ ids = []
117
+ if isinstance(tokens, (str, bytes)):
118
+ if tokens in self.special_tokens:
119
+ return self.special_tokens[tokens]
120
+ else:
121
+ return self.mergeable_ranks.get(tokens)
122
+ for token in tokens:
123
+ if token in self.special_tokens:
124
+ ids.append(self.special_tokens[token])
125
+ else:
126
+ ids.append(self.mergeable_ranks.get(token))
127
+ return ids
128
+
129
+ def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int:
130
+ if not special_tokens and new_tokens:
131
+ raise ValueError('Adding regular tokens is not supported')
132
+ for token in new_tokens:
133
+ surface_form = token.content if isinstance(token, AddedToken) else token
134
+ if surface_form not in SPECIAL_TOKENS:
135
+ raise ValueError('Adding unknown special tokens is not supported')
136
+ return 0
137
+
138
+ def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]:
139
+ """
140
+ Save only the vocabulary of the tokenizer (vocabulary).
141
+
142
+ Returns:
143
+ `Tuple(str)`: Paths to the files saved.
144
+ """
145
+ file_path = os.path.join(save_directory, "qwen.tiktoken")
146
+ with open(file_path, "w", encoding="utf8") as w:
147
+ for k, v in self.mergeable_ranks.items():
148
+ line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n"
149
+ w.write(line)
150
+ return (file_path,)
151
+
152
+ def tokenize(
153
+ self,
154
+ text: str,
155
+ allowed_special: Union[Set, str] = "all",
156
+ disallowed_special: Union[Collection, str] = (),
157
+ **kwargs,
158
+ ) -> List[Union[bytes, str]]:
159
+ """
160
+ Converts a string in a sequence of tokens.
161
+
162
+ Args:
163
+ text (`str`):
164
+ The sequence to be encoded.
165
+ allowed_special (`Literal["all"]` or `set`):
166
+ The surface forms of the tokens to be encoded as special tokens in regular texts.
167
+ Default to "all".
168
+ disallowed_special (`Literal["all"]` or `Collection`):
169
+ The surface forms of the tokens that should not be in regular texts and trigger errors.
170
+ Default to an empty tuple.
171
+
172
+ kwargs (additional keyword arguments, *optional*):
173
+ Will be passed to the underlying model specific encode method.
174
+
175
+ Returns:
176
+ `List[bytes|str]`: The list of tokens.
177
+ """
178
+ tokens = []
179
+ text = unicodedata.normalize("NFC", text)
180
+
181
+ # this implementation takes a detour: text -> token id -> token surface forms
182
+ for t in self.tokenizer.encode(
183
+ text, allowed_special=allowed_special, disallowed_special=disallowed_special
184
+ ):
185
+ tokens.append(self.decoder[t])
186
+ return tokens
187
+
188
+ def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
189
+ """
190
+ Converts a sequence of tokens in a single string.
191
+ """
192
+ text = ""
193
+ temp = b""
194
+ for t in tokens:
195
+ if isinstance(t, str):
196
+ if temp:
197
+ text += temp.decode("utf-8", errors=self.errors)
198
+ temp = b""
199
+ text += t
200
+ elif isinstance(t, bytes):
201
+ temp += t
202
+ else:
203
+ raise TypeError("token should only be of type types or str")
204
+ if temp:
205
+ text += temp.decode("utf-8", errors=self.errors)
206
+ return text
207
+
208
+ @property
209
+ def vocab_size(self):
210
+ return self.tokenizer.n_vocab
211
+
212
+ def _convert_id_to_token(self, index: int) -> Union[bytes, str]:
213
+ """Converts an id to a token, special tokens included"""
214
+ if index in self.decoder:
215
+ return self.decoder[index]
216
+ raise ValueError("unknown ids")
217
+
218
+ def _convert_token_to_id(self, token: Union[bytes, str]) -> int:
219
+ """Converts a token to an id using the vocab, special tokens included"""
220
+ if token in self.special_tokens:
221
+ return self.special_tokens[token]
222
+ if token in self.mergeable_ranks:
223
+ return self.mergeable_ranks[token]
224
+ raise ValueError("unknown token")
225
+
226
+ def _tokenize(self, text: str, **kwargs):
227
+ """
228
+ Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based
229
+ vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).
230
+
231
+ Do NOT take care of added tokens.
232
+ """
233
+ raise NotImplementedError
234
+
235
+ def _decode(
236
+ self,
237
+ token_ids: Union[int, List[int]],
238
+ skip_special_tokens: bool = False,
239
+ errors: str = None,
240
+ **kwargs,
241
+ ) -> str:
242
+ if isinstance(token_ids, int):
243
+ token_ids = [token_ids]
244
+ if skip_special_tokens:
245
+ token_ids = [i for i in token_ids if i < self.eod_id]
246
+ return self.tokenizer.decode(token_ids, errors=errors or self.errors)
vocab/qwen/Qwen-7B-Chat/tokenizer_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_max_length": 8192,
3
+ "tokenizer_class": "QWenTokenizer",
4
+ "auto_map": {
5
+ "AutoTokenizer": [
6
+ "tokenization_qwen.QWenTokenizer",
7
+ null
8
+ ]
9
+ }
10
+ }
vocab/qwen/__init__.py CHANGED
@@ -1,13 +1,18 @@
1
  """
2
-
3
  依赖 torch tiktoken
4
  依赖 transformer 4.31.0 及以上,
 
 
5
  """
6
 
 
7
  from transformers import AutoTokenizer
 
 
8
 
9
  # 请注意:分词器默认行为已更改为默认关闭特殊token攻击防护。
10
- tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen-VL-Chat", trust_remote_code=True)
 
11
 
12
  def test():
13
  encoding = tokenizer.encode("测试华为手机10086 8个空格")
 
1
  """
 
2
  依赖 torch tiktoken
3
  依赖 transformer 4.31.0 及以上,
4
+
5
+ https://huggingface.co/tangger/Qwen-7B-Chat Qwen官方模型临时下架了,这个是备份
6
  """
7
 
8
+ import os
9
  from transformers import AutoTokenizer
10
+ CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
11
+ TOKENIZER_DIR = os.path.join(CURRENT_DIR, "Qwen-7B-Chat")
12
 
13
  # 请注意:分词器默认行为已更改为默认关闭特殊token攻击防护。
14
+ # tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen-VL-Chat", trust_remote_code=True)
15
+ tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR, trust_remote_code=True)
16
 
17
  def test():
18
  encoding = tokenizer.encode("测试华为手机10086 8个空格")