Update processors

#14
image_processing_minicpmv.py ADDED
@@ -0,0 +1,408 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional, Union, Dict, Any
2
+
3
+ import torch
4
+ import math
5
+ import PIL.Image
6
+ import PIL.ImageSequence
7
+ import numpy as np
8
+ import PIL
9
+ from PIL import Image
10
+
11
+ from transformers.utils import TensorType, requires_backends, is_torch_dtype, is_torch_device
12
+ from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
13
+ from transformers import AutoImageProcessor
14
+ from transformers.image_transforms import to_channel_dimension_format
15
+ from transformers.image_utils import (
16
+ ImageInput,
17
+ make_list_of_images,
18
+ valid_images,
19
+ is_torch_tensor,
20
+ to_numpy_array,
21
+ infer_channel_dimension_format,
22
+ ChannelDimension
23
+ )
24
+
25
+
26
+ def recursive_converter(converter, value):
27
+ if isinstance(value, list):
28
+ new_value = []
29
+ for v in value:
30
+ new_value += [recursive_converter(converter, v)]
31
+ return new_value
32
+ else:
33
+ return converter(value)
34
+
35
+
36
+ class MiniCPMVBatchFeature(BatchFeature):
37
+ r"""
38
+ Extend from BatchFeature for supporting various image size
39
+ """
40
+ def __init__(self, data: Optional[Dict[str, Any]] = None, tensor_type: Union[None, str, TensorType] = None):
41
+ super().__init__(data)
42
+ self.convert_to_tensors(tensor_type=tensor_type)
43
+
44
+ def convert_to_tensors(self, tensor_type: Optional[Union[str, TensorType]] = None):
45
+ if tensor_type is None:
46
+ return self
47
+
48
+ is_tensor, as_tensor = self._get_is_as_tensor_fns(tensor_type)
49
+
50
+ def converter(value):
51
+ try:
52
+ if not is_tensor(value):
53
+ tensor = as_tensor(value)
54
+ return tensor
55
+ return value
56
+ except: # noqa E722
57
+ if key == "overflowing_values":
58
+ raise ValueError("Unable to create tensor returning overflowing values of different lengths. ")
59
+ raise ValueError(
60
+ "Unable to create tensor, you should probably activate padding "
61
+ "with 'padding=True' to have batched tensors with the same length."
62
+ )
63
+
64
+
65
+ for key, value in self.items():
66
+ self[key] = recursive_converter(converter, value)
67
+ return self
68
+
69
+ def to(self, *args, **kwargs) -> "MiniCPMVBatchFeature":
70
+ requires_backends(self, ["torch"])
71
+ import torch
72
+
73
+ def cast_tensor(v):
74
+ # check if v is a floating point
75
+ if v is None:
76
+ return None
77
+ if torch.is_floating_point(v):
78
+ # cast and send to device
79
+ return v.to(*args, **kwargs)
80
+ elif device is not None:
81
+ return v.to(device=device)
82
+ else:
83
+ return v
84
+
85
+ new_data = {}
86
+ device = kwargs.get("device")
87
+ # Check if the args are a device or a dtype
88
+ if device is None and len(args) > 0:
89
+ # device should be always the first argument
90
+ arg = args[0]
91
+ if is_torch_dtype(arg):
92
+ # The first argument is a dtype
93
+ pass
94
+ elif isinstance(arg, str) or is_torch_device(arg) or isinstance(arg, int):
95
+ device = arg
96
+ else:
97
+ # it's something else
98
+ raise ValueError(f"Attempting to cast a BatchFeature to type {str(arg)}. This is not supported.")
99
+ # We cast only floating point tensors to avoid issues with tokenizers casting `LongTensor` to `FloatTensor`
100
+ for k, v in self.items():
101
+ new_data[k] = recursive_converter(cast_tensor, v)
102
+ self.data = new_data
103
+ return self
104
+
105
+
106
+ class MiniCPMVImageProcessor(BaseImageProcessor):
107
+ model_input_names = ["pixel_values"]
108
+
109
+ def __init__(
110
+ self,
111
+ max_slice_nums=9,
112
+ scale_resolution=448,
113
+ patch_size=14,
114
+ **kwargs):
115
+ super().__init__(**kwargs)
116
+ self.max_slice_nums = max_slice_nums
117
+ self.scale_resolution = scale_resolution
118
+ self.patch_size = patch_size
119
+ self.image_feature_size = kwargs.pop("image_feature_size", 64)
120
+ self.im_start_token = kwargs.pop("im_start", "<image>")
121
+ self.im_end_token = kwargs.pop("im_end", "</image>")
122
+ self.slice_start_token = kwargs.pop("slice_start", "<slice>")
123
+ self.slice_end_token = kwargs.pop("slice_end", "</slice>")
124
+ self.unk_token = kwargs.pop("unk", "<unk>")
125
+ self.mean = np.array(kwargs.pop("norm_mean", [0.5, 0.5, 0.5]))
126
+ self.std = np.array(kwargs.pop("norm_std", [0.5, 0.5, 0.5]))
127
+ self.version = kwargs.pop("version", 2.0)
128
+
129
+ def ensure_divide(self, length, patch_size):
130
+ return max(round(length / patch_size) * patch_size, patch_size)
131
+
132
+ def find_best_resize(self,
133
+ original_size,
134
+ scale_resolution,
135
+ patch_size,
136
+ allow_upscale=False):
137
+ width, height = original_size
138
+ if (width * height >
139
+ scale_resolution * scale_resolution) or allow_upscale:
140
+ r = width / height
141
+ height = int(scale_resolution / math.sqrt(r))
142
+ width = int(height * r)
143
+ best_width = self.ensure_divide(width, patch_size)
144
+ best_height = self.ensure_divide(height, patch_size)
145
+ return (best_width, best_height)
146
+
147
+ def get_refine_size(self,
148
+ original_size,
149
+ grid,
150
+ scale_resolution,
151
+ patch_size,
152
+ allow_upscale=False):
153
+ width, height = original_size
154
+ grid_x, grid_y = grid
155
+
156
+ refine_width = self.ensure_divide(width, grid_x)
157
+ refine_height = self.ensure_divide(height, grid_y)
158
+
159
+ grid_width = refine_width / grid_x
160
+ grid_height = refine_height / grid_y
161
+
162
+ best_grid_size = self.find_best_resize((grid_width, grid_height),
163
+ scale_resolution,
164
+ patch_size,
165
+ allow_upscale=allow_upscale)
166
+ refine_size = (best_grid_size[0] * grid_x, best_grid_size[1] * grid_y)
167
+ return refine_size
168
+
169
+ def split_to_patches(self, image, grid):
170
+ patches = []
171
+ width, height = image.size
172
+ grid_x = int(width / grid[0])
173
+ grid_y = int(height / grid[1])
174
+ for i in range(0, height, grid_y):
175
+ images = []
176
+ for j in range(0, width, grid_x):
177
+ box = (j, i, j + grid_x, i + grid_y)
178
+ patch = image.crop(box)
179
+ images.append(patch)
180
+ patches.append(images)
181
+ return patches
182
+
183
+ def slice_image(
184
+ self, image, max_slice_nums=9, scale_resolution=448, patch_size=14, never_split=False
185
+ ):
186
+ original_size = image.size
187
+ original_width, original_height = original_size
188
+ log_ratio = math.log(original_width / original_height)
189
+ ratio = original_width * original_height / (scale_resolution * scale_resolution)
190
+ multiple = min(math.ceil(ratio), max_slice_nums)
191
+
192
+ source_image = None
193
+ best_grid = None
194
+ patches = []
195
+
196
+ if multiple <= 1 or never_split:
197
+ # dont need to slice, upsample
198
+ best_size = self.find_best_resize(
199
+ original_size, scale_resolution, patch_size, allow_upscale=True
200
+ )
201
+ source_image = image.resize(best_size, resample=Image.Resampling.BICUBIC)
202
+ else:
203
+ candidate_split_grids_nums = []
204
+ for i in [multiple - 1, multiple, multiple + 1]:
205
+ if i == 1 or i > max_slice_nums:
206
+ continue
207
+ candidate_split_grids_nums.append(i)
208
+
209
+ # source image, down-sampling and ensure divided by patch_size
210
+ best_resize = self.find_best_resize(original_size, scale_resolution, patch_size)
211
+ source_image = image.copy().resize(best_resize, resample=Image.Resampling.BICUBIC)
212
+ candidate_grids = []
213
+
214
+ # find best grid
215
+ for split_grids_nums in candidate_split_grids_nums:
216
+ m = 1
217
+ while m <= split_grids_nums:
218
+ if split_grids_nums % m == 0:
219
+ candidate_grids.append([m, split_grids_nums // m])
220
+ m += 1
221
+
222
+ best_grid = [1, 1]
223
+ min_error = float("inf")
224
+ for grid in candidate_grids:
225
+ error = abs(log_ratio - math.log(grid[0] / grid[1]))
226
+ if error < min_error:
227
+ best_grid = grid
228
+ min_error = error
229
+
230
+ refine_size = self.get_refine_size(
231
+ original_size, best_grid, scale_resolution, patch_size, allow_upscale=True
232
+ )
233
+
234
+ refine_image = image.resize(refine_size, resample=Image.Resampling.BICUBIC)
235
+ patches = self.split_to_patches(refine_image, best_grid)
236
+
237
+ return source_image, patches, best_grid
238
+
239
+ def get_grid_placeholder(self, grid):
240
+ if grid is None:
241
+ return ""
242
+ image_placeholder = (
243
+ self.im_start_token
244
+ + self.unk_token * self.image_feature_size
245
+ + self.im_end_token
246
+ )
247
+
248
+ cols = grid[0]
249
+ rows = grid[1]
250
+ slices = []
251
+ for i in range(rows):
252
+ lines = []
253
+ for j in range(cols):
254
+ lines.append(image_placeholder)
255
+ slices.append("".join(lines))
256
+
257
+ slice_placeholder = self.slice_start_token + "\n".join(slices) + self.slice_end_token
258
+ return slice_placeholder
259
+
260
+ def get_sliced_images(self, image):
261
+ slice_images = []
262
+
263
+ source_image, patches, sliced_grid = self.slice_image(
264
+ image,
265
+ self.max_slice_nums, # default: 9
266
+ self.scale_resolution, # default: 448
267
+ self.patch_size # default: 14
268
+ )
269
+ slice_images.append(source_image)
270
+
271
+ if len(patches) > 0:
272
+ for i in range(len(patches)):
273
+ for j in range(len(patches[0])):
274
+ slice_images.append(patches[i][j])
275
+ return slice_images
276
+
277
+ def get_sliced_grid(self, image_size):
278
+ original_width, original_height = image_size
279
+ log_ratio = math.log(original_width / original_height)
280
+ ratio = original_width * original_height / (self.scale_resolution * self.scale_resolution)
281
+ multiple = min(math.ceil(ratio), self.max_slice_nums)
282
+ if multiple <= 1:
283
+ return None
284
+ candidate_split_grids_nums = []
285
+ for i in [multiple - 1, multiple, multiple + 1]:
286
+ if i == 1 or i > self.max_slice_nums:
287
+ continue
288
+ candidate_split_grids_nums.append(i)
289
+
290
+ candidate_grids = []
291
+ for split_grids_nums in candidate_split_grids_nums:
292
+ m = 1
293
+ while m <= split_grids_nums:
294
+ if split_grids_nums % m == 0:
295
+ candidate_grids.append([m, split_grids_nums // m])
296
+ m += 1
297
+
298
+ best_grid = [1, 1]
299
+ min_error = float("inf")
300
+ for grid in candidate_grids:
301
+ error = abs(log_ratio - math.log(grid[0] / grid[1]))
302
+ if error < min_error:
303
+ best_grid = grid
304
+ min_error = error
305
+
306
+ return best_grid
307
+
308
+ def get_slice_image_placeholder(self, image_size):
309
+ grid = self.get_sliced_grid(image_size=image_size)
310
+ return (
311
+ self.im_start_token
312
+ + self.unk_token * self.image_feature_size
313
+ + self.im_end_token
314
+ ) + self.get_grid_placeholder(grid=grid) + "\n"
315
+
316
+ def to_pil_image(self, image, rescale=None) -> PIL.Image.Image:
317
+ """
318
+ Converts `image` to a PIL Image. Optionally rescales it and puts the channel dimension back as the last axis if
319
+ needed.
320
+
321
+ Args:
322
+ image (`PIL.Image.Image` or `numpy.ndarray` or `torch.Tensor`):
323
+ The image to convert to the PIL Image format.
324
+ rescale (`bool`, *optional*):
325
+ Whether or not to apply the scaling factor (to make pixel values integers between 0 and 255). Will
326
+ default to `True` if the image type is a floating type, `False` otherwise.
327
+ """
328
+ if isinstance(image, PIL.Image.Image):
329
+ return image
330
+ if is_torch_tensor(image):
331
+ image = image.numpy()
332
+
333
+ if isinstance(image, np.ndarray):
334
+ if rescale is None:
335
+ # rescale default to the array being of floating type.
336
+ rescale = isinstance(image.flat[0], np.floating)
337
+ # If the channel as been moved to first dim, we put it back at the end.
338
+ if image.ndim == 3 and image.shape[0] in [1, 3]:
339
+ image = image.transpose(1, 2, 0)
340
+ if rescale:
341
+ image = image * 255
342
+ image = image.astype(np.uint8)
343
+ return PIL.Image.fromarray(image)
344
+ return image
345
+
346
+ def reshape_by_patch(self, image):
347
+ """
348
+ :param image: shape [3, H, W]
349
+ :param patch_size:
350
+ :return: [3, patch_size, HW/patch_size]
351
+ """
352
+ image = torch.from_numpy(image)
353
+ patch_size = self.patch_size
354
+ patches = torch.nn.functional.unfold(
355
+ image,
356
+ (patch_size, patch_size),
357
+ stride=(patch_size, patch_size)
358
+ )
359
+
360
+ patches = patches.reshape(image.size(0), patch_size, patch_size, -1)
361
+ patches = patches.permute(0, 1, 3, 2).reshape(image.size(0), patch_size, -1)
362
+ return patches.numpy()
363
+
364
+ def preprocess(
365
+ self,
366
+ images: ImageInput,
367
+ do_pad: Optional[bool] = True, # TODO: add pad for MiniCPM-Llama3-V-2_5
368
+ return_tensors: Optional[Union[str, TensorType]] = None
369
+ ) -> MiniCPMVBatchFeature:
370
+ images = make_list_of_images(images)
371
+
372
+ if not valid_images(images):
373
+ raise ValueError(
374
+ "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
375
+ "torch.Tensor, tf.Tensor or jax.ndarray."
376
+ )
377
+
378
+ images = [self.to_pil_image(image).convert("RGB") for image in images]
379
+ input_data_format = infer_channel_dimension_format(np.array(images[0]))
380
+
381
+ new_images = []
382
+ image_sizes = [image.size for image in images]
383
+ tgt_sizes = []
384
+ for image in images:
385
+ image_patches = self.get_sliced_images(image)
386
+ image_patches = [to_numpy_array(image).astype(np.float32) / 255 for image in image_patches]
387
+ image_patches = [
388
+ self.normalize(image=image, mean=self.mean, std=self.std, input_data_format=input_data_format)
389
+ for image in image_patches
390
+ ]
391
+ image_patches = [
392
+ to_channel_dimension_format(image, ChannelDimension.FIRST, input_channel_dim=input_data_format)
393
+ for image in image_patches
394
+ ]
395
+ patches_tgt_sizes = [
396
+ np.array((image.shape[1] // self.patch_size, image.shape[2] // self.patch_size))
397
+ for image in image_patches
398
+ ]
399
+ patches_tgt_sizes = np.vstack(patches_tgt_sizes)
400
+
401
+ new_images += [image_patches]
402
+ tgt_sizes += [patches_tgt_sizes]
403
+
404
+ return MiniCPMVBatchFeature(
405
+ data={"pixel_values": new_images, "image_sizes": image_sizes, "tgt_sizes": tgt_sizes}, tensor_type=return_tensors
406
+ )
407
+
408
+ AutoImageProcessor.register("MiniCPMVImageProcessor", MiniCPMVImageProcessor)
modeling_minicpmv.py CHANGED
@@ -7,7 +7,6 @@ import torchvision
7
  from PIL import Image
8
  from timm.data import IMAGENET_INCEPTION_MEAN, IMAGENET_INCEPTION_STD
9
  from torchvision import transforms
10
- from transformers import LlamaTokenizer
11
 
12
  from .configuration_minicpm import MiniCPMVConfig
13
  from .modeling_minicpm import MiniCPMForCausalLM, MiniCPMPreTrainedModel
@@ -67,11 +66,23 @@ class MiniCPMV(MiniCPMVPreTrainedModel):
67
  )
68
 
69
  def get_input_embeddings(self):
70
- return self.llm.get_input_embeddings()
71
 
72
  def set_input_embeddings(self, value):
73
  self.llm.embed_tokens = value
74
 
 
 
 
 
 
 
 
 
 
 
 
 
75
  def get_vision_embedding(self, pixel_values):
76
  res = []
77
  dtype = self.vpm.pos_embed.data.dtype
@@ -118,7 +129,7 @@ class MiniCPMV(MiniCPMVPreTrainedModel):
118
  cur_vs_hs = vision_hidden_states[i]
119
  if len(cur_vs_hs) > 0:
120
  cur_vllm_emb = vllm_embedding[i]
121
- cur_image_bound = data["image_bound"][i]
122
  if len(cur_image_bound) > 0:
123
  image_indices = torch.stack(
124
  [
@@ -150,59 +161,6 @@ class MiniCPMV(MiniCPMVPreTrainedModel):
150
  **kwargs
151
  )
152
 
153
- def _convert_to_tensors(
154
- self, tokenizer, input_str, max_inp_length: Optional[int] = None
155
- ):
156
- if tokenizer.add_bos_token:
157
- input_ids = tokenizer.encode(input_str)
158
- else:
159
- input_ids = [tokenizer.bos_id] + tokenizer.encode(input_str)
160
- if max_inp_length is not None:
161
- input_ids = input_ids[:max_inp_length]
162
- input_ids = torch.tensor(input_ids, dtype=torch.int32)
163
-
164
- image_start_tokens = torch.where(input_ids == tokenizer.im_start_id)[0]
165
- # 跳过 im_start
166
- image_start_tokens += 1
167
- image_end_tokens = torch.where(input_ids == tokenizer.im_end_id)[0]
168
- valid_image_nums = max(len(image_start_tokens), len(image_end_tokens))
169
- image_bound = torch.hstack(
170
- [
171
- image_start_tokens[:valid_image_nums].unsqueeze(-1),
172
- image_end_tokens[:valid_image_nums].unsqueeze(-1),
173
- ]
174
- )
175
-
176
- model_input = {}
177
- model_input["input_ids"] = input_ids.unsqueeze(0).to(self.device)
178
- model_input["image_bound"] = image_bound
179
-
180
- return model_input
181
-
182
- def _process_list(
183
- self, tokenizer, data_list: List[str], max_inp_length: Optional[int] = None
184
- ):
185
- pad_keys = ["input_ids"]
186
- input_tensors = []
187
- for data in data_list:
188
- input_tensors.append(
189
- self._convert_to_tensors(tokenizer, data, max_inp_length)
190
- )
191
- padded = {}
192
- for key in pad_keys:
193
- padded[key] = pad(input_tensors, key, padding_side="left").to(self.device)
194
- padded["image_bound"] = [i["image_bound"] for i in input_tensors]
195
- return padded
196
-
197
- def _decode(self, inputs_embeds, tokenizer, **kwargs):
198
- output = self.llm.generate(
199
- inputs_embeds=inputs_embeds,
200
- pad_token_id=0,
201
- eos_token_id=tokenizer.eos_token_id,
202
- **kwargs
203
- )
204
- return self._decode_text(output, tokenizer)
205
-
206
  def _decode_text(self, result_ids, tokenizer):
207
  result_text = []
208
  for result in result_ids:
@@ -214,87 +172,52 @@ class MiniCPMV(MiniCPMVPreTrainedModel):
214
  result_text.append(tokenizer.decode(result).strip())
215
  return result_text
216
 
217
- def slice_image(self, image):
218
- return slice_image(
219
- image,
220
- self.config.max_slice_nums,
221
- self.config.scale_resolution,
222
- self.config.patch_size,
223
- )
224
-
225
- def get_slice_image_placeholder(self, image, tokenizer):
226
- image_placeholder = (
227
- tokenizer.im_start
228
- + tokenizer.unk_token * self.config.query_num
229
- + tokenizer.im_end
230
- )
231
-
232
- slice_images = []
233
-
234
- source_image, patches, best_grid = slice_image(
235
- image,
236
- self.config.max_slice_nums,
237
- self.config.scale_resolution,
238
- self.config.patch_size,
239
  )
240
-
241
- slice_images.append(source_image)
242
- final_placeholder = image_placeholder
243
-
244
- if len(patches) > 0:
245
- for i in range(len(patches)):
246
- for j in range(len(patches[0])):
247
- slice_images.append(patches[i][j])
248
-
249
- final_placeholder += get_grid_placeholder(
250
- tokenizer, best_grid, self.config.query_num
251
- )
252
-
253
- return slice_images, final_placeholder
254
 
255
  def generate(
256
  self,
257
- data_list=None,
258
- img_list=None,
 
 
 
259
  tokenizer=None,
260
- max_inp_length: Optional[int] = None,
261
  vision_hidden_states=None,
262
- return_vision_hidden_states=False,
263
  **kwargs
264
  ):
265
-
266
- assert data_list is not None
267
- bs = len(data_list)
268
  if img_list == None:
269
  img_list = [[] for i in range(bs)]
270
  assert bs == len(img_list)
271
 
272
- model_inputs = self._process_list(tokenizer, data_list, max_inp_length)
273
-
274
  if vision_hidden_states is None:
275
  pixel_values = []
276
  for i in range(bs):
277
  img_inps = []
278
  for img in img_list[i]:
279
- img_inps.append(self.transform(img).to(self.device))
280
- if img_inps:
281
- pixel_values.append(img_inps)
282
- else:
283
- pixel_values.append([])
284
- model_inputs["pixel_values"] = pixel_values
285
- else:
286
- model_inputs["vision_hidden_states"] = vision_hidden_states
287
-
288
- with torch.inference_mode():
289
- (
290
- model_inputs["inputs_embeds"],
291
- vision_hidden_states,
292
- ) = self.get_vllm_embedding(model_inputs)
293
-
294
- result = self._decode(model_inputs["inputs_embeds"], tokenizer, **kwargs)
295
-
296
- if return_vision_hidden_states:
297
- return result, vision_hidden_states
298
 
299
  return result
300
 
@@ -304,6 +227,7 @@ class MiniCPMV(MiniCPMVPreTrainedModel):
304
  msgs,
305
  context,
306
  tokenizer,
 
307
  vision_hidden_states=None,
308
  max_new_tokens=1024,
309
  sampling=True,
@@ -313,34 +237,9 @@ class MiniCPMV(MiniCPMVPreTrainedModel):
313
  if isinstance(msgs, str):
314
  msgs = json.loads(msgs)
315
  # msgs to prompt
316
- prompt = ""
317
- for i, msg in enumerate(msgs):
318
- role = msg["role"]
319
- content = msg["content"]
320
- assert role in ["user", "assistant"]
321
- if i == 0:
322
- if image is None:
323
- images = []
324
- else:
325
- assert role == "user", "The role of first msg should be user"
326
- if self.config.slice_mode:
327
- images, final_placeholder = self.get_slice_image_placeholder(
328
- image, tokenizer
329
- )
330
- content = final_placeholder + "\n" + content
331
- else:
332
- images = [image]
333
- content = (
334
- tokenizer.im_start
335
- + tokenizer.unk_token * self.config.query_num
336
- + tokenizer.im_end
337
- + "\n"
338
- + content
339
- )
340
- prompt += "<用户>" if role == "user" else "<AI>"
341
- prompt += content
342
- prompt += "<AI>"
343
- final_input = prompt
344
 
345
  if sampling:
346
  generation_config = {
@@ -359,235 +258,18 @@ class MiniCPMV(MiniCPMVPreTrainedModel):
359
  generation_config.update(
360
  (k, kwargs[k]) for k in generation_config.keys() & kwargs.keys()
361
  )
362
-
363
  with torch.inference_mode():
364
- res, vision_hidden_states = self.generate(
365
- data_list=[final_input],
366
- max_inp_length=max_inp_length,
367
- img_list=[images],
368
  tokenizer=tokenizer,
369
  max_new_tokens=max_new_tokens,
370
  vision_hidden_states=vision_hidden_states,
371
- return_vision_hidden_states=True,
372
- **generation_config
373
  )
 
374
  answer = res[0]
375
  context = msgs.copy()
376
  context.append({"role": "assistant", "content": answer})
377
 
378
  return answer, context, generation_config
379
-
380
-
381
- class LlamaTokenizerWrapper(LlamaTokenizer):
382
- def __init__(self, **kwargs):
383
- super().__init__(**kwargs)
384
- self.im_start = "<image>"
385
- self.im_end = "</image>"
386
- self.ref_start = "<ref>"
387
- self.ref_end = "</ref>"
388
- self.box_start = "<box>"
389
- self.box_end = "</box>"
390
- self.quad_start = "<quad>"
391
- self.quad_end = "</quad>"
392
- self.point_start = "<point>"
393
- self.point_end = "</point>"
394
- self.slice_start = "<slice>"
395
- self.slice_end = "</slice>"
396
-
397
- @property
398
- def eos_id(self):
399
- return self.sp_model.eos_id()
400
-
401
- @property
402
- def bos_id(self):
403
- return self.sp_model.bos_id()
404
-
405
- @property
406
- def unk_id(self):
407
- return self.sp_model.unk_id()
408
-
409
- @property
410
- def im_start_id(self):
411
- return self._convert_token_to_id(self.im_start)
412
-
413
- @property
414
- def im_end_id(self):
415
- return self._convert_token_to_id(self.im_end)
416
-
417
-
418
- def pad(orig_items, key, max_length=None, padding_value=0, padding_side="left"):
419
- items = []
420
- if isinstance(orig_items[0][key], list):
421
- assert isinstance(orig_items[0][key][0], torch.Tensor)
422
- for it in orig_items:
423
- for tr in it[key]:
424
- items.append({key: tr})
425
- else:
426
- assert isinstance(orig_items[0][key], torch.Tensor)
427
- items = orig_items
428
-
429
- batch_size = len(items)
430
- shape = items[0][key].shape
431
- dim = len(shape)
432
- assert dim <= 3
433
- if max_length is None:
434
- max_length = 0
435
- max_length = max(max_length, max(item[key].shape[-1] for item in items))
436
- min_length = min(item[key].shape[-1] for item in items)
437
- dtype = items[0][key].dtype
438
-
439
- if dim == 1:
440
- return torch.cat([item[key] for item in items], dim=0)
441
- elif dim == 2:
442
- if max_length == min_length:
443
- return torch.cat([item[key] for item in items], dim=0)
444
- tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value
445
- else:
446
- tensor = (
447
- torch.zeros((batch_size, max_length, shape[-1]), dtype=dtype)
448
- + padding_value
449
- )
450
-
451
- for i, item in enumerate(items):
452
- if dim == 2:
453
- if padding_side == "left":
454
- tensor[i, -len(item[key][0]) :] = item[key][0].clone()
455
- else:
456
- tensor[i, : len(item[key][0])] = item[key][0].clone()
457
- elif dim == 3:
458
- if padding_side == "left":
459
- tensor[i, -len(item[key][0]) :, :] = item[key][0].clone()
460
- else:
461
- tensor[i, : len(item[key][0]), :] = item[key][0].clone()
462
-
463
- return tensor
464
-
465
-
466
- def slice_image(
467
- image, max_slice_nums=9, scale_resolution=448, patch_size=14, never_split=False
468
- ):
469
- original_size = image.size
470
- original_width, original_height = original_size
471
- log_ratio = math.log(original_width / original_height)
472
- ratio = original_width * original_height / (scale_resolution * scale_resolution)
473
- multiple = min(math.ceil(ratio), max_slice_nums)
474
-
475
- source_image = None
476
- best_grid = None
477
- patches = []
478
-
479
- if multiple <= 1 or never_split:
480
- # dont need to slice, upsample
481
- best_size = find_best_resize(
482
- original_size, scale_resolution, patch_size, allow_upscale=True
483
- )
484
- source_image = image.resize(best_size, Image.Resampling.BICUBIC)
485
- else:
486
- candidate_split_grids_nums = []
487
- for i in [multiple - 1, multiple, multiple + 1]:
488
- if i == 1 or i > max_slice_nums:
489
- continue
490
- candidate_split_grids_nums.append(i)
491
-
492
- # source image, down-sampling and ensure divided by patch_size
493
- best_resize = find_best_resize(original_size, scale_resolution, patch_size)
494
- source_image = image.copy().resize(best_resize, Image.Resampling.BICUBIC)
495
- candidate_grids = []
496
-
497
- # find best grid
498
- for split_grids_nums in candidate_split_grids_nums:
499
- m = 1
500
- while m <= split_grids_nums:
501
- if split_grids_nums % m == 0:
502
- candidate_grids.append([m, split_grids_nums // m])
503
- m += 1
504
-
505
- best_grid = [1, 1]
506
- min_error = float("inf")
507
- for grid in candidate_grids:
508
- error = abs(log_ratio - math.log(grid[0] / grid[1]))
509
- if error < min_error:
510
- best_grid = grid
511
- min_error = error
512
-
513
- refine_size = get_refine_size(
514
- original_size, best_grid, scale_resolution, patch_size, allow_upscale=True
515
- )
516
-
517
- refine_image = image.resize(refine_size, Image.Resampling.BICUBIC)
518
- patches = split_to_patches(refine_image, best_grid)
519
-
520
- return source_image, patches, best_grid
521
-
522
-
523
- def ensure_divide(length, patch_size):
524
- return max(round(length / patch_size) * patch_size, patch_size)
525
-
526
-
527
- def find_best_resize(original_size, scale_resolution, patch_size, allow_upscale=False):
528
- width, height = original_size
529
- if (width * height > scale_resolution * scale_resolution) or allow_upscale:
530
- r = width / height
531
- height = int(scale_resolution / math.sqrt(r))
532
- width = int(height * r)
533
- best_width = ensure_divide(width, patch_size)
534
- best_height = ensure_divide(height, patch_size)
535
- return (best_width, best_height)
536
-
537
-
538
- def get_refine_size(
539
- original_size, grid, scale_resolution, patch_size, allow_upscale=False
540
- ):
541
- width, height = original_size
542
- grid_x, grid_y = grid
543
-
544
- refine_width = ensure_divide(width, grid_x)
545
- refine_height = ensure_divide(height, grid_y)
546
-
547
- grid_width = refine_width / grid_x
548
- grid_height = refine_height / grid_y
549
-
550
- best_grid_size = find_best_resize(
551
- (grid_width, grid_height),
552
- scale_resolution,
553
- patch_size,
554
- allow_upscale=allow_upscale,
555
- )
556
-
557
- refine_size = (best_grid_size[0] * grid_x, best_grid_size[1] * grid_y)
558
-
559
- return refine_size
560
-
561
-
562
- def split_to_patches(image, grid):
563
- patches = []
564
- width, height = image.size
565
- grid_x = int(width / grid[0])
566
- grid_y = int(height / grid[1])
567
-
568
- for i in range(0, height, grid_y):
569
- images = []
570
- for j in range(0, width, grid_x):
571
- box = (j, i, j + grid_x, i + grid_y)
572
- patch = image.crop(box)
573
- images.append(patch)
574
- patches.append(images)
575
-
576
- return patches
577
-
578
-
579
- def get_grid_placeholder(tokenizer, grid, query_num):
580
- image_placeholder = (
581
- tokenizer.im_start + tokenizer.unk_token * query_num + tokenizer.im_end
582
- )
583
-
584
- cols = grid[0]
585
- rows = grid[1]
586
- slices = []
587
- for i in range(rows):
588
- lines = []
589
- for j in range(cols):
590
- lines.append(image_placeholder)
591
- slices.append("".join(lines))
592
- slice_placeholder = tokenizer.slice_start + "\n".join(slices) + tokenizer.slice_end
593
- return slice_placeholder
 
7
  from PIL import Image
8
  from timm.data import IMAGENET_INCEPTION_MEAN, IMAGENET_INCEPTION_STD
9
  from torchvision import transforms
 
10
 
11
  from .configuration_minicpm import MiniCPMVConfig
12
  from .modeling_minicpm import MiniCPMForCausalLM, MiniCPMPreTrainedModel
 
66
  )
67
 
68
  def get_input_embeddings(self):
69
+ return self.llm.embed_tokens
70
 
71
  def set_input_embeddings(self, value):
72
  self.llm.embed_tokens = value
73
 
74
+ def get_output_embeddings(self):
75
+ return self.llm.lm_head
76
+
77
+ def set_output_embeddings(self, new_embeddings):
78
+ self.llm.lm_head = new_embeddings
79
+
80
+ def set_decoder(self, decoder):
81
+ self.llm = decoder
82
+
83
+ def get_decoder(self):
84
+ return self.llm
85
+
86
  def get_vision_embedding(self, pixel_values):
87
  res = []
88
  dtype = self.vpm.pos_embed.data.dtype
 
129
  cur_vs_hs = vision_hidden_states[i]
130
  if len(cur_vs_hs) > 0:
131
  cur_vllm_emb = vllm_embedding[i]
132
+ cur_image_bound = data["image_bounds"][i]
133
  if len(cur_image_bound) > 0:
134
  image_indices = torch.stack(
135
  [
 
161
  **kwargs
162
  )
163
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
164
  def _decode_text(self, result_ids, tokenizer):
165
  result_text = []
166
  for result in result_ids:
 
172
  result_text.append(tokenizer.decode(result).strip())
173
  return result_text
174
 
175
+ def _decode(self, inputs_embeds, tokenizer, **kwargs):
176
+ output = self.llm.generate(
177
+ inputs_embeds=inputs_embeds,
178
+ pad_token_id=0,
179
+ eos_token_id=tokenizer.eos_token_id if tokenizer is not None else kwargs.pop("eos_token_id", 2),
180
+ **kwargs
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
181
  )
182
+ return output
 
 
 
 
 
 
 
 
 
 
 
 
 
183
 
184
  def generate(
185
  self,
186
+ input_ids,
187
+ pixel_values=None,
188
+ image_sizes=[],
189
+ image_bounds=[],
190
+ tgt_sizes=[],
191
  tokenizer=None,
 
192
  vision_hidden_states=None,
 
193
  **kwargs
194
  ):
195
+ bs = len(input_ids)
196
+ img_list = pixel_values
 
197
  if img_list == None:
198
  img_list = [[] for i in range(bs)]
199
  assert bs == len(img_list)
200
 
 
 
201
  if vision_hidden_states is None:
202
  pixel_values = []
203
  for i in range(bs):
204
  img_inps = []
205
  for img in img_list[i]:
206
+ img_inps.append(img.to(self.device, self.dtype))
207
+ pixel_values.append(img_inps)
208
+
209
+ # with torch.inference_mode():
210
+ (
211
+ input_embeds,
212
+ vision_hidden_states,
213
+ ) = self.get_vllm_embedding({
214
+ "input_ids": input_ids,
215
+ "pixel_values": pixel_values,
216
+ "image_sizes": image_sizes,
217
+ "image_bounds": image_bounds,
218
+ "tgt_sizes": tgt_sizes
219
+ })
220
+ result = self._decode(input_embeds, tokenizer, **kwargs)
 
 
 
 
221
 
222
  return result
223
 
 
227
  msgs,
228
  context,
229
  tokenizer,
230
+ processor,
231
  vision_hidden_states=None,
232
  max_new_tokens=1024,
233
  sampling=True,
 
237
  if isinstance(msgs, str):
238
  msgs = json.loads(msgs)
239
  # msgs to prompt
240
+
241
+ prompt = processor.tokenizer.apply_chat_template(msgs)
242
+ inputs = processor(prompt, [image], return_tensors="pt").to(self.device)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
243
 
244
  if sampling:
245
  generation_config = {
 
258
  generation_config.update(
259
  (k, kwargs[k]) for k in generation_config.keys() & kwargs.keys()
260
  )
 
261
  with torch.inference_mode():
262
+ res = self.generate(
263
+ **inputs,
 
 
264
  tokenizer=tokenizer,
265
  max_new_tokens=max_new_tokens,
266
  vision_hidden_states=vision_hidden_states,
267
+ **generation_config,
 
268
  )
269
+ res = self._decode_text(res, tokenizer)
270
  answer = res[0]
271
  context = msgs.copy()
272
  context.append({"role": "assistant", "content": answer})
273
 
274
  return answer, context, generation_config
275
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
preprocessor_config.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "image_processor_type": "MiniCPMVImageProcessor",
3
+ "auto_map": {
4
+ "AutoProcessor": "processing_minicpmv.MiniCPMVProcessor",
5
+ "AutoImageProcessor": "image_processing_minicpmv.MiniCPMVImageProcessor"
6
+ },
7
+ "processor_class": "MiniCPMVProcessor",
8
+ "max_slice_nums": 9,
9
+ "scale_resolution": 448,
10
+ "patch_size": 14,
11
+ "image_feature_size": 64,
12
+ "im_start": "<image>",
13
+ "im_end": "</image>",
14
+ "slice_start": "<slice>",
15
+ "slice_end": "</slice>",
16
+ "unk": "<unk>",
17
+ "norm_mean": [0.5, 0.5, 0.5],
18
+ "norm_std": [0.5, 0.5, 0.5]
19
+ }
processing_minicpmv.py ADDED
@@ -0,0 +1,146 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # coding=utf-8
2
+ # Copyright 2024 The HuggingFace Inc. team.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """
16
+ Processor class for MiniCPMV.
17
+ """
18
+
19
+ from typing import List, Optional, Union
20
+ import torch
21
+ import re
22
+
23
+ from transformers.image_utils import ImageInput
24
+ from transformers.processing_utils import ProcessorMixin
25
+ from transformers.tokenization_utils_base import PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
26
+ from transformers.utils import TensorType
27
+
28
+ from .image_processing_minicpmv import MiniCPMVBatchFeature
29
+
30
+
31
+ class MiniCPMVProcessor(ProcessorMixin):
32
+ r"""
33
+ Constructs a MiniCPMV processor which wraps a MiniCPMV image processor and a MiniCPMV tokenizer into a single processor.
34
+
35
+ [`MiniCPMVProcessor`] offers all the functionalities of [`MiniCPMVImageProcessor`] and [`LlamaTokenizerWrapper`]. See the
36
+ [`~MiniCPMVProcessor.__call__`] and [`~MiniCPMVProcessor.decode`] for more information.
37
+
38
+ Args:
39
+ image_processor ([`MiniCPMVImageProcessor`], *optional*):
40
+ The image processor is a required input.
41
+ tokenizer ([`LlamaTokenizerWrapper`], *optional*):
42
+ The tokenizer is a required input.
43
+ """
44
+ attributes = ["image_processor", "tokenizer"]
45
+ image_processor_class = "AutoImageProcessor"
46
+ tokenizer_class = "AutoTokenizer"
47
+
48
+ def __init__(self, image_processor=None, tokenizer=None):
49
+ super().__init__(image_processor, tokenizer)
50
+
51
+ def __call__(
52
+ self,
53
+ text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]],
54
+ images: ImageInput = None,
55
+ padding: Union[bool, str, PaddingStrategy] = False,
56
+ max_length: Optional[int] = None,
57
+ do_pad: Optional[bool] = True,
58
+ return_tensors: Optional[Union[str, TensorType]] = TensorType.PYTORCH,
59
+ ) -> MiniCPMVBatchFeature:
60
+ if images is not None:
61
+ image_inputs = self.image_processor(images, do_pad=do_pad, return_tensors=return_tensors)
62
+ return self._convert_images_texts_to_inputs(image_inputs, text, max_length=max_length, return_tensors=return_tensors)
63
+
64
+ # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode with CLIP->Llama
65
+ def batch_decode(self, *args, **kwargs):
66
+ """
67
+ This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
68
+ refer to the docstring of this method for more information.
69
+ """
70
+ output_ids = args[0]
71
+ result_text = []
72
+ for result in output_ids:
73
+ result = result[result != 0]
74
+ if result[0] == self.tokenizer.bos_id:
75
+ result = result[1:]
76
+ if result[-1] == self.tokenizer.eos_id:
77
+ result = result[:-1]
78
+ result_text.append(self.tokenizer.decode(result, *args[1:], **kwargs).strip())
79
+ return result_text
80
+ # return self.tokenizer.batch_decode(*args, **kwargs)
81
+
82
+ # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode with CLIP->Llama
83
+ def decode(self, *args, **kwargs):
84
+ """
85
+ This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
86
+ the docstring of this method for more information.
87
+ """
88
+ result = args[0]
89
+ result = result[result != 0]
90
+ if result[0] == self.tokenizer.bos_id:
91
+ result = result[1:]
92
+ if result[-1] == self.tokenizer.eos_id:
93
+ result = result[:-1]
94
+ return self.tokenizer.decode(result, *args[1:], **kwargs).strip()
95
+
96
+ def _convert(
97
+ self, input_str, max_inp_length: Optional[int] = None
98
+ ):
99
+ if self.tokenizer.add_bos_token:
100
+ input_ids = self.tokenizer.encode(input_str)
101
+ else:
102
+ input_ids = [self.tokenizer.bos_id] + self.tokenizer.encode(input_str)
103
+ if max_inp_length is not None:
104
+ input_ids = input_ids[:max_inp_length]
105
+ input_ids = torch.tensor(input_ids, dtype=torch.int32)
106
+
107
+ image_start_tokens = torch.where(input_ids == self.tokenizer.im_start_id)[0]
108
+ image_start_tokens += 1
109
+ image_end_tokens = torch.where(input_ids == self.tokenizer.im_end_id)[0]
110
+ valid_image_nums = max(len(image_start_tokens), len(image_end_tokens))
111
+ image_bounds = torch.hstack(
112
+ [
113
+ image_start_tokens[:valid_image_nums].unsqueeze(-1),
114
+ image_end_tokens[:valid_image_nums].unsqueeze(-1),
115
+ ]
116
+ )
117
+ return input_ids.unsqueeze(0), image_bounds
118
+
119
+ def _convert_images_texts_to_inputs(self, images, texts, do_pad=False, truncation=None, max_length=None, return_tensors=None):
120
+ if not len(images):
121
+ model_inputs = self.tokenizer(texts, return_tensors=return_tensors, padding=do_pad, truncation=truncation, max_length=max_length)
122
+ return MiniCPMVBatchFeature(data={**model_inputs})
123
+
124
+ pattern = "(<image>./</image>)"
125
+ images, image_sizes = images["pixel_values"], images["image_sizes"]
126
+
127
+ image_tags = re.findall(pattern, texts)
128
+ assert len(image_tags) <= 1 and len(image_sizes) == 1
129
+ text_chunks = texts.split(pattern)
130
+ final_texts = text_chunks[0] + self.image_processor.get_slice_image_placeholder(image_sizes[0]) \
131
+ + text_chunks[1] + "<AI>"
132
+ input_ids, image_bounds = self._convert(final_texts, max_length)
133
+
134
+ return MiniCPMVBatchFeature(data={
135
+ "input_ids": input_ids,
136
+ "pixel_values": images,
137
+ "image_sizes": [image_sizes],
138
+ "image_bounds": [image_bounds]
139
+ }, tensor_type=return_tensors)
140
+
141
+ @property
142
+ # Copied from transformers.models.clip.processing_clip.CLIPProcessor.model_input_names
143
+ def model_input_names(self):
144
+ tokenizer_input_names = self.tokenizer.model_input_names
145
+ image_processor_input_names = self.image_processor.model_input_names
146
+ return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
tokenization_minicpmv.py ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+
3
+ from transformers import LlamaTokenizer
4
+
5
+
6
+ class MiniCPMVTokenizer(LlamaTokenizer):
7
+ def __init__(self, **kwargs):
8
+ super().__init__(**kwargs)
9
+ self.im_start = "<image>"
10
+ self.im_end = "</image>"
11
+ self.ref_start = "<ref>"
12
+ self.ref_end = "</ref>"
13
+ self.box_start = "<box>"
14
+ self.box_end = "</box>"
15
+ self.quad_start = "<quad>"
16
+ self.quad_end = "</quad>"
17
+ self.point_start = "<point>"
18
+ self.point_end = "</point>"
19
+ self.slice_start = "<slice>"
20
+ self.slice_end = "</slice>"
21
+
22
+ @property
23
+ def eos_id(self):
24
+ return self.sp_model.eos_id()
25
+
26
+ @property
27
+ def bos_id(self):
28
+ return self.sp_model.bos_id()
29
+
30
+ @property
31
+ def unk_id(self):
32
+ return self.sp_model.unk_id()
33
+
34
+ @property
35
+ def im_start_id(self):
36
+ return self._convert_token_to_id(self.im_start)
37
+
38
+ @property
39
+ def im_end_id(self):
40
+ return self._convert_token_to_id(self.im_end)
41
+
42
+ def apply_chat_template(self,
43
+ conversation,
44
+ add_image_msg: bool=True):
45
+ if isinstance(conversation, str):
46
+ conversation = json.loads(conversation)
47
+
48
+ prompt = ""
49
+ for i, msg in enumerate(conversation):
50
+ role = msg["role"]
51
+ content = msg["content"]
52
+ assert role in ["user", "assistant"]
53
+ if i == 0:
54
+ assert role == "user", "The role of first msg should be user"
55
+ if add_image_msg is True and "(<image>./</image>)" not in content:
56
+ content = "(<image>./</image>)" + content
57
+ prompt += "<用户>" if role == "user" else "<AI>"
58
+ prompt += content
59
+ prompt += "<AI>"
60
+ return prompt
61
+
tokenizer_config.json CHANGED
@@ -139,7 +139,7 @@
139
  ],
140
  "auto_map": {
141
  "AutoTokenizer": [
142
- "modeling_minicpmv.LlamaTokenizerWrapper",
143
  null
144
  ]
145
  },
@@ -152,7 +152,7 @@
152
  "padding_side": "right",
153
  "sp_model_kwargs": {},
154
  "spaces_between_special_tokens": false,
155
- "tokenizer_class": "LlamaTokenizerWrapper",
156
  "truncation_side": "right",
157
  "unk_token": "<unk>",
158
  "use_default_system_prompt": false
 
139
  ],
140
  "auto_map": {
141
  "AutoTokenizer": [
142
+ "tokenization_minicpmv.MiniCPMVTokenizer",
143
  null
144
  ]
145
  },
 
152
  "padding_side": "right",
153
  "sp_model_kwargs": {},
154
  "spaces_between_special_tokens": false,
155
+ "tokenizer_class": "MiniCPMVTokenizer",
156
  "truncation_side": "right",
157
  "unk_token": "<unk>",
158
  "use_default_system_prompt": false