Upload modeling_mplug_owl2.py with huggingface_hub
Browse files- modeling_mplug_owl2.py +2 -2
modeling_mplug_owl2.py
CHANGED
@@ -275,14 +275,14 @@ class MPLUGOwl2LlamaForCausalLM(LlamaForCausalLM, MPLUGOwl2MetaForCausalLM):
|
|
275 |
input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to(self.device)
|
276 |
with torch.inference_mode():
|
277 |
image_tensor = self.image_processor.preprocess(images, return_tensors="pt")["pixel_values"].half().to(self.device)
|
278 |
-
output_logits =
|
279 |
images=image_tensor)["logits"][:,-1, self.preferential_ids_]
|
280 |
return torch.softmax(output_logits, -1) @ self.weight_tensor
|
281 |
else:
|
282 |
video = [[expand2square(frame, tuple(int(x*255) for x in self.image_processor.image_mean)) for frame in vid] for vid in images]
|
283 |
with torch.inference_mode():
|
284 |
video_tensors = [self.image_processor.preprocess(vid, return_tensors="pt")["pixel_values"].half().to(self.model.device) for vid in video]
|
285 |
-
output_logits = self
|
286 |
images=video_tensors)["logits"][:,-1, self.preferential_ids_]
|
287 |
return torch.softmax(output_logits, -1) @ self.weight_tensor
|
288 |
|
|
|
275 |
input_ids = tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to(self.device)
|
276 |
with torch.inference_mode():
|
277 |
image_tensor = self.image_processor.preprocess(images, return_tensors="pt")["pixel_values"].half().to(self.device)
|
278 |
+
output_logits = self(input_ids.repeat(image_tensor.shape[0], 1),
|
279 |
images=image_tensor)["logits"][:,-1, self.preferential_ids_]
|
280 |
return torch.softmax(output_logits, -1) @ self.weight_tensor
|
281 |
else:
|
282 |
video = [[expand2square(frame, tuple(int(x*255) for x in self.image_processor.image_mean)) for frame in vid] for vid in images]
|
283 |
with torch.inference_mode():
|
284 |
video_tensors = [self.image_processor.preprocess(vid, return_tensors="pt")["pixel_values"].half().to(self.model.device) for vid in video]
|
285 |
+
output_logits = self(self.input_ids.repeat(len(video_tensors), 1),
|
286 |
images=video_tensors)["logits"][:,-1, self.preferential_ids_]
|
287 |
return torch.softmax(output_logits, -1) @ self.weight_tensor
|
288 |
|