Create handler.py

handler.py

@@ -0,0 +1,34 @@
+from io import BytesIO
+from typing import Any, List, Dict
+
+from PIL import Image
+from transformers import AutoModel, AutoTokenizer
+
+
+class EndpointHandler():
+    def __init__(self, path=""):
+        # Use a pipeline as a high-level helper
+        model_name = "SwordElucidator/MiniCPM-Llama3-V-2_5-int4"
+        model = AutoModel.from_pretrained(model_name, trust_remote_code=True)
+        tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+        model.eval()
+        self.model = model
+        self.tokenizer = tokenizer
+
+    def __call__(self, data: Any) -> List[List[Dict[str, float]]]:
+        image_bytes = data.pop("image_bytes", None)
+        question = data.pop("question", None)
+        image = Image.open(BytesIO(image_bytes))
+
+        msgs = [{'role': 'user', 'content': question}]
+
+        res = self.model.chat(
+            image=image,
+            msgs=msgs,
+            tokenizer=self.tokenizer,
+            sampling=True,  # if sampling=False, beam_search will be used by default
+            temperature=0.7,
+            # system_prompt='' # pass system_prompt if needed
+        )
+
+        return res