Pixtral vLLM

.idea/Pixal-Inference.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/Pixal-Inference.iml" filepath="$PROJECT_DIR$/.idea/Pixal-Inference.iml" />
+    </modules>
+  </component>
+</project>

.idea/workspace.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="PropertiesComponent">{}</component>
+</project>

app.py

@@ -0,0 +1,78 @@
+from vllm import LLM, SamplingParams
+import gradio as gr
+repo_id = "mistral-community/pixtral-12b-240910" #Replace to the model you would like to use
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
+
+# @spaces.GPU #[uncomment to use ZeroGPU]
+def infer(image_url, prompt, progress=gr.Progress(track_tqdm=True)):
+    # tokenize image urls and text
+    llm = LLM(model="mistralai/Pixtral-12B-2409")  # Name or path of your model
+    messages = [
+        {
+            "role": "user",
+            "content": [{"type": "text", "text": prompt}, {"type": "image_url", "image_url": {"url": image_url}}]
+        },
+    ]
+
+    outputs = llm.chat(messages, sampling_params=sampling_params)
+
+    print(outputs[0].outputs[0].text)
+    return outputs
+
+
+example_images = ["https://picsum.photos/id/237/200/300"]
+example_prompts = ["What do you see in this image?"]
+
+css = """
+#col-container {
+    margin: 0 auto;
+    max-width: 640px;
+}
+"""
+
+with gr.Blocks(css=css) as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.Markdown(f"""
+        # Text-to-Image Gradio Template
+        """)
+
+        with gr.Row():
+            prompt = gr.Text(
+                label="Prompt",
+                show_label=False,
+                max_lines=2,
+                placeholder="Enter your prompt",
+                container=False,
+            )
+
+            image_url = gr.Text(
+                label="Image URL",
+                show_label=False,
+                max_lines=1,
+                placeholder="Enter your image URL",
+                container=False,
+            )
+
+            run_button = gr.Button("Run", scale=0)
+
+        result = gr.Textbox(
+            show_label=False
+        )
+
+        gr.Examples(
+            examples=example_images,
+            inputs=[image_url]
+        )
+
+        gr.Examples(
+            examples=example_prompts,
+            inputs=[prompt]
+        )
+    gr.on(
+        triggers=[run_button.click, image_url.submit, prompt.submit],
+        fn=infer,
+        inputs=[image_url, prompt],
+        outputs=[result]
+    )
+
+demo.queue().launch()

requirements.txt

@@ -0,0 +1 @@
+vllm==0.6.1