Overwrite the `eos_token_id` for generation, avoiding the endless generation issue that happens only with the HF converted models

demo.ipynb

@@ -2,25 +2,22 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [
     {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/export/share/anasawadalla/miniconda3/envs/xgenmm-release-clone/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n",
-      "A new version of the following files was downloaded from https://huggingface.co/Salesforce/xgen-mm-phi3-mini-instruct-interleave-r-v1.5:\n",
-      "- modeling_xgenmm.py\n",
-      ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n",
-      "Downloading shards: 100%|██████████| 4/4 [08:00<00:00, 120.13s/it]\n",
-      "Loading checkpoint shards: 100%|██████████| 4/4 [00:01<00:00,  2.65it/s]\n",
-      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
-      "A new version of the following files was downloaded from https://huggingface.co/Salesforce/xgen-mm-phi3-mini-instruct-interleave-r-v1.5:\n",
-      "- image_processing_blip_3.py\n",
-      ". Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.\n"
-     ]
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "7e3b39b749f9427cbb75c404056185a4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
     }
    ],
    "source": [
@@ -36,7 +33,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -59,18 +56,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [],
    "source": [
     "model = model.to('cuda')\n",
     "model.eval()\n",
-    "tokenizer.padding_side = \"left\""
+    "tokenizer.padding_side = \"left\"\n",
+    "tokenizer.eos_token = '<|end|>'"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [
     {
@@ -101,14 +99,6 @@
      },
      "output_type": "display_data"
     },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/export/share/anasawadalla/miniconda3/envs/xgenmm-release-clone/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:515: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.05` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.\n",
-      "  warnings.warn(\n"
-     ]
-    },
     {
      "name": "stdout",
      "output_type": "stream",
@@ -236,6 +226,7 @@
     "                inputs[name] = value.cuda()\n",
     "        generated_text = model.generate(**inputs, image_size=[image_sizes],\n",
     "                                        pad_token_id=tokenizer.pad_token_id,\n",
+    "                                        eos_token_id=tokenizer.eos_token_id,\n",
     "                                        temperature=0.05,\n",
     "                                        do_sample=False, max_new_tokens=1024, top_p=None, num_beams=1,\n",
     "                                        )\n",