add vllm version

src/display/utils.py

@@ -4,6 +4,7 @@ from enum import Enum
 import pandas as pd
 
 from src.about import Tasks, TaskType
+from src.envs import VLLM_CURRENT_VERSION
 
 
 def fields(raw_class):
@@ -169,6 +170,16 @@ class Backend(Enum):
             return Backend.Unknown
 
 
+class VllmVersion(Enum):
+    current = ModelDetails(VLLM_CURRENT_VERSION)
+    Unknown = ModelDetails("?")
+
+    def from_str(version):
+        if version == VLLM_CURRENT_VERSION:
+            return VllmVersion.current
+        return VllmVersion.Unknown
+
+
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 TYPES = [c.type for c in fields(AutoEvalColumn)]

src/envs.py

@@ -22,4 +22,7 @@ EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
 EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 
+# vllm version
+VLLM_CURRENT_VERSION = "v0.6.3"
+
 API = HfApi(token=TOKEN)

src/submission/submit.py

@@ -4,7 +4,7 @@ from datetime import datetime, timezone
 
 from src.display.formatting import styled_error, styled_message, styled_warning
 from src.display.utils import Version
-from src.envs import API, EVAL_REQUESTS_PATH, QUEUE_REPO, TOKEN
+from src.envs import API, EVAL_REQUESTS_PATH, QUEUE_REPO, TOKEN, VLLM_CURRENT_VERSION
 from src.submission.check_validity import already_submitted_models, check_model_card, get_model_size, is_model_on_hub
 
 REQUESTED_MODELS = None
@@ -24,11 +24,14 @@ def add_new_eval(
         REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
 
     current_version = Version.v1_4_1.value.name
+    current_vllm_version = VLLM_CURRENT_VERSION
 
     # バージョン情報を含めた重複チェック
-    submission_id = f"{model}_{precision}_{add_special_tokens}_{current_version}"
+    submission_id = f"{model}_{precision}_{add_special_tokens}_{current_version}_{current_vllm_version}"
     if submission_id in REQUESTED_MODELS:
-        return styled_warning(f"This model has already been evaluated with llm-jp-eval version {current_version}")
+        return styled_warning(
+            f"This model has already been evaluated with llm-jp-eval version {current_version} and vllm version {current_vllm_version}"
+        )
 
     user_name = ""
     model_path = model
@@ -85,6 +88,7 @@ def add_new_eval(
         "private": False,
         "add_special_tokens": add_special_tokens,
         "llm_jp_eval_version": current_version,
+        "vllm_version": current_vllm_version,
     }
 
     # Check for duplicate submission
@@ -94,7 +98,9 @@ def add_new_eval(
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{add_special_tokens}.json"
+    out_path = (
+        f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{add_special_tokens}_{current_vllm_version}.json"
+    )
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))