add tanuki

api_endpoints.json

@@ -194,6 +194,19 @@
         "text-arena": true,
         "vision-arena": false
     },
+    "team-hatakeyama-phase2/Tanuki-8x8B-dpo-v1.0-AWQ": {
+        "model_name": "team-hatakeyama-phase2/Tanuki-8x8B-dpo-v1.0-AWQ",
+        "api_type": "openai-custom-tanuki",
+        "api_end": "Tanuki-8x8B-dpo",
+        "env_api_key": "VLLM_API_KEY",
+        "anony_only": false,
+        "recommended_config": {
+            "temperature": 0.7,
+            "top_p": 1.0
+        },
+        "text-arena": true,
+        "vision-arena": false
+    },
     "llm-jp-3-13b-instruct-Q8_0.gguf": {
         "model_name": "llm-jp-3-13b-instruct-Q8_0.gguf",
         "api_type": "openai-llmjp3",

local/local_setup

@@ -37,7 +37,7 @@ python -m vllm.entrypoints.openai.api_server --model tokyotech-llm/Llama-3.1-Swa
 
 #tanuki 8x8b
 export CUDA_VISIBLE_DEVICES=1
-python -m vllm.entrypoints.openai.api_server --model team-hatakeyama-phase2/Tanuki-8x8B-dpo-v1.0-AWQ --max-model-len 4096 --port 8020 --gpu-memory-utilization 0.2 --trust-remote-code --quantization awq --api-key $VLLM_API_KEY
+python -m vllm.entrypoints.openai.api_server --model team-hatakeyama-phase2/Tanuki-8x8B-dpo-v1.0-AWQ --max-model-len 4096 --port 8020 --gpu-memory-utilization 0.35 --trust-remote-code --quantization awq --api-key $VLLM_API_KEY
 
 #########################
 #launch ngrok

local/nginx

@@ -1,14 +1,15 @@
 #sudo vi /etc/nginx/sites-available/default
+#sudo systemctl stop nginx
 #sudo systemctl restart nginx
 
 
 server {
-    listen 8765;  # 一つのポートでまとめる
+listen 8765;  # 一つのポートでまとめる
 
-    location /swallow70/ {proxy_pass http://localhost:8019/v1/;}
-    location /llm-jp-13b/ {proxy_pass http://localhost:8016/v1/;}
-    location /Tanuki-8B-dpo/ {proxy_pass http://localhost:8012/v1/;}
-    location /calm3-22b-chat/ {proxy_pass http://localhost:8011/v1/;}
-    location /Swallow-8B/ {proxy_pass http://localhost:8010/v1/;}
+location /swallow70/ {proxy_pass http://localhost:8019/v1/;}
+location /llm-jp-13b/ {proxy_pass http://localhost:8016/v1/;}
+location /Tanuki-8B-dpo/ {proxy_pass http://localhost:8012/v1/;}
+location /calm3-22b-chat/ {proxy_pass http://localhost:8011/v1/;}
+location /Swallow-8B/ {proxy_pass http://localhost:8010/v1/;}
 }