feat: try to use llama.cpp server to load gguf model

Dockerfile

@@ -1,42 +1,27 @@
-FROM debian:bookworm-slim
+ARG LLAMA_CPP_VERSION
+
+FROM intel/oneapi-runtime:latest
 
 WORKDIR /app
 
-RUN apt-get update && apt-get install -y apt-transport-https ca-certificates wget gpg curl \
-    && wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null \
-    && echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list \
-    && apt-get update && apt-get install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y --no-install-recommends \
     aria2 \
-    nginx \
-    curl \
     unzip \
-    intel-oneapi-mkl \
+    clinfo \
     && rm -rf /var/lib/apt/lists/*
 
-RUN aria2c -c -x16 https://github.com/MZWNET/actions/releases/download/nitro-v0.3.21/nitro-0.3.21-linux-amd64-intel-mkl-avx2.zip \
-    && unzip nitro-0.3.21-linux-amd64-intel-mkl-avx2.zip -d /app \
-    && chmod +x /app/nitro \
-    && rm -rf nitro-0.3.21-linux-amd64-intel-mkl-avx2.zip
-
-RUN mkdir -p /etc/nginx/
-
-COPY nginx.conf /etc/nginx/nginx.conf
-
-RUN echo '#!/bin/sh\n\
-(source /opt/intel/oneapi/setvars.sh && /app/nitro) &\n\
-nginx\n\
-sleep 5\n\
-curl -X POST http://127.0.0.1:3928/inferences/llamacpp/loadmodel \\\n\
-  -H "Content-Type: application/json" \\\n\
-  -d @/app/model/loadmodel.json\n\
-wait' > /start.sh && chmod +x /start.sh
+RUN aria2c -c -x16 https://github.com/MZWNET/actions/releases/download/llama_cpp-$(LLAMA_CPP_VERSION)/llama-$(LLAMA_CPP_VERSION)-bin-linux-avx2-intel-mkl-x64.zip \
+    && unzip llama-$(LLAMA_CPP_VERSION)-bin-linux-avx2-intel-mkl-x64.zip -d /app/llama.cpp \
+    && mv /app/llama.cpp/server /app/ \
+    && chmod +x /app/server \
+    && rm -rf llama-$(LLAMA_CPP_VERSION)-bin-linux-avx2-intel-mkl-x64.zip llama.cpp
 
-COPY loadmodel.json /app/model/loadmodel.json
+RUN clinfo -l
 
 ENV PATH="/app:${PATH}"
 
 ENV HF_HOME="/data/.huggingface"
 
-EXPOSE 80
+EXPOSE 1145
 
-CMD ["/start.sh"]
+CMD ["server", "-m ~/.cache/huggingface/hub/mzwing/AquilaChat2-7B-16K-GGUF/AquilaChat2-7B-16K.Q8_0.gguf -t $(nproc) -c 16384 -a AquilaChat2-7B-16K --port 1145 --api-key sk-1145141919810"]

README.md

@@ -1,19 +1,21 @@
 ---
-title: Nitro AquilaChat2-7B-16K-GGUF
+title: AquilaChat2-7B-16K-GGUF
 emoji: 👀
 colorFrom: blue
 colorTo: blue
 sdk: docker
 pinned: false
 license: mit
-short_description: AquilaChat2-7B-16K-GGUF, loaded by nitro.
+short_description: AquilaChat2-7B-16K-GGUF, loaded by llama.cpp server.
 app_port: 1145
 custom_headers:
   cross-origin-embedder-policy: credentialless
   cross-origin-opener-policy: same-origin
   cross-origin-resource-policy: cross-origin
+models:
+  - mzwing/AquilaChat2-7B-16K-GGUF
 preload_from_hub:
   - mzwing/AquilaChat2-7B-16K-GGUF AquilaChat2-7B-16K.Q8_0.gguf
 ---
 
-AquilaChat2-7B-16K-GGUF, loaded by nitro.
+AquilaChat2-7B-16K-GGUF, loaded by llama.cpp server.

loadmodel.json

@@ -1,4 +0,0 @@
-{
-  "llama_model_path": "~/.cache/huggingface/hub/mzwing/AquilaChat2-7B-16K-GGUF/AquilaChat2-7B-16K.Q8_0.gguf",
-  "ctx_len": 16384
-}

nginx.conf

@@ -1,41 +0,0 @@
-worker_processes auto;
-
-events {
-  worker_connections 1024;
-}
-
-http {
-  include mime.types;
-  default_type application/octet-stream;
-
-  sendfile on;
-  keepalive_timeout 65;
-
-  proxy_connect_timeout 600;
-  proxy_send_timeout 600;
-  proxy_read_timeout 600;
-  send_timeout 600;
-
-  map $http_upgrade $connection_upgrade {
-    default upgrade;
-    '' close;
-  }
-
-  server {
-    listen 1145;
-
-    location / {
-      proxy_pass http://127.0.0.1:3928;
-      proxy_set_header Host $host;
-      proxy_set_header X-Real-IP $remote_addr;
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header Upgrade $http_upgrade;
-      proxy_set_header Connection $connection_upgrade;
-    }
-
-    location ~ ^/(inferences/llamacpp/loadmodel/|inferences/llamacpp/unloadmodel/) {
-      deny all;
-      return 403;
-    }
-  }
-}