Merge branch 'main' of hf.co:catid/cat-llama-3-70b-awq-q128-w4-gemm

README.md

@@ -10,7 +10,9 @@ git clone https://huggingface.co/catid/cat-llama-3-70b-awq-q128-w4-gemm
 
 conda create -n vllm70 python=3.10 -y && conda activate vllm70
 
-pip install git+https://github.com/vllm-project/vllm.git
+pip install -U git+https://github.com/vllm-project/vllm.git
 
-python -m vllm.entrypoints.openai.api_server --model cat-llama-3-70b-awq-q128-w4-gemm --tensor-parallel-size 2 --gpu-memory-utilization 0.95
-```
+python -m vllm.entrypoints.openai.api_server --model cat-llama-3-70b-awq-q128-w4-gemm --tensor-parallel-size 2 --gpu-memory-utilization 0.935
+```
+
+Sadly this *barely* doesn't fit by ~300MB or so.