load model.

V3.py

@@ -1,31 +1,34 @@
-import os
+from transformers import AutoModelForCausalLM, AutoTokenizer
 
-from transformers import AutoTokenizer
-from vllm import LLM, SamplingParams
-from huggingface_hub import snapshot_download
-
-model_path = "happzy2633/qwen2.5-7b-ins-v3"
-
-tokenizer = AutoTokenizer.from_pretrained(model_path)
-sampling_params = SamplingParams(temperature=0.7, top_p=0.8, repetition_penalty=1.05, max_tokens=8192)
-llm = LLM(model=model_path)
-
-def api_call_batch(batch_messages):
-    text_list = [
-        tokenizer.apply_chat_template(conversation=messages, tokenize=False, add_generation_prompt=True, return_tensors='pt')
-        for messages in batch_messages
-    ]
-    outputs = llm.generate(text_list, sampling_params)
-    result = [output.outputs[0].text for output in outputs]
-    return result
+model_name = "happzy2633/qwen2.5-7b-ins-v3"
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
+    torch_dtype="auto",
+    device_map="auto"
+)
+tokenizer = AutoTokenizer.from_pretrained(model_name)
 
 def api_call(messages):
-    return api_call_batch([messages])[0]
+    text = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True
+    )
+    model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
+    generated_ids = model.generate(
+        **model_inputs,
+        max_new_tokens=512
+    )
+    generated_ids = [
+        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+    ]
+    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+    return response
 
 def call_gpt(history, prompt):
     return api_call(history+[{"role":"user", "content":prompt}])
 
 if __name__ == "__main__":
     messages = [{"role":"user", "content":"你是谁？"}]
+    print(api_call(messages))
     breakpoint()
-    print(api_call_batch([messages]*4))

requirements.txt

@@ -1,65 +1,38 @@
+accelerate==0.34.2
 aiofiles==23.2.1
-aiohappyeyeballs==2.3.6
-aiohttp==3.10.3
-aiosignal==1.3.1
 annotated-types==0.7.0
-anyio==4.4.0
-async-timeout==4.0.3
-attrs==24.2.0
-blinker==1.8.2
-certifi==2024.7.4
+anyio==4.6.0
+certifi==2024.8.30
 charset-normalizer==3.3.2
 click==8.1.7
-cloudpickle==3.0.0
-cmake==3.30.2
 contourpy==1.3.0
 cycler==0.12.1
-datasets==2.21.0
-dill==0.3.8
-diskcache==5.6.3
 distro==1.9.0
 exceptiongroup==1.2.2
-fastapi==0.112.1
+fastapi==0.115.0
 ffmpy==0.4.0
-filelock==3.15.4
-Flask==3.0.3
-Flask-Cors==4.0.1
+filelock==3.16.1
 fonttools==4.54.1
-frozenlist==1.4.1
-fsspec==2024.6.1
+fsspec==2024.9.0
 gradio==4.44.1
 gradio_client==1.3.0
 h11==0.14.0
-httpcore==1.0.5
-httptools==0.6.1
-httpx==0.27.0
-huggingface-hub==0.24.5
-idna==3.7
+httpcore==1.0.6
+httpx==0.27.2
+huggingface-hub==0.25.1
+idna==3.10
 importlib_resources==6.4.5
-interegular==0.3.3
-itsdangerous==2.2.0
 Jinja2==3.1.4
 jiter==0.5.0
-jsonschema==4.23.0
-jsonschema-specifications==2023.12.1
 kiwisolver==1.4.7
-lark==1.2.2
-llvmlite==0.43.0
-lm-format-enforcer==0.10.3
 loguru==0.7.2
 markdown-it-py==3.0.0
 MarkupSafe==2.1.5
 matplotlib==3.9.2
 mdurl==0.1.2
 mpmath==1.3.0
-msgpack==1.0.8
-multidict==6.0.5
-multiprocess==0.70.16
-nest-asyncio==1.6.0
 networkx==3.3
-ninja==1.11.1.1
-numba==0.60.0
-numpy==1.26.4
+numpy==2.1.1
 nvidia-cublas-cu12==12.1.3.1
 nvidia-cuda-cupti-cu12==12.1.105
 nvidia-cuda-nvrtc-cu12==12.1.105
@@ -69,70 +42,45 @@ nvidia-cufft-cu12==11.0.2.54
 nvidia-curand-cu12==10.3.2.106
 nvidia-cusolver-cu12==11.4.5.107
 nvidia-cusparse-cu12==12.1.0.106
-nvidia-ml-py==12.560.30
 nvidia-nccl-cu12==2.20.5
-nvidia-nvjitlink-cu12==12.6.20
+nvidia-nvjitlink-cu12==12.6.77
 nvidia-nvtx-cu12==12.1.105
-openai==1.40.8
+openai==1.51.0
 orjson==3.10.7
-outlines==0.0.46
 packaging==24.1
-pandas==2.2.2
+pandas==2.2.3
 pillow==10.4.0
-prometheus-fastapi-instrumentator==7.0.0
-prometheus_client==0.20.0
-protobuf==5.27.3
 psutil==6.0.0
-py-cpuinfo==9.0.0
-pyairports==2.1.1
-pyarrow==17.0.0
-pycountry==24.6.1
-pydantic==2.8.2
-pydantic_core==2.20.1
+pydantic==2.9.2
+pydantic_core==2.23.4
 pydub==0.25.1
-pyext==0.7
 Pygments==2.18.0
 pyparsing==3.1.4
 python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
 python-multipart==0.0.12
-pytz==2024.1
+pytz==2024.2
 PyYAML==6.0.2
-pyzmq==26.1.0
-ray==2.34.0
-referencing==0.35.1
-regex==2024.7.24
+regex==2024.9.11
 requests==2.32.3
 rich==13.9.1
-rpds-py==0.20.0
 ruff==0.6.8
-safetensors==0.4.4
+safetensors==0.4.5
 semantic-version==2.10.0
-sentencepiece==0.2.0
 shellingham==1.5.4
 six==1.16.0
 sniffio==1.3.1
-starlette==0.38.2
-sympy==1.13.2
-tiktoken==0.7.0
-tokenizers==0.19.1
+starlette==0.38.6
+sympy==1.13.3
+tokenizers==0.20.0
 tomlkit==0.12.0
-torch==2.4.0
-torchvision==0.19.0
+torch==2.4.1
 tqdm==4.66.5
-transformers==4.44.0
+transformers==4.45.1
 triton==3.0.0
 typer==0.12.5
 typing_extensions==4.12.2
-tzdata==2024.1
-urllib3==2.2.2
-uvicorn==0.30.6
-uvloop==0.20.0
-vllm==0.5.4
-vllm-flash-attn==2.6.1
-watchfiles==0.23.0
+tzdata==2024.2
+urllib3==2.2.3
+uvicorn==0.31.0
 websockets==12.0
-Werkzeug==3.0.3
-xformers==0.0.27.post2
-xxhash==3.4.1
-yarl==1.9.4