File size: 5,439 Bytes
9dc0e21 8988bbf 931d3ff 8988bbf f0929ee 9dc0e21 c38b609 e74047c 973bde6 41bb1cf 8988bbf f29252d 5799733 459fbe3 9dc0e21 b099d9e 931d3ff b099d9e 931d3ff 459fbe3 5658533 b420ebd c38b609 931d3ff c38b609 f0929ee 5658533 c38b609 973bde6 e74047c 459fbe3 e74047c 5799733 e74047c c38b609 b099d9e d47c36f e52ef2a d47c36f e74047c b099d9e e74047c b099d9e 9dc0e21 b099d9e 3ac04fa b6dd571 b099d9e e74047c b6dd571 b099d9e e74047c b099d9e e52ef2a 9dc0e21 459fbe3 8988bbf 9dc0e21 7d0829c 0064c50 aa1923f 0064c50 7d0829c 0064c50 d47c36f aa1923f 0064c50 d47c36f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 |
"""
https://github.com/abetlen/llama-cpp-python/blob/main/examples/gradio_chat/local.py
https://github.com/awinml/llama-cpp-python-bindings
python convert_hf_to_gguf.py --outtype f16 Qwen1.5-0.5B-Chat
python convert_hf_to_gguf.py /workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct/
./llama-cli -m /workspace/xusong/huggingface/models/Qwen1.5-0.5B-Chat/Qwen1.5-0.5B-Chat-F16.gguf -p "I believe the meaning of life is" -n 128
./llama-cli -m /workspace/xusong/huggingface/models/Qwen1.5-0.5B-Chat/Qwen1.5-0.5B-Chat-F16.gguf -f prompt.txt -n 128
./llama-cli -m /workspace/xusong/huggingface/models/Qwen1.5-0.5B-Chat/Qwen1.5-0.5B-Chat-F16.gguf -p "You are a helpful assistant" -cnv
## reference
- https://github.com/langchain-ai/langchain/blob/master/libs/community/langchain_community/llms/llamacpp.py
- https://github.com/abetlen/llama-cpp-python/blob/main/examples/gradio_chat/server.py
- https://github.com/abetlen/llama-cpp-python/blob/main/llama_cpp/server/app.py
"""
import json
import copy
import os
from models.base_model import Simulator
import llama_cpp
# import llama_cpp.llama_tokenizer
from transformers import AutoTokenizer
from utils.logging_util import logger
import config
class Qwen2Simulator(Simulator):
def __init__(self):
local_path = "/workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct-GGUF/qwen2-0_5b-instruct-fp16.gguf"
if os.path.exists(local_path):
self.hf_tokenizer = AutoTokenizer.from_pretrained(
"/workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct/")
self.llm = llama_cpp.Llama( # n_ctx, n_threads
model_path=local_path,
# 默认的tokenizer有bug，tokenize后的id不同
tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer(self.hf_tokenizer),
n_ctx=config.MAX_SEQUENCE_LENGTH, #
# n_threads=None, # 默认会根据cpu数来设置 n_threads
# use_mlock=True,
verbose=True,
)
else:
self.hf_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct")
self.llm = llama_cpp.Llama.from_pretrained(
repo_id="Qwen/Qwen2-0.5B-Instruct-GGUF",
tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer(self.hf_tokenizer),
filename="*fp16.gguf",
n_ctx=config.MAX_SEQUENCE_LENGTH,
# use_mlock=True,
verbose=False,
)
logger.info(f"llm has been initialized: {self.llm}, "
f"n_threads={self.llm.n_threads}, n_ctx={self.llm.n_ctx}, "
f"env[CACHE]={os.environ.get('CACHE', None)}")
self.generation_kwargs = dict(
temperature=config.DEFAULT_TEMPERATURE,
top_p=config.DEFAULT_TOP_P,
top_k=config.DEFAULT_TOP_K,
max_tokens=config.DEFAULT_MAX_TOKENS,
repeat_penalty=1.1,
# qwen2-0.5b-chat 有时内容生成结束没有<|im_end|>，直接跟 <|im_start|>
stop=[
"<|im_end|>",
"<|im_start|>",
"<|endoftext|>",
],
)
def tokenize(self, text):
return self.llm.tokenize(text.encode("utf-8"))
def generate(self, history, stream=True):
if history[-1]['role'] in ["user"]:
start_tokens = self.tokenize("<|im_start|>assistant\n")
elif history[-1]['role'] in ["assistant", "system"]:
start_tokens = self.tokenize("<|im_start|>user\n")
input_ids = []
for message in history:
if "tokens" not in message:
message["tokens"] = self.tokenize(message["content"])
input_ids += self.tokenize(f"<|im_start|>{message['role']}\n") \
+ message["tokens"] \
+ self.tokenize("<|im_end|>\n")
input_ids += start_tokens
if stream:
return self._stream_generate(input_ids)
else:
return self._generate(input_ids)
def _stream_generate(self, input_ids):
logger.info(f"generation_kwargs {self.generation_kwargs}")
# self.llm.generate .set_cache .last_n_tokens_size .reset .ctx ._ctx
output = self.llm.create_completion(
input_ids,
stream=True,
**self.generation_kwargs
)
# TODO: 检测finish reason，如果是length，则shift，并继续生成。
# TODO: 返回 token_id,
for out in output:
stream = copy.deepcopy(out)
if stream["choices"][0]["finish_reason"] is None:
yield stream["choices"][0]["completion_text"], stream["choices"][0]["completion_tokens"]
bot = Qwen2Simulator()
if __name__ == "__main__":
messages = [{"role": "system", "content": "你是一个导游。"}]
generated_tokens = None
print("######## requesting", messages)
for generated_text, generated_tokens in bot.generate(messages, stream=True):
print(generated_text, generated_tokens)
for i in range(3):
messages.append({"role": "user" if i % 2 == 0 else "assistant", "content": generated_text, "tokens": generated_tokens})
print("######## requesting", messages)
for generated_text, generated_tokens in bot.generate(messages, stream=True):
pass
# print(generated_text, all_tokens)
|