""" ## convert to gguf python convert_hf_to_gguf.py /workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct/ ## predict ./llama-cli -m /workspace/xusong/huggingface/models/Qwen1.5-0.5B-Chat/Qwen1.5-0.5B-Chat-F16.gguf -p "I believe the meaning of life is" -n 128 ./llama-cli -m /workspace/xusong/huggingface/models/Qwen1.5-0.5B-Chat/Qwen1.5-0.5B-Chat-F16.gguf -f prompt.txt -n 128 ./llama-cli -m /workspace/xusong/huggingface/models/Qwen1.5-0.5B-Chat/Qwen1.5-0.5B-Chat-F16.gguf -p "You are a helpful assistant" -cnv ## timing **重庆GPU服务器，cache为空 ** llama_print_timings: load time = 1711.48 ms llama_print_timings: sample time = 73.89 ms / 41 runs ( 1.80 ms per token, 554.84 tokens per second) llama_print_timings: prompt eval time = 2621.25 ms / 5 tokens ( 524.25 ms per token, 1.91 tokens per second) # 0.2-0.5秒/token llama_print_timings: eval time = 1430.91 ms / 40 runs ( 35.77 ms per token, 27.95 tokens per second) llama_print_timings: total time = 4848.09 ms / 45 tokens llama_print_timings: load time = 1939.72 ms llama_print_timings: sample time = 286.69 ms / 170 runs ( 1.69 ms per token, 592.99 tokens per second) llama_print_timings: prompt eval time = 0.00 ms / 0 tokens ( -nan ms per token, -nan tokens per second) # warmup后，加速明显。 llama_print_timings: eval time = 5737.50 ms / 170 runs ( 33.75 ms per token, 29.63 tokens per second) llama_print_timings: total time = 8219.82 ms / 170 tokens **hf-space，cache为空 （关闭GGML_BLAS） ** ----------- llama_print_timings: load time = 28230.06 ms llama_print_timings: sample time = 147.58 ms / 8 runs ( 18.45 ms per token, 54.21 tokens per second) # 18ms/token llama_print_timings: prompt eval time = 28864.82 ms / 5 tokens ( 5772.96 ms per token, 0.17 tokens per second) # 5.7s/token llama_print_timings: eval time = 1557.94 ms / 7 runs ( 222.56 ms per token, 4.49 tokens per second) llama_print_timings: total time = 30753.48 ms / 12 tokens **hf-space，cache为空 （开启GGML_BLAS）** ----------- llama_print_timings: load time = 27347.29 ms llama_print_timings: sample time = 82.53 ms / 26 runs ( 3.17 ms per token, 315.05 tokens per second) # 3ms/token llama_print_timings: prompt eval time = 28855.64 ms / 9 tokens ( 3206.18 ms per token, 0.31 tokens per second) # 3s/token llama_print_timings: eval time = 9810.01 ms / 25 runs ( 392.40 ms per token, 2.55 tokens per second) llama_print_timings: total time = 39073.77 ms / 34 tokens llama_print_timings: load time = 27347.29 ms llama_print_timings: sample time = 272.12 ms / 96 runs ( 2.83 ms per token, 352.79 tokens per second) # 2.8ms/token llama_print_timings: prompt eval time = 0.00 ms / 0 tokens ( -nan ms per token, -nan tokens per second) llama_print_timings: eval time = 19974.85 ms / 96 runs ( 208.07 ms per token, 4.81 tokens per second) llama_print_timings: total time = 22517.08 ms / 96 tokens ## TODO: - 解决warmup慢的问题 - 支持cache，并提前对所有预设system进行cache。 ## reference - https://github.com/abetlen/llama-cpp-python/blob/main/examples/gradio_chat/local.py - https://github.com/awinml/llama-cpp-python-bindings - https://github.com/langchain-ai/langchain/blob/master/libs/community/langchain_community/llms/llamacpp.py - https://github.com/abetlen/llama-cpp-python/blob/main/examples/gradio_chat/server.py - https://github.com/abetlen/llama-cpp-python/blob/main/llama_cpp/server/model.py - https://github.com/abetlen/llama-cpp-python/blob/main/llama_cpp/server/app.py """ import json import copy import os from models.base_model import Simulator import llama_cpp from transformers import AutoTokenizer from utils.logging_util import logger import config class Qwen2Simulator(Simulator): def __init__(self): local_path = "/workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct-GGUF/qwen2-0_5b-instruct-fp16.gguf" if os.path.exists(local_path): self.hf_tokenizer = AutoTokenizer.from_pretrained( "/workspace/xusong/huggingface/models/Qwen2-0.5B-Instruct/") self.llm = llama_cpp.Llama( # n_ctx, n_threads model_path=local_path, # 默认的tokenizer有bug，tokenize后的id不同 tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer(self.hf_tokenizer), n_ctx=config.MAX_SEQUENCE_LENGTH, # # n_threads=None, # 默认会根据cpu数来设置 n_threads # use_mlock=True, verbose=True, ) else: self.hf_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B-Instruct") self.llm = llama_cpp.Llama.from_pretrained( repo_id="Qwen/Qwen2-0.5B-Instruct-GGUF", tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer(self.hf_tokenizer), filename="*fp16.gguf", n_ctx=config.MAX_SEQUENCE_LENGTH, # use_mlock=True, verbose=True, ) logger.info(f"llm has been initialized: {self.llm}, " f"n_threads={self.llm.n_threads}, n_ctx={self.llm.n_ctx}, " f"env[CACHE]={os.environ.get('CACHE', None)}") self.stop_words = [ "<|im_end|>", "<|im_start|>", "<|endoftext|>", ] self.stop_tokens = self.tokenize("".join(self.stop_words)) self.generation_kwargs = dict( temperature=config.DEFAULT_TEMPERATURE, top_p=config.DEFAULT_TOP_P, top_k=config.DEFAULT_TOP_K, max_tokens=config.DEFAULT_MAX_NEW_TOKENS, repeat_penalty=1.1, # qwen2-0.5b-chat 有时内容生成结束没有<|im_end|>，直接跟 <|im_start|> stop=self.stop_words, ) self.user_start_tokens = self.tokenize("<|im_start|>user\n") self.assistant_start_tokens = self.tokenize("<|im_start|>assistant\n") # self.llm.generate .set_cache .last_n_tokens_size .reset .ctx ._ctx # cache = llama_cpp.LlamaDiskCache(capacity_bytes=cache_size) cache = llama_cpp.LlamaRAMCache(capacity_bytes=2 << 30) # 2G self.llm.set_cache(cache) def tokenize(self, text): return self.llm.tokenize(text.encode("utf-8")) def detokenize(self, tokens): return self.llm.detokenize(tokens).decode("utf-8") def strip_stoptokens(self, tokens): while tokens and tokens[0] in self.stop_tokens: logger.info(f"head-striping {tokens[0]} {self.detokenize([tokens[0]])}") tokens.pop(0) while tokens and tokens[-1] in self.stop_tokens: logger.info(f"tail-striping {tokens[-1]} {self.detokenize([tokens[-1]])}") tokens.pop() return tokens def generate(self, history, stream=True): """ 额外前向：remains 5 to forward "<|im_end|>\n<|im_start|>assistant\n" :param history: :param stream: :return: """ if history[-1]['role'] in ["user"]: start_tokens = self.assistant_start_tokens suffix_tokens = self.user_start_tokens elif history[-1]['role'] in ["assistant", "system"]: start_tokens = self.user_start_tokens suffix_tokens = self.assistant_start_tokens input_ids = [] for message in history: if "tokens" not in message: # tokens message["tokens"] = self.tokenize(message["content"]) input_ids += self.tokenize(f"<|im_start|>{message['role']}\n") \ + message["tokens"] \ + self.tokenize("<|im_end|>\n") input_ids += start_tokens if stream: return self._stream_generate(input_ids, suffix_tokens) else: return self._generate(input_ids) def _stream_generate(self, input_ids, suffix_tokens=None): logger.info(f"generation_kwargs {self.generation_kwargs}") output = self.llm.create_completion( input_ids, stream=True, **self.generation_kwargs ) # TODO: 检测finish reason，如果是length，则shift，并继续生成。 # TODO: 返回 token_id, for out in output: stream = copy.deepcopy(out) if stream["choices"][0]["finish_reason"] is None: yield stream["choices"][0]["completion_text"], stream["choices"][0]["completion_tokens"] else: logger.info( f'finish_reason {stream["choices"][0]["finish_reason"]} with text: {stream["choices"][0]["text"]}') # self.post_cache(suffix_tokens) def pre_cache_system(self, system_list): """ warmup for system prompt :param system_list: :return: """ logger.info(f"cache size {self.llm.cache.cache_size}") for system_prompt in system_list: logger.info(f"pre caching '{system_prompt}'") input_ids = self.tokenize(f"<|im_start|>system\n{system_prompt}<|im_end|>\n<|im_start|>user\n") output = self.llm.create_completion( input_ids, stream=False, max_tokens=1, top_k=1 ) logger.info(f"cache size {self.llm.cache.cache_size}") # disable cache after llama_cpp.LlamaRAMCache.__setitem__ = lambda *args: None llama_cpp.Llama.save_state = lambda *args: None def post_cache(self, suffix_tokens): """ warmup for next turn generation :param suffix_tokens: :return: """ if suffix_tokens: logger.info(f"before warmup: n_tokens = {self.llm.n_tokens}") self.llm.eval([151645, 198] + suffix_tokens) # <|im_end|>\n logger.info(f"after warmup: n_tokens = {self.llm.n_tokens}") bot = Qwen2Simulator() if __name__ == "__main__": messages = [{"role": "system", "content": "你是一个导游。"}] generated_tokens = None print("######## requesting", messages) for generated_text, generated_tokens in bot.generate(messages, stream=True): print(generated_text, generated_tokens) for i in range(3): generated_tokens = bot.strip_stoptokens(generated_tokens) messages.append( {"role": "user" if i % 2 == 0 else "assistant", "content": generated_text, "tokens": generated_tokens}) print("######## requesting", messages) for generated_text, generated_tokens in bot.generate(messages, stream=True): pass # print(generated_text, all_tokens)