{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "^C\n", "Note: you may need to restart the kernel to use updated packages.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "[notice] A new release of pip is available: 23.1.2 -> 24.2\n", "[notice] To update, run: python.exe -m pip install --upgrade pip\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Collecting llama-cpp-python\n", " Downloading llama_cpp_python-0.2.90.tar.gz (63.8 MB)\n", " 0.0/63.8 MB ? eta -:--:--\n", " 0.0/63.8 MB 960.0 kB/s eta 0:01:07\n", " 0.2/63.8 MB 2.1 MB/s eta 0:00:31\n", " 0.4/63.8 MB 2.9 MB/s eta 0:00:23\n", " 0.7/63.8 MB 3.8 MB/s eta 0:00:17\n", " 1.1/63.8 MB 5.0 MB/s eta 0:00:13\n", " 1.3/63.8 MB 4.9 MB/s eta 0:00:13\n", " - 1.8/63.8 MB 5.6 MB/s eta 0:00:12\n", " - 1.9/63.8 MB 5.1 MB/s eta 0:00:13\n", " - 2.1/63.8 MB 5.2 MB/s eta 0:00:12\n", " - 2.6/63.8 MB 5.8 MB/s eta 0:00:11\n", " - 3.0/63.8 MB 5.9 MB/s eta 0:00:11\n", " -- 3.5/63.8 MB 6.3 MB/s eta 0:00:10\n", " -- 3.9/63.8 MB 6.6 MB/s eta 0:00:10\n", " -- 4.5/63.8 MB 7.0 MB/s eta 0:00:09\n", " --- 4.9/63.8 MB 7.1 MB/s eta 0:00:09\n", " --- 5.1/63.8 MB 7.1 MB/s eta 0:00:09\n", " --- 5.5/63.8 MB 7.0 MB/s eta 0:00:09\n", " --- 5.9/63.8 MB 7.1 MB/s eta 0:00:09\n", " ---- 6.4/63.8 MB 7.3 MB/s eta 0:00:08\n", " ---- 6.7/63.8 MB 7.3 MB/s eta 0:00:08\n", " ---- 7.2/63.8 MB 7.4 MB/s eta 0:00:08\n", " ---- 7.5/63.8 MB 7.4 MB/s eta 0:00:08\n", " ----- 8.0/63.8 MB 7.6 MB/s eta 0:00:08\n", " ----- 8.6/63.8 MB 7.7 MB/s eta 0:00:08\n", " ----- 9.1/63.8 MB 7.8 MB/s eta 0:00:07\n", " ------ 9.6/63.8 MB 8.0 MB/s eta 0:00:07\n", " ------ 9.9/63.8 MB 7.9 MB/s eta 0:00:07\n", " ------ 10.6/63.8 MB 8.7 MB/s eta 0:00:07\n", " ------ 11.1/63.8 MB 9.0 MB/s eta 0:00:06\n", " ------- 11.5/63.8 MB 9.1 MB/s eta 0:00:06\n", " ------- 11.9/63.8 MB 9.1 MB/s eta 0:00:06\n", " ------- 12.6/63.8 MB 9.8 MB/s eta 0:00:06\n", " -------- 13.1/63.8 MB 9.8 MB/s eta 0:00:06\n", " -------- 13.5/63.8 MB 9.8 MB/s eta 0:00:06\n", " -------- 13.9/63.8 MB 9.8 MB/s eta 0:00:06\n", " -------- 14.3/63.8 MB 9.8 MB/s eta 0:00:06\n", " --------- 14.8/63.8 MB 9.6 MB/s eta 0:00:06\n", " --------- 15.4/63.8 MB 10.2 MB/s eta 0:00:05\n", " --------- 16.1/63.8 MB 10.4 MB/s eta 0:00:05\n", " ---------- 16.6/63.8 MB 10.4 MB/s eta 0:00:05\n", " ---------- 16.6/63.8 MB 9.9 MB/s eta 0:00:05\n", " ---------- 17.0/63.8 MB 10.1 MB/s eta 0:00:05\n", " ---------- 17.7/63.8 MB 10.4 MB/s eta 0:00:05\n", " ----------- 18.4/63.8 MB 10.6 MB/s eta 0:00:05\n", " ----------- 18.9/63.8 MB 10.6 MB/s eta 0:00:05\n", " ----------- 19.4/63.8 MB 10.6 MB/s eta 0:00:05\n", " ------------ 19.9/63.8 MB 10.7 MB/s eta 0:00:05\n", " ------------ 20.3/63.8 MB 10.7 MB/s eta 0:00:05\n", " ------------ 20.8/63.8 MB 10.4 MB/s eta 0:00:05\n", " ------------- 21.4/63.8 MB 10.6 MB/s eta 0:00:05\n", " ------------- 21.9/63.8 MB 10.7 MB/s eta 0:00:04\n", " ------------- 22.5/63.8 MB 10.7 MB/s eta 0:00:04\n", " ------------- 22.9/63.8 MB 10.6 MB/s eta 0:00:04\n", " -------------- 23.5/63.8 MB 10.6 MB/s eta 0:00:04\n", " -------------- 23.7/63.8 MB 10.4 MB/s eta 0:00:04\n", " -------------- 24.1/63.8 MB 10.6 MB/s eta 0:00:04\n", " -------------- 24.4/63.8 MB 10.4 MB/s eta 0:00:04\n", " --------------- 24.8/63.8 MB 10.2 MB/s eta 0:00:04\n", " --------------- 25.2/63.8 MB 10.2 MB/s eta 0:00:04\n", " ---------------- 25.7/63.8 MB 9.9 MB/s eta 0:00:04\n", " ---------------- 26.2/63.8 MB 9.9 MB/s eta 0:00:04\n", " ---------------- 26.6/63.8 MB 9.8 MB/s eta 0:00:04\n", " ---------------- 26.9/63.8 MB 10.1 MB/s eta 0:00:04\n", " ---------------- 27.3/63.8 MB 10.1 MB/s eta 0:00:04\n", " ----------------- 27.8/63.8 MB 9.9 MB/s eta 0:00:04\n", " ----------------- 28.3/63.8 MB 9.8 MB/s eta 0:00:04\n", " ----------------- 28.7/63.8 MB 9.6 MB/s eta 0:00:04\n", " ------------------ 29.0/63.8 MB 9.4 MB/s eta 0:00:04\n", " ------------------ 29.2/63.8 MB 9.2 MB/s eta 0:00:04\n", " ------------------ 29.7/63.8 MB 9.4 MB/s eta 0:00:04\n", " ------------------ 30.1/63.8 MB 9.1 MB/s eta 0:00:04\n", " ------------------- 30.3/63.8 MB 8.8 MB/s eta 0:00:04\n", " ------------------- 30.8/63.8 MB 9.0 MB/s eta 0:00:04\n", " ------------------- 31.0/63.8 MB 8.7 MB/s eta 0:00:04\n", " ------------------- 31.3/63.8 MB 8.5 MB/s eta 0:00:04\n", " ------------------- 31.6/63.8 MB 8.4 MB/s eta 0:00:04\n", " -------------------- 32.0/63.8 MB 8.4 MB/s eta 0:00:04\n", " -------------------- 32.4/63.8 MB 8.3 MB/s eta 0:00:04\n", " -------------------- 32.9/63.8 MB 8.3 MB/s eta 0:00:04\n", " -------------------- 33.3/63.8 MB 8.3 MB/s eta 0:00:04\n", " --------------------- 33.7/63.8 MB 8.1 MB/s eta 0:00:04\n", " --------------------- 34.2/63.8 MB 8.3 MB/s eta 0:00:04\n", " --------------------- 34.7/63.8 MB 8.5 MB/s eta 0:00:04\n", " ---------------------- 35.1/63.8 MB 8.5 MB/s eta 0:00:04\n", " ---------------------- 35.5/63.8 MB 8.5 MB/s eta 0:00:04\n", " ---------------------- 36.0/63.8 MB 8.4 MB/s eta 0:00:04\n", " ---------------------- 36.3/63.8 MB 8.3 MB/s eta 0:00:04\n", " ----------------------- 36.8/63.8 MB 8.4 MB/s eta 0:00:04\n", " ----------------------- 37.2/63.8 MB 8.5 MB/s eta 0:00:04\n", " ----------------------- 37.6/63.8 MB 8.5 MB/s eta 0:00:04\n", " ----------------------- 38.1/63.8 MB 8.4 MB/s eta 0:00:04\n", " ------------------------ 38.6/63.8 MB 8.6 MB/s eta 0:00:03\n", " ------------------------ 39.2/63.8 MB 8.7 MB/s eta 0:00:03\n", " ------------------------ 39.6/63.8 MB 8.8 MB/s eta 0:00:03\n", " ------------------------- 40.1/63.8 MB 9.0 MB/s eta 0:00:03\n", " ------------------------- 40.5/63.8 MB 9.2 MB/s eta 0:00:03\n", " ------------------------- 41.0/63.8 MB 9.2 MB/s eta 0:00:03\n", " ------------------------- 41.4/63.8 MB 9.4 MB/s eta 0:00:03\n", " -------------------------- 41.7/63.8 MB 9.2 MB/s eta 0:00:03\n", " -------------------------- 42.3/63.8 MB 9.6 MB/s eta 0:00:03\n", " -------------------------- 42.8/63.8 MB 9.6 MB/s eta 0:00:03\n", " --------------------------- 43.3/63.8 MB 9.6 MB/s eta 0:00:03\n", " --------------------------- 43.6/63.8 MB 9.6 MB/s eta 0:00:03\n", " --------------------------- 44.2/63.8 MB 9.8 MB/s eta 0:00:02\n", " ---------------------------- 44.7/63.8 MB 9.8 MB/s eta 0:00:02\n", " ---------------------------- 45.2/63.8 MB 9.9 MB/s eta 0:00:02\n", " ---------------------------- 45.6/63.8 MB 9.9 MB/s eta 0:00:02\n", " ---------------------------- 46.1/63.8 MB 9.8 MB/s eta 0:00:02\n", " ----------------------------- 46.5/63.8 MB 9.8 MB/s eta 0:00:02\n", " ----------------------------- 46.9/63.8 MB 9.9 MB/s eta 0:00:02\n", " ----------------------------- 47.4/63.8 MB 9.9 MB/s eta 0:00:02\n", " ----------------------------- 47.9/63.8 MB 10.1 MB/s eta 0:00:02\n", " ----------------------------- 48.4/63.8 MB 10.1 MB/s eta 0:00:02\n", " ------------------------------ 48.8/63.8 MB 9.9 MB/s eta 0:00:02\n", " ------------------------------ 49.4/63.8 MB 9.9 MB/s eta 0:00:02\n", " ------------------------------- 49.8/63.8 MB 9.8 MB/s eta 0:00:02\n", " ------------------------------- 50.2/63.8 MB 9.8 MB/s eta 0:00:02\n", " ------------------------------- 50.8/63.8 MB 9.9 MB/s eta 0:00:02\n", " -------------------------------- 51.2/63.8 MB 9.8 MB/s eta 0:00:02\n", " -------------------------------- 51.5/63.8 MB 9.8 MB/s eta 0:00:02\n", " -------------------------------- 52.0/63.8 MB 9.9 MB/s eta 0:00:02\n", " -------------------------------- 52.4/63.8 MB 9.9 MB/s eta 0:00:02\n", " --------------------------------- 52.9/63.8 MB 9.9 MB/s eta 0:00:02\n", " --------------------------------- 53.4/63.8 MB 9.9 MB/s eta 0:00:02\n", " --------------------------------- 53.8/63.8 MB 9.9 MB/s eta 0:00:02\n", " ---------------------------------- 54.4/63.8 MB 9.9 MB/s eta 0:00:01\n", " ---------------------------------- 54.7/63.8 MB 9.6 MB/s eta 0:00:01\n", " ---------------------------------- 55.2/63.8 MB 9.8 MB/s eta 0:00:01\n", " ---------------------------------- 55.5/63.8 MB 9.6 MB/s eta 0:00:01\n", " ----------------------------------- 56.0/63.8 MB 9.6 MB/s eta 0:00:01\n", " ----------------------------------- 56.4/63.8 MB 9.8 MB/s eta 0:00:01\n", " ----------------------------------- 57.1/63.8 MB 9.9 MB/s eta 0:00:01\n", " ----------------------------------- 57.7/63.8 MB 10.1 MB/s eta 0:00:01\n", " ------------------------------------ 58.1/63.8 MB 9.9 MB/s eta 0:00:01\n", " ------------------------------------ 58.6/63.8 MB 9.9 MB/s eta 0:00:01\n", " ------------------------------------ 59.1/63.8 MB 10.1 MB/s eta 0:00:01\n", " ------------------------------------ 59.7/63.8 MB 10.1 MB/s eta 0:00:01\n", " ------------------------------------ 60.2/63.8 MB 10.1 MB/s eta 0:00:01\n", " ------------------------------------- 60.8/63.8 MB 10.2 MB/s eta 0:00:01\n", " ------------------------------------- 61.3/63.8 MB 10.2 MB/s eta 0:00:01\n", " ------------------------------------- 61.9/63.8 MB 10.6 MB/s eta 0:00:01\n", " ------------------------------------- 61.9/63.8 MB 10.6 MB/s eta 0:00:01\n", " -------------------------------------- 62.3/63.8 MB 10.1 MB/s eta 0:00:01\n", " -------------------------------------- 62.9/63.8 MB 10.2 MB/s eta 0:00:01\n", " -------------------------------------- 63.2/63.8 MB 10.1 MB/s eta 0:00:01\n", " --------------------------------------- 63.8/63.8 MB 9.9 MB/s eta 0:00:01\n", " --------------------------------------- 63.8/63.8 MB 9.9 MB/s eta 0:00:01\n", " --------------------------------------- 63.8/63.8 MB 9.9 MB/s eta 0:00:01\n", " ---------------------------------------- 63.8/63.8 MB 8.8 MB/s eta 0:00:00\n", " Installing build dependencies: started\n", " Installing build dependencies: finished with status 'done'\n", " Getting requirements to build wheel: started\n", " Getting requirements to build wheel: finished with status 'done'\n", " Preparing metadata (pyproject.toml): started\n", " Preparing metadata (pyproject.toml): finished with status 'done'\n", "Requirement already satisfied: typing-extensions>=4.5.0 in c:\\users\\bldng\\dev\\demo_human_gpt\\.venv\\lib\\site-packages (from llama-cpp-python) (4.12.2)\n", "Requirement already satisfied: numpy>=1.20.0 in c:\\users\\bldng\\dev\\demo_human_gpt\\.venv\\lib\\site-packages (from llama-cpp-python) (2.1.1)\n", "Collecting diskcache>=5.6.1 (from llama-cpp-python)\n", " Downloading diskcache-5.6.3-py3-none-any.whl (45 kB)\n", " 0.0/45.5 kB ? eta -:--:--\n", " ---------------------------------------- 45.5/45.5 kB ? eta 0:00:00\n", "Requirement already satisfied: jinja2>=2.11.3 in c:\\users\\bldng\\dev\\demo_human_gpt\\.venv\\lib\\site-packages (from llama-cpp-python) (3.1.4)\n", "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\bldng\\dev\\demo_human_gpt\\.venv\\lib\\site-packages (from jinja2>=2.11.3->llama-cpp-python) (2.1.5)\n", "Building wheels for collected packages: llama-cpp-python\n", " Building wheel for llama-cpp-python (pyproject.toml): started\n", " Building wheel for llama-cpp-python (pyproject.toml): still running...\n", " Building wheel for llama-cpp-python (pyproject.toml): finished with status 'done'\n", " Created wheel for llama-cpp-python: filename=llama_cpp_python-0.2.90-cp311-cp311-win_amd64.whl size=3132437 sha256=b3123f77abab7606d9e29442b91118b19ce2419f505ffe71f599da6060482312\n", " Stored in directory: c:\\users\\bldng\\appdata\\local\\pip\\cache\\wheels\\f8\\bd\\3d\\dcb25387572a6fe6c9c2953597c4c4d6dd985a425475da444d\n", "Successfully built llama-cpp-python\n", "Installing collected packages: diskcache, llama-cpp-python\n", "Successfully installed diskcache-5.6.3 llama-cpp-python-0.2.90\n" ] } ], "source": [ "%pip install llama-cpp-python" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "llama_model_loader: loaded meta data with 40 key-value pairs and 197 tensors from C:\\Users\\bldng\\.cache\\huggingface\\hub\\models--bartowski--Phi-3.5-mini-instruct-GGUF\\snapshots\\5c8d5381f90b6ca4348f090238be34eada23d192\\.\\Phi-3.5-mini-instruct-IQ3_XS.gguf (version GGUF V3 (latest))\n", "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n", "llama_model_loader: - kv 0: general.architecture str = phi3\n", "llama_model_loader: - kv 1: general.type str = model\n", "llama_model_loader: - kv 2: general.name str = Phi 3.5 Mini Instruct\n", "llama_model_loader: - kv 3: general.finetune str = instruct\n", "llama_model_loader: - kv 4: general.basename str = Phi-3.5\n", "llama_model_loader: - kv 5: general.size_label str = mini\n", "llama_model_loader: - kv 6: general.license str = mit\n", "llama_model_loader: - kv 7: general.license.link str = https://huggingface.co/microsoft/Phi-...\n", "llama_model_loader: - kv 8: general.tags arr[str,3] = [\"nlp\", \"code\", \"text-generation\"]\n", "llama_model_loader: - kv 9: general.languages arr[str,1] = [\"multilingual\"]\n", "llama_model_loader: - kv 10: phi3.context_length u32 = 131072\n", "llama_model_loader: - kv 11: phi3.rope.scaling.original_context_length u32 = 4096\n", "llama_model_loader: - kv 12: phi3.embedding_length u32 = 3072\n", "llama_model_loader: - kv 13: phi3.feed_forward_length u32 = 8192\n", "llama_model_loader: - kv 14: phi3.block_count u32 = 32\n", "llama_model_loader: - kv 15: phi3.attention.head_count u32 = 32\n", "llama_model_loader: - kv 16: phi3.attention.head_count_kv u32 = 32\n", "llama_model_loader: - kv 17: phi3.attention.layer_norm_rms_epsilon f32 = 0.000010\n", "llama_model_loader: - kv 18: phi3.rope.dimension_count u32 = 96\n", "llama_model_loader: - kv 19: phi3.rope.freq_base f32 = 10000.000000\n", "llama_model_loader: - kv 20: general.file_type u32 = 22\n", "llama_model_loader: - kv 21: phi3.attention.sliding_window u32 = 262144\n", "llama_model_loader: - kv 22: phi3.rope.scaling.attn_factor f32 = 1.190238\n", "llama_model_loader: - kv 23: tokenizer.ggml.model str = llama\n", "llama_model_loader: - kv 24: tokenizer.ggml.pre str = default\n", "llama_model_loader: - kv 25: tokenizer.ggml.tokens arr[str,32064] = [\"\", \"\", \"\", \"<0x00>\", \"<...\n", "llama_model_loader: - kv 26: tokenizer.ggml.scores arr[f32,32064] = [-1000.000000, -1000.000000, -1000.00...\n", "llama_model_loader: - kv 27: tokenizer.ggml.token_type arr[i32,32064] = [3, 3, 4, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n", "llama_model_loader: - kv 28: tokenizer.ggml.bos_token_id u32 = 1\n", "llama_model_loader: - kv 29: tokenizer.ggml.eos_token_id u32 = 32000\n", "llama_model_loader: - kv 30: tokenizer.ggml.unknown_token_id u32 = 0\n", "llama_model_loader: - kv 31: tokenizer.ggml.padding_token_id u32 = 32000\n", "llama_model_loader: - kv 32: tokenizer.ggml.add_bos_token bool = false\n", "llama_model_loader: - kv 33: tokenizer.ggml.add_eos_token bool = false\n", "llama_model_loader: - kv 34: tokenizer.chat_template str = {% for message in messages %}{% if me...\n", "llama_model_loader: - kv 35: general.quantization_version u32 = 2\n", "llama_model_loader: - kv 36: quantize.imatrix.file str = /models_out/Phi-3.5-mini-instruct-GGU...\n", "llama_model_loader: - kv 37: quantize.imatrix.dataset str = /training_dir/calibration_datav3.txt\n", "llama_model_loader: - kv 38: quantize.imatrix.entries_count i32 = 128\n", "llama_model_loader: - kv 39: quantize.imatrix.chunks_count i32 = 151\n", "llama_model_loader: - type f32: 67 tensors\n", "llama_model_loader: - type q6_K: 1 tensors\n", "llama_model_loader: - type iq3_xxs: 24 tensors\n", "llama_model_loader: - type iq3_s: 105 tensors\n", "llm_load_vocab: special tokens cache size = 14\n", "llm_load_vocab: token to piece cache size = 0.1685 MB\n", "llm_load_print_meta: format = GGUF V3 (latest)\n", "llm_load_print_meta: arch = phi3\n", "llm_load_print_meta: vocab type = SPM\n", "llm_load_print_meta: n_vocab = 32064\n", "llm_load_print_meta: n_merges = 0\n", "llm_load_print_meta: vocab_only = 0\n", "llm_load_print_meta: n_ctx_train = 131072\n", "llm_load_print_meta: n_embd = 3072\n", "llm_load_print_meta: n_layer = 32\n", "llm_load_print_meta: n_head = 32\n", "llm_load_print_meta: n_head_kv = 32\n", "llm_load_print_meta: n_rot = 96\n", "llm_load_print_meta: n_swa = 262144\n", "llm_load_print_meta: n_embd_head_k = 96\n", "llm_load_print_meta: n_embd_head_v = 96\n", "llm_load_print_meta: n_gqa = 1\n", "llm_load_print_meta: n_embd_k_gqa = 3072\n", "llm_load_print_meta: n_embd_v_gqa = 3072\n", "llm_load_print_meta: f_norm_eps = 0.0e+00\n", "llm_load_print_meta: f_norm_rms_eps = 1.0e-05\n", "llm_load_print_meta: f_clamp_kqv = 0.0e+00\n", "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n", "llm_load_print_meta: f_logit_scale = 0.0e+00\n", "llm_load_print_meta: n_ff = 8192\n", "llm_load_print_meta: n_expert = 0\n", "llm_load_print_meta: n_expert_used = 0\n", "llm_load_print_meta: causal attn = 1\n", "llm_load_print_meta: pooling type = 0\n", "llm_load_print_meta: rope type = 2\n", "llm_load_print_meta: rope scaling = linear\n", "llm_load_print_meta: freq_base_train = 10000.0\n", "llm_load_print_meta: freq_scale_train = 1\n", "llm_load_print_meta: n_ctx_orig_yarn = 4096\n", "llm_load_print_meta: rope_finetuned = unknown\n", "llm_load_print_meta: ssm_d_conv = 0\n", "llm_load_print_meta: ssm_d_inner = 0\n", "llm_load_print_meta: ssm_d_state = 0\n", "llm_load_print_meta: ssm_dt_rank = 0\n", "llm_load_print_meta: ssm_dt_b_c_rms = 0\n", "llm_load_print_meta: model type = 3B\n", "llm_load_print_meta: model ftype = IQ3_XS - 3.3 bpw\n", "llm_load_print_meta: model params = 3.82 B\n", "llm_load_print_meta: model size = 1.51 GiB (3.40 BPW) \n", "llm_load_print_meta: general.name = Phi 3.5 Mini Instruct\n", "llm_load_print_meta: BOS token = 1 ''\n", "llm_load_print_meta: EOS token = 32000 '<|endoftext|>'\n", "llm_load_print_meta: UNK token = 0 ''\n", "llm_load_print_meta: PAD token = 32000 '<|endoftext|>'\n", "llm_load_print_meta: LF token = 13 '<0x0A>'\n", "llm_load_print_meta: EOT token = 32007 '<|end|>'\n", "llm_load_print_meta: max token length = 48\n", "llm_load_tensors: ggml ctx size = 0.10 MiB\n", "llm_load_tensors: CPU buffer size = 1549.18 MiB\n", ".......................................................................................\n", "llama_new_context_with_model: n_ctx = 512\n", "llama_new_context_with_model: n_batch = 512\n", "llama_new_context_with_model: n_ubatch = 512\n", "llama_new_context_with_model: flash_attn = 0\n", "llama_new_context_with_model: freq_base = 10000.0\n", "llama_new_context_with_model: freq_scale = 1\n", "llama_kv_cache_init: CPU KV buffer size = 192.00 MiB\n", "llama_new_context_with_model: KV self size = 192.00 MiB, K (f16): 96.00 MiB, V (f16): 96.00 MiB\n", "llama_new_context_with_model: CPU output buffer size = 0.12 MiB\n", "llama_new_context_with_model: CPU compute buffer size = 83.01 MiB\n", "llama_new_context_with_model: graph nodes = 1286\n", "llama_new_context_with_model: graph splits = 1\n", "AVX = 1 | AVX_VNNI = 0 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | AVX512_BF16 = 0 | FMA = 1 | NEON = 0 | SVE = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 0 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | MATMUL_INT8 = 0 | LLAMAFILE = 1 | \n", "Model metadata: {'general.name': 'Phi 3.5 Mini Instruct', 'general.architecture': 'phi3', 'general.type': 'model', 'general.basename': 'Phi-3.5', 'general.finetune': 'instruct', 'general.size_label': 'mini', 'general.license': 'mit', 'general.license.link': 'https://huggingface.co/microsoft/Phi-3.5-mini-instruct/resolve/main/LICENSE', 'phi3.attention.head_count_kv': '32', 'phi3.context_length': '131072', 'phi3.rope.scaling.original_context_length': '4096', 'phi3.embedding_length': '3072', 'tokenizer.ggml.model': 'llama', 'phi3.feed_forward_length': '8192', 'phi3.block_count': '32', 'phi3.attention.head_count': '32', 'phi3.attention.layer_norm_rms_epsilon': '0.000010', 'phi3.rope.dimension_count': '96', 'tokenizer.chat_template': \"{% for message in messages %}{% if message['role'] == 'system' and message['content'] %}{{'<|system|>\\n' + message['content'] + '<|end|>\\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\\n' + message['content'] + '<|end|>\\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\\n' + message['content'] + '<|end|>\\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\\n' }}{% else %}{{ eos_token }}{% endif %}\", 'phi3.rope.freq_base': '10000.000000', 'tokenizer.ggml.eos_token_id': '32000', 'general.file_type': '22', 'tokenizer.ggml.add_eos_token': 'false', 'phi3.attention.sliding_window': '262144', 'phi3.rope.scaling.attn_factor': '1.190238', 'tokenizer.ggml.pre': 'default', 'general.quantization_version': '2', 'tokenizer.ggml.bos_token_id': '1', 'tokenizer.ggml.unknown_token_id': '0', 'tokenizer.ggml.padding_token_id': '32000', 'tokenizer.ggml.add_bos_token': 'false', 'quantize.imatrix.chunks_count': '151', 'quantize.imatrix.file': '/models_out/Phi-3.5-mini-instruct-GGUF/Phi-3.5-mini-instruct.imatrix', 'quantize.imatrix.dataset': '/training_dir/calibration_datav3.txt', 'quantize.imatrix.entries_count': '128'}\n", "Available chat formats from metadata: chat_template.default\n", "Using gguf chat template: {% for message in messages %}{% if message['role'] == 'system' and message['content'] %}{{'<|system|>\n", "' + message['content'] + '<|end|>\n", "'}}{% elif message['role'] == 'user' %}{{'<|user|>\n", "' + message['content'] + '<|end|>\n", "'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n", "' + message['content'] + '<|end|>\n", "'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n", "' }}{% else %}{{ eos_token }}{% endif %}\n", "Using chat eos_token: <|endoftext|>\n", "Using chat bos_token: \n" ] } ], "source": [ "from llama_cpp import Llama\n", "\n", "llm = Llama.from_pretrained(\n", "\trepo_id=\"bartowski/Phi-3.5-mini-instruct-GGUF\",\n", "\tfilename=\"Phi-3.5-mini-instruct-IQ3_XS.gguf\",\n", ")\n", "\n" ] }, { "cell_type": "code", "execution_count": 41, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Test 1\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Llama.generate: 195 prefix-match hit, remaining 22 prompt tokens to eval\n", "\n", "llama_print_timings: load time = 1880.42 ms\n", "llama_print_timings: sample time = 0.27 ms / 9 runs ( 0.03 ms per token, 33333.33 tokens per second)\n", "llama_print_timings: prompt eval time = 1371.67 ms / 22 tokens ( 62.35 ms per token, 16.04 tokens per second)\n", "llama_print_timings: eval time = 791.79 ms / 8 runs ( 98.97 ms per token, 10.10 tokens per second)\n", "llama_print_timings: total time = 2166.71 ms / 30 tokens\n", "Llama.generate: 196 prefix-match hit, remaining 26 prompt tokens to eval\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "<|user|>\n", "What is the capital of France?<|end|>\n", "<|assistant|>\n", "The capital of France is Paris<|end|>\n", "<|user|>\n", " \n", "<|end\n", "Test 2\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "llama_print_timings: load time = 1880.42 ms\n", "llama_print_timings: sample time = 0.30 ms / 9 runs ( 0.03 ms per token, 29801.32 tokens per second)\n", "llama_print_timings: prompt eval time = 1561.16 ms / 26 tokens ( 60.04 ms per token, 16.65 tokens per second)\n", "llama_print_timings: eval time = 787.52 ms / 8 runs ( 98.44 ms per token, 10.16 tokens per second)\n", "llama_print_timings: total time = 2352.22 ms / 34 tokens\n", "Llama.generate: 195 prefix-match hit, remaining 23 prompt tokens to eval\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "<|user|>\n", "What is 3*4?<|end|>\n", "<|assistant|>\n", "The result of 3*4 is 12<|end|>\n", "<|user|>\n", " \n", "<|end\n", "Test 3\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "llama_print_timings: load time = 1880.42 ms\n", "llama_print_timings: sample time = 0.21 ms / 7 runs ( 0.03 ms per token, 32863.85 tokens per second)\n", "llama_print_timings: prompt eval time = 1388.34 ms / 23 tokens ( 60.36 ms per token, 16.57 tokens per second)\n", "llama_print_timings: eval time = 578.12 ms / 6 runs ( 96.35 ms per token, 10.38 tokens per second)\n", "llama_print_timings: total time = 1969.21 ms / 29 tokens\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "<|user|>\n", "What it the capital?<|end|>\n", "<|assistant|>\n", "Please specify which country you are referring to.<|end|>\n", "<|user|>\n", " \n", "The capital of Italy\n" ] } ], "source": [ "syspropmt=r\"\"\"\n", "<|system|>\n", "The User will make an inquiry to the assistant.\n", "Fullfill the users inquiry.\n", "The User will write a message with the content \"<|endtile|>\" if his inquiry is fullfilled.\n", "The User will never have more than one inquiry in one conversation.\n", "The User will never complete his own inquiry.\n", "All conversations will end with \"<|endtile|>\".\n", "After each User message is one assistant response.\n", "There can never be more than one assistant response in succession.\n", "The end of each user and assistant message is marked with \"<|end|>\".\n", "\n", "\n", "Example:\n", "<|user|>\n", "What is the capital?<|end|>\n", "<|assistant|>\n", "Could you please specify which capital you are referring to?<|end|>\n", "<|user|>\n", "The capital of France<|end|>\n", "<|assistant|>\n", "The capital of France is Paris\n", "<|user|>\n", "<|endtile|><|end|>\n", "\"\"\".strip()\n", "tests=[\n", "\"\"\"\n", "<|user|>\n", "What is the capital of France?<|end|>\n", "<|assistant|>\n", "The capital of France is Paris<|end|>\n", "<|user|>\n", "\"\"\".strip(),#Should terminate the conversation with <|endtile|>\n", "\"\"\"\n", "<|user|>\n", "What is 3*4?<|end|>\n", "<|assistant|>\n", "The result of 3*4 is 12<|end|>\n", "<|user|>\n", "\"\"\".strip(),#Should terminate the conversation with <|endtile|>\n", "\"\"\"\n", "<|user|>\n", "What it the capital?<|end|>\n", "<|assistant|>\n", "Please specify which country you are referring to.<|end|>\n", "<|user|>\n", "\"\"\".strip(),#Should specify the country\n", "]\n", "for num,test in enumerate(tests):\n", " print(f\"Test {num+1}\")\n", " output=llm(syspropmt+test, stop=[\"\\n \",\"?\\n\",\".\\n\",\"tile|>\"],max_tokens=100)[\"choices\"][0][\"text\"]#Only stop at tile|> to see if the conv got terminated\n", " print(test)\n", " print(output)" ] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.4" } }, "nbformat": 4, "nbformat_minor": 2 }