it run on colab cpu
it run on colab cpu
download part1
from llama_cpp import Llama
llm = Llama.from_pretrained(
repo_id="bartowski/DeepSeek-V2.5-GGUF",
filename="DeepSeek-V2.5-IQ1_M/DeepSeek-V2.5-IQ1_M-00001-of-00002.gguf",
)
llm.create_chat_completion(
messages = [
{
"role": "user",
"content": "What is the capital of France?"
}
]
)
download part2
from llama_cpp import Llama
llm = Llama.from_pretrained(
repo_id="bartowski/DeepSeek-V2.5-GGUF",
filename="DeepSeek-V2.5-IQ1_M/DeepSeek-V2.5-IQ1_M-00002-of-00002.gguf",
)
llm.create_chat_completion(
messages = [
{
"role": "user",
"content": "What is the capital of France?"
}
]
)
run model with two part without merge
from llama_cpp import Llama
llm = Llama.from_pretrained(
repo_id="bartowski/DeepSeek-V2.5-GGUF",
filename="DeepSeek-V2.5-IQ1_M/DeepSeek-V2.5-IQ1_M-00001-of-00002.gguf",
)
llm.create_chat_completion(
messages = [
{
"role": "user",
"content": "What is the capital of France?"
}
]
)
llama_model_loader: additional 1 GGUFs metadata loaded.
llama_model_loader: loaded meta data with 53 key-value pairs and 959 tensors from /root/.cache/huggingface/hub/models--bartowski--DeepSeek-V2.5-GGUF/snapshots/c5fe4383c9155f148939a5bd5a308eba249a42e1/DeepSeek-V2.5-IQ1_M/DeepSeek-V2.5-IQ1_M-00001-of-00002.gguf (version GGUF V3 (latest))
llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.
llama_model_loader: - kv 0: general.architecture str = deepseek2
llama_model_loader: - kv 1: general.type str = model
llama_model_loader: - kv 2: general.name str = DeepSeek V2.5
llama_model_loader: - kv 3: general.version str = V2.5
llama_model_loader: - kv 4: general.basename str = DeepSeek
llama_model_loader: - kv 5: general.size_label str = 160x14B
llama_model_loader: - kv 6: general.license str = other
llama_model_loader: - kv 7: general.license.name str = deepseek
llama_model_loader: - kv 8: general.license.link str = https://github.com/deepseek-ai/DeepSe...
llama_model_loader: - kv 9: deepseek2.block_count u32 = 60
llama_model_loader: - kv 10: deepseek2.context_length u32 = 163840
llama_model_loader: - kv 11: deepseek2.embedding_length u32 = 5120
llama_model_loader: - kv 12: deepseek2.feed_forward_length u32 = 12288
llama_model_loader: - kv 13: deepseek2.attention.head_count u32 = 128
llama_model_loader: - kv 14: deepseek2.attention.head_count_kv u32 = 128
llama_model_loader: - kv 15: deepseek2.rope.freq_base f32 = 10000.000000
llama_model_loader: - kv 16: deepseek2.attention.layer_norm_rms_epsilon f32 = 0.000001
llama_model_loader: - kv 17: deepseek2.expert_used_count u32 = 6
llama_model_loader: - kv 18: general.file_type u32 = 31
llama_model_loader: - kv 19: deepseek2.leading_dense_block_count u32 = 1
llama_model_loader: - kv 20: deepseek2.vocab_size u32 = 102400
llama_model_loader: - kv 21: deepseek2.attention.q_lora_rank u32 = 1536
llama_model_loader: - kv 22: deepseek2.attention.kv_lora_rank u32 = 512
llama_model_loader: - kv 23: deepseek2.attention.key_length u32 = 192
llama_model_loader: - kv 24: deepseek2.attention.value_length u32 = 128
llama_model_loader: - kv 25: deepseek2.expert_feed_forward_length u32 = 1536
llama_model_loader: - kv 26: deepseek2.expert_count u32 = 160
llama_model_loader: - kv 27: deepseek2.expert_shared_count u32 = 2
llama_model_loader: - kv 28: deepseek2.expert_weights_scale f32 = 16.000000
llama_model_loader: - kv 29: deepseek2.rope.dimension_count u32 = 64
llama_model_loader: - kv 30: deepseek2.rope.scaling.type str = yarn
llama_model_loader: - kv 31: deepseek2.rope.scaling.factor f32 = 40.000000
llama_model_loader: - kv 32: deepseek2.rope.scaling.original_context_length u32 = 4096
llama_model_loader: - kv 33: deepseek2.rope.scaling.yarn_log_multiplier f32 = 0.100000
llama_model_loader: - kv 34: tokenizer.ggml.model str = gpt2
llama_model_loader: - kv 35: tokenizer.ggml.pre str = deepseek-llm
llama_model_loader: - kv 36: tokenizer.ggml.tokens arr[str,102400] = ["!", """, "#", "$", "%", "&", "'", ...
llama_model_loader: - kv 37: tokenizer.ggml.token_type arr[i32,102400] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
llama_model_loader: - kv 38: tokenizer.ggml.merges arr[str,99757] = ["Ġ Ġ", "Ġ t", "Ġ a", "i n", "h e...
llama_model_loader: - kv 39: tokenizer.ggml.bos_token_id u32 = 100000
llama_model_loader: - kv 40: tokenizer.ggml.eos_token_id u32 = 100001
llama_model_loader: - kv 41: tokenizer.ggml.padding_token_id u32 = 100001
llama_model_loader: - kv 42: tokenizer.ggml.add_bos_token bool = true
llama_model_loader: - kv 43: tokenizer.ggml.add_eos_token bool = false
llama_model_loader: - kv 44: tokenizer.chat_template str = {% if not add_generation_prompt is de...
llama_model_loader: - kv 45: general.quantization_version u32 = 2
llama_model_loader: - kv 46: quantize.imatrix.file str = /models_out/DeepSeek-V2.5-GGUF/DeepSe...
llama_model_loader: - kv 47: quantize.imatrix.dataset str = /training_dir/calibration_datav3.txt
llama_model_loader: - kv 48: quantize.imatrix.entries_count i32 = 716
llama_model_loader: - kv 49: quantize.imatrix.chunks_count i32 = 139
llama_model_loader: - kv 50: split.no u16 = 0
llama_model_loader: - kv 51: split.count u16 = 2
llama_model_loader: - kv 52: split.tensors.count i32 = 959
llama_model_loader: - type f32: 300 tensors
llama_model_loader: - type q2_K: 8 tensors
llama_model_loader: - type q5_K: 1 tensors
llama_model_loader: - type iq2_xxs: 60 tensors
llama_model_loader: - type iq1_m: 590 tensors
llm_load_vocab: special tokens cache size = 18
llm_load_vocab: token to piece cache size = 0.6411 MB
llm_load_print_meta: format = GGUF V3 (latest)
llm_load_print_meta: arch = deepseek2
llm_load_print_meta: vocab type = BPE
llm_load_print_meta: n_vocab = 102400
llm_load_print_meta: n_merges = 99757
llm_load_print_meta: vocab_only = 0
llm_load_print_meta: n_ctx_train = 163840
llm_load_print_meta: n_embd = 5120
llm_load_print_meta: n_layer = 60
llm_load_print_meta: n_head = 128
llm_load_print_meta: n_head_kv = 128
llm_load_print_meta: n_rot = 64
llm_load_print_meta: n_swa = 0
llm_load_print_meta: n_embd_head_k = 192
llm_load_print_meta: n_embd_head_v = 128
llm_load_print_meta: n_gqa = 1
llm_load_print_meta: n_embd_k_gqa = 24576
llm_load_print_meta: n_embd_v_gqa = 16384
llm_load_print_meta: f_norm_eps = 0.0e+00
llm_load_print_meta: f_norm_rms_eps = 1.0e-06
llm_load_print_meta: f_clamp_kqv = 0.0e+00
llm_load_print_meta: f_max_alibi_bias = 0.0e+00
llm_load_print_meta: f_logit_scale = 0.0e+00
llm_load_print_meta: n_ff = 12288
llm_load_print_meta: n_expert = 160
llm_load_print_meta: n_expert_used = 6
llm_load_print_meta: causal attn = 1
llm_load_print_meta: pooling type = 0
llm_load_print_meta: rope type = 0
llm_load_print_meta: rope scaling = yarn
llm_load_print_meta: freq_base_train = 10000.0
llm_load_print_meta: freq_scale_train = 0.025
llm_load_print_meta: n_ctx_orig_yarn = 4096
llm_load_print_meta: rope_finetuned = unknown
llm_load_print_meta: ssm_d_conv = 0
llm_load_print_meta: ssm_d_inner = 0
llm_load_print_meta: ssm_d_state = 0
llm_load_print_meta: ssm_dt_rank = 0
llm_load_print_meta: ssm_dt_b_c_rms = 0
llm_load_print_meta: model type = 236B
llm_load_print_meta: model ftype = IQ1_M - 1.75 bpw
llm_load_print_meta: model params = 235.74 B
llm_load_print_meta: model size = 49.06 GiB (1.79 BPW)
llm_load_print_meta: general.name = DeepSeek V2.5
llm_load_print_meta: BOS token = 100000 '<｜begin▁of▁sentence｜>'
llm_load_print_meta: EOS token = 100001 '<｜end▁of▁sentence｜>'
llm_load_print_meta: PAD token = 100001 '<｜end▁of▁sentence｜>'
llm_load_print_meta: LF token = 126 'Ä'
llm_load_print_meta: max token length = 256
llm_load_print_meta: n_layer_dense_lead = 1
llm_load_print_meta: n_lora_q = 1536
llm_load_print_meta: n_lora_kv = 512
llm_load_print_meta: n_ff_exp = 1536
llm_load_print_meta: n_expert_shared = 2
llm_load_print_meta: expert_weights_scale = 16.0
llm_load_print_meta: rope_yarn_log_mul = 0.1000
llm_load_tensors: ggml ctx size = 0.40 MiB
llm_load_tensors: CPU buffer size = 38026.44 MiB
llm_load_tensors: CPU buffer size = 12211.82 MiB
....................................................................................................
llama_new_context_with_model: n_ctx = 512
llama_new_context_with_model: n_batch = 512
llama_new_context_with_model: n_ubatch = 512
llama_new_context_with_model: flash_attn = 0
llama_new_context_with_model: freq_base = 10000.0
llama_new_context_with_model: freq_scale = 0.025
llama_kv_cache_init: CPU KV buffer size = 2400.00 MiB
llama_new_context_with_model: KV self size = 2400.00 MiB, K (f16): 1440.00 MiB, V (f16): 960.00 MiB
llama_new_context_with_model: CPU output buffer size = 0.39 MiB
llama_new_context_with_model: CPU compute buffer size = 271.01 MiB
llama_new_context_with_model: graph nodes = 4480
llama_new_context_with_model: graph splits = 1
AVX = 1 | AVX_VNNI = 0 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | AVX512_BF16 = 0 | FMA = 1 | NEON = 0 | SVE = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 0 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | MATMUL_INT8 = 0 | LLAMAFILE = 1 |
Model metadata: {'split.tensors.count': '959', 'split.count': '2', 'split.no': '0', 'quantize.imatrix.chunks_count': '139', 'quantize.imatrix.entries_count': '716', 'quantize.imatrix.dataset': '/training_dir/calibration_datav3.txt', 'general.quantization_version': '2', 'tokenizer.ggml.add_eos_token': 'false', 'tokenizer.ggml.padding_token_id': '100001', 'deepseek2.rope.scaling.yarn_log_multiplier': '0.100000', 'deepseek2.rope.scaling.original_context_length': '4096', 'deepseek2.rope.scaling.factor': '40.000000', 'deepseek2.rope.dimension_count': '64', 'deepseek2.feed_forward_length': '12288', 'quantize.imatrix.file': '/models_out/DeepSeek-V2.5-GGUF/DeepSeek-V2.5.imatrix', 'deepseek2.attention.head_count': '128', 'deepseek2.rope.scaling.type': 'yarn', 'deepseek2.attention.q_lora_rank': '1536', 'tokenizer.chat_template': "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %} {%- if message['role'] == 'system' %} {% set ns.system_prompt = message['content'] %} {%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %} {%- if message['role'] == 'user' %} {%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}} {%- endif %} {%- if message['role'] == 'assistant' and message['content'] is none %} {%- set ns.is_tool = false -%} {%- for tool in message['tool_calls']%} {%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + 'json' + '\\n' + tool['function']['arguments'] + '\\n' + '' + '<｜tool▁call▁end｜>'}} {%- set ns.is_first = true -%} {%- else %}{{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + 'json' + '\\n' + tool['function']['arguments'] + '\\n' + '' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}} {%- endif %} {%- endfor %} {%- endif %} {%- if message['role'] == 'assistant' and message['content'] is not none %} {%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}} {%- set ns.is_tool = false -%} {%- else %}{{'<｜Assistant｜>' + message['content'] + '<｜end▁of▁sentence｜>'}} {%- endif %} {%- endif %} {%- if message['role'] == 'tool' %} {%- set ns.is_tool = true -%} {%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}} {%- set ns.is_output_first = false %} {%- else %}{{'\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}} {%- endif %} {%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}", 'deepseek2.context_length': '163840', 'deepseek2.attention.head_count_kv': '128', 'deepseek2.block_count': '60', 'general.license': 'other', 'tokenizer.ggml.pre': 'deepseek-llm', 'general.license.link': 'https://github.com/deepseek-ai/DeepSeek-V2/blob/main/LICENSE-MODEL', 'tokenizer.ggml.model': 'gpt2', 'deepseek2.expert_weights_scale': '16.000000', 'general.license.name': 'deepseek', 'deepseek2.leading_dense_block_count': '1', 'deepseek2.attention.kv_lora_rank': '512', 'tokenizer.ggml.add_bos_token': 'true', 'general.size_label': '160x14B', 'general.type': 'model', 'deepseek2.expert_count': '160', 'deepseek2.embedding_length': '5120', 'deepseek2.expert_used_count': '6', 'deepseek2.attention.key_length': '192', 'tokenizer.ggml.bos_token_id': '100000', 'general.name': 'DeepSeek V2.5', 'general.basename': 'DeepSeek', 'deepseek2.attention.layer_norm_rms_epsilon': '0.000001', 'general.architecture': 'deepseek2', 'general.file_type': '31', 'deepseek2.vocab_size': '102400', 'tokenizer.ggml.eos_token_id': '100001', 'deepseek2.expert_shared_count': '2', 'deepseek2.rope.freq_base': '10000.000000', 'deepseek2.attention.value_length': '128', 'general.version': 'V2.5', 'deepseek2.expert_feed_forward_length': '1536'}
Available chat formats from metadata: chat_template.default
Using gguf chat template: {% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %} {%- if message['role'] == 'system' %} {% set ns.system_prompt = message['content'] %} {%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %} {%- if message['role'] == 'user' %} {%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}} {%- endif %} {%- if message['role'] == 'assistant' and message['content'] is none %} {%- set ns.is_tool = false -%} {%- for tool in message['tool_calls']%} {%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + 'json' + '\n' + tool['function']['arguments'] + '\n' + '' + '<｜tool▁call▁end｜>'}} {%- set ns.is_first = true -%} {%- else %}{{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + 'json' + '\n' + tool['function']['arguments'] + '\n' + '' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}} {%- endif %} {%- endfor %} {%- endif %} {%- if message['role'] == 'assistant' and message['content'] is not none %} {%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}} {%- set ns.is_tool = false -%} {%- else %}{{'<｜Assistant｜>' + message['content'] + '<｜end▁of▁sentence｜>'}} {%- endif %} {%- endif %} {%- if message['role'] == 'tool' %} {%- set ns.is_tool = true -%} {%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}} {%- set ns.is_output_first = false %} {%- else %}{{'\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}} {%- endif %} {%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}
Using chat eos_token: <｜end▁of▁sentence｜>
Using chat bos_token: <｜begin▁of▁sentence｜>
llama_print_timings: load time = 129846.84 ms
llama_print_timings: sample time = 1.31 ms / 8 runs ( 0.16 ms per token, 6116.21 tokens per second)
llama_print_timings: prompt eval time = 129846.01 ms / 10 tokens (12984.60 ms per token, 0.08 tokens per second)
llama_print_timings: eval time = 214371.47 ms / 7 runs (30624.50 ms per token, 0.03 tokens per second)
llama_print_timings: total time = 344311.20 ms / 17 tokens
{'id': 'chatcmpl-e84e6cc4-e2e9-4a33-aec4-5aed81e6fd9a',
'object': 'chat.completion',
'created': 1726357254,
'model': '/root/.cache/huggingface/hub/models--bartowski--DeepSeek-V2.5-GGUF/snapshots/c5fe4383c9155f148939a5bd5a308eba249a42e1/DeepSeek-V2.5-IQ1_M/DeepSeek-V2.5-IQ1_M-00001-of-00002.gguf',
'choices': [{'index': 0,
'message': {'role': 'assistant',
'content': 'The capital of France is Paris.'},
'logprobs': None,
'finish_reason': 'stop'}],
'usage': {'prompt_tokens': 10, 'completion_tokens': 7, 'total_tokens': 17}}