Initial commit

logs.txt

@@ -0,0 +1,261 @@
+/home/ruihang/Workspace/miniconda3/envs/python311/bin/python -m mlc_llm gen_config /models/Llama-2-7b-chat-hf --quantization q0f16 --conv-template llama-2 --output /tmp/tmp8ew5r2yr
+[2024-05-22 00:30:05] INFO auto_config.py:115: Found model configuration: /models/Llama-2-7b-chat-hf/config.json
+[2024-05-22 00:30:05] INFO auto_config.py:153: Found model type: llama. Use `--model-type` to override.
+[2024-05-22 00:30:05] INFO llama_model.py:52: context_window_size not found in config.json. Falling back to max_position_embeddings (4096)
+[2024-05-22 00:30:05] INFO llama_model.py:72: prefill_chunk_size defaults to 2048
+[2024-05-22 00:30:05] INFO config.py:106: Overriding max_batch_size from 1 to 80
+[2024-05-22 00:30:05] INFO gen_config.py:255: [generation_config.json] Setting bos_token_id: 1
+[2024-05-22 00:30:05] INFO gen_config.py:255: [generation_config.json] Setting eos_token_id: 2
+[2024-05-22 00:30:05] INFO gen_config.py:255: [generation_config.json] Setting pad_token_id: 0
+[2024-05-22 00:30:05] INFO gen_config.py:255: [generation_config.json] Setting temperature: 0.6
+[2024-05-22 00:30:05] INFO gen_config.py:255: [generation_config.json] Setting top_p: 0.9
+[2024-05-22 00:30:05] INFO gen_config.py:267: Found tokenizer config: /models/Llama-2-7b-chat-hf/tokenizer.model. Copying to /tmp/tmp8ew5r2yr/tokenizer.model
+[2024-05-22 00:30:05] INFO gen_config.py:267: Found tokenizer config: /models/Llama-2-7b-chat-hf/tokenizer.json. Copying to /tmp/tmp8ew5r2yr/tokenizer.json
+[2024-05-22 00:30:05] INFO gen_config.py:269: Not found tokenizer config: /models/Llama-2-7b-chat-hf/vocab.json
+[2024-05-22 00:30:05] INFO gen_config.py:269: Not found tokenizer config: /models/Llama-2-7b-chat-hf/merges.txt
+[2024-05-22 00:30:05] INFO gen_config.py:269: Not found tokenizer config: /models/Llama-2-7b-chat-hf/added_tokens.json
+[2024-05-22 00:30:05] INFO gen_config.py:267: Found tokenizer config: /models/Llama-2-7b-chat-hf/tokenizer_config.json. Copying to /tmp/tmp8ew5r2yr/tokenizer_config.json
+[2024-05-22 00:30:05] INFO gen_config.py:80: [System default] Setting presence_penalty: 0.0
+[2024-05-22 00:30:05] INFO gen_config.py:80: [System default] Setting frequency_penalty: 0.0
+[2024-05-22 00:30:05] INFO gen_config.py:80: [System default] Setting repetition_penalty: 1.0
+[2024-05-22 00:30:05] INFO gen_config.py:80: [System default] Setting mean_gen_len: 128
+[2024-05-22 00:30:05] INFO gen_config.py:80: [System default] Setting max_gen_len: 512
+[2024-05-22 00:30:05] INFO gen_config.py:80: [System default] Setting shift_fill_factor: 0.3
+[2024-05-22 00:30:05] INFO gen_config.py:335: Dumping configuration file to: /tmp/tmp8ew5r2yr/mlc-chat-config.json
+/home/ruihang/Workspace/miniconda3/envs/python311/bin/python -m mlc_llm convert_weight /models/Llama-2-7b-chat-hf --quantization q0f16 --source-format auto --output /tmp/tmp8ew5r2yr
+[2024-05-22 00:30:06] INFO auto_config.py:115: Found model configuration: /models/Llama-2-7b-chat-hf/config.json
+[2024-05-22 00:30:06] INFO auto_device.py:79: Found device: cuda:0
+[2024-05-22 00:30:06] INFO auto_device.py:79: Found device: cuda:1
+[2024-05-22 00:30:07] INFO auto_device.py:88: Not found device: rocm:0
+[2024-05-22 00:30:08] INFO auto_device.py:88: Not found device: metal:0
+[2024-05-22 00:30:09] INFO auto_device.py:79: Found device: vulkan:0
+[2024-05-22 00:30:09] INFO auto_device.py:79: Found device: vulkan:1
+[2024-05-22 00:30:09] INFO auto_device.py:79: Found device: vulkan:2
+[2024-05-22 00:30:10] INFO auto_device.py:79: Found device: opencl:0
+[2024-05-22 00:30:10] INFO auto_device.py:79: Found device: opencl:1
+[2024-05-22 00:30:10] INFO auto_device.py:35: Using device: cuda:0
+[2024-05-22 00:30:10] INFO auto_weight.py:70: Finding weights in: /models/Llama-2-7b-chat-hf
+[2024-05-22 00:30:10] INFO auto_weight.py:120: Found source weight format: huggingface-torch. Source configuration: /models/Llama-2-7b-chat-hf/pytorch_model.bin.index.json
+[2024-05-22 00:30:10] INFO auto_weight.py:143: Found source weight format: huggingface-safetensor. Source configuration: /models/Llama-2-7b-chat-hf/model.safetensors.index.json
+[2024-05-22 00:30:10] INFO auto_weight.py:106: Using source weight configuration: /models/Llama-2-7b-chat-hf/pytorch_model.bin.index.json. Use `--source` to override.
+[2024-05-22 00:30:10] INFO auto_weight.py:110: Using source weight format: huggingface-torch. Use `--source-format` to override.
+[2024-05-22 00:30:10] INFO auto_config.py:153: Found model type: llama. Use `--model-type` to override.
+[2024-05-22 00:30:10] INFO llama_model.py:52: context_window_size not found in config.json. Falling back to max_position_embeddings (4096)
+[2024-05-22 00:30:10] INFO llama_model.py:72: prefill_chunk_size defaults to 2048
+Weight conversion with arguments:
+  --config          /models/Llama-2-7b-chat-hf/config.json
+  --quantization    NoQuantize(name='q0f16', kind='no-quant', model_dtype='float16')
+  --model-type      llama
+  --device          cuda:0
+  --source          /models/Llama-2-7b-chat-hf/pytorch_model.bin.index.json
+  --source-format   huggingface-torch
+  --output          /tmp/tmp8ew5r2yr
+Start storing to cache /tmp/tmp8ew5r2yr
+
  0%|          | 0/195 [00:00<?, ?it/s]
                                       
+
  0%|          | 0/195 [00:00<?, ?it/s]
                                       
+
  0%|          | 0/195 [00:08<?, ?it/s]
  1%|          | 1/195 [00:09<29:48,  9.22s/it]
                                               
+
  1%|          | 1/195 [00:09<29:48,  9.22s/it]
                                               
+
  1%|          | 1/195 [00:09<29:48,  9.22s/it]
  2%|▏         | 3/195 [00:09<07:58,  2.49s/it]
                                               
+
  2%|▏         | 3/195 [00:09<07:58,  2.49s/it]
  2%|▏         | 4/195 [00:10<06:17,  1.98s/it]
                                               
+
  2%|▏         | 4/195 [00:10<06:17,  1.98s/it]
                                               
+
  2%|▏         | 4/195 [00:10<06:17,  1.98s/it]
  3%|▎         | 6/195 [00:10<03:28,  1.10s/it]
                                               
+
  3%|▎         | 6/195 [00:10<03:28,  1.10s/it]
  4%|▎         | 7/195 [00:11<02:38,  1.19it/s]
                                               
+
  4%|▎         | 7/195 [00:11<02:38,  1.19it/s]
                                               
+
  4%|▎         | 7/195 [00:11<02:38,  1.19it/s]
  5%|▍         | 9/195 [00:11<01:39,  1.87it/s]
                                               
+
  5%|▍         | 9/195 [00:11<01:39,  1.87it/s]
  5%|▌         | 10/195 [00:12<02:00,  1.54it/s]
                                                
+
  5%|▌         | 10/195 [00:12<02:00,  1.54it/s]
                                                
+
  5%|▌         | 10/195 [00:12<02:00,  1.54it/s]
  6%|▌         | 12/195 [00:12<01:24,  2.15it/s]
                                                
+
  6%|▌         | 12/195 [00:12<01:24,  2.15it/s]
  7%|▋         | 13/195 [00:12<01:09,  2.61it/s]
                                                
+
  7%|▋         | 13/195 [00:12<01:09,  2.61it/s]
                                                
+
  7%|▋         | 13/195 [00:12<01:09,  2.61it/s]
  8%|▊         | 15/195 [00:13<00:50,  3.53it/s]
                                                
+
  8%|▊         | 15/195 [00:13<00:50,  3.53it/s]
  8%|▊         | 16/195 [00:13<01:15,  2.36it/s]
                                                
+
  8%|▊         | 16/195 [00:13<01:15,  2.36it/s]
                                                
+
  8%|▊         | 16/195 [00:14<01:15,  2.36it/s]
  9%|▉         | 18/195 [00:14<00:58,  3.04it/s]
                                                
+
  9%|▉         | 18/195 [00:14<00:58,  3.04it/s]
 10%|▉         | 19/195 [00:14<00:49,  3.57it/s]
                                                
+
 10%|▉         | 19/195 [00:14<00:49,  3.57it/s]
                                                
+
 10%|▉         | 19/195 [00:14<00:49,  3.57it/s]
 11%|█         | 21/195 [00:14<00:38,  4.53it/s]
                                                
+
 11%|█         | 21/195 [00:14<00:38,  4.53it/s]
 11%|█▏        | 22/195 [00:15<00:55,  3.11it/s]
                                                
+
 11%|█▏        | 22/195 [00:15<00:55,  3.11it/s]
                                                
+
 11%|█▏        | 22/195 [00:15<00:55,  3.11it/s]
 12%|█▏        | 24/195 [00:15<00:44,  3.87it/s]
                                                
+
 12%|█▏        | 24/195 [00:15<00:44,  3.87it/s]
 13%|█▎        | 25/195 [00:15<00:38,  4.45it/s]
                                                
+
 13%|█▎        | 25/195 [00:15<00:38,  4.45it/s]
                                                
+
 13%|█▎        | 25/195 [00:15<00:38,  4.45it/s]
 14%|█▍        | 27/195 [00:16<00:31,  5.38it/s]
                                                
+
 14%|█▍        | 27/195 [00:16<00:31,  5.38it/s]
 14%|█▍        | 28/195 [00:16<00:53,  3.14it/s]
                                                
+
 14%|█▍        | 28/195 [00:16<00:53,  3.14it/s]
                                                
+
 14%|█▍        | 28/195 [00:16<00:53,  3.14it/s]
 15%|█▌        | 30/195 [00:17<00:42,  3.89it/s]
                                                
+
 15%|█▌        | 30/195 [00:17<00:42,  3.89it/s]
 16%|█▌        | 31/195 [00:17<00:36,  4.47it/s]
                                                
+
 16%|█▌        | 31/195 [00:17<00:36,  4.47it/s]
                                                
+
 16%|█▌        | 31/195 [00:17<00:36,  4.47it/s]
 17%|█▋        | 33/195 [00:17<00:32,  4.97it/s]
                                                
+
 17%|█▋        | 33/195 [00:17<00:32,  4.97it/s]
 17%|█▋        | 34/195 [00:18<00:51,  3.12it/s]
                                                
+
 17%|█▋        | 34/195 [00:18<00:51,  3.12it/s]
                                                
+
 17%|█▋        | 34/195 [00:18<00:51,  3.12it/s]
 18%|█▊        | 36/195 [00:18<00:42,  3.77it/s]
                                                
+
 18%|█▊        | 36/195 [00:18<00:42,  3.77it/s]
                                                
+
 18%|█▊        | 36/195 [00:18<00:42,  3.77it/s]
 19%|█▉        | 38/195 [00:18<00:29,  5.26it/s]
                                                
+
 19%|█▉        | 38/195 [00:18<00:29,  5.26it/s]
 20%|██        | 39/195 [00:19<00:31,  5.02it/s]
                                                
+
 20%|██        | 39/195 [00:19<00:31,  5.02it/s]
 21%|██        | 40/195 [00:19<00:46,  3.35it/s]
                                                
+
 21%|██        | 40/195 [00:19<00:46,  3.35it/s]
                                                
+
 21%|██        | 40/195 [00:19<00:46,  3.35it/s]
 22%|██▏       | 42/195 [00:19<00:37,  4.12it/s]
                                                
+
 22%|██▏       | 42/195 [00:19<00:37,  4.12it/s]
 22%|██▏       | 43/195 [00:20<00:32,  4.72it/s]
                                                
+
 22%|██▏       | 43/195 [00:20<00:32,  4.72it/s]
                                                
+
 22%|██▏       | 43/195 [00:20<00:32,  4.72it/s]
 23%|██▎       | 45/195 [00:20<00:26,  5.62it/s]
                                                
+
 23%|██▎       | 45/195 [00:20<00:26,  5.62it/s]
 24%|██▎       | 46/195 [00:21<00:44,  3.35it/s]
                                                
+
 24%|██▎       | 46/195 [00:21<00:44,  3.35it/s]
                                                
+
 24%|██▎       | 46/195 [00:21<00:44,  3.35it/s]
 25%|██▍       | 48/195 [00:21<00:35,  4.11it/s]
                                                
+
 25%|██▍       | 48/195 [00:21<00:35,  4.11it/s]
 25%|██▌       | 49/195 [00:21<00:31,  4.70it/s]
                                                
+
 25%|██▌       | 49/195 [00:21<00:31,  4.70it/s]
                                                
+
 25%|██▌       | 49/195 [00:21<00:31,  4.70it/s]
                                                
+
 25%|██▌       | 49/195 [00:21<00:31,  4.70it/s]
                                                
+
 25%|██▌       | 49/195 [00:58<00:31,  4.70it/s]
 26%|██▌       | 51/195 [00:58<17:18,  7.21s/it]
                                                
+
 26%|██▌       | 51/195 [00:58<17:18,  7.21s/it]
                                                
+
 26%|██▌       | 51/195 [00:58<17:18,  7.21s/it]
 27%|██▋       | 53/195 [00:59<11:13,  4.74s/it]
                                                
+
 27%|██▋       | 53/195 [00:59<11:13,  4.74s/it]
 28%|██▊       | 54/195 [01:00<09:23,  4.00s/it]
                                                
+
 28%|██▊       | 54/195 [01:00<09:23,  4.00s/it]
                                                
+
 28%|██▊       | 54/195 [01:00<09:23,  4.00s/it]
 29%|██▊       | 56/195 [01:00<06:03,  2.61s/it]
                                                
+
 29%|██▊       | 56/195 [01:00<06:03,  2.61s/it]
 29%|██▉       | 57/195 [01:00<04:50,  2.10s/it]
                                                
+
 29%|██▉       | 57/195 [01:00<04:50,  2.10s/it]
                                                
+
 29%|██▉       | 57/195 [01:00<04:50,  2.10s/it]
 30%|███       | 59/195 [01:01<03:07,  1.38s/it]
                                                
+
 30%|███       | 59/195 [01:01<03:07,  1.38s/it]
 31%|███       | 60/195 [01:01<02:47,  1.24s/it]
                                                
+
 31%|███       | 60/195 [01:01<02:47,  1.24s/it]
                                                
+
 31%|███       | 60/195 [01:01<02:47,  1.24s/it]
 32%|███▏      | 62/195 [01:02<01:52,  1.18it/s]
                                                
+
 32%|███▏      | 62/195 [01:02<01:52,  1.18it/s]
 32%|███▏      | 63/195 [01:02<01:31,  1.44it/s]
                                                
+
 32%|███▏      | 63/195 [01:02<01:31,  1.44it/s]
                                                
+
 32%|███▏      | 63/195 [01:02<01:31,  1.44it/s]
 33%|███▎      | 65/195 [01:02<01:03,  2.04it/s]
                                                
+
 33%|███▎      | 65/195 [01:02<01:03,  2.04it/s]
 34%|███▍      | 66/195 [01:03<01:08,  1.90it/s]
                                                
+
 34%|███▍      | 66/195 [01:03<01:08,  1.90it/s]
                                                
+
 34%|███▍      | 66/195 [01:03<01:08,  1.90it/s]
 35%|███▍      | 68/195 [01:03<00:48,  2.61it/s]
                                                
+
 35%|███▍      | 68/195 [01:03<00:48,  2.61it/s]
                                                
+
 35%|███▍      | 68/195 [01:03<00:48,  2.61it/s]
                                                
+
 35%|███▍      | 68/195 [01:03<00:48,  2.61it/s]
 36%|███▋      | 71/195 [01:03<00:32,  3.82it/s]
                                                
+
 36%|███▋      | 71/195 [01:03<00:32,  3.82it/s]
 37%|███▋      | 72/195 [01:04<00:37,  3.31it/s]
                                                
+
 37%|███▋      | 72/195 [01:04<00:37,  3.31it/s]
                                                
+
 37%|███▋      | 72/195 [01:04<00:37,  3.31it/s]
 38%|███▊      | 74/195 [01:04<00:30,  4.02it/s]
                                                
+
 38%|███▊      | 74/195 [01:04<00:30,  4.02it/s]
                                                
+
 38%|███▊      | 74/195 [01:04<00:30,  4.02it/s]
                                                
+
 38%|███▊      | 74/195 [01:04<00:30,  4.02it/s]
 39%|███▉      | 77/195 [01:04<00:22,  5.26it/s]
                                                
+
 39%|███▉      | 77/195 [01:05<00:22,  5.26it/s]
 40%|████      | 78/195 [01:05<00:27,  4.18it/s]
                                                
+
 40%|████      | 78/195 [01:05<00:27,  4.18it/s]
                                                
+
 40%|████      | 78/195 [01:05<00:27,  4.18it/s]
 41%|████      | 80/195 [01:05<00:23,  4.83it/s]
                                                
+
 41%|████      | 80/195 [01:05<00:23,  4.83it/s]
                                                
+
 41%|████      | 80/195 [01:05<00:23,  4.83it/s]
                                                
+
 41%|████      | 80/195 [01:05<00:23,  4.83it/s]
 43%|████▎     | 83/195 [01:06<00:18,  6.01it/s]
                                                
+
 43%|████▎     | 83/195 [01:06<00:18,  6.01it/s]
 43%|████▎     | 84/195 [01:06<00:24,  4.57it/s]
                                                
+
 43%|████▎     | 84/195 [01:06<00:24,  4.57it/s]
                                                
+
 43%|████▎     | 84/195 [01:06<00:24,  4.57it/s]
 44%|████▍     | 86/195 [01:06<00:20,  5.19it/s]
                                                
+
 44%|████▍     | 86/195 [01:06<00:20,  5.19it/s]
                                                
+
 44%|████▍     | 86/195 [01:06<00:20,  5.19it/s]
                                                
+
 44%|████▍     | 86/195 [01:06<00:20,  5.19it/s]
 46%|████▌     | 89/195 [01:07<00:16,  6.37it/s]
                                                
+
 46%|████▌     | 89/195 [01:07<00:16,  6.37it/s]
 46%|████▌     | 90/195 [01:07<00:21,  4.80it/s]
                                                
+
 46%|████▌     | 90/195 [01:07<00:21,  4.80it/s]
                                                
+
 46%|████▌     | 90/195 [01:07<00:21,  4.80it/s]
 47%|████▋     | 92/195 [01:07<00:19,  5.41it/s]
                                                
+
 47%|████▋     | 92/195 [01:07<00:19,  5.41it/s]
                                                
+
 47%|████▋     | 92/195 [01:07<00:19,  5.41it/s]
                                                
+
 47%|████▋     | 92/195 [01:07<00:19,  5.41it/s]
 49%|████▊     | 95/195 [01:08<00:15,  6.51it/s]
                                                
+
 49%|████▊     | 95/195 [01:08<00:15,  6.51it/s]
 49%|████▉     | 96/195 [01:08<00:20,  4.81it/s]
                                                
+
 49%|████▉     | 96/195 [01:08<00:20,  4.81it/s]
                                                
+
 49%|████▉     | 96/195 [01:08<00:20,  4.81it/s]
 50%|█████     | 98/195 [01:08<00:17,  5.41it/s]
                                                
+
 50%|█████     | 98/195 [01:08<00:17,  5.41it/s]
                                                
+
 50%|█████     | 98/195 [01:09<00:17,  5.41it/s]
                                                
+
 50%|█████     | 98/195 [01:09<00:17,  5.41it/s]
 52%|█████▏    | 101/195 [01:09<00:14,  6.58it/s]
                                                 
+
 52%|█████▏    | 101/195 [01:09<00:14,  6.58it/s]
 52%|█████▏    | 102/195 [01:09<00:18,  4.92it/s]
                                                 
+
 52%|█████▏    | 102/195 [01:09<00:18,  4.92it/s]
                                                 
+
 52%|█████▏    | 102/195 [01:09<00:18,  4.92it/s]
 53%|█████▎    | 104/195 [01:10<00:16,  5.44it/s]
                                                 
+
 53%|█████▎    | 104/195 [01:10<00:16,  5.44it/s]
                                                 
+
 53%|█████▎    | 104/195 [01:10<00:16,  5.44it/s]
                                                 
+
 53%|█████▎    | 104/195 [01:10<00:16,  5.44it/s]
 55%|█████▍    | 107/195 [01:10<00:13,  6.60it/s]
                                                 
+
 55%|█████▍    | 107/195 [01:10<00:13,  6.60it/s]
 55%|█████▌    | 108/195 [01:10<00:17,  4.94it/s]
                                                 
+
 55%|█████▌    | 108/195 [01:10<00:17,  4.94it/s]
                                                 
+
 55%|█████▌    | 108/195 [01:10<00:17,  4.94it/s]
 56%|█████▋    | 110/195 [01:11<00:15,  5.47it/s]
                                                 
+
 56%|█████▋    | 110/195 [01:11<00:15,  5.47it/s]
                                                 
+
 56%|█████▋    | 110/195 [01:11<00:15,  5.47it/s]
                                                 
+
 56%|█████▋    | 110/195 [01:11<00:15,  5.47it/s]
 58%|█████▊    | 113/195 [01:11<00:12,  6.62it/s]
                                                 
+
 58%|█████▊    | 113/195 [01:11<00:12,  6.62it/s]
 58%|█████▊    | 114/195 [01:11<00:16,  4.95it/s]
                                                 
+
 58%|█████▊    | 114/195 [01:11<00:16,  4.95it/s]
                                                 
+
 58%|█████▊    | 114/195 [01:11<00:16,  4.95it/s]
 59%|█████▉    | 116/195 [01:12<00:14,  5.55it/s]
                                                 
+
 59%|█████▉    | 116/195 [01:12<00:14,  5.55it/s]
                                                 
+
 59%|█████▉    | 116/195 [01:12<00:14,  5.55it/s]
                                                 
+
 59%|█████▉    | 116/195 [01:12<00:14,  5.55it/s]
 61%|██████    | 119/195 [01:12<00:11,  6.63it/s]
                                                 
+
 61%|██████    | 119/195 [01:12<00:11,  6.63it/s]
 62%|██████▏   | 120/195 [01:12<00:15,  4.78it/s]
                                                 
+
 62%|██████▏   | 120/195 [01:12<00:15,  4.78it/s]
                                                 
+
 62%|██████▏   | 120/195 [01:13<00:15,  4.78it/s]
 63%|██████▎   | 122/195 [01:13<00:13,  5.33it/s]
                                                 
+
 63%|██████▎   | 122/195 [01:13<00:13,  5.33it/s]
                                                 
+
 63%|██████▎   | 122/195 [01:13<00:13,  5.33it/s]
                                                 
+
 63%|██████▎   | 122/195 [01:13<00:13,  5.33it/s]
 64%|██████▍   | 125/195 [01:13<00:10,  6.42it/s]
                                                 
+
 64%|██████▍   | 125/195 [01:13<00:10,  6.42it/s]
 65%|██████▍   | 126/195 [01:14<00:14,  4.77it/s]
                                                 
+
 65%|██████▍   | 126/195 [01:14<00:14,  4.77it/s]
                                                 
+
 65%|██████▍   | 126/195 [01:14<00:14,  4.77it/s]
 66%|██████▌   | 128/195 [01:14<00:12,  5.32it/s]
                                                 
+
 66%|██████▌   | 128/195 [01:14<00:12,  5.32it/s]
                                                 
+
 66%|██████▌   | 128/195 [01:14<00:12,  5.32it/s]
                                                 
+
 66%|██████▌   | 128/195 [01:14<00:12,  5.32it/s]
 67%|██████▋   | 131/195 [01:14<00:09,  6.50it/s]
                                                 
+
 67%|██████▋   | 131/195 [01:14<00:09,  6.50it/s]
 68%|██████▊   | 132/195 [01:15<00:12,  4.89it/s]
                                                 
+
 68%|██████▊   | 132/195 [01:15<00:12,  4.89it/s]
                                                 
+
 68%|██████▊   | 132/195 [01:15<00:12,  4.89it/s]
 69%|██████▊   | 134/195 [01:15<00:11,  5.50it/s]
                                                 
+
 69%|██████▊   | 134/195 [01:15<00:11,  5.50it/s]
                                                 
+
 69%|██████▊   | 134/195 [01:15<00:11,  5.50it/s]
                                                 
+
 69%|██████▊   | 134/195 [01:15<00:11,  5.50it/s]
 70%|███████   | 137/195 [01:15<00:08,  6.58it/s]
                                                 
+
 70%|███████   | 137/195 [01:15<00:08,  6.58it/s]
 71%|███████   | 138/195 [01:16<00:11,  4.80it/s]
                                                 
+
 71%|███████   | 138/195 [01:16<00:11,  4.80it/s]
                                                 
+
 71%|███████   | 138/195 [01:16<00:11,  4.80it/s]
 72%|███████▏  | 140/195 [01:16<00:10,  5.42it/s]
                                                 
+
 72%|███████▏  | 140/195 [01:16<00:10,  5.42it/s]
                                                 
+
 72%|███████▏  | 140/195 [01:16<00:10,  5.42it/s]
                                                 
+
 72%|███████▏  | 140/195 [01:16<00:10,  5.42it/s]
 73%|███████▎  | 143/195 [01:16<00:07,  6.51it/s]
                                                 
+
 73%|███████▎  | 143/195 [01:16<00:07,  6.51it/s]
 74%|███████▍  | 144/195 [01:17<00:10,  4.87it/s]
                                                 
+
 74%|███████▍  | 144/195 [01:17<00:10,  4.87it/s]
                                                 
+
 74%|███████▍  | 144/195 [01:17<00:10,  4.87it/s]
 75%|███████▍  | 146/195 [01:17<00:09,  5.41it/s]
                                                 
+
 75%|███████▍  | 146/195 [01:17<00:09,  5.41it/s]
                                                 
+
 75%|███████▍  | 146/195 [01:17<00:09,  5.41it/s]
                                                 
+
 75%|███████▍  | 146/195 [01:17<00:09,  5.41it/s]
 76%|███████▋  | 149/195 [01:17<00:06,  6.58it/s]
                                                 
+
 76%|███████▋  | 149/195 [01:17<00:06,  6.58it/s]
 77%|███████▋  | 150/195 [01:18<00:09,  4.82it/s]
                                                 
+
 77%|███████▋  | 150/195 [01:18<00:09,  4.82it/s]
                                                 
+
 77%|███████▋  | 150/195 [01:18<00:09,  4.82it/s]
 78%|███████▊  | 152/195 [01:18<00:07,  5.44it/s]
                                                 
+
 78%|███████▊  | 152/195 [01:18<00:07,  5.44it/s]
                                                 
+
 78%|███████▊  | 152/195 [01:18<00:07,  5.44it/s]
                                                 
+
 78%|███████▊  | 152/195 [01:18<00:07,  5.44it/s]
 79%|███████▉  | 155/195 [01:18<00:06,  6.53it/s]
                                                 
+
 79%|███████▉  | 155/195 [01:19<00:06,  6.53it/s]
 80%|████████  | 156/195 [01:19<00:08,  4.72it/s]
                                                 
+
 80%|████████  | 156/195 [01:19<00:08,  4.72it/s]
                                                 
+
 80%|████████  | 156/195 [01:19<00:08,  4.72it/s]
 81%|████████  | 158/195 [01:19<00:06,  5.33it/s]
                                                 
+
 81%|████████  | 158/195 [01:19<00:06,  5.33it/s]
                                                 
+
 81%|████████  | 158/195 [01:19<00:06,  5.33it/s]
                                                 
+
 81%|████████  | 158/195 [01:19<00:06,  5.33it/s]
 83%|████████▎ | 161/195 [01:20<00:05,  6.51it/s]
                                                 
+
 83%|████████▎ | 161/195 [01:20<00:05,  6.51it/s]
 83%|████████▎ | 162/195 [01:20<00:06,  4.83it/s]
                                                 
+
 83%|████████▎ | 162/195 [01:20<00:06,  4.83it/s]
                                                 
+
 83%|████████▎ | 162/195 [01:20<00:06,  4.83it/s]
 84%|████████▍ | 164/195 [01:20<00:05,  5.43it/s]
                                                 
+
 84%|████████▍ | 164/195 [01:20<00:05,  5.43it/s]
                                                 
+
 84%|████████▍ | 164/195 [01:20<00:05,  5.43it/s]
                                                 
+
 84%|████████▍ | 164/195 [01:20<00:05,  5.43it/s]
 86%|████████▌ | 167/195 [01:21<00:04,  6.52it/s]
                                                 
+
 86%|████████▌ | 167/195 [01:21<00:04,  6.52it/s]
 86%|████████▌ | 168/195 [01:21<00:05,  4.74it/s]
                                                 
+
 86%|████████▌ | 168/195 [01:21<00:05,  4.74it/s]
                                                 
+
 86%|████████▌ | 168/195 [01:21<00:05,  4.74it/s]
 87%|████████▋ | 170/195 [01:21<00:04,  5.35it/s]
                                                 
+
 87%|████████▋ | 170/195 [01:21<00:04,  5.35it/s]
                                                 
+
 87%|████████▋ | 170/195 [01:21<00:04,  5.35it/s]
                                                 
+
 87%|████████▋ | 170/195 [01:22<00:04,  5.35it/s]
 89%|████████▊ | 173/195 [01:22<00:03,  6.53it/s]
                                                 
+
 89%|████████▊ | 173/195 [01:22<00:03,  6.53it/s]
 89%|████████▉ | 174/195 [01:22<00:04,  4.84it/s]
                                                 
+
 89%|████████▉ | 174/195 [01:22<00:04,  4.84it/s]
                                                 
+
 89%|████████▉ | 174/195 [01:22<00:04,  4.84it/s]
 90%|█████████ | 176/195 [01:22<00:03,  5.38it/s]
                                                 
+
 90%|█████████ | 176/195 [01:22<00:03,  5.38it/s]
                                                 
+
 90%|█████████ | 176/195 [01:23<00:03,  5.38it/s]
                                                 
+
 90%|█████████ | 176/195 [01:23<00:03,  5.38it/s]
 92%|█████████▏| 179/195 [01:23<00:02,  6.56it/s]
                                                 
+
 92%|█████████▏| 179/195 [01:23<00:02,  6.56it/s]
 92%|█████████▏| 180/195 [01:23<00:03,  4.82it/s]
                                                 
+
 92%|█████████▏| 180/195 [01:23<00:03,  4.82it/s]
                                                 
+
 92%|█████████▏| 180/195 [01:23<00:03,  4.82it/s]
 93%|█████████▎| 182/195 [01:24<00:02,  5.43it/s]
                                                 
+
 93%|█████████▎| 182/195 [01:24<00:02,  5.43it/s]
                                                 
+
 93%|█████████▎| 182/195 [01:24<00:02,  5.43it/s]
                                                 
+
 93%|█████████▎| 182/195 [01:24<00:02,  5.43it/s]
 95%|█████████▍| 185/195 [01:24<00:01,  5.69it/s]
                                                 
+
 95%|█████████▍| 185/195 [01:24<00:01,  5.69it/s]
 95%|█████████▌| 186/195 [01:25<00:02,  4.42it/s]
                                                 
+
 95%|█████████▌| 186/195 [01:25<00:02,  4.42it/s]
                                                 
+
 95%|█████████▌| 186/195 [01:25<00:02,  4.42it/s]
 96%|█████████▋| 188/195 [01:25<00:01,  5.07it/s]
                                                 
+
 96%|█████████▋| 188/195 [01:25<00:01,  5.07it/s]
                                                 
+
 96%|█████████▋| 188/195 [01:25<00:01,  5.07it/s]
                                                 
+
 96%|█████████▋| 188/195 [01:25<00:01,  5.07it/s]
 98%|█████████▊| 191/195 [01:25<00:00,  6.28it/s]
                                                 
+
 98%|█████████▊| 191/195 [01:25<00:00,  6.28it/s]
 98%|█████████▊| 192/195 [01:26<00:00,  4.79it/s]
                                                 
+
 98%|█████████▊| 192/195 [01:26<00:00,  4.79it/s]
                                                 
+
 98%|█████████▊| 192/195 [01:26<00:00,  4.79it/s]
 99%|█████████▉| 194/195 [01:26<00:00,  5.34it/s]
                                                 
+
 99%|█████████▉| 194/195 [01:26<00:00,  5.34it/s]
+[2024-05-22 00:31:37] INFO huggingface_loader.py:196: Unloading HF weight file: /models/Llama-2-7b-chat-hf/pytorch_model-00001-of-00002.bin
+[2024-05-22 00:31:38] INFO stats.py:76: Time usage: HF loading: 44.115 sec; Pre-quantization mapping: 7.475 sec; Quantization: 0.000 sec
+[2024-05-22 00:31:38] INFO stats.py:90: RAM usage: Peak RAM: 9.291 GB. Total bytes loaded from disk: 12.551 GB
+[2024-05-22 00:31:38] INFO convert_weight.py:155: Parameter size after quantization: 12.551 GB
+[2024-05-22 00:31:38] INFO convert_weight.py:160: Total parameters: 6,738,415,616
+[2024-05-22 00:31:38] INFO convert_weight.py:161: Bits per parameter: 16.000
+[2024-05-22 00:31:38] INFO convert_weight.py:166: Saved to directory: /tmp/tmp8ew5r2yr
+
+All finished, 131 total shards committed, record saved to /tmp/tmp8ew5r2yr/ndarray-cache.json

mlc-chat-config.json

@@ -0,0 +1,77 @@
+{
+  "model_type": "llama",
+  "quantization": "q0f16",
+  "model_config": {
+    "hidden_size": 4096,
+    "intermediate_size": 11008,
+    "num_attention_heads": 32,
+    "num_hidden_layers": 32,
+    "rms_norm_eps": 1e-06,
+    "vocab_size": 32000,
+    "position_embedding_base": 10000,
+    "context_window_size": 4096,
+    "prefill_chunk_size": 2048,
+    "num_key_value_heads": 32,
+    "head_dim": 128,
+    "tensor_parallel_shards": 1,
+    "max_batch_size": 80
+  },
+  "vocab_size": 32000,
+  "context_window_size": 4096,
+  "sliding_window_size": -1,
+  "prefill_chunk_size": 2048,
+  "attention_sink_size": -1,
+  "tensor_parallel_shards": 1,
+  "mean_gen_len": 128,
+  "max_gen_len": 512,
+  "shift_fill_factor": 0.3,
+  "temperature": 0.6,
+  "presence_penalty": 0.0,
+  "frequency_penalty": 0.0,
+  "repetition_penalty": 1.0,
+  "top_p": 0.9,
+  "conv_template": {
+    "name": "llama-2",
+    "system_template": "[INST] <<SYS>>\n{system_message}\n<</SYS>>\n\n",
+    "system_message": "You are a helpful, respectful and honest assistant.",
+    "system_prefix_token_ids": [
+      1
+    ],
+    "add_role_after_system_message": false,
+    "roles": {
+      "user": "<s>[INST]",
+      "assistant": "[/INST]",
+      "tool": "[INST]"
+    },
+    "role_templates": {
+      "user": "{user_message}",
+      "assistant": "{assistant_message}",
+      "tool": "{tool_message}"
+    },
+    "messages": [],
+    "seps": [
+      " ",
+      " </s>"
+    ],
+    "role_content_sep": " ",
+    "role_empty_sep": " ",
+    "stop_str": [
+      "[INST]"
+    ],
+    "stop_token_ids": [
+      2
+    ],
+    "function_string": "",
+    "use_function_calling": false
+  },
+  "pad_token_id": 0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "tokenizer_files": [
+    "tokenizer.model",
+    "tokenizer.json",
+    "tokenizer_config.json"
+  ],
+  "token_table_postproc_method": "byte_fallback",
+  "version": "0.1.0"
+}

params_shard_0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c82acf0f390411b8f3e071702ab6cb79d05d6b2b894af831001f25d32e0ab1e1
+size 262144000

params_shard_1.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:106e8a821011d724b7f1241642ecf78879e826319f95c3cb286f8cda78edfbd4
+size 90177536

params_shard_10.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe1e3fafa7c030fabc2555a4aec0f69754ccea75810b99b6cb25032481828cf6
+size 180355072

params_shard_100.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f86376cd89a5d6c2871f92df82ee8a8ac2fb3719bd76990f7a8c3751ced05d9e
+size 100663296

params_shard_101.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a23209c92a4de08f01f590e86c05550d4f9cd7b09026cf7db3889fa2f7631292
+size 33554432

params_shard_102.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5c431da9b0c87ae1ea137823781c5f59bd7a3a4208235c03bbdddeba1a50bd1
+size 90177536

params_shard_103.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:890556b34cf64071ae1e2d963246e721228abf101f1beaf668330689aa411e60
+size 180355072

params_shard_104.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7ea55b7ecb31513ecddbaf5d2af31677c680c8c1df70e2fe461f65d3097e58b
+size 100663296

params_shard_105.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd25f92eb7d9989192ee70823f46221d2e34021a40299703a0aff4da3fe577b6
+size 33554432

params_shard_106.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fb5921bffeec01dcbde3337439cbbe7a17af324342c4692f829717fa635f95b
+size 90177536

params_shard_107.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:412a16b7382f43671eca72726d7c1c6aa5c5457dc31d21d3ca0322c6e874510f
+size 180355072

params_shard_108.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1c0fbc97262d1f66587d3cdab0d6ba796030dc5aa3bb39e1940548d25798b66
+size 100663296

params_shard_109.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecdc141700f1211748c51581929cd3ef008f0cf90984d8b99c72d511fe933228
+size 33554432

params_shard_11.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a91698047e5e2cdf10352fbae0a543b00f0b5ee5f5641f14e4a5dfcbae38b87
+size 100663296

params_shard_110.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f7663ae3cb01aa5ef0f5faf0c3501e5866c4e5b00f431fb5870027d26dec1b4
+size 90177536

params_shard_111.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6f27eca17673b962330affcf8be33f6376443cda6246b669d6d22abbe8400d2
+size 180355072

params_shard_112.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcb7fe8fff62cdd3ad71c57834f4968e1f0535ed8c22796c9740955c57aa9358
+size 100663296

params_shard_113.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91b72108e1ee3f8a888e23f76dcdf65d70ae78b27bef12a31c71199d5a109071
+size 33554432

params_shard_114.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99834e9dc3779a64632e87d38dccd40bec3cfd29433ae68b1fd56408cee35dbd
+size 90177536

params_shard_115.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b54b37221472531b9617451ed64277ac6127cd527d8a152b9cf6524baadd579d
+size 180355072

params_shard_116.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8254d40a5ff399a0d4d69311a918c0c28b38905dcf88881f82351d7cffaca307
+size 100663296

params_shard_117.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1175e3d5178225b867ad5744133215c98f62f7a7dc7da2e449a2d1a5dc1b77f
+size 33554432

params_shard_118.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8da1458efd7b705a77bd0fb2462a4ca699b99f499e62f5df060cf2c4d24367a4
+size 90177536

params_shard_119.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:308fd51e6030ac6a1aa00d594646a3461259f106c64921319f4e5a9f4672eb4d
+size 180355072

params_shard_12.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dfd67a74420433e411bae645b94f80e59b159727635b819bd9580d292c8ea33
+size 33554432

params_shard_120.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:454f659581268e99fa93293fdb1ed6eb1b0d6ddf57dfd50964d6afa0ee465312
+size 100663296

params_shard_121.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:866706c3393eb7d12e673f5cadbe0999809e21b5513caedcece80bf8228de2a0
+size 33554432

params_shard_122.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bcff9547e262f5a79d8c44f19f8744e3574458f253fa36ae804eace6ebb48ab
+size 90177536

params_shard_123.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2a1035caf4b9073d2a67b3493b1b9392bef7d274814f14ac2d7e140108abeba
+size 180355072

params_shard_124.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f49cb1d10398bf5e6512839fad0d6e55bd3e08139064b38fa881c605b2f5f17d
+size 100663296

params_shard_125.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02686f613907af9428296a15bc1a50f137c3176979756bcaef84fd535326ffa9
+size 33554432

params_shard_126.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fac7c9231aeb4cbf40c86d2efa8dbfdae0f935678e89d0ed1e5932843b46a63b
+size 90177536

params_shard_127.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:537a0ae2ef4d887c43193ee94748dca11b8b497ec3445521f370c368f9a8db22
+size 180355072

params_shard_128.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cecbb255e4aa37e122de9783c67676c7404447e155f041df09e256e9b1c0eba
+size 100663296

params_shard_129.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40cbdf9e6fc55d6db597c1997c557f1be3d7528bfb9f5e749cde4efa11de8b99
+size 33554432

params_shard_13.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:415c8fce0ef3057da704a879a037349d482953604d5f1b13d0094ead6f8bbff8
+size 90177536

params_shard_130.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5313166331b4b3cbbdf6e6eb14575e543271969c134affeb8c24b0819c46e999
+size 532480

params_shard_14.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c068d9a20dc11e42a1758a204d5089bc448e533e06ca91ae8f088e8c81784301
+size 180355072

params_shard_15.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42359fda2ba2830d26593cde8177763c7b230f8c06cd2be170748384b051a59b
+size 100663296

params_shard_16.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf39454db1d0ea4263c5932bc08cf8d20fe9362e91f2df99994ce686e0783c9a
+size 33554432

params_shard_17.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d27836197f2185bf5e156879999766a58a174ca68ee82d3ac7d1dc3cd71ada71
+size 90177536

params_shard_18.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d60cf0ee0435346f25a60771f1f7160369c59cca4e7a4c4f6cf3fe6a2df920e2
+size 180355072

params_shard_19.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2825c85d632bffcd7b09242da7c7fba10b678827db4af10255f7c0d4f82e72d5
+size 100663296

params_shard_2.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ead039b62eb49d02d56d611e4c4683dbbcd30d53860ac4c015108040cebc3546
+size 180355072

params_shard_20.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79565c3ce00cd083ac33beef8c1d48e4342a16fc77456514b7ccf2e1e2325f1d
+size 33554432

params_shard_21.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2d349cad6eeca2eaa53c09059a6812ccfd31546711e60fdac425ecdeb8245f9
+size 90177536

params_shard_22.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5424bb60b1830416c670a528e0a3c8cfdac621edee256fadfc9e521c0d017b6d
+size 180355072