Hugging Face's logo

neuralmagic
/
Qwen2-72B-Instruct-quantized.w8a16

Text Generation
Transformers
Safetensors
English
qwen2
conversational
text-generation-inference
Inference Endpoints
8-bit precision
gptq
Model card Files Community
alexmarques commited on
Commit
4444d8e
•
1 Parent(s): 33107a0

Upload Qwen2ForCausalLM

Browse files
Files changed (19) hide show
  1. config.json +52 -0
  2. generation_config.json +6 -0
  3. model-00001-of-00016.safetensors +3 -0
  4. model-00002-of-00016.safetensors +3 -0
  5. model-00003-of-00016.safetensors +3 -0
  6. model-00004-of-00016.safetensors +3 -0
  7. model-00005-of-00016.safetensors +3 -0
  8. model-00006-of-00016.safetensors +3 -0
  9. model-00007-of-00016.safetensors +3 -0
  10. model-00008-of-00016.safetensors +3 -0
  11. model-00009-of-00016.safetensors +3 -0
  12. model-00010-of-00016.safetensors +3 -0
  13. model-00011-of-00016.safetensors +3 -0
  14. model-00012-of-00016.safetensors +3 -0
  15. model-00013-of-00016.safetensors +3 -0
  16. model-00014-of-00016.safetensors +3 -0
  17. model-00015-of-00016.safetensors +3 -0
  18. model-00016-of-00016.safetensors +3 -0
  19. model.safetensors.index.json +0 -0
config.json ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "Qwen__Qwen2-72B-Instruct",
3
+ "architectures": [
4
+ "Qwen2ForCausalLM"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 151643,
8
+ "eos_token_id": 151645,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 8192,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 29568,
13
+ "max_position_embeddings": 32768,
14
+ "max_window_layers": 80,
15
+ "model_type": "qwen2",
16
+ "num_attention_heads": 64,
17
+ "num_hidden_layers": 80,
18
+ "num_key_value_heads": 8,
19
+ "quantization_config": {
20
+ "batch_size": 1,
21
+ "bits": 8,
22
+ "block_name_to_quantize": null,
23
+ "cache_block_outputs": true,
24
+ "damp_percent": 0.01,
25
+ "dataset": null,
26
+ "desc_act": false,
27
+ "exllama_config": {
28
+ "version": 1
29
+ },
30
+ "group_size": -1,
31
+ "max_input_length": null,
32
+ "model_seqlen": null,
33
+ "module_name_preceding_first_block": null,
34
+ "modules_in_block_to_quantize": null,
35
+ "pad_token_id": null,
36
+ "quant_method": "gptq",
37
+ "sym": true,
38
+ "tokenizer": null,
39
+ "true_sequential": true,
40
+ "use_cuda_fp16": false,
41
+ "use_exllama": true
42
+ },
43
+ "rms_norm_eps": 1e-06,
44
+ "rope_theta": 1000000.0,
45
+ "sliding_window": 131072,
46
+ "tie_word_embeddings": false,
47
+ "torch_dtype": "float16",
48
+ "transformers_version": "4.42.1",
49
+ "use_cache": true,
50
+ "use_sliding_window": false,
51
+ "vocab_size": 152064
52
+ }
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 151643,
4
+ "eos_token_id": 151645,
5
+ "transformers_version": "4.42.1"
6
+ }
model-00001-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:500228de365dfc731e54e2bc17b241abedde7e88081168ab126e9ff71f3d1b2d
3
+ size 4883902320
model-00002-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d9ffe23fb4ed73f2c4c3978a09d7028a2f2268b30fdb468a9d6bb70365af94dd
3
+ size 4785015912
model-00003-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4418358c7ac778353bfda32eef5b873fa4a0e09d23c44bd5ae745b2e1d20c500
3
+ size 4876143456
model-00004-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2d1222619a09a326e0113b7bd47eb9ed842451e05f8d28abddc4a355e907b70e
3
+ size 4785016104
model-00005-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7677b742e75a9fc38663be38419a6fa85a34bd1e2cb3663c83a7a8bd91202f4
3
+ size 4876143520
model-00006-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d0ca2a748e001e08b7833a62d0e3a59fab38d68e6c4ac032b38aebd525550f84
3
+ size 4785016104
model-00007-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1846b64b3696d2f82923e35c48ba81f0336b5ce733baa9689c8c519c4e5161a2
3
+ size 4876143520
model-00008-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4cb50bb7faecc6065caabf63b83e68b62ffb163d8515eba6e4fa37467341c1e
3
+ size 4785016104
model-00009-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7bbd802a8943f06ca34ad71ecc0354eb567ea583160fe94e420a774e9b35b246
3
+ size 4876143520
model-00010-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0de5c79ebc6dace096b2ac4fff2f3ee3e49258ab94f3ab9450ba3d8054a5c6b0
3
+ size 4785016104
model-00011-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:acff0f789229397a7e57e0b214f06d9ac306a7c8014825149c3ee36400cd3f84
3
+ size 4876143520
model-00012-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1bf854b3b919703e5536cb9f8fcde2d84470b18c935fb288fb69cc9252bed79d
3
+ size 4785016104
model-00013-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d65ee4cd7ca96c0343ee289da509eb3cc230a59049ba55dfce35c7bcacf5f22
3
+ size 4876143520
model-00014-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c008ae752456c0069bd841ccccf5ebce380cfa1a4e09630934c07d1ba26814d8
3
+ size 4785016104
model-00015-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:51b469210db38874e4e84ac97150d9ea60fe18d4fb5c57d436f6e343fd1367dc
3
+ size 4876143520
model-00016-of-00016.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e963b6398502a5693fe0f29691ff92c5910966af3e00b7e11f393a79dd252892
3
+ size 2733809128
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff