Hugging Face's logo

neuralmagic
/
Meta-Llama-3.1-405B-Instruct-FP8

Text Generation
Transformers
Safetensors
llama
fp8
vllm
conversational
text-generation-inference
Inference Endpoints
compressed-tensors
Model card Files Community
2
Lin-K76 commited on
Commit
de1282d
•
1 Parent(s): 919cfdf

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. config.json +73 -0
  2. generation_config.json +10 -0
  3. model-00001-of-00086.safetensors +3 -0
  4. model-00002-of-00086.safetensors +3 -0
  5. model-00003-of-00086.safetensors +3 -0
  6. model-00004-of-00086.safetensors +3 -0
  7. model-00005-of-00086.safetensors +3 -0
  8. model-00006-of-00086.safetensors +3 -0
  9. model-00007-of-00086.safetensors +3 -0
  10. model-00008-of-00086.safetensors +3 -0
  11. model-00009-of-00086.safetensors +3 -0
  12. model-00010-of-00086.safetensors +3 -0
  13. model-00011-of-00086.safetensors +3 -0
  14. model-00012-of-00086.safetensors +3 -0
  15. model-00013-of-00086.safetensors +3 -0
  16. model-00014-of-00086.safetensors +3 -0
  17. model-00015-of-00086.safetensors +3 -0
  18. model-00016-of-00086.safetensors +3 -0
  19. model-00017-of-00086.safetensors +3 -0
  20. model-00018-of-00086.safetensors +3 -0
  21. model-00019-of-00086.safetensors +3 -0
  22. model-00020-of-00086.safetensors +3 -0
  23. model-00021-of-00086.safetensors +3 -0
  24. model-00022-of-00086.safetensors +3 -0
  25. model-00023-of-00086.safetensors +3 -0
  26. model-00024-of-00086.safetensors +3 -0
  27. model-00025-of-00086.safetensors +3 -0
  28. model-00026-of-00086.safetensors +3 -0
  29. model-00027-of-00086.safetensors +3 -0
  30. model-00028-of-00086.safetensors +3 -0
  31. model-00029-of-00086.safetensors +3 -0
  32. model-00030-of-00086.safetensors +3 -0
  33. model-00031-of-00086.safetensors +3 -0
  34. model-00032-of-00086.safetensors +3 -0
  35. model-00033-of-00086.safetensors +3 -0
  36. model-00034-of-00086.safetensors +3 -0
  37. model-00035-of-00086.safetensors +3 -0
  38. model-00036-of-00086.safetensors +3 -0
  39. model-00037-of-00086.safetensors +3 -0
  40. model-00038-of-00086.safetensors +3 -0
  41. model-00039-of-00086.safetensors +3 -0
  42. model-00040-of-00086.safetensors +3 -0
  43. model-00041-of-00086.safetensors +3 -0
  44. model-00042-of-00086.safetensors +3 -0
  45. model-00043-of-00086.safetensors +3 -0
  46. model-00044-of-00086.safetensors +3 -0
  47. model-00045-of-00086.safetensors +3 -0
  48. model-00046-of-00086.safetensors +3 -0
  49. model-00047-of-00086.safetensors +3 -0
  50. model-00048-of-00086.safetensors +3 -0
config.json ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/home/meta-llama/Meta-Llama-3.1-405B-Instruct",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 128000,
9
+ "compression_config": {
10
+ "config_groups": {
11
+ "group_0": {
12
+ "input_activations": {
13
+ "block_structure": null,
14
+ "dynamic": false,
15
+ "group_size": null,
16
+ "num_bits": 8,
17
+ "observer": "minmax",
18
+ "observer_kwargs": {},
19
+ "strategy": "tensor",
20
+ "symmetric": true,
21
+ "type": "float"
22
+ },
23
+ "output_activations": null,
24
+ "targets": [
25
+ "Linear"
26
+ ],
27
+ "weights": {
28
+ "block_structure": null,
29
+ "dynamic": false,
30
+ "group_size": null,
31
+ "num_bits": 8,
32
+ "observer": "minmax",
33
+ "observer_kwargs": {},
34
+ "strategy": "tensor",
35
+ "symmetric": true,
36
+ "type": "float"
37
+ }
38
+ }
39
+ },
40
+ "format": "naive-quantized",
41
+ "global_compression_ratio": 1.4646996824138383,
42
+ "ignore": [
43
+ "lm_head"
44
+ ],
45
+ "kv_cache_scheme": null,
46
+ "quant_method": "compressed-tensors",
47
+ "quantization_status": "frozen"
48
+ },
49
+ "eos_token_id": [
50
+ 128001,
51
+ 128008,
52
+ 128009
53
+ ],
54
+ "hidden_act": "silu",
55
+ "hidden_size": 16384,
56
+ "initializer_range": 0.02,
57
+ "intermediate_size": 53248,
58
+ "max_position_embeddings": 131072,
59
+ "mlp_bias": false,
60
+ "model_type": "llama",
61
+ "num_attention_heads": 128,
62
+ "num_hidden_layers": 126,
63
+ "num_key_value_heads": 16,
64
+ "pretraining_tp": 1,
65
+ "rms_norm_eps": 1e-05,
66
+ "rope_scaling": null,
67
+ "rope_theta": 500000.0,
68
+ "tie_word_embeddings": false,
69
+ "torch_dtype": "float16",
70
+ "transformers_version": "4.43.1",
71
+ "use_cache": true,
72
+ "vocab_size": 128256
73
+ }
generation_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 128000,
4
+ "eos_token_id": [
5
+ 128001,
6
+ 128008,
7
+ 128009
8
+ ],
9
+ "transformers_version": "4.43.1"
10
+ }
model-00001-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:616328cbdca2d4ebc13b7ffbe09a5df2f6d0b549d095ee8d1783479a40128236
3
+ size 4806673936
model-00002-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:38bb1639011c9df1d24c6fa1b526132e7cb3ad131b63b7dd8870f08f2ba175e1
3
+ size 4966124652
model-00003-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2618e56962e7f70edb444705f3cbfab569d0576a22c69b2cb03c607bbd431d7c
3
+ size 4697755992
model-00004-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa74b8afd0e2bb2ebff69445f9ebbc5d1ec708d2fae057439ed132cb164b8841
3
+ size 4966124652
model-00005-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5557f88eee4a8a9a8cbef8e6422c7e2083547085439d871984e0e19b5a68aa9d
3
+ size 4697755992
model-00006-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4af2d12eda9209bc92672aa5533ca37836f937ee814402ba73b347d9fc4cfba8
3
+ size 4966124652
model-00007-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ebe318ef5bbf5a076aebf8c78a8f16eff7b3f969bb2ef7c6a58d7f0c4237be0e
3
+ size 4697755992
model-00008-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7f69dce8e0c5c2cf395ae2f6433d5470d559dd0bb14a74d3ce66324445a393a4
3
+ size 4966124588
model-00009-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:92b344e127d65179557a4a8dc7be2620bd9b9734b05f97516740b6bb8172db83
3
+ size 4697756032
model-00010-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:99efb3448e97cc379804d810a46860459f96d3323bf8ccd1e86905bc7d69b404
3
+ size 4966124684
model-00011-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59497dcd34f369c66b95ec95fa86659cc3760c88d89e939b4fff39f35d387f98
3
+ size 4697756032
model-00012-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:520a24c2bf6455d1d6bee0b7f5e659bb471ae51074ace12dfbef85b2ef7e98eb
3
+ size 4966124684
model-00013-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:89432b63bc6e6f28ca6647fa6b974915e211d31d00af2699e3010b74053bde29
3
+ size 4697756032
model-00014-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ccfb16edb3b78bd62dea6d46da3644523cd6ff738fb74defba529fa93b4c52ce
3
+ size 4966124684
model-00015-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3a7962c9e8e99a8a234da28e7e42cbc94cefa53afb9e8a66924358c5e14f426c
3
+ size 4697756032
model-00016-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c5f11d746cc2b48e0ac54d08088e023bcc63fb5d285235e0724b465f6a15a499
3
+ size 4966124684
model-00017-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cfee1c99296a8a256d62811285b24778a4d781a6372688ffd9b306efb716758a
3
+ size 4697756032
model-00018-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:be5e2f98b80e71d3043d961a086b1d36875ebfe413f338ba9358100e036956dd
3
+ size 4966124684
model-00019-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:494e22b17c27b76eb4585ea28e43be276010fa8bc594be10b4be73a7bf4bd939
3
+ size 4697756032
model-00020-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bddd428ef6fb1aef6ec8cafd1cb0ea89433e188ed27aa86f20faf36ab38012c7
3
+ size 4966124684
model-00021-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4e1758c4924d10b81c13775c253408a3604fd29ccb7eab12df10e51f25191a36
3
+ size 4697756032
model-00022-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6a1bdaecdd48e5d63e12ad3658db303c95996df3232fc095ffaece733c2e4a87
3
+ size 4966124684
model-00023-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6a4e094c5f2fc777b013280279b6d5f69412f685127d462234e99eb47f115f93
3
+ size 4697756032
model-00024-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cbc0e311ab6297595e36202ca6eb8fd96af8b5537d41328f2925cef9b26c2fa9
3
+ size 4966124684
model-00025-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30d0df43f777043ad4a918082e0c22ce1236295b0175253b6e2488363790c687
3
+ size 4697756032
model-00026-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05bbc5876fc5c0b209abf1e72a7188113ef3a91c6fa54a648b60e9b5c534988c
3
+ size 4966124684
model-00027-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:389a0f47438a7766a42363c0748c946346d7276cfd968415fa8e9b65c41357bc
3
+ size 4697756032
model-00028-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f3e1d9c986c5868b9c317fb7f3d37b31592148ad66f79e03974a0f83d606d2bb
3
+ size 4966124684
model-00029-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b24842d025f0006b1cb769d498785dd496356c1f2fb788361583a75adfaab3ab
3
+ size 4697756032
model-00030-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac27dbf0d82739f449172bf11912245f7db052c5ad5a1c0791a8f90f32f8f1ed
3
+ size 4966124684
model-00031-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:933d875e72c38d79941ad6cbff29c4bba1054a72c46c1815aedc2cd0413e3c31
3
+ size 4697756032
model-00032-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7fd10afe42445a5098d5cc807eca68f79a01f39fe0b98604c1e837195df361b8
3
+ size 4966124684
model-00033-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:27fab3ea3c4db43f8a3f629e2912b73750a50186b4bca225b6b07b4cd2d40d31
3
+ size 4697756032
model-00034-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b876189a00c26a689e642165fd976e7527469086b9b89fbae6cb8843ab21a05
3
+ size 4966124684
model-00035-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44aa3c3bfb52ee17964c0b3a95ad32edd99de38315f760a2282369195de3c269
3
+ size 4697756032
model-00036-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b0860847e065244ee8755d76df037a90010fde35f3250955245cce6dfd137430
3
+ size 4966124684
model-00037-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dc1ea36c61476466a0600a63bba7520cbc0a2bf037474c2079cdc1548a90714a
3
+ size 4697756032
model-00038-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d4ebda9eb2be494b31fed14751a2377af97e35a0699f2c2390e2adaf96d387e2
3
+ size 4966124684
model-00039-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ef90571c7e19762c23e30abb19851a2d8f71ae8def9f7cb7df9fc0c59f9a8172
3
+ size 4697756032
model-00040-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:26df5f4840b86dd8385eb6072aaf772ab917ac73d95eea24686e12789b9d14cd
3
+ size 4966124684
model-00041-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2cc8c25deadffacde37fbcd16788a6bcc4d9b5513923614a9155ff0fed5c12dc
3
+ size 4697756032
model-00042-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a289658b67c5b656161df1e26f2fad1c7011eb360f856ce58e9e3cf07178c80d
3
+ size 4966124684
model-00043-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aba8bc716319cea8b0c4c2303d76de8f86952a56ca8ebff74cb0231b37c2f01d
3
+ size 4697756032
model-00044-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e6e6a255e9c8b69e7124804ed1a7daf56d75ffdc8822cb8c32bfff0974235c47
3
+ size 4966124684
model-00045-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:543fc8812eb7cc36406f50ae65b17c3b0f7848ddc8565498ab646a3b60c6121c
3
+ size 4697756032
model-00046-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b52dadcab7fd81ff8763d34bd879469926e82b40847c8421cfc11a9db483d4a9
3
+ size 4966124684
model-00047-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9f18bb35c727ff8d6c975ce07732b9596b715de7eded4dc136b55b77d56e690
3
+ size 4697756032
model-00048-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4be16e3d6a3a77b5e7ca3a8088c8ceb171c098a1ae86c02955f0826754dbf20f
3
+ size 4966124684