Hugging Face's logo

neuralmagic
/
Meta-Llama-3.1-405B-Instruct-FP8-dynamic

Text Generation
Transformers
Safetensors
llama
fp8
vllm
conversational
text-generation-inference
Inference Endpoints
compressed-tensors
Model card Files Community
Lin-K76 commited on
Commit
90da764
•
1 Parent(s): 656ec87

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. config.json +79 -0
  2. generation_config.json +10 -0
  3. model-00001-of-00086.safetensors +3 -0
  4. model-00002-of-00086.safetensors +3 -0
  5. model-00003-of-00086.safetensors +3 -0
  6. model-00004-of-00086.safetensors +3 -0
  7. model-00005-of-00086.safetensors +3 -0
  8. model-00006-of-00086.safetensors +3 -0
  9. model-00007-of-00086.safetensors +3 -0
  10. model-00008-of-00086.safetensors +3 -0
  11. model-00009-of-00086.safetensors +3 -0
  12. model-00010-of-00086.safetensors +3 -0
  13. model-00011-of-00086.safetensors +3 -0
  14. model-00012-of-00086.safetensors +3 -0
  15. model-00013-of-00086.safetensors +3 -0
  16. model-00014-of-00086.safetensors +3 -0
  17. model-00015-of-00086.safetensors +3 -0
  18. model-00016-of-00086.safetensors +3 -0
  19. model-00017-of-00086.safetensors +3 -0
  20. model-00018-of-00086.safetensors +3 -0
  21. model-00019-of-00086.safetensors +3 -0
  22. model-00020-of-00086.safetensors +3 -0
  23. model-00021-of-00086.safetensors +3 -0
  24. model-00022-of-00086.safetensors +3 -0
  25. model-00023-of-00086.safetensors +3 -0
  26. model-00024-of-00086.safetensors +3 -0
  27. model-00025-of-00086.safetensors +3 -0
  28. model-00026-of-00086.safetensors +3 -0
  29. model-00027-of-00086.safetensors +3 -0
  30. model-00028-of-00086.safetensors +3 -0
  31. model-00029-of-00086.safetensors +3 -0
  32. model-00030-of-00086.safetensors +3 -0
  33. model-00031-of-00086.safetensors +3 -0
  34. model-00032-of-00086.safetensors +3 -0
  35. model-00033-of-00086.safetensors +3 -0
  36. model-00034-of-00086.safetensors +3 -0
  37. model-00035-of-00086.safetensors +3 -0
  38. model-00036-of-00086.safetensors +3 -0
  39. model-00037-of-00086.safetensors +3 -0
  40. model-00038-of-00086.safetensors +3 -0
  41. model-00039-of-00086.safetensors +3 -0
  42. model-00040-of-00086.safetensors +3 -0
  43. model-00041-of-00086.safetensors +3 -0
  44. model-00042-of-00086.safetensors +3 -0
  45. model-00043-of-00086.safetensors +3 -0
  46. model-00044-of-00086.safetensors +3 -0
  47. model-00045-of-00086.safetensors +3 -0
  48. model-00046-of-00086.safetensors +3 -0
  49. model-00047-of-00086.safetensors +3 -0
  50. model-00048-of-00086.safetensors +3 -0
config.json ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/home/meta-llama/Meta-Llama-3.1-405B-Instruct",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 128000,
9
+ "compression_config": {
10
+ "config_groups": {
11
+ "group_0": {
12
+ "input_activations": {
13
+ "block_structure": null,
14
+ "dynamic": true,
15
+ "group_size": null,
16
+ "num_bits": 8,
17
+ "observer": "memoryless",
18
+ "observer_kwargs": {},
19
+ "strategy": "token",
20
+ "symmetric": true,
21
+ "type": "float"
22
+ },
23
+ "output_activations": null,
24
+ "targets": [
25
+ "Linear"
26
+ ],
27
+ "weights": {
28
+ "block_structure": null,
29
+ "dynamic": false,
30
+ "group_size": null,
31
+ "num_bits": 8,
32
+ "observer": "minmax",
33
+ "observer_kwargs": {},
34
+ "strategy": "channel",
35
+ "symmetric": true,
36
+ "type": "float"
37
+ }
38
+ }
39
+ },
40
+ "format": "naive-quantized",
41
+ "global_compression_ratio": 1.240844659615461,
42
+ "ignore": [
43
+ "lm_head"
44
+ ],
45
+ "kv_cache_scheme": null,
46
+ "quant_method": "compressed-tensors",
47
+ "quantization_status": "frozen"
48
+ },
49
+ "eos_token_id": [
50
+ 128001,
51
+ 128008,
52
+ 128009
53
+ ],
54
+ "hidden_act": "silu",
55
+ "hidden_size": 16384,
56
+ "initializer_range": 0.02,
57
+ "intermediate_size": 53248,
58
+ "max_position_embeddings": 131072,
59
+ "mlp_bias": false,
60
+ "model_type": "llama",
61
+ "num_attention_heads": 128,
62
+ "num_hidden_layers": 126,
63
+ "num_key_value_heads": 16,
64
+ "pretraining_tp": 1,
65
+ "rms_norm_eps": 1e-05,
66
+ "rope_scaling": {
67
+ "factor": 8.0,
68
+ "high_freq_factor": 4.0,
69
+ "low_freq_factor": 1.0,
70
+ "original_max_position_embeddings": 8192,
71
+ "rope_type": "llama3"
72
+ },
73
+ "rope_theta": 500000.0,
74
+ "tie_word_embeddings": false,
75
+ "torch_dtype": "float16",
76
+ "transformers_version": "4.43.1",
77
+ "use_cache": true,
78
+ "vocab_size": 128256
79
+ }
generation_config.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 128000,
4
+ "eos_token_id": [
5
+ 128001,
6
+ 128008,
7
+ 128009
8
+ ],
9
+ "transformers_version": "4.43.1"
10
+ }
model-00001-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8f3fec6d9144a4213d6f260abc1083ed2a9ec48af78397ec0dda2e2b7807b78b
3
+ size 4806747216
model-00002-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:25f36078f1d264a97a1cf73cf1fec08dd4c06048531dc4a6b55dfb07ac371766
3
+ size 4966656256
model-00003-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a14dc56bf2923aeac2d13b3c0a6c020e9b23ca93ed852cebf13f9db6767bde9b
3
+ size 4698180768
model-00004-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ada176ef239a7d2393bc8a3a5d301b89e6a2bd0148d6052a7bf6a490792d7b5e
3
+ size 4966656256
model-00005-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fdbd3757bbbda4e2be402129d3593f006c1e735415c754d9fcf2890ff1192e80
3
+ size 4698180768
model-00006-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:349b7fb2a4c24587e982d0457a0077f02042ece0ffb29b67d158adce055bb8e1
3
+ size 4966656256
model-00007-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7b6b8797102efa93e0aa2ba0300e5ab8dd1e7274ab7422b237e7f5043befd867
3
+ size 4698180768
model-00008-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a9e7f8fafee928725817564be4466f7771aa7c9e4b5e9051b8dc476b3458060a
3
+ size 4966656272
model-00009-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:adce843e67ad2b0eb8205d54278e50bf5f8fd01e21defacb2cb8dd6749c44daf
3
+ size 4698180792
model-00010-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:10fd0cccdcff514e2ec1667afff8219d07fafec05e06308993a6c7ee032ca0a2
3
+ size 4966656280
model-00011-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bfdc38214d0f96360f5a47530ce611cafe290e4cbd844f2d459eeba4fcf56503
3
+ size 4698180792
model-00012-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d972f00809ad3e4c97f445a4977b2678c49c89f83d14f0b0abcf4379ad513d1
3
+ size 4966656280
model-00013-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b59e13e67b36f3159fc43d3965d93f940e05356c613b3b2abf58ffd40ccd65a8
3
+ size 4698180792
model-00014-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7fbf90d7e35b8b9c202fdd46ac1a112c815e4c3a9671cd1a23cd557ff78ccaa5
3
+ size 4966656280
model-00015-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c40a1a2bae05be0e781874df1a9fc03da2cca25d7ec3333464363bba2dfd25f0
3
+ size 4698180792
model-00016-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dcc5da3ccf73d178aff4401b5eadfe4eccb3cca1c0dade3a6250ef3076ce8752
3
+ size 4966656280
model-00017-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3a514ae2434340942fd4343533a1b7bfad2eb37392e53f013e7a2cdf1df12377
3
+ size 4698180792
model-00018-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e2faa42ba86b1b50c201a339ce19ce330654a3d226cdd5dce1d4fc87b878b81b
3
+ size 4966656280
model-00019-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:297ea850e686a022534ce9b5b6c81cca8867827852138a2f2588b19124a76891
3
+ size 4698180792
model-00020-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:876d58eb88cd52bfc7e2672e17fd8e1bd7a46283260820c047fb6df9349d18c2
3
+ size 4966656280
model-00021-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:831ad616d65564f3ed7429bccf9b3ef88853ffeae64528e8efeff0e60925350e
3
+ size 4698180792
model-00022-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3222e08a25f9972e7b1a9dd9b8ab2215b474ee340fb17c093efb2304c111e62
3
+ size 4966656280
model-00023-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:abf0608042fbc6cb166b6bca8e58161293b9b1a38a8b4e71749aeff3f99f3dff
3
+ size 4698180792
model-00024-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0347ef75d0c678260b374ca829247f9a33aa7960015d9a7cf0ee5793d3c891be
3
+ size 4966656280
model-00025-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7e2a70af5f9646484ddd871f95d81f72c5f0956b8d4bbc11a461af94aa318489
3
+ size 4698180792
model-00026-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d93a74c75a64bf42ba603dc9b2e05529ca9ba33f52c1c2d607f52f8354a51dc4
3
+ size 4966656280
model-00027-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:52919f0d5db6b660bf6ee515a2f73eea013779dd17dc8a85f9c72df938bf8d7d
3
+ size 4698180792
model-00028-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd02efccd4c4a319a6da83a6201460801d0aa84ace0ac93408de221d4029ab93
3
+ size 4966656280
model-00029-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1b473710d73e21213410aea503295979bdbbd2d92da7a36f5eda40310f33277e
3
+ size 4698180792
model-00030-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:97d5da10966020b9e03eba9123d48c46cc77d3ba36a81dbe43544dbc8ea3832d
3
+ size 4966656280
model-00031-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:519d2cadd161664e3168a1296e809e5d617719ae41923c9e85be3b01cc418d4c
3
+ size 4698180792
model-00032-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eb593a8a265429245a202bf7d5ca4ce988a102360f70e975e1fd07fd78e003e7
3
+ size 4966656280
model-00033-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:156e5ee80f97740f6a2f019b8e1cbae17f2392c68c860a8bc441cb2dcda371d2
3
+ size 4698180792
model-00034-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e77042287eb50587a9aefe778d9f3b7564a37da1e45529f69959c8256163f67
3
+ size 4966656280
model-00035-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:db2e7d59523f6b08471586c5cbbb234ad21adad9d4ee0f5851708ac8de270ec3
3
+ size 4698180792
model-00036-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ceaf0869d2fba7c9d49c5db5c10ed82ff7718861cff93ba2fb4b590ca72d45ba
3
+ size 4966656280
model-00037-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aae1e5ab385d8ba092e96971d995bb3071d861349848d1a9a2376d2d60f3bda5
3
+ size 4698180792
model-00038-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0f9ea705f2934194c8f1699eaef37acc010f395c213fc23753cc20ba67cfa0a4
3
+ size 4966656280
model-00039-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4c58e81c92b6bbe114071fd766ebcb851dcaa74f356a51512c6c798d04997d84
3
+ size 4698180792
model-00040-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:74f2b6d4e97f89194672b461a599174273859d03d1c7f70dca8708b76f3cce35
3
+ size 4966656280
model-00041-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8358e040388815d43a4be54862ae5f40dc114005091af6c77088e9bcf531d8e3
3
+ size 4698180792
model-00042-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dde4a8dab2b53f377e68cd668f53f9f363f0b73055ca006e2189a5df93cebb9b
3
+ size 4966656280
model-00043-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59a5c59efcb08e3b47b91710dd0dac3b0c2fcdb8944da88cc97bc782408ccf00
3
+ size 4698180792
model-00044-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4720b2cde8ca31a5ea20ba2d869bdbb92c04ff80f50630fb58acfac27de698e0
3
+ size 4966656280
model-00045-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e8348376122b82bdb5c6f4eafa4445e02e93b4964d593bba989e61863e33862
3
+ size 4698180792
model-00046-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12540e5e8238d67bb85c8266ed5065b7785bd9bfd37b69163e971e6b302fecc7
3
+ size 4966656280
model-00047-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ce64e95785ba6edb38741f677b89700449bf99af37d57f10604d8a80b25ba98a
3
+ size 4698180792
model-00048-of-00086.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1bc12e310109263cd11c0ffbc0899f86ac71c93f0cc47e60077f0e700a894571
3
+ size 4966656280