Hugging Face's logo

ibivibiv
/
giant-hydra-moe-240b

Text Generation
Transformers
Safetensors
English
mixtral
Mixture of Experts
moerge
text-generation-inference
Inference Endpoints
Model card Files Community
4
ibivibiv commited on
Commit
4f1200d
•
1 Parent(s): 7864508

Upload MixtralForCausalLM

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. config.json +34 -0
  2. generation_config.json +8 -0
  3. model-00001-of-00201.safetensors +3 -0
  4. model-00002-of-00201.safetensors +3 -0
  5. model-00003-of-00201.safetensors +3 -0
  6. model-00004-of-00201.safetensors +3 -0
  7. model-00005-of-00201.safetensors +3 -0
  8. model-00006-of-00201.safetensors +3 -0
  9. model-00007-of-00201.safetensors +3 -0
  10. model-00008-of-00201.safetensors +3 -0
  11. model-00009-of-00201.safetensors +3 -0
  12. model-00010-of-00201.safetensors +3 -0
  13. model-00011-of-00201.safetensors +3 -0
  14. model-00012-of-00201.safetensors +3 -0
  15. model-00013-of-00201.safetensors +3 -0
  16. model-00014-of-00201.safetensors +3 -0
  17. model-00015-of-00201.safetensors +3 -0
  18. model-00016-of-00201.safetensors +3 -0
  19. model-00017-of-00201.safetensors +3 -0
  20. model-00018-of-00201.safetensors +3 -0
  21. model-00019-of-00201.safetensors +3 -0
  22. model-00020-of-00201.safetensors +3 -0
  23. model-00021-of-00201.safetensors +3 -0
  24. model-00022-of-00201.safetensors +3 -0
  25. model-00023-of-00201.safetensors +3 -0
  26. model-00024-of-00201.safetensors +3 -0
  27. model-00025-of-00201.safetensors +3 -0
  28. model-00026-of-00201.safetensors +3 -0
  29. model-00027-of-00201.safetensors +3 -0
  30. model-00028-of-00201.safetensors +3 -0
  31. model-00029-of-00201.safetensors +3 -0
  32. model-00030-of-00201.safetensors +3 -0
  33. model-00031-of-00201.safetensors +3 -0
  34. model-00032-of-00201.safetensors +3 -0
  35. model-00033-of-00201.safetensors +3 -0
  36. model-00034-of-00201.safetensors +3 -0
  37. model-00035-of-00201.safetensors +3 -0
  38. model-00036-of-00201.safetensors +3 -0
  39. model-00037-of-00201.safetensors +3 -0
  40. model-00038-of-00201.safetensors +3 -0
  41. model-00039-of-00201.safetensors +3 -0
  42. model-00040-of-00201.safetensors +3 -0
  43. model-00041-of-00201.safetensors +3 -0
  44. model-00042-of-00201.safetensors +3 -0
  45. model-00043-of-00201.safetensors +3 -0
  46. model-00044-of-00201.safetensors +3 -0
  47. model-00045-of-00201.safetensors +3 -0
  48. model-00046-of-00201.safetensors +3 -0
  49. model-00047-of-00201.safetensors +3 -0
  50. model-00048-of-00201.safetensors +3 -0
config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "./gianthydra",
3
+ "architectures": [
4
+ "MixtralForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "eos_token_id": 2,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 8192,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 28672,
14
+ "max_position_embeddings": 4096,
15
+ "model_type": "mixtral",
16
+ "num_attention_heads": 64,
17
+ "num_experts_per_tok": 2,
18
+ "num_hidden_layers": 80,
19
+ "num_key_value_heads": 8,
20
+ "num_local_experts": 4,
21
+ "output_router_logits": false,
22
+ "pad_token_id": 0,
23
+ "pretraining_tp": 1,
24
+ "rms_norm_eps": 1e-05,
25
+ "rope_scaling": null,
26
+ "rope_theta": 10000.0,
27
+ "router_aux_loss_coef": 0.001,
28
+ "sliding_window": null,
29
+ "tie_word_embeddings": false,
30
+ "torch_dtype": "float32",
31
+ "transformers_version": "4.37.2",
32
+ "use_cache": false,
33
+ "vocab_size": 32000
34
+ }
generation_config.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "pad_token_id": 0,
6
+ "transformers_version": "4.37.2",
7
+ "use_cache": false
8
+ }
model-00001-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:510176a48d72902ac5ecb133257286150049b0dd61f56c4ab1b0dc792c2eddb9
3
+ size 4471260248
model-00002-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ca1d723e8af07415a60a5ed59fe26841558d463c7fa18bb1397d66715e3afec6
3
+ size 4697621160
model-00003-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70b9ff8d8387de3432a3f691924e78a4f71812c6acb6e52f39820b30e08c8a23
3
+ size 4362274128
model-00004-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:90d54ac75e956eac7cbc7d2c5c1d491b4ca0fc7de3bd73b4bff4e313667ee35b
3
+ size 4697621160
model-00005-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cdac281289add341da239eee538d139d334af921d29a12885462be7391b7a6e3
3
+ size 4697621160
model-00006-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9ee21f0110b662a3e699bcc01f61f103e38d22d08e7f2f0aa468b80540aa3013
3
+ size 4362274128
model-00007-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a6daf30af231e1d2184c53726f6f99aafa7da6811e5ec2d8e1b279d4aa950910
3
+ size 4697621160
model-00008-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ff61de5aba6feee62fa32c33125f3ea147ad6d2f486093ec2f27266c20b500f
3
+ size 4999677048
model-00009-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6344d7c963091c486b7ed07e100b838484b5bce79dcd73efc5e79de3a8ec1cb2
3
+ size 4999742472
model-00010-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:174da2e4365d1ed5e09e4e0129f599ac6f6ce7d0772e638ca9fefb0b3628e8e9
3
+ size 4697621160
model-00011-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c7e41230bbc6f75a2da7e7a26ffcc4d411132a1955b30511f9a44a3b1f70c4e
3
+ size 4362274128
model-00012-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:52813b32615f5e77bc170fe69e0a5c96feb2ea994ad0f4d69a7a27e5877b4255
3
+ size 4697621160
model-00013-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c3411dfbe187eb9f27d9ede2faa51d9bf6501cae55b5c45fcdd53018c94400d2
3
+ size 4999677048
model-00014-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3607b0a308a6d0a6e7f1cc82c88195ba9650760d547fed64a294084bbe8c0903
3
+ size 4999742472
model-00015-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e8b4cbc134e17e8d4761b2b93ee827f0dbaac072c4006b5e133835a1af3820b8
3
+ size 4697621160
model-00016-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:387438f39f6d93f67d76806e0e1ee08fadf34cced4c58d8517827de99f8ac910
3
+ size 4362274128
model-00017-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:acc7d1efccfb33e7dee4312bd4b267c1c58ca4202c24402048b5f7597b4823f7
3
+ size 4697621160
model-00018-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:faabc9f5889c35689f15380c5570051b1ff97ad555a9aa5e4b1acfdba378b3ee
3
+ size 4999677048
model-00019-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0c491dd97738beb840e2be54b18b4e4ac361e75bfa4ee2a93e36931db73b9d76
3
+ size 4999742472
model-00020-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c3cf13479f07de09e166369906dedc73e8a4c4aa1b8dd8bf9c912bda7f97dad7
3
+ size 4697621160
model-00021-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eb3b2b9164e1fe8dfd1798b85284192041ee7bc4819e39472e7c1441c32d2c7b
3
+ size 4362274128
model-00022-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7c0e26dd8f9e9e4d09ad0bd3111910d4c3a10efeb1d10ea9a39f82099c61b134
3
+ size 4697621160
model-00023-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ebf6a944439c121af7a10f9f7f38ab48e314b90743ff2ab1d47dd865d06a60fe
3
+ size 4999677048
model-00024-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:622a7b55ea6b40dc825a3261834080bb0b48b5f748594528adca93c4bd258027
3
+ size 4999742472
model-00025-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:280f460e27719b2d4bbc86f23ced27c4b7e1ecc01a9ec902c8c1a6d8483572b8
3
+ size 4697621160
model-00026-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b91b968ce35564a5a793d153cc41acc3ebe34cc1c2978bd16447018b1a8cb00
3
+ size 4362274136
model-00027-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:470c380514cc20e7cd763b9bc42e1b0858242c4fadba6581f9d4838903c6bd27
3
+ size 4697621168
model-00028-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:482edf2b7e445f9829c0ab2d6f9ce30f4fc92ee524a66e8843cf1230db5f8b3c
3
+ size 4999677056
model-00029-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e1ac89f7c7b7ceb8c9ae29771395607a501f845dfadfdeaefeb08521606f9520
3
+ size 4999742480
model-00030-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7335ac561a2732ca8e1a933256b591723adc81213eca6582e7585697b46736c6
3
+ size 4697621168
model-00031-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:849cf8779c007f802b1c00491bc9443055a718d5fb079c72d8ce47d1633b4858
3
+ size 4362274144
model-00032-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e67ded99b4290ae672e255160725659f9d895fbccac23584246bbfbbceadbd54
3
+ size 4697621168
model-00033-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a488f2af2430764200760f2cf7b015dc1f9b1c8f257182de28f24f8b476a6e02
3
+ size 4999677056
model-00034-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7a261940d3cdfc7ec7874722eb8a92a3088bbdb217bd916af80e05351ec16fa4
3
+ size 4999742480
model-00035-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:12681f117d6fda8f0147bee0ea4ca39a7e7195d8ab3f2c899e819f11e0192917
3
+ size 4697621168
model-00036-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ca6d8bf5f153aa79addaedb6a5676f588e9b96f00413ed871dbbe77ee17fc3b7
3
+ size 4362274144
model-00037-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61305834df92f9db558276beccc0ffea8368f60fe5f5e866f7bd3fda726bc55c
3
+ size 4697621168
model-00038-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3a1975d6701a159ea79fb46dcb6879e227b3f0f9e470882544f37065a25268f
3
+ size 4999677056
model-00039-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:96ea3217d2d650785c19e9ee7c15bcc79b4d26306c06ee84e03e2da301579d8a
3
+ size 4999742480
model-00040-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:73a581abee18103482961cd171f97118ee7de6a0808783d5cfb498118f44fb74
3
+ size 4697621168
model-00041-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8296e2f117ed7ed953c18c5c1605d50505a3e8e11c6545cf76e108f67b11ebea
3
+ size 4362274144
model-00042-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:589a45e0722415cb79fe6cadc9b1d0ecc30f998c2ee0928dc7ba385b69e135d7
3
+ size 4697621168
model-00043-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:08e1765cfb4c65d6e898e3d567f0dfb985167117ab5dafd1f773b3250ed5f409
3
+ size 4999677056
model-00044-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c739a03f221cb5653856d1c4842b6fc225e28f52f5573d85fc032b5a1687a77f
3
+ size 4999742480
model-00045-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:760ef5a75be203f04d5a823afb56095a92e906678acb2343316cffea94b1d6d6
3
+ size 4697621168
model-00046-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:776d0670887a11c19c3818d5edaadd51a95c516a97626f386ce11d4756cf84d1
3
+ size 4362274144
model-00047-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:739a9e9f62b3d10f47344534d44659914d1ef052c8fd1c431bbd1aa603350b79
3
+ size 4697621168
model-00048-of-00201.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:16d24b31e322d96479f3337daa952ed65ca487b7cd69043f00796c0bf1048db2
3
+ size 4999677056