Hugging Face's logo

ISTA-DASLab
/
switch-c-2048_qmoe

Transformers
switch_transformers
Inference Endpoints
Model card Files Community
efrantar commited on
Commit
fb3616a
•
1 Parent(s): ae20e25

Added files

Browse files
Files changed (32) hide show
  1. config.json +41 -0
  2. exp01.pt +3 -0
  3. exp02.pt +3 -0
  4. exp03.pt +3 -0
  5. exp04.pt +3 -0
  6. exp05.pt +3 -0
  7. exp06.pt +3 -0
  8. exp07.pt +3 -0
  9. exp08.pt +3 -0
  10. exp09.pt +3 -0
  11. exp10.pt +3 -0
  12. exp11.pt +3 -0
  13. exp12.pt +3 -0
  14. exp13.pt +3 -0
  15. exp14.pt +3 -0
  16. exp15.pt +3 -0
  17. exp16.pt +3 -0
  18. exp17.pt +3 -0
  19. exp18.pt +3 -0
  20. exp19.pt +3 -0
  21. exp20.pt +3 -0
  22. exp21.pt +3 -0
  23. exp22.pt +3 -0
  24. exp23.pt +3 -0
  25. exp24.pt +3 -0
  26. exp25.pt +3 -0
  27. exp26.pt +3 -0
  28. exp27.pt +3 -0
  29. exp28.pt +3 -0
  30. exp29.pt +3 -0
  31. noexp.pt +3 -0
  32. sizes.pt +3 -0
config.json ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_router_probs": false,
3
+ "d_ff": 6144,
4
+ "d_kv": 64,
5
+ "d_model": 2080,
6
+ "decoder_sparse_step": 0,
7
+ "decoder_start_token_id": 0,
8
+ "dense_act_fn": "relu",
9
+ "dropout_rate": 0.1,
10
+ "encoder_sparse_step": 1,
11
+ "eos_token_id": 1,
12
+ "expert_capacity": 1024,
13
+ "feed_forward_proj": "relu",
14
+ "initializer_factor": 1.0,
15
+ "is_encoder_decoder": true,
16
+ "is_full_sparse": true,
17
+ "is_gated_act": false,
18
+ "layer_norm_epsilon": 1e-06,
19
+ "model_type": "switch_transformers",
20
+ "num_decoder_layers": 15,
21
+ "num_experts": 2048,
22
+ "num_heads": 30,
23
+ "num_layers": 15,
24
+ "num_sparse_decoder_layers": 15,
25
+ "num_sparse_encoder_layers": 15,
26
+ "pad_token_id": 0,
27
+ "relative_attention_max_distance": 128,
28
+ "relative_attention_num_buckets": 32,
29
+ "router_aux_loss_coef": 0.001,
30
+ "router_bias": false,
31
+ "router_dtype": "float32",
32
+ "router_ignore_padding_tokens": false,
33
+ "router_jitter_noise": 0,
34
+ "router_type": "tokens_masked",
35
+ "router_z_loss_coef": 0.001,
36
+ "tie_word_embeddings": false,
37
+ "torch_dtype": "bfloat16",
38
+ "transformers_version": "4.28.0.dev0",
39
+ "use_cache": false,
40
+ "vocab_size": 32128
41
+ }
exp01.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d96011756494a5e5e000633ecbc8d4e3b3df24b11d25ba40347571d3d5d3b457
3
+ size 4919670301
exp02.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:020c0f9aec2a4794fa6f9938c04a5ced4fcf648751f4c5ebe7d869e40c09ee14
3
+ size 4932289313
exp03.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bf036e46f4d40d5a6cfe5e7151075458329e32dffa9afbf848b7f73d624f2ac4
3
+ size 4918181749
exp04.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:983085f3784131472e2e734e29fd7ac1bdbf91f68fcb2590289e165f6ef04e69
3
+ size 4974027793
exp05.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70fe2d1e6e38951a3078097e36486c71633f7f4501c6b5069c3b47f37d7be209
3
+ size 5018120781
exp06.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:931ce9a88ed4d25526538c15ea85ba86f6b9ed06eb44734bb16be16d99e8f0a4
3
+ size 5120976549
exp07.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8b7c8fdfd48c117569c3acdb6ed167c9a56081732f036cd16cb7c08ef78794a0
3
+ size 5199253445
exp08.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf2dec0f600557eb87f43fa9c82a4cee667ed1958b5b515b4fdf42e381b84ced
3
+ size 5187820301
exp09.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d6ee018473d55d0906f244ced2d67e2166c670ace4830b941f9f0f595e9a7c49
3
+ size 5150116045
exp10.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:98684032b8c89ebe84809426f41a9779c3b9f6a68665022965eba651113ae8cd
3
+ size 5194854181
exp11.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:acec7c1735cfac2764ecea56fed65a8eadfcaaeaa95ed73c582490b23d2917e1
3
+ size 5175204181
exp12.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9907b478b3968c065dd239ba249bf130f59a216bdce47f56c4944e041f3b80ad
3
+ size 5217034609
exp13.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ecd885a964dcbb079db3430f2e1c41e02b911580bcf85f246a23899c7d433103
3
+ size 5258123161
exp14.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:330fa2e37c936dafce7be66a776a2550fdc0ecfbe1dac4be7bf73fa4315f592c
3
+ size 5299198909
exp15.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:30f23f06f68deead7224f43dafdf424d32ae228f9d9f7467e104dbc03d7230f2
3
+ size 4161038981
exp16.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5e34ef3953a3dfc0f37b1ab60f68af20d3d3ef0153f75da302d001d1e671bdf5
3
+ size 4401268213
exp17.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:208460f050db6f6c3fc1a7d38d471eb0296077fc0ecfb3351ca5bbd0fea67300
3
+ size 5177699649
exp18.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9a1aeb3e94d9c4915ffcd9dd053d468510b4fb608bfdb6504870c0b852cf7fb9
3
+ size 5339639953
exp19.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4ed130cf37fb4f9683e088f38110a8855c2c47dc0d7d28b25bb4a7a3ef548f53
3
+ size 5274397253
exp20.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6e259a3986811db05d983b8ca1cba6adfeb3a9bfb16f1aa7ded24a43992c1166
3
+ size 5392869237
exp21.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1bc90b4d71ff431c412594776120411eb23b7c1ad80551151e4e23d789d6ccf8
3
+ size 5377198145
exp22.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:259c9ae6255201faf82a1814809d1319a166c49c4a906b49d36e6c853536623a
3
+ size 5633498997
exp23.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:65a3a68f72ab00e18a9180022378f9045f8b731072858280873ba6ffe8247569
3
+ size 5620589109
exp24.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d48d4fdf0d59e981d42686339ae53370c94f9471120fcd902bb4d7579769f73d
3
+ size 5627159825
exp25.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:648debfddb5b8d2fff954b6e6e2f0a89e9b541e35ff833661c099c73bedeb870
3
+ size 5666074493
exp26.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44f5493fa7e28d5375fdf03acec670c42873fb5d44d0ed58eeeae8ad9f3c89dd
3
+ size 5632836909
exp27.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c79d1bb49ac856b412ce76048a63ea8cbc814cefcf7464b199c73112ec81709d
3
+ size 5638474585
exp28.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:114c42cf270d60aa224d82e2aba830453d50e80fb0548215e9945c0616a6a2dc
3
+ size 5635378089
exp29.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1d4c48d716703581f1d2233201d067904221d144ee62f7e0c345c357b67cee17
3
+ size 5549490917
noexp.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c8f1404153cf9e11de520773c491209ad2b6766397698b56aaafa848647545f2
3
+ size 1983881265
sizes.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a311aeb2f7ced7d67b3a51a1fc989331b89664749b07dfc5fc890b6bcd2263e4
3
+ size 7961451