diff --git a/L2L_fineweb-100b_N1.1e08_D1.4e09_C9.6e17/README.md b/L2L_fineweb-100b_N1.1e08_D1.4e09_C9.6e17/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-100b_N1.1e08_D1.4e09_C9.6e17/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-100b_N1.1e08_D1.4e09_C9.6e17/config.json b/L2L_fineweb-100b_N1.1e08_D1.4e09_C9.6e17/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41baa877f3632c674fd6e77965ab3ee5a2a819e1 --- /dev/null +++ b/L2L_fineweb-100b_N1.1e08_D1.4e09_C9.6e17/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 704, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 11, + "n_kv_heads": null, + "n_layers": 11, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_fineweb-100b_N1.5e09_D5.6e09_C4.8e19/README.md b/L2L_fineweb-100b_N1.5e09_D5.6e09_C4.8e19/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-100b_N1.5e09_D5.6e09_C4.8e19/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-100b_N1.5e09_D5.6e09_C4.8e19/config.json b/L2L_fineweb-100b_N1.5e09_D5.6e09_C4.8e19/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b0ac0ef6bc85f96b3d24ba507bc08973d74c3d3a --- /dev/null +++ b/L2L_fineweb-100b_N1.5e09_D5.6e09_C4.8e19/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 1920, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 30, + "n_kv_heads": null, + "n_layers": 30, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_fineweb-100b_N1.6e08_D9.9e08_C9.6e17/README.md b/L2L_fineweb-100b_N1.6e08_D9.9e08_C9.6e17/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-100b_N1.6e08_D9.9e08_C9.6e17/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-100b_N1.6e08_D9.9e08_C9.6e17/config.json b/L2L_fineweb-100b_N1.6e08_D9.9e08_C9.6e17/config.json new file mode 100644 index 0000000000000000000000000000000000000000..eefab0956111af04f5cf1d2f3741df6009e664c0 --- /dev/null +++ b/L2L_fineweb-100b_N1.6e08_D9.9e08_C9.6e17/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 832, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 13, + "n_kv_heads": null, + "n_layers": 13, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_fineweb-100b_N2.3e08_D1.5e09_C2.1e18/README.md b/L2L_fineweb-100b_N2.3e08_D1.5e09_C2.1e18/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-100b_N2.3e08_D1.5e09_C2.1e18/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-100b_N2.3e08_D1.5e09_C2.1e18/config.json b/L2L_fineweb-100b_N2.3e08_D1.5e09_C2.1e18/config.json new file mode 100644 index 0000000000000000000000000000000000000000..75ecc120927724265e2dea013d0855357144913b --- /dev/null +++ b/L2L_fineweb-100b_N2.3e08_D1.5e09_C2.1e18/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 960, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 15, + "n_kv_heads": null, + "n_layers": 15, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_fineweb-100b_N2.3e08_D7.0e08_C9.6e17/README.md b/L2L_fineweb-100b_N2.3e08_D7.0e08_C9.6e17/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-100b_N2.3e08_D7.0e08_C9.6e17/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-100b_N2.3e08_D7.0e08_C9.6e17/config.json b/L2L_fineweb-100b_N2.3e08_D7.0e08_C9.6e17/config.json new file mode 100644 index 0000000000000000000000000000000000000000..75ecc120927724265e2dea013d0855357144913b --- /dev/null +++ b/L2L_fineweb-100b_N2.3e08_D7.0e08_C9.6e17/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 960, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 15, + "n_kv_heads": null, + "n_layers": 15, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_fineweb-100b_N3.1e08_D2.4e09_C4.6e18/README.md b/L2L_fineweb-100b_N3.1e08_D2.4e09_C4.6e18/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-100b_N3.1e08_D2.4e09_C4.6e18/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-100b_N3.1e08_D2.4e09_C4.6e18/config.json b/L2L_fineweb-100b_N3.1e08_D2.4e09_C4.6e18/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3835200c4d862cef6491b75dd50221796f727638 --- /dev/null +++ b/L2L_fineweb-100b_N3.1e08_D2.4e09_C4.6e18/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 1088, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 17, + "n_kv_heads": null, + "n_layers": 17, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_fineweb-100b_N4.2e08_D8.4e08_C2.1e18/README.md b/L2L_fineweb-100b_N4.2e08_D8.4e08_C2.1e18/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-100b_N4.2e08_D8.4e08_C2.1e18/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-100b_N4.2e08_D8.4e08_C2.1e18/config.json b/L2L_fineweb-100b_N4.2e08_D8.4e08_C2.1e18/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ccc1be91ae6e3a3a5db6d7bbc9f81e132ab878d0 --- /dev/null +++ b/L2L_fineweb-100b_N4.2e08_D8.4e08_C2.1e18/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 1216, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 19, + "n_kv_heads": null, + "n_layers": 19, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_fineweb-100b_N6.1e08_D1.2e09_C4.6e18/README.md b/L2L_fineweb-100b_N6.1e08_D1.2e09_C4.6e18/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-100b_N6.1e08_D1.2e09_C4.6e18/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-100b_N6.1e08_D1.2e09_C4.6e18/config.json b/L2L_fineweb-100b_N6.1e08_D1.2e09_C4.6e18/config.json new file mode 100644 index 0000000000000000000000000000000000000000..bcd719639ab49b75ca716ab2c2986e97b96845f2 --- /dev/null +++ b/L2L_fineweb-100b_N6.1e08_D1.2e09_C4.6e18/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 1408, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 22, + "n_kv_heads": null, + "n_layers": 22, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N1.3e08_D1.2e10_C1.0e19/model.safetensors b/L2L_fineweb-edu-100b_N1.3e08_D1.2e10_C1.0e19/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a543a7adb5c16a023d5e3858df639eb7ec4cdad5 --- /dev/null +++ b/L2L_fineweb-edu-100b_N1.3e08_D1.2e10_C1.0e19/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea00f30941c1e0f0c2115513640697e05de3b0677781bd20ee2ced3c3c8c4c50 +size 536507728 diff --git a/L2L_fineweb-edu-100b_N1.3e08_D5.4e08_C4.4e17/README.md b/L2L_fineweb-edu-100b_N1.3e08_D5.4e08_C4.4e17/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-edu-100b_N1.3e08_D5.4e08_C4.4e17/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N1.3e08_D5.4e08_C4.4e17/config.json b/L2L_fineweb-edu-100b_N1.3e08_D5.4e08_C4.4e17/config.json new file mode 100644 index 0000000000000000000000000000000000000000..7fee56c0ab7ba49c7eef3e6cbba1ec901da83d98 --- /dev/null +++ b/L2L_fineweb-edu-100b_N1.3e08_D5.4e08_C4.4e17/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 768, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 12, + "n_kv_heads": null, + "n_layers": 12, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N1.6e08_D2.3e10_C2.2e19/README.md b/L2L_fineweb-edu-100b_N1.6e08_D2.3e10_C2.2e19/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-edu-100b_N1.6e08_D2.3e10_C2.2e19/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N1.6e08_D2.3e10_C2.2e19/config.json b/L2L_fineweb-edu-100b_N1.6e08_D2.3e10_C2.2e19/config.json new file mode 100644 index 0000000000000000000000000000000000000000..eefab0956111af04f5cf1d2f3741df6009e664c0 --- /dev/null +++ b/L2L_fineweb-edu-100b_N1.6e08_D2.3e10_C2.2e19/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 832, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 13, + "n_kv_heads": null, + "n_layers": 13, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N1.9e08_D1.8e09_C2.1e18/model.safetensors b/L2L_fineweb-edu-100b_N1.9e08_D1.8e09_C2.1e18/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c62ff0bfc593506f5bf37ccd8dea3ec02369c28f --- /dev/null +++ b/L2L_fineweb-edu-100b_N1.9e08_D1.8e09_C2.1e18/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89920129e18dd1d11a50ff27c0032e39f8bff6aed9396c001ec292c44dc9f9da +size 769084968 diff --git a/L2L_fineweb-edu-100b_N2.3e08_D1.6e10_C2.2e19/README.md b/L2L_fineweb-edu-100b_N2.3e08_D1.6e10_C2.2e19/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-edu-100b_N2.3e08_D1.6e10_C2.2e19/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N2.3e08_D1.6e10_C2.2e19/config.json b/L2L_fineweb-edu-100b_N2.3e08_D1.6e10_C2.2e19/config.json new file mode 100644 index 0000000000000000000000000000000000000000..75ecc120927724265e2dea013d0855357144913b --- /dev/null +++ b/L2L_fineweb-edu-100b_N2.3e08_D1.6e10_C2.2e19/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 960, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 15, + "n_kv_heads": null, + "n_layers": 15, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N2.7e08_D6.0e08_C9.6e17/README.md b/L2L_fineweb-edu-100b_N2.7e08_D6.0e08_C9.6e17/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-edu-100b_N2.7e08_D6.0e08_C9.6e17/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N2.7e08_D6.0e08_C9.6e17/config.json b/L2L_fineweb-edu-100b_N2.7e08_D6.0e08_C9.6e17/config.json new file mode 100644 index 0000000000000000000000000000000000000000..13cb78433c2752f971a3e07be8753b97f7077d57 --- /dev/null +++ b/L2L_fineweb-edu-100b_N2.7e08_D6.0e08_C9.6e17/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 1024, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 16, + "n_kv_heads": null, + "n_layers": 16, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N3.3e09_D5.0e10_C1.0e21/README.md b/L2L_fineweb-edu-100b_N3.3e09_D5.0e10_C1.0e21/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-edu-100b_N3.3e09_D5.0e10_C1.0e21/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N3.3e09_D5.0e10_C1.0e21/config.json b/L2L_fineweb-edu-100b_N3.3e09_D5.0e10_C1.0e21/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9e5ee84b664de716a40321f3ecdf1eec8b7e33f4 --- /dev/null +++ b/L2L_fineweb-edu-100b_N3.3e09_D5.0e10_C1.0e21/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 2560, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 40, + "n_kv_heads": null, + "n_layers": 40, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N3.5e07_D9.5e08_C2.0e17/README.md b/L2L_fineweb-edu-100b_N3.5e07_D9.5e08_C2.0e17/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-edu-100b_N3.5e07_D9.5e08_C2.0e17/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N3.5e07_D9.5e08_C2.0e17/config.json b/L2L_fineweb-edu-100b_N3.5e07_D9.5e08_C2.0e17/config.json new file mode 100644 index 0000000000000000000000000000000000000000..115dfdf55c2ceb0f1879029732bf838f04e74f21 --- /dev/null +++ b/L2L_fineweb-edu-100b_N3.5e07_D9.5e08_C2.0e17/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 384, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 6, + "n_kv_heads": null, + "n_layers": 6, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N4.2e08_D1.8e09_C4.6e18/README.md b/L2L_fineweb-edu-100b_N4.2e08_D1.8e09_C4.6e18/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-edu-100b_N4.2e08_D1.8e09_C4.6e18/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N4.2e08_D1.8e09_C4.6e18/config.json b/L2L_fineweb-edu-100b_N4.2e08_D1.8e09_C4.6e18/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ccc1be91ae6e3a3a5db6d7bbc9f81e132ab878d0 --- /dev/null +++ b/L2L_fineweb-edu-100b_N4.2e08_D1.8e09_C4.6e18/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 1216, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 19, + "n_kv_heads": null, + "n_layers": 19, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N7.3e07_D1.0e09_C4.4e17/model.safetensors b/L2L_fineweb-edu-100b_N7.3e07_D1.0e09_C4.4e17/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fff6e4e8d6ff217aa6657818af082c34f73db5fb --- /dev/null +++ b/L2L_fineweb-edu-100b_N7.3e07_D1.0e09_C4.4e17/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c13f3f083a5aa028bb8761c72057167ecda3a724fd677b3a5b6c5db29faaef9 +size 290876416 diff --git a/L2L_fineweb-edu-100b_N9.0e07_D3.9e09_C2.1e18/README.md b/L2L_fineweb-edu-100b_N9.0e07_D3.9e09_C2.1e18/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_fineweb-edu-100b_N9.0e07_D3.9e09_C2.1e18/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_fineweb-edu-100b_N9.0e07_D3.9e09_C2.1e18/config.json b/L2L_fineweb-edu-100b_N9.0e07_D3.9e09_C2.1e18/config.json new file mode 100644 index 0000000000000000000000000000000000000000..305c6219e9d60aa18eab0d74691079e359ce6816 --- /dev/null +++ b/L2L_fineweb-edu-100b_N9.0e07_D3.9e09_C2.1e18/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 640, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 10, + "n_kv_heads": null, + "n_layers": 10, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_proof-pile-2_N1.1e08_D1.4e09_C9.6e17/README.md b/L2L_proof-pile-2_N1.1e08_D1.4e09_C9.6e17/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_proof-pile-2_N1.1e08_D1.4e09_C9.6e17/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_proof-pile-2_N1.1e08_D1.4e09_C9.6e17/config.json b/L2L_proof-pile-2_N1.1e08_D1.4e09_C9.6e17/config.json new file mode 100644 index 0000000000000000000000000000000000000000..41baa877f3632c674fd6e77965ab3ee5a2a819e1 --- /dev/null +++ b/L2L_proof-pile-2_N1.1e08_D1.4e09_C9.6e17/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 704, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 11, + "n_kv_heads": null, + "n_layers": 11, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_proof-pile-2_N1.1e08_D1.5e10_C1.0e19/model.safetensors b/L2L_proof-pile-2_N1.1e08_D1.5e10_C1.0e19/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0eba86612a90250e93a761eb0d61c662641c8fe5 --- /dev/null +++ b/L2L_proof-pile-2_N1.1e08_D1.5e10_C1.0e19/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b756c6dd8d5ee75c2fe4a083e0c402e0e214dc790e7c38490f435c5ecb792a64 +size 442045664 diff --git a/L2L_proof-pile-2_N1.5e09_D5.6e09_C4.8e19/README.md b/L2L_proof-pile-2_N1.5e09_D5.6e09_C4.8e19/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_proof-pile-2_N1.5e09_D5.6e09_C4.8e19/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_proof-pile-2_N1.5e09_D5.6e09_C4.8e19/config.json b/L2L_proof-pile-2_N1.5e09_D5.6e09_C4.8e19/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b0ac0ef6bc85f96b3d24ba507bc08973d74c3d3a --- /dev/null +++ b/L2L_proof-pile-2_N1.5e09_D5.6e09_C4.8e19/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 1920, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 30, + "n_kv_heads": null, + "n_layers": 30, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_proof-pile-2_N2.0e07_D3.7e09_C4.4e17/README.md b/L2L_proof-pile-2_N2.0e07_D3.7e09_C4.4e17/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_proof-pile-2_N2.0e07_D3.7e09_C4.4e17/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_proof-pile-2_N2.0e07_D3.7e09_C4.4e17/config.json b/L2L_proof-pile-2_N2.0e07_D3.7e09_C4.4e17/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cacd3da0cfff31756238a0a22f7bd7b455cefe6c --- /dev/null +++ b/L2L_proof-pile-2_N2.0e07_D3.7e09_C4.4e17/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 256, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 4, + "n_kv_heads": null, + "n_layers": 4, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_proof-pile-2_N2.7e08_D6.0e08_C9.6e17/model.safetensors b/L2L_proof-pile-2_N2.7e08_D6.0e08_C9.6e17/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..10e4b5e2161c2fe5cbe01ae361b83eacb27d841b --- /dev/null +++ b/L2L_proof-pile-2_N2.7e08_D6.0e08_C9.6e17/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:839ddf2af0b1aed70a7959af84b7680554c64b29e86db85a295b95e27d49d98b +size 1067730840 diff --git a/L2L_proof-pile-2_N4.8e08_D3.5e09_C1.0e19/README.md b/L2L_proof-pile-2_N4.8e08_D3.5e09_C1.0e19/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_proof-pile-2_N4.8e08_D3.5e09_C1.0e19/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_proof-pile-2_N4.8e08_D3.5e09_C1.0e19/config.json b/L2L_proof-pile-2_N4.8e08_D3.5e09_C1.0e19/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1ecddeacf8ac1c859c33a3bc4e79e4fbc8a22ae9 --- /dev/null +++ b/L2L_proof-pile-2_N4.8e08_D3.5e09_C1.0e19/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 1280, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 20, + "n_kv_heads": null, + "n_layers": 20, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N1.3e08_D2.6e09_C2.1e18/model.safetensors b/L2L_slimpajama-chunk1_N1.3e08_D2.6e09_C2.1e18/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c7aa194d27f581bc23391771f4223eacf8099a5f --- /dev/null +++ b/L2L_slimpajama-chunk1_N1.3e08_D2.6e09_C2.1e18/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:031c3db10708a92238e858ec7d865a065cc77ef0480eeefcc81af0af6de37781 +size 536507728 diff --git a/L2L_slimpajama-chunk1_N1.6e08_D2.2e09_C2.1e18/model.safetensors b/L2L_slimpajama-chunk1_N1.6e08_D2.2e09_C2.1e18/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c62bc937b4901b760e4e651cb9af13919a3232ba --- /dev/null +++ b/L2L_slimpajama-chunk1_N1.6e08_D2.2e09_C2.1e18/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdc87a4e6da2bca86209ccc1264f7e6cd6d47a7157178053a6e976a9d5427d58 +size 645127616 diff --git a/L2L_slimpajama-chunk1_N1.6e08_D4.5e08_C4.4e17/README.md b/L2L_slimpajama-chunk1_N1.6e08_D4.5e08_C4.4e17/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_slimpajama-chunk1_N1.6e08_D4.5e08_C4.4e17/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N1.6e08_D4.5e08_C4.4e17/config.json b/L2L_slimpajama-chunk1_N1.6e08_D4.5e08_C4.4e17/config.json new file mode 100644 index 0000000000000000000000000000000000000000..eefab0956111af04f5cf1d2f3741df6009e664c0 --- /dev/null +++ b/L2L_slimpajama-chunk1_N1.6e08_D4.5e08_C4.4e17/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 832, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 13, + "n_kv_heads": null, + "n_layers": 13, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N1.6e08_D9.9e08_C9.6e17/README.md b/L2L_slimpajama-chunk1_N1.6e08_D9.9e08_C9.6e17/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_slimpajama-chunk1_N1.6e08_D9.9e08_C9.6e17/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N1.6e08_D9.9e08_C9.6e17/config.json b/L2L_slimpajama-chunk1_N1.6e08_D9.9e08_C9.6e17/config.json new file mode 100644 index 0000000000000000000000000000000000000000..eefab0956111af04f5cf1d2f3741df6009e664c0 --- /dev/null +++ b/L2L_slimpajama-chunk1_N1.6e08_D9.9e08_C9.6e17/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 832, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 13, + "n_kv_heads": null, + "n_layers": 13, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N1.9e08_D4.0e09_C4.6e18/model.safetensors b/L2L_slimpajama-chunk1_N1.9e08_D4.0e09_C4.6e18/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..129572e9c3a887f61307252586b438d45f02b8e6 --- /dev/null +++ b/L2L_slimpajama-chunk1_N1.9e08_D4.0e09_C4.6e18/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cde5e26f3bafb98bf2a7484b678621fbb43ab06eef89a56fe4d06d69597c9b52 +size 769084968 diff --git a/L2L_slimpajama-chunk1_N2.3e08_D7.0e08_C9.6e17/README.md b/L2L_slimpajama-chunk1_N2.3e08_D7.0e08_C9.6e17/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_slimpajama-chunk1_N2.3e08_D7.0e08_C9.6e17/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N2.3e08_D7.0e08_C9.6e17/config.json b/L2L_slimpajama-chunk1_N2.3e08_D7.0e08_C9.6e17/config.json new file mode 100644 index 0000000000000000000000000000000000000000..75ecc120927724265e2dea013d0855357144913b --- /dev/null +++ b/L2L_slimpajama-chunk1_N2.3e08_D7.0e08_C9.6e17/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 960, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 15, + "n_kv_heads": null, + "n_layers": 15, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N4.2e08_D8.8e09_C2.2e19/README.md b/L2L_slimpajama-chunk1_N4.2e08_D8.8e09_C2.2e19/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_slimpajama-chunk1_N4.2e08_D8.8e09_C2.2e19/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N4.2e08_D8.8e09_C2.2e19/config.json b/L2L_slimpajama-chunk1_N4.2e08_D8.8e09_C2.2e19/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ccc1be91ae6e3a3a5db6d7bbc9f81e132ab878d0 --- /dev/null +++ b/L2L_slimpajama-chunk1_N4.2e08_D8.8e09_C2.2e19/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 1216, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 19, + "n_kv_heads": null, + "n_layers": 19, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N4.6e07_D7.3e08_C2.0e17/README.md b/L2L_slimpajama-chunk1_N4.6e07_D7.3e08_C2.0e17/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_slimpajama-chunk1_N4.6e07_D7.3e08_C2.0e17/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N4.6e07_D7.3e08_C2.0e17/config.json b/L2L_slimpajama-chunk1_N4.6e07_D7.3e08_C2.0e17/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6ddcaca5a561d50cfa15ad13fe288fd743cf7562 --- /dev/null +++ b/L2L_slimpajama-chunk1_N4.6e07_D7.3e08_C2.0e17/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 448, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 7, + "n_kv_heads": null, + "n_layers": 7, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N5.4e08_D6.8e09_C2.2e19/README.md b/L2L_slimpajama-chunk1_N5.4e08_D6.8e09_C2.2e19/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_slimpajama-chunk1_N5.4e08_D6.8e09_C2.2e19/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N5.4e08_D6.8e09_C2.2e19/config.json b/L2L_slimpajama-chunk1_N5.4e08_D6.8e09_C2.2e19/config.json new file mode 100644 index 0000000000000000000000000000000000000000..bac9bb4f890ea6cd4a9d8006d807ace3d623f0c0 --- /dev/null +++ b/L2L_slimpajama-chunk1_N5.4e08_D6.8e09_C2.2e19/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 1344, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 21, + "n_kv_heads": null, + "n_layers": 21, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N5.8e07_D5.8e08_C2.0e17/README.md b/L2L_slimpajama-chunk1_N5.8e07_D5.8e08_C2.0e17/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_slimpajama-chunk1_N5.8e07_D5.8e08_C2.0e17/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N5.8e07_D5.8e08_C2.0e17/config.json b/L2L_slimpajama-chunk1_N5.8e07_D5.8e08_C2.0e17/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9edd0fa4cd96d0459cf987588de667ef40cd5c07 --- /dev/null +++ b/L2L_slimpajama-chunk1_N5.8e07_D5.8e08_C2.0e17/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 512, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 8, + "n_kv_heads": null, + "n_layers": 8, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N9.0e07_D1.8e09_C9.6e17/README.md b/L2L_slimpajama-chunk1_N9.0e07_D1.8e09_C9.6e17/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_slimpajama-chunk1_N9.0e07_D1.8e09_C9.6e17/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N9.0e07_D1.8e09_C9.6e17/config.json b/L2L_slimpajama-chunk1_N9.0e07_D1.8e09_C9.6e17/config.json new file mode 100644 index 0000000000000000000000000000000000000000..305c6219e9d60aa18eab0d74691079e359ce6816 --- /dev/null +++ b/L2L_slimpajama-chunk1_N9.0e07_D1.8e09_C9.6e17/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 640, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 10, + "n_kv_heads": null, + "n_layers": 10, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_slimpajama-chunk1_N9.7e08_D8.3e09_C4.8e19/config.json b/L2L_slimpajama-chunk1_N9.7e08_D8.3e09_C4.8e19/config.json new file mode 100644 index 0000000000000000000000000000000000000000..634f954d05ede293355e1f8db2b2eef6c5b0a2df --- /dev/null +++ b/L2L_slimpajama-chunk1_N9.7e08_D8.3e09_C4.8e19/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 1664, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 26, + "n_kv_heads": null, + "n_layers": 26, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_smollm-corpus_N1.9e08_D4.0e09_C4.6e18/README.md b/L2L_smollm-corpus_N1.9e08_D4.0e09_C4.6e18/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_smollm-corpus_N1.9e08_D4.0e09_C4.6e18/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_smollm-corpus_N1.9e08_D4.0e09_C4.6e18/config.json b/L2L_smollm-corpus_N1.9e08_D4.0e09_C4.6e18/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f728e6ee6c2b2cc150965adb6bf78b3f7d1f19e1 --- /dev/null +++ b/L2L_smollm-corpus_N1.9e08_D4.0e09_C4.6e18/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 896, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 14, + "n_kv_heads": null, + "n_layers": 14, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_smollm-corpus_N2.3e08_D1.5e09_C2.1e18/model.safetensors b/L2L_smollm-corpus_N2.3e08_D1.5e09_C2.1e18/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f0a942ef0c2916e3bce9e9690ae46b318d4b39b8 --- /dev/null +++ b/L2L_smollm-corpus_N2.3e08_D1.5e09_C2.1e18/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed19a2322a1fe35bd1259f7a940aa1b574759e70a86927dc37b32725b4fca71d +size 909559448 diff --git a/L2L_smollm-corpus_N4.8e08_D3.5e09_C1.0e19/model.safetensors b/L2L_smollm-corpus_N4.8e08_D3.5e09_C1.0e19/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7072f07d86852288a830402f194d58d6a68a48fc --- /dev/null +++ b/L2L_smollm-corpus_N4.8e08_D3.5e09_C1.0e19/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1494dc12c1be72594996ef48a263905949b4a0e6de1ef407f2660c48a4a725e +size 1900976584 diff --git a/L2L_smollm-corpus_N5.4e08_D3.1e09_C1.0e19/README.md b/L2L_smollm-corpus_N5.4e08_D3.1e09_C1.0e19/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_smollm-corpus_N5.4e08_D3.1e09_C1.0e19/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_smollm-corpus_N5.4e08_D3.1e09_C1.0e19/config.json b/L2L_smollm-corpus_N5.4e08_D3.1e09_C1.0e19/config.json new file mode 100644 index 0000000000000000000000000000000000000000..bac9bb4f890ea6cd4a9d8006d807ace3d623f0c0 --- /dev/null +++ b/L2L_smollm-corpus_N5.4e08_D3.1e09_C1.0e19/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 1344, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 21, + "n_kv_heads": null, + "n_layers": 21, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_smollm-corpus_N5.8e07_D6.0e09_C2.1e18/README.md b/L2L_smollm-corpus_N5.8e07_D6.0e09_C2.1e18/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_smollm-corpus_N5.8e07_D6.0e09_C2.1e18/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_smollm-corpus_N5.8e07_D6.0e09_C2.1e18/config.json b/L2L_smollm-corpus_N5.8e07_D6.0e09_C2.1e18/config.json new file mode 100644 index 0000000000000000000000000000000000000000..9edd0fa4cd96d0459cf987588de667ef40cd5c07 --- /dev/null +++ b/L2L_smollm-corpus_N5.8e07_D6.0e09_C2.1e18/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 512, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 8, + "n_kv_heads": null, + "n_layers": 8, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_smollm-corpus_N7.3e07_D1.0e10_C4.6e18/model.safetensors b/L2L_smollm-corpus_N7.3e07_D1.0e10_C4.6e18/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..851b7e1ff8327b1fc771d6eb58bc72ae122dd64a --- /dev/null +++ b/L2L_smollm-corpus_N7.3e07_D1.0e10_C4.6e18/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf28a1f8e93c806784162ee941e503afcb76ff7afda19238a131276209064f14 +size 290876416 diff --git a/L2L_smollm-corpus_N9.0e07_D8.5e09_C4.6e18/README.md b/L2L_smollm-corpus_N9.0e07_D8.5e09_C4.6e18/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_smollm-corpus_N9.0e07_D8.5e09_C4.6e18/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_smollm-corpus_N9.0e07_D8.5e09_C4.6e18/config.json b/L2L_smollm-corpus_N9.0e07_D8.5e09_C4.6e18/config.json new file mode 100644 index 0000000000000000000000000000000000000000..305c6219e9d60aa18eab0d74691079e359ce6816 --- /dev/null +++ b/L2L_smollm-corpus_N9.0e07_D8.5e09_C4.6e18/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 640, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 10, + "n_kv_heads": null, + "n_layers": 10, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_starcoder_N1.6e08_D1.0e10_C1.0e19/README.md b/L2L_starcoder_N1.6e08_D1.0e10_C1.0e19/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_starcoder_N1.6e08_D1.0e10_C1.0e19/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_starcoder_N1.6e08_D1.0e10_C1.0e19/config.json b/L2L_starcoder_N1.6e08_D1.0e10_C1.0e19/config.json new file mode 100644 index 0000000000000000000000000000000000000000..eefab0956111af04f5cf1d2f3741df6009e664c0 --- /dev/null +++ b/L2L_starcoder_N1.6e08_D1.0e10_C1.0e19/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 832, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 13, + "n_kv_heads": null, + "n_layers": 13, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_starcoder_N1.6e08_D2.2e09_C2.1e18/README.md b/L2L_starcoder_N1.6e08_D2.2e09_C2.1e18/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_starcoder_N1.6e08_D2.2e09_C2.1e18/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_starcoder_N1.6e08_D2.2e09_C2.1e18/config.json b/L2L_starcoder_N1.6e08_D2.2e09_C2.1e18/config.json new file mode 100644 index 0000000000000000000000000000000000000000..eefab0956111af04f5cf1d2f3741df6009e664c0 --- /dev/null +++ b/L2L_starcoder_N1.6e08_D2.2e09_C2.1e18/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 832, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 13, + "n_kv_heads": null, + "n_layers": 13, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_starcoder_N1.9e08_D1.8e09_C2.1e18/README.md b/L2L_starcoder_N1.9e08_D1.8e09_C2.1e18/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_starcoder_N1.9e08_D1.8e09_C2.1e18/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_starcoder_N1.9e08_D1.8e09_C2.1e18/config.json b/L2L_starcoder_N1.9e08_D1.8e09_C2.1e18/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f728e6ee6c2b2cc150965adb6bf78b3f7d1f19e1 --- /dev/null +++ b/L2L_starcoder_N1.9e08_D1.8e09_C2.1e18/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 896, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 14, + "n_kv_heads": null, + "n_layers": 14, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_starcoder_N1.9e08_D8.7e09_C1.0e19/README.md b/L2L_starcoder_N1.9e08_D8.7e09_C1.0e19/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_starcoder_N1.9e08_D8.7e09_C1.0e19/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_starcoder_N1.9e08_D8.7e09_C1.0e19/config.json b/L2L_starcoder_N1.9e08_D8.7e09_C1.0e19/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f728e6ee6c2b2cc150965adb6bf78b3f7d1f19e1 --- /dev/null +++ b/L2L_starcoder_N1.9e08_D8.7e09_C1.0e19/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 896, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 14, + "n_kv_heads": null, + "n_layers": 14, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file diff --git a/L2L_starcoder_N4.2e08_D1.8e09_C4.6e18/model.safetensors b/L2L_starcoder_N4.2e08_D1.8e09_C4.6e18/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7e214b1461475271ead7bfdaf3fb946d643540d0 --- /dev/null +++ b/L2L_starcoder_N4.2e08_D1.8e09_C4.6e18/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b219da3cd4bf475582b93bb437202d1984d818ca77834e08a567ef86f7f72c0 +size 1660221752 diff --git a/L2L_starcoder_N4.2e08_D8.4e08_C2.1e18/README.md b/L2L_starcoder_N4.2e08_D8.4e08_C2.1e18/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23bb8239b30ad636a1b592db0346c4753dc5325d --- /dev/null +++ b/L2L_starcoder_N4.2e08_D8.4e08_C2.1e18/README.md @@ -0,0 +1,9 @@ +--- +tags: +- model_hub_mixin +- pytorch_model_hub_mixin +--- + +This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration: +- Library: [More Information Needed] +- Docs: [More Information Needed] \ No newline at end of file diff --git a/L2L_starcoder_N4.2e08_D8.4e08_C2.1e18/config.json b/L2L_starcoder_N4.2e08_D8.4e08_C2.1e18/config.json new file mode 100644 index 0000000000000000000000000000000000000000..ccc1be91ae6e3a3a5db6d7bbc9f81e132ab878d0 --- /dev/null +++ b/L2L_starcoder_N4.2e08_D8.4e08_C2.1e18/config.json @@ -0,0 +1,39 @@ +{ + "activation_type": "gelu", + "alibi": false, + "alibi_bias_max": 8.0, + "attention_dropout": 0.0, + "attention_layer_norm": true, + "attention_layer_norm_with_affine": true, + "bias_for_layer_norm": false, + "block_group_size": 1, + "block_type": "sequential", + "clip_qkv": null, + "d_model": 1216, + "embedding_dropout": 0.0, + "embedding_size": 32000, + "eos_token_id": 2, + "flash_attention": false, + "include_bias": false, + "init_cutoff_factor": null, + "init_device": "cpu", + "init_fn": "mitchell", + "init_std": 0.02, + "layer_norm_type": "default", + "layer_norm_with_affine": true, + "max_sequence_length": 512, + "mlp_hidden_size": null, + "mlp_ratio": 4, + "multi_query_attention": false, + "n_heads": 19, + "n_kv_heads": null, + "n_layers": 19, + "pad_token_id": 2, + "precision": "amp_bf16", + "residual_dropout": 0.0, + "rope": true, + "rope_full_precision": true, + "scale_logits": false, + "vocab_size": 32000, + "weight_tying": false +} \ No newline at end of file