init

README.md

@@ -1,3 +1,45 @@
 ---
 license: apache-2.0
+datasets:
+- BelleGroup/train_2M_CN
+- BelleGroup/train_3.5M_CN
+- BelleGroup/train_1M_CN
+- BelleGroup/train_0.5M_CN
+- BelleGroup/school_math_0.25M
+language:
+- zh
 ---
+
+## GoGPT
+
+基于中文指令数据微调BLOOM
+![img.png](resources/img.png)
+> 训练第一轮足够了，后续第二轮和第三轮提升不大
+
+- 🚀多样性指令数据
+- 🚀筛选高质量中文数据
+
+| 模型名字       | 参数量    | 模型地址 |
+|------------|--------|------|
+| gogpt-560m | 5.6亿参数 | 🤗[golaxy/gogpt-560m](https://huggingface.co/golaxy/gogpt-560m) |
+| gogpt-3b   | 30亿参数  | 🤗[golaxy/gogpt-3b](https://huggingface.co/golaxy/gogpt-3b) |
+
+
+## 测试效果
+![img.png](resources/test1.png)
+![img.png](resources/test2.png)
+![img.png](resources/test3.png)
+![img.png](resources/test4.png)
+![img.png](resources/test5.png)
+![img.png](resources/test6.png)
+
+
+## TODO
+- 进行RLFH训练
+- 后续加入中英平行语料
+
+## 感谢
+
+- [@hz大佬-zero_nlp](https://github.com/yuanzhoulvpi2017/zero_nlp)
+- [stanford_alpaca](https://github.com/tatsu-lab/stanford_alpaca)
+- [Belle数据](https://huggingface.co/BelleGroup)

config.json

@@ -0,0 +1,33 @@
+{
+  "_name_or_path": "/data/searchgpt/pretrained_models/bloomz-7b1",
+  "apply_residual_connection_post_layernorm": false,
+  "architectures": [
+    "BloomForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attention_softmax_in_fp32": true,
+  "bias_dropout_fusion": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_dropout": 0.0,
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "masked_softmax_fusion": true,
+  "model_type": "bloom",
+  "n_head": 32,
+  "n_inner": null,
+  "n_layer": 30,
+  "offset_alibi": 100,
+  "pad_token_id": 3,
+  "pretraining_tp": 4,
+  "seq_length": 2048,
+  "skip_bias_add": true,
+  "skip_bias_add_qkv": false,
+  "slow_but_exact": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.29.1",
+  "unk_token_id": 0,
+  "use_cache": true,
+  "vocab_size": 250880
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 3,
+  "transformers_version": "4.29.1"
+}

pytorch_model-00001-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a72a6bc9c041a82d23021b7e82e192133fda78a281cfce99d0da37f025a25264
+size 9950529693

pytorch_model-00002-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ff339e549c11d2c7b497f585f69288901c9570a9c8f3d80767510915b6afcfa
+size 9733446763

pytorch_model-00003-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e1d5ea75238737f3588211896cd747ef6f3766eae60131d11fef023a2732406
+size 8592227069

pytorch_model-00004-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fff7dd5d55258a0ad0f5fb85aa4b26cc31a66793276829417558102e90c8db7
+size 4110418922

pytorch_model.bin.index.json

@@ -0,0 +1,373 @@
+{
+  "metadata": {
+    "total_size": 32386482176
+  },
+  "weight_map": {
+    "lm_head.weight": "pytorch_model-00004-of-00004.bin",
+    "transformer.h.0.input_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.0.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.0.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.0.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.0.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.0.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.0.post_attention_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.0.self_attention.dense.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.0.self_attention.dense.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.0.self_attention.query_key_value.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.0.self_attention.query_key_value.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.1.input_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.1.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.1.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.1.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.1.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.1.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.1.post_attention_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.1.self_attention.dense.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.1.self_attention.dense.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.1.self_attention.query_key_value.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.1.self_attention.query_key_value.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.10.input_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.10.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.10.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.10.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.10.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.10.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.10.post_attention_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.10.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.10.self_attention.dense.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.10.self_attention.dense.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.10.self_attention.query_key_value.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.10.self_attention.query_key_value.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.11.input_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.11.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.11.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.11.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.11.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.11.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.11.post_attention_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.11.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.11.self_attention.dense.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.11.self_attention.dense.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.11.self_attention.query_key_value.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.11.self_attention.query_key_value.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.12.input_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.12.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.12.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.12.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.12.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.12.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.12.post_attention_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.12.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.12.self_attention.dense.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.12.self_attention.dense.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.12.self_attention.query_key_value.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.12.self_attention.query_key_value.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.13.input_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.13.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.13.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.13.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.13.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.13.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.13.post_attention_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.13.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.13.self_attention.dense.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.13.self_attention.dense.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.13.self_attention.query_key_value.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.13.self_attention.query_key_value.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.14.input_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.14.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.14.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.14.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.14.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.14.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.14.post_attention_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.14.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.14.self_attention.dense.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.14.self_attention.dense.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.14.self_attention.query_key_value.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.14.self_attention.query_key_value.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.15.input_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.15.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.15.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.15.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.15.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.15.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.15.post_attention_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.15.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.15.self_attention.dense.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.15.self_attention.dense.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.15.self_attention.query_key_value.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.15.self_attention.query_key_value.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.16.input_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.16.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.16.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.16.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.16.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.16.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.16.post_attention_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.16.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.16.self_attention.dense.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.16.self_attention.dense.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.16.self_attention.query_key_value.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.16.self_attention.query_key_value.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.17.input_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.17.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.17.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.17.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.17.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.17.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.17.post_attention_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.17.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.17.self_attention.dense.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.17.self_attention.dense.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.17.self_attention.query_key_value.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.17.self_attention.query_key_value.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.18.input_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.18.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.18.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.18.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.18.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.18.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.18.post_attention_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.18.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.18.self_attention.dense.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.18.self_attention.dense.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.18.self_attention.query_key_value.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.18.self_attention.query_key_value.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.19.input_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.19.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.19.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.19.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.19.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.19.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.19.post_attention_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.19.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.19.self_attention.dense.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.19.self_attention.dense.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.19.self_attention.query_key_value.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.19.self_attention.query_key_value.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.2.input_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.2.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.2.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.2.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.2.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.2.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.2.post_attention_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.2.self_attention.dense.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.2.self_attention.dense.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.2.self_attention.query_key_value.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.2.self_attention.query_key_value.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.20.input_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.20.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.20.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.20.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.20.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.20.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.20.post_attention_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.20.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.20.self_attention.dense.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.20.self_attention.dense.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.20.self_attention.query_key_value.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.20.self_attention.query_key_value.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.21.input_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.21.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.21.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.21.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.21.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.21.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.21.post_attention_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.21.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.21.self_attention.dense.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.21.self_attention.dense.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.21.self_attention.query_key_value.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.21.self_attention.query_key_value.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.22.input_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.22.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.22.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.22.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.22.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.22.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.22.post_attention_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.22.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.22.self_attention.dense.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.22.self_attention.dense.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.22.self_attention.query_key_value.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.22.self_attention.query_key_value.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.23.input_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.23.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.23.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.23.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.23.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.23.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.23.post_attention_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.23.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.23.self_attention.dense.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.23.self_attention.dense.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.23.self_attention.query_key_value.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.23.self_attention.query_key_value.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.24.input_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.24.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.24.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.24.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.24.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.24.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.24.post_attention_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.24.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.24.self_attention.dense.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.24.self_attention.dense.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.24.self_attention.query_key_value.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.24.self_attention.query_key_value.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.25.input_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.25.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.25.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.25.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.25.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.25.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.25.post_attention_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.25.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.25.self_attention.dense.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.25.self_attention.dense.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.25.self_attention.query_key_value.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.25.self_attention.query_key_value.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.26.input_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.26.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.26.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.26.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.26.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.26.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.26.post_attention_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.26.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.26.self_attention.dense.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.26.self_attention.dense.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.26.self_attention.query_key_value.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.26.self_attention.query_key_value.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.27.input_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.27.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.27.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.27.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.27.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.27.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.27.post_attention_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.27.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.27.self_attention.dense.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.27.self_attention.dense.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.27.self_attention.query_key_value.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.27.self_attention.query_key_value.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.28.input_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.28.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.28.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.28.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.28.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.28.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.28.post_attention_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.28.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.28.self_attention.dense.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.28.self_attention.dense.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.28.self_attention.query_key_value.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.28.self_attention.query_key_value.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.29.input_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.29.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.29.mlp.dense_4h_to_h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.29.mlp.dense_4h_to_h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.29.mlp.dense_h_to_4h.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.29.mlp.dense_h_to_4h.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.29.post_attention_layernorm.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.29.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.29.self_attention.dense.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.29.self_attention.dense.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.29.self_attention.query_key_value.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.29.self_attention.query_key_value.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.h.3.input_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.3.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.3.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.3.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.3.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.3.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.3.post_attention_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.3.self_attention.dense.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.3.self_attention.dense.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.3.self_attention.query_key_value.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.3.self_attention.query_key_value.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.4.input_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.4.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.4.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.4.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.4.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.4.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.4.post_attention_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.4.self_attention.dense.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.4.self_attention.dense.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.4.self_attention.query_key_value.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.4.self_attention.query_key_value.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.5.input_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.5.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.5.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.5.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.5.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.5.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.5.post_attention_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.5.self_attention.dense.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.5.self_attention.dense.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.5.self_attention.query_key_value.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.5.self_attention.query_key_value.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.6.input_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.6.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.6.mlp.dense_4h_to_h.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.6.mlp.dense_4h_to_h.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.6.mlp.dense_h_to_4h.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.6.mlp.dense_h_to_4h.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.6.post_attention_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.6.self_attention.dense.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.6.self_attention.dense.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.6.self_attention.query_key_value.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.6.self_attention.query_key_value.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.7.input_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.7.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.7.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.7.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.7.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.7.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.7.post_attention_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.7.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.7.self_attention.dense.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.7.self_attention.dense.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.7.self_attention.query_key_value.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.7.self_attention.query_key_value.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.h.8.input_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.8.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.8.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.8.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.8.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.8.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.8.post_attention_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.8.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.8.self_attention.dense.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.8.self_attention.dense.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.8.self_attention.query_key_value.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.8.self_attention.query_key_value.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.9.input_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.9.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.9.mlp.dense_4h_to_h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.9.mlp.dense_4h_to_h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.9.mlp.dense_h_to_4h.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.9.mlp.dense_h_to_4h.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.9.post_attention_layernorm.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.9.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.9.self_attention.dense.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.9.self_attention.dense.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.9.self_attention.query_key_value.bias": "pytorch_model-00002-of-00004.bin",
+    "transformer.h.9.self_attention.query_key_value.weight": "pytorch_model-00002-of-00004.bin",
+    "transformer.ln_f.bias": "pytorch_model-00003-of-00004.bin",
+    "transformer.ln_f.weight": "pytorch_model-00003-of-00004.bin",
+    "transformer.word_embeddings.weight": "pytorch_model-00001-of-00004.bin",
+    "transformer.word_embeddings_layernorm.bias": "pytorch_model-00001-of-00004.bin",
+    "transformer.word_embeddings_layernorm.weight": "pytorch_model-00001-of-00004.bin"
+  }
+}

rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ff49e9fd7e8b782779b5a9b561ac7a88294f19be62c93dbc40ca668ebb70f03
+size 15607

rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d84f3e88e80f41ffd1056b31a40d0a1847fb2899262c9576d595c53d8aa2554
+size 15607

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5506a968d27f892baf0373f593bf8bd1a757192458971ae527f4321ff2712049
+size 627

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "padding_side": "right",
+  "tokenizer_class": "BloomTokenizer",
+  "unk_token": "<unk>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8699628644be8597ae194dde3004da69b09c19a15957abca399f1c4abb9afc40
+size 4091