Model save

README.md

@@ -2,31 +2,28 @@
 license: apache-2.0
 base_model: AmberYifan/mistral-safe-sft-full
 tags:
-- alignment-handbook
 - generated_from_trainer
-datasets:
-- AmberYifan/sft-spin-kcenter-5k
 model-index:
-- name: sft-spin-kcenter-5k
+- name: mistral-sft-kcenter-5k
   results: []
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-# sft-spin-kcenter-5k
+# mistral-sft-kcenter-5k
 
-This model is a fine-tuned version of [AmberYifan/mistral-safe-sft-full](https://huggingface.co/AmberYifan/mistral-safe-sft-full) on the AmberYifan/sft-spin-kcenter-5k dataset.
+This model is a fine-tuned version of [AmberYifan/mistral-safe-sft-full](https://huggingface.co/AmberYifan/mistral-safe-sft-full) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.0181
-- Rewards/real: 1.4812
-- Rewards/generated: 1.4157
-- Rewards/accuracies: 0.0812
-- Rewards/margins: 0.0655
-- Logps/generated: -57.7408
-- Logps/real: -16.8189
-- Logits/generated: -2.6921
-- Logits/real: -2.4190
+- Loss: 0.9619
+- Rewards/real: 0.2019
+- Rewards/generated: -0.1441
+- Rewards/accuracies: 0.8906
+- Rewards/margins: 0.3460
+- Logps/generated: -259.6614
+- Logps/real: -210.6393
+- Logits/generated: -2.6617
+- Logits/real: -2.6095
 
 ## Model description
 
@@ -62,7 +59,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/real | Rewards/generated | Rewards/accuracies | Rewards/margins | Logps/generated | Logps/real | Logits/generated | Logits/real |
 |:-------------:|:------:|:----:|:---------------:|:------------:|:-----------------:|:------------------:|:---------------:|:---------------:|:----------:|:----------------:|:-----------:|
-| 0.3429        | 0.6369 | 100  | 1.0181          | 1.4812       | 1.4157            | 0.0812             | 0.0655          | -57.7408        | -16.8189   | -2.6921          | -2.4190     |
+| 0.0           | 0.6369 | 100  | 0.9619          | 0.2019       | -0.1441           | 0.8906             | 0.3460          | -259.6614       | -210.6393  | -2.6617          | -2.6095     |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.3886139180250229,
-    "train_runtime": 1518.6555,
+    "train_loss": 0.03164779691052703,
+    "train_runtime": 1178.5993,
     "train_samples": 5000,
-    "train_samples_per_second": 3.292,
-    "train_steps_per_second": 0.103
+    "train_samples_per_second": 4.242,
+    "train_steps_per_second": 0.133
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.3886139180250229,
-    "train_runtime": 1518.6555,
+    "train_loss": 0.03164779691052703,
+    "train_runtime": 1178.5993,
     "train_samples": 5000,
-    "train_samples_per_second": 3.292,
-    "train_steps_per_second": 0.103
+    "train_samples_per_second": 4.242,
+    "train_steps_per_second": 0.133
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.006369426751592357,
-      "grad_norm": 436.30790272840835,
+      "grad_norm": 2514.601414684904,
       "learning_rate": 3.125e-08,
-      "logits/generated": -2.574023723602295,
-      "logits/real": -2.507308006286621,
-      "logps/generated": -396.6226806640625,
-      "logps/real": -228.6820831298828,
-      "loss": 1.1036,
+      "logits/generated": -2.661752223968506,
+      "logits/real": -2.483980894088745,
+      "logps/generated": -429.17132568359375,
+      "logps/real": -342.051025390625,
+      "loss": 1.3612,
       "rewards/accuracies": 0.0,
       "rewards/generated": 0.0,
       "rewards/margins": 0.0,
@@ -25,253 +25,253 @@
     },
     {
       "epoch": 0.06369426751592357,
-      "grad_norm": 86.39288206954664,
+      "grad_norm": 0.31574006570864244,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/generated": -2.6174476146698,
-      "logits/real": -2.577033042907715,
-      "logps/generated": -314.8096923828125,
-      "logps/real": -233.916259765625,
-      "loss": 0.8432,
-      "rewards/accuracies": 0.8055555820465088,
-      "rewards/generated": -1.1551631689071655,
-      "rewards/margins": 1.29329252243042,
-      "rewards/real": 0.13812927901744843,
+      "logits/generated": -2.668369770050049,
+      "logits/real": -2.424091339111328,
+      "logps/generated": -450.4893798828125,
+      "logps/real": -144.69952392578125,
+      "loss": 0.4008,
+      "rewards/accuracies": 0.8888888955116272,
+      "rewards/generated": -2.1318063735961914,
+      "rewards/margins": 21.866958618164062,
+      "rewards/real": 19.735153198242188,
       "step": 10
     },
     {
       "epoch": 0.12738853503184713,
-      "grad_norm": 23.096946473336036,
+      "grad_norm": 0.005520241230176221,
       "learning_rate": 4.858156028368794e-07,
-      "logits/generated": -2.934861421585083,
-      "logits/real": -2.8925399780273438,
-      "logps/generated": -313.32012939453125,
-      "logps/real": -235.418701171875,
-      "loss": 0.4872,
+      "logits/generated": -2.692908763885498,
+      "logits/real": -2.329023599624634,
+      "logps/generated": -478.9815368652344,
+      "logps/real": -0.014682354405522346,
+      "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/generated": -3.896754741668701,
-      "rewards/margins": 6.176419258117676,
-      "rewards/real": 2.2796645164489746,
+      "rewards/generated": -4.981024742126465,
+      "rewards/margins": 39.18465805053711,
+      "rewards/real": 34.203636169433594,
       "step": 20
     },
     {
       "epoch": 0.1910828025477707,
-      "grad_norm": 23.454945687282052,
+      "grad_norm": 0.002789639064368334,
       "learning_rate": 4.50354609929078e-07,
-      "logits/generated": -2.9848690032958984,
-      "logits/real": -2.9865798950195312,
-      "logps/generated": -304.69854736328125,
-      "logps/real": -152.80667114257812,
-      "loss": 0.3868,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/generated": -3.0515201091766357,
-      "rewards/margins": 7.164666652679443,
-      "rewards/real": 4.113146781921387,
+      "logits/generated": -2.705698013305664,
+      "logits/real": -2.342294216156006,
+      "logps/generated": -486.1914978027344,
+      "logps/real": -0.004650969058275223,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -5.702020645141602,
+      "rewards/margins": 39.90666198730469,
+      "rewards/real": 34.20463943481445,
       "step": 30
     },
     {
       "epoch": 0.25477707006369427,
-      "grad_norm": 56.59285724233864,
+      "grad_norm": 0.0019297231936813803,
       "learning_rate": 4.148936170212766e-07,
-      "logits/generated": -2.821133852005005,
-      "logits/real": -2.894505262374878,
-      "logps/generated": -349.6683044433594,
-      "logps/real": -201.2012176513672,
-      "loss": 0.4438,
+      "logits/generated": -2.7122185230255127,
+      "logits/real": -2.348639965057373,
+      "logps/generated": -489.88970947265625,
+      "logps/real": -0.002834505634382367,
+      "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/generated": -5.3700079917907715,
-      "rewards/margins": 9.444425582885742,
-      "rewards/real": 4.074416637420654,
+      "rewards/generated": -6.071843147277832,
+      "rewards/margins": 40.27666473388672,
+      "rewards/real": 34.2048225402832,
       "step": 40
     },
     {
       "epoch": 0.3184713375796178,
-      "grad_norm": 26.344052963980364,
+      "grad_norm": 0.0014726852980064813,
       "learning_rate": 3.7943262411347514e-07,
-      "logits/generated": -3.041593074798584,
-      "logits/real": -2.9904277324676514,
-      "logps/generated": -335.09942626953125,
-      "logps/real": -158.0336151123047,
-      "loss": 0.37,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/generated": -3.180644989013672,
-      "rewards/margins": 8.93952751159668,
-      "rewards/real": 5.758882999420166,
+      "logits/generated": -2.71730375289917,
+      "logits/real": -2.3547444343566895,
+      "logps/generated": -492.354248046875,
+      "logps/real": -0.0020735759753733873,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -6.318298816680908,
+      "rewards/margins": 40.523193359375,
+      "rewards/real": 34.20489501953125,
       "step": 50
     },
     {
       "epoch": 0.3821656050955414,
-      "grad_norm": 47.871426260084256,
+      "grad_norm": 0.0012108227405915682,
       "learning_rate": 3.4397163120567375e-07,
-      "logits/generated": -2.9896843433380127,
-      "logits/real": -3.0180420875549316,
-      "logps/generated": -302.9552307128906,
-      "logps/real": -140.7099151611328,
-      "loss": 0.3324,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/generated": -3.2430217266082764,
-      "rewards/margins": 9.301046371459961,
-      "rewards/real": 6.0580244064331055,
+      "logits/generated": -2.7216479778289795,
+      "logits/real": -2.359346866607666,
+      "logps/generated": -493.62725830078125,
+      "logps/real": -0.0016389258671551943,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -6.445591926574707,
+      "rewards/margins": 40.65053176879883,
+      "rewards/real": 34.20494079589844,
       "step": 60
     },
     {
       "epoch": 0.445859872611465,
-      "grad_norm": 24.288240218254995,
+      "grad_norm": 0.0010092968797445821,
       "learning_rate": 3.085106382978723e-07,
-      "logits/generated": -3.0595779418945312,
-      "logits/real": -2.9718518257141113,
-      "logps/generated": -328.691650390625,
-      "logps/real": -140.92660522460938,
-      "loss": 0.333,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/generated": -3.642324447631836,
-      "rewards/margins": 10.118036270141602,
-      "rewards/real": 6.47571325302124,
+      "logits/generated": -2.7243549823760986,
+      "logits/real": -2.3620219230651855,
+      "logps/generated": -494.82586669921875,
+      "logps/real": -0.0013623478589579463,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -6.565457820892334,
+      "rewards/margins": 40.77042770385742,
+      "rewards/real": 34.20496368408203,
       "step": 70
     },
     {
       "epoch": 0.5095541401273885,
-      "grad_norm": 14.83120429808784,
+      "grad_norm": 0.0008924667007579917,
       "learning_rate": 2.730496453900709e-07,
-      "logits/generated": -2.996634006500244,
-      "logits/real": -3.0401620864868164,
-      "logps/generated": -322.6135559082031,
-      "logps/real": -159.5257110595703,
-      "loss": 0.3328,
-      "rewards/accuracies": 0.9624999761581421,
-      "rewards/generated": -2.347047805786133,
-      "rewards/margins": 10.133938789367676,
-      "rewards/real": 7.786890506744385,
+      "logits/generated": -2.726926565170288,
+      "logits/real": -2.362954616546631,
+      "logps/generated": -495.93377685546875,
+      "logps/real": -0.0011800352949649096,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -6.676251411437988,
+      "rewards/margins": 40.88123321533203,
+      "rewards/real": 34.204986572265625,
       "step": 80
     },
     {
       "epoch": 0.5732484076433121,
-      "grad_norm": 16.65278970025503,
+      "grad_norm": 0.0008155704567801252,
       "learning_rate": 2.375886524822695e-07,
-      "logits/generated": -3.0264060497283936,
-      "logits/real": -2.979038715362549,
-      "logps/generated": -360.536376953125,
-      "logps/real": -162.26622009277344,
-      "loss": 0.3387,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/generated": -4.7210164070129395,
-      "rewards/margins": 12.409219741821289,
-      "rewards/real": 7.68820333480835,
+      "logits/generated": -2.72874116897583,
+      "logits/real": -2.366927146911621,
+      "logps/generated": -496.6890563964844,
+      "logps/real": -0.0010528427083045244,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -6.751776218414307,
+      "rewards/margins": 40.95677185058594,
+      "rewards/real": 34.204994201660156,
       "step": 90
     },
     {
       "epoch": 0.6369426751592356,
-      "grad_norm": 7.324308799399964,
+      "grad_norm": 0.000725474761841383,
       "learning_rate": 2.0212765957446807e-07,
-      "logits/generated": -2.968139410018921,
-      "logits/real": -3.0011098384857178,
-      "logps/generated": -325.79107666015625,
-      "logps/real": -162.4396514892578,
-      "loss": 0.3429,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/generated": -2.580188512802124,
-      "rewards/margins": 10.546457290649414,
-      "rewards/real": 7.966268062591553,
+      "logits/generated": -2.7310385704040527,
+      "logits/real": -2.369218349456787,
+      "logps/generated": -497.72625732421875,
+      "logps/real": -0.000954283110331744,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -6.855503082275391,
+      "rewards/margins": 41.06051254272461,
+      "rewards/real": 34.20500946044922,
       "step": 100
     },
     {
       "epoch": 0.6369426751592356,
-      "eval_logits/generated": -2.6921370029449463,
-      "eval_logits/real": -2.418954372406006,
-      "eval_logps/generated": -57.74082565307617,
-      "eval_logps/real": -16.818851470947266,
-      "eval_loss": 1.018080234527588,
-      "eval_rewards/accuracies": 0.0812101885676384,
-      "eval_rewards/generated": 1.4157419204711914,
-      "eval_rewards/margins": 0.06545643508434296,
-      "eval_rewards/real": 1.4811984300613403,
-      "eval_runtime": 325.124,
-      "eval_samples_per_second": 15.379,
-      "eval_steps_per_second": 0.483,
+      "eval_logits/generated": -2.661693572998047,
+      "eval_logits/real": -2.609511137008667,
+      "eval_logps/generated": -259.66143798828125,
+      "eval_logps/real": -210.63931274414062,
+      "eval_loss": 0.9619492292404175,
+      "eval_rewards/accuracies": 0.890625,
+      "eval_rewards/generated": -0.14410093426704407,
+      "eval_rewards/margins": 0.3460058867931366,
+      "eval_rewards/real": 0.20190495252609253,
+      "eval_runtime": 37.6272,
+      "eval_samples_per_second": 13.288,
+      "eval_steps_per_second": 0.425,
       "step": 100
     },
     {
       "epoch": 0.7006369426751592,
-      "grad_norm": 44.79401383111945,
+      "grad_norm": 0.0006953242722311237,
       "learning_rate": 1.6666666666666665e-07,
-      "logits/generated": -2.966001033782959,
-      "logits/real": -2.9780120849609375,
-      "logps/generated": -347.74993896484375,
-      "logps/real": -158.44979858398438,
-      "loss": 0.365,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/generated": -3.206289768218994,
-      "rewards/margins": 11.313484191894531,
-      "rewards/real": 8.107194900512695,
+      "logits/generated": -2.732168436050415,
+      "logits/real": -2.370328426361084,
+      "logps/generated": -497.7915954589844,
+      "logps/real": -0.0008821273222565651,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -6.86203145980835,
+      "rewards/margins": 41.06704330444336,
+      "rewards/real": 34.20501708984375,
       "step": 110
     },
     {
       "epoch": 0.7643312101910829,
-      "grad_norm": 7.15542526632935,
+      "grad_norm": 0.0006463359840785572,
       "learning_rate": 1.3120567375886523e-07,
-      "logits/generated": -2.9361348152160645,
-      "logits/real": -2.991243839263916,
-      "logps/generated": -340.3627014160156,
-      "logps/real": -145.40615844726562,
-      "loss": 0.3243,
+      "logits/generated": -2.7338597774505615,
+      "logits/real": -2.3714773654937744,
+      "logps/generated": -498.8688049316406,
+      "logps/real": -0.0008279000176116824,
+      "loss": 0.0,
       "rewards/accuracies": 1.0,
-      "rewards/generated": -3.4838199615478516,
-      "rewards/margins": 10.616706848144531,
-      "rewards/real": 7.132887363433838,
+      "rewards/generated": -6.969751834869385,
+      "rewards/margins": 41.174774169921875,
+      "rewards/real": 34.205020904541016,
       "step": 120
     },
     {
       "epoch": 0.8280254777070064,
-      "grad_norm": 28.617693401838523,
+      "grad_norm": 0.0006205498614208318,
       "learning_rate": 9.574468085106382e-08,
-      "logits/generated": -2.880622386932373,
-      "logits/real": -2.873990774154663,
-      "logps/generated": -284.91949462890625,
-      "logps/real": -130.95306396484375,
-      "loss": 0.3022,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/generated": -1.7718982696533203,
-      "rewards/margins": 8.987956047058105,
-      "rewards/real": 7.216057777404785,
+      "logits/generated": -2.7348275184631348,
+      "logits/real": -2.3729214668273926,
+      "logps/generated": -498.8643493652344,
+      "logps/real": -0.0007825180655345321,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -6.969304084777832,
+      "rewards/margins": 41.17433547973633,
+      "rewards/real": 34.20502471923828,
       "step": 130
     },
     {
       "epoch": 0.89171974522293,
-      "grad_norm": 11.2855453725489,
+      "grad_norm": 0.0005942730744268325,
       "learning_rate": 6.02836879432624e-08,
-      "logits/generated": -2.893979549407959,
-      "logits/real": -2.919368028640747,
-      "logps/generated": -334.3547058105469,
-      "logps/real": -154.1964569091797,
-      "loss": 0.3389,
-      "rewards/accuracies": 0.9750000238418579,
-      "rewards/generated": -3.628894090652466,
-      "rewards/margins": 11.9329252243042,
-      "rewards/real": 8.304032325744629,
+      "logits/generated": -2.7360432147979736,
+      "logits/real": -2.374084949493408,
+      "logps/generated": -499.4146423339844,
+      "logps/real": -0.0007567574502900243,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -7.024338722229004,
+      "rewards/margins": 41.229366302490234,
+      "rewards/real": 34.20502471923828,
       "step": 140
     },
     {
       "epoch": 0.9554140127388535,
-      "grad_norm": 31.41907953329688,
+      "grad_norm": 0.0005956400007297931,
       "learning_rate": 2.4822695035460993e-08,
-      "logits/generated": -2.864701747894287,
-      "logits/real": -2.9138083457946777,
-      "logps/generated": -315.903564453125,
-      "logps/real": -138.75643920898438,
-      "loss": 0.3255,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/generated": -2.897620677947998,
-      "rewards/margins": 9.965813636779785,
-      "rewards/real": 7.068192958831787,
+      "logits/generated": -2.73626708984375,
+      "logits/real": -2.374666690826416,
+      "logps/generated": -499.5492248535156,
+      "logps/real": -0.0007398539455607533,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -7.0377936363220215,
+      "rewards/margins": 41.242820739746094,
+      "rewards/real": 34.20502471923828,
       "step": 150
     },
     {
       "epoch": 1.0,
       "step": 157,
       "total_flos": 0.0,
-      "train_loss": 0.3886139180250229,
-      "train_runtime": 1518.6555,
-      "train_samples_per_second": 3.292,
-      "train_steps_per_second": 0.103
+      "train_loss": 0.03164779691052703,
+      "train_runtime": 1178.5993,
+      "train_samples_per_second": 4.242,
+      "train_steps_per_second": 0.133
     }
   ],
   "logging_steps": 10,