Model save

README.md

@@ -2,31 +2,28 @@
 license: apache-2.0
 base_model: AmberYifan/mistral-safe-sft-full
 tags:
-- alignment-handbook
 - generated_from_trainer
-datasets:
-- AmberYifan/sft-spin-kcenter-5k
 model-index:
-- name: sft-spin-kcenter-5k
+- name: mistral-sft-kcenter-5k
   results: []
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-# sft-spin-kcenter-5k
+# mistral-sft-kcenter-5k
 
-This model is a fine-tuned version of [AmberYifan/mistral-safe-sft-full](https://huggingface.co/AmberYifan/mistral-safe-sft-full) on the AmberYifan/sft-spin-kcenter-5k dataset.
+This model is a fine-tuned version of [AmberYifan/mistral-safe-sft-full](https://huggingface.co/AmberYifan/mistral-safe-sft-full) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9619
-- Rewards/real: 0.2019
-- Rewards/generated: -0.1441
-- Rewards/accuracies: 0.8906
-- Rewards/margins: 0.3460
-- Logps/generated: -259.6614
-- Logps/real: -210.6393
-- Logits/generated: -2.6617
-- Logits/real: -2.6095
+- Loss: 0.1925
+- Rewards/real: 5.5330
+- Rewards/generated: -4.6681
+- Rewards/accuracies: 0.9922
+- Rewards/margins: 10.2011
+- Logps/generated: -304.9019
+- Logps/real: -157.3288
+- Logits/generated: -2.8831
+- Logits/real: -2.8163
 
 ## Model description
 
@@ -62,7 +59,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/real | Rewards/generated | Rewards/accuracies | Rewards/margins | Logps/generated | Logps/real | Logits/generated | Logits/real |
 |:-------------:|:------:|:----:|:---------------:|:------------:|:-----------------:|:------------------:|:---------------:|:---------------:|:----------:|:----------------:|:-----------:|
-| 0.0           | 0.6369 | 100  | 0.9619          | 0.2019       | -0.1441           | 0.8906             | 0.3460          | -259.6614       | -210.6393  | -2.6617          | -2.6095     |
+| 0.1804        | 0.6369 | 100  | 0.1925          | 5.5330       | -4.6681           | 0.9922             | 10.2011         | -304.9019       | -157.3288  | -2.8831          | -2.8163     |
 
 
 ### Framework versions

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.03164779691052703,
-    "train_runtime": 1178.5993,
+    "train_loss": 0.23225101619769054,
+    "train_runtime": 1278.722,
     "train_samples": 5000,
-    "train_samples_per_second": 4.242,
-    "train_steps_per_second": 0.133
+    "train_samples_per_second": 3.91,
+    "train_steps_per_second": 0.123
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.03164779691052703,
-    "train_runtime": 1178.5993,
+    "train_loss": 0.23225101619769054,
+    "train_runtime": 1278.722,
     "train_samples": 5000,
-    "train_samples_per_second": 4.242,
-    "train_steps_per_second": 0.133
+    "train_samples_per_second": 3.91,
+    "train_steps_per_second": 0.123
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.006369426751592357,
-      "grad_norm": 2514.601414684904,
+      "grad_norm": 553.6689195387686,
       "learning_rate": 3.125e-08,
-      "logits/generated": -2.661752223968506,
-      "logits/real": -2.483980894088745,
-      "logps/generated": -429.17132568359375,
-      "logps/real": -342.051025390625,
-      "loss": 1.3612,
+      "logits/generated": -2.5852508544921875,
+      "logits/real": -2.6413676738739014,
+      "logps/generated": -325.0230407714844,
+      "logps/real": -285.6551513671875,
+      "loss": 0.9368,
       "rewards/accuracies": 0.0,
       "rewards/generated": 0.0,
       "rewards/margins": 0.0,
@@ -25,253 +25,253 @@
     },
     {
       "epoch": 0.06369426751592357,
-      "grad_norm": 0.31574006570864244,
+      "grad_norm": 114.25554694751857,
       "learning_rate": 3.1249999999999997e-07,
-      "logits/generated": -2.668369770050049,
-      "logits/real": -2.424091339111328,
-      "logps/generated": -450.4893798828125,
-      "logps/real": -144.69952392578125,
-      "loss": 0.4008,
-      "rewards/accuracies": 0.8888888955116272,
-      "rewards/generated": -2.1318063735961914,
-      "rewards/margins": 21.866958618164062,
-      "rewards/real": 19.735153198242188,
+      "logits/generated": -2.6055305004119873,
+      "logits/real": -2.5509183406829834,
+      "logps/generated": -363.4412536621094,
+      "logps/real": -227.83956909179688,
+      "loss": 0.6399,
+      "rewards/accuracies": 0.7916666865348816,
+      "rewards/generated": -1.371843934059143,
+      "rewards/margins": 1.6683220863342285,
+      "rewards/real": 0.2964780032634735,
       "step": 10
     },
     {
       "epoch": 0.12738853503184713,
-      "grad_norm": 0.005520241230176221,
+      "grad_norm": 32.95472745041124,
       "learning_rate": 4.858156028368794e-07,
-      "logits/generated": -2.692908763885498,
-      "logits/real": -2.329023599624634,
-      "logps/generated": -478.9815368652344,
-      "logps/real": -0.014682354405522346,
-      "loss": 0.0,
+      "logits/generated": -2.859358072280884,
+      "logits/real": -2.843193531036377,
+      "logps/generated": -321.1373596191406,
+      "logps/real": -200.79653930664062,
+      "loss": 0.2658,
       "rewards/accuracies": 1.0,
-      "rewards/generated": -4.981024742126465,
-      "rewards/margins": 39.18465805053711,
-      "rewards/real": 34.203636169433594,
+      "rewards/generated": -4.476873874664307,
+      "rewards/margins": 5.808846950531006,
+      "rewards/real": 1.3319734334945679,
       "step": 20
     },
     {
       "epoch": 0.1910828025477707,
-      "grad_norm": 0.002789639064368334,
+      "grad_norm": 15.376345388496501,
       "learning_rate": 4.50354609929078e-07,
-      "logits/generated": -2.705698013305664,
-      "logits/real": -2.342294216156006,
-      "logps/generated": -486.1914978027344,
-      "logps/real": -0.004650969058275223,
-      "loss": 0.0,
+      "logits/generated": -2.8705520629882812,
+      "logits/real": -2.8971784114837646,
+      "logps/generated": -368.5296325683594,
+      "logps/real": -192.18089294433594,
+      "loss": 0.2291,
       "rewards/accuracies": 1.0,
-      "rewards/generated": -5.702020645141602,
-      "rewards/margins": 39.90666198730469,
-      "rewards/real": 34.20463943481445,
+      "rewards/generated": -5.970882415771484,
+      "rewards/margins": 9.524211883544922,
+      "rewards/real": 3.553328275680542,
       "step": 30
     },
     {
       "epoch": 0.25477707006369427,
-      "grad_norm": 0.0019297231936813803,
+      "grad_norm": 32.240493773919496,
       "learning_rate": 4.148936170212766e-07,
-      "logits/generated": -2.7122185230255127,
-      "logits/real": -2.348639965057373,
-      "logps/generated": -489.88970947265625,
-      "logps/real": -0.002834505634382367,
-      "loss": 0.0,
-      "rewards/accuracies": 1.0,
-      "rewards/generated": -6.071843147277832,
-      "rewards/margins": 40.27666473388672,
-      "rewards/real": 34.2048225402832,
+      "logits/generated": -2.9980812072753906,
+      "logits/real": -2.925062656402588,
+      "logps/generated": -352.6733093261719,
+      "logps/real": -184.546142578125,
+      "loss": 0.2192,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/generated": -5.158407688140869,
+      "rewards/margins": 9.533308029174805,
+      "rewards/real": 4.374899864196777,
       "step": 40
     },
     {
       "epoch": 0.3184713375796178,
-      "grad_norm": 0.0014726852980064813,
+      "grad_norm": 22.443882761667297,
       "learning_rate": 3.7943262411347514e-07,
-      "logits/generated": -2.71730375289917,
-      "logits/real": -2.3547444343566895,
-      "logps/generated": -492.354248046875,
-      "logps/real": -0.0020735759753733873,
-      "loss": 0.0,
+      "logits/generated": -2.9874608516693115,
+      "logits/real": -2.901106357574463,
+      "logps/generated": -366.24774169921875,
+      "logps/real": -178.15200805664062,
+      "loss": 0.1918,
       "rewards/accuracies": 1.0,
-      "rewards/generated": -6.318298816680908,
-      "rewards/margins": 40.523193359375,
-      "rewards/real": 34.20489501953125,
+      "rewards/generated": -5.891494274139404,
+      "rewards/margins": 10.835145950317383,
+      "rewards/real": 4.94365119934082,
       "step": 50
     },
     {
       "epoch": 0.3821656050955414,
-      "grad_norm": 0.0012108227405915682,
+      "grad_norm": 27.90941399161424,
       "learning_rate": 3.4397163120567375e-07,
-      "logits/generated": -2.7216479778289795,
-      "logits/real": -2.359346866607666,
-      "logps/generated": -493.62725830078125,
-      "logps/real": -0.0016389258671551943,
-      "loss": 0.0,
+      "logits/generated": -2.9581241607666016,
+      "logits/real": -2.923001527786255,
+      "logps/generated": -340.056884765625,
+      "logps/real": -175.193603515625,
+      "loss": 0.2088,
       "rewards/accuracies": 1.0,
-      "rewards/generated": -6.445591926574707,
-      "rewards/margins": 40.65053176879883,
-      "rewards/real": 34.20494079589844,
+      "rewards/generated": -4.940432548522949,
+      "rewards/margins": 10.540928840637207,
+      "rewards/real": 5.6004958152771,
       "step": 60
     },
     {
       "epoch": 0.445859872611465,
-      "grad_norm": 0.0010092968797445821,
+      "grad_norm": 19.61548013874602,
       "learning_rate": 3.085106382978723e-07,
-      "logits/generated": -2.7243549823760986,
-      "logits/real": -2.3620219230651855,
-      "logps/generated": -494.82586669921875,
-      "logps/real": -0.0013623478589579463,
-      "loss": 0.0,
-      "rewards/accuracies": 1.0,
-      "rewards/generated": -6.565457820892334,
-      "rewards/margins": 40.77042770385742,
-      "rewards/real": 34.20496368408203,
+      "logits/generated": -2.950155735015869,
+      "logits/real": -2.920978546142578,
+      "logps/generated": -374.2133483886719,
+      "logps/real": -176.62344360351562,
+      "loss": 0.1973,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/generated": -5.149114608764648,
+      "rewards/margins": 11.470724105834961,
+      "rewards/real": 6.321610450744629,
       "step": 70
     },
     {
       "epoch": 0.5095541401273885,
-      "grad_norm": 0.0008924667007579917,
+      "grad_norm": 7.313023868270791,
       "learning_rate": 2.730496453900709e-07,
-      "logits/generated": -2.726926565170288,
-      "logits/real": -2.362954616546631,
-      "logps/generated": -495.93377685546875,
-      "logps/real": -0.0011800352949649096,
-      "loss": 0.0,
-      "rewards/accuracies": 1.0,
-      "rewards/generated": -6.676251411437988,
-      "rewards/margins": 40.88123321533203,
-      "rewards/real": 34.204986572265625,
+      "logits/generated": -2.891042709350586,
+      "logits/real": -2.816384792327881,
+      "logps/generated": -343.1810607910156,
+      "logps/real": -187.75689697265625,
+      "loss": 0.1921,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -4.03465461730957,
+      "rewards/margins": 11.060041427612305,
+      "rewards/real": 7.025385856628418,
       "step": 80
     },
     {
       "epoch": 0.5732484076433121,
-      "grad_norm": 0.0008155704567801252,
+      "grad_norm": 5.902065577304844,
       "learning_rate": 2.375886524822695e-07,
-      "logits/generated": -2.72874116897583,
-      "logits/real": -2.366927146911621,
-      "logps/generated": -496.6890563964844,
-      "logps/real": -0.0010528427083045244,
-      "loss": 0.0,
-      "rewards/accuracies": 1.0,
-      "rewards/generated": -6.751776218414307,
-      "rewards/margins": 40.95677185058594,
-      "rewards/real": 34.204994201660156,
+      "logits/generated": -2.7973499298095703,
+      "logits/real": -2.7673676013946533,
+      "logps/generated": -361.3358154296875,
+      "logps/real": -168.44692993164062,
+      "loss": 0.1829,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/generated": -4.653961658477783,
+      "rewards/margins": 10.882070541381836,
+      "rewards/real": 6.228109359741211,
       "step": 90
     },
     {
       "epoch": 0.6369426751592356,
-      "grad_norm": 0.000725474761841383,
+      "grad_norm": 21.378621598556965,
       "learning_rate": 2.0212765957446807e-07,
-      "logits/generated": -2.7310385704040527,
-      "logits/real": -2.369218349456787,
-      "logps/generated": -497.72625732421875,
-      "logps/real": -0.000954283110331744,
-      "loss": 0.0,
+      "logits/generated": -2.8296706676483154,
+      "logits/real": -2.783637762069702,
+      "logps/generated": -385.32916259765625,
+      "logps/real": -161.2016143798828,
+      "loss": 0.1804,
       "rewards/accuracies": 1.0,
-      "rewards/generated": -6.855503082275391,
-      "rewards/margins": 41.06051254272461,
-      "rewards/real": 34.20500946044922,
+      "rewards/generated": -5.7995285987854,
+      "rewards/margins": 12.244100570678711,
+      "rewards/real": 6.444572448730469,
       "step": 100
     },
     {
       "epoch": 0.6369426751592356,
-      "eval_logits/generated": -2.661693572998047,
-      "eval_logits/real": -2.609511137008667,
-      "eval_logps/generated": -259.66143798828125,
-      "eval_logps/real": -210.63931274414062,
-      "eval_loss": 0.9619492292404175,
-      "eval_rewards/accuracies": 0.890625,
-      "eval_rewards/generated": -0.14410093426704407,
-      "eval_rewards/margins": 0.3460058867931366,
-      "eval_rewards/real": 0.20190495252609253,
-      "eval_runtime": 37.6272,
-      "eval_samples_per_second": 13.288,
-      "eval_steps_per_second": 0.425,
+      "eval_logits/generated": -2.8830642700195312,
+      "eval_logits/real": -2.8163247108459473,
+      "eval_logps/generated": -304.90185546875,
+      "eval_logps/real": -157.32879638671875,
+      "eval_loss": 0.19246906042099,
+      "eval_rewards/accuracies": 0.9921875,
+      "eval_rewards/generated": -4.66813850402832,
+      "eval_rewards/margins": 10.201096534729004,
+      "eval_rewards/real": 5.532957077026367,
+      "eval_runtime": 40.3701,
+      "eval_samples_per_second": 12.385,
+      "eval_steps_per_second": 0.396,
       "step": 100
     },
     {
       "epoch": 0.7006369426751592,
-      "grad_norm": 0.0006953242722311237,
+      "grad_norm": 4.836243072704085,
       "learning_rate": 1.6666666666666665e-07,
-      "logits/generated": -2.732168436050415,
-      "logits/real": -2.370328426361084,
-      "logps/generated": -497.7915954589844,
-      "logps/real": -0.0008821273222565651,
-      "loss": 0.0,
+      "logits/generated": -2.814863920211792,
+      "logits/real": -2.813689708709717,
+      "logps/generated": -349.199951171875,
+      "logps/real": -167.05197143554688,
+      "loss": 0.1732,
       "rewards/accuracies": 1.0,
-      "rewards/generated": -6.86203145980835,
-      "rewards/margins": 41.06704330444336,
-      "rewards/real": 34.20501708984375,
+      "rewards/generated": -4.092279434204102,
+      "rewards/margins": 10.799135208129883,
+      "rewards/real": 6.706856727600098,
       "step": 110
     },
     {
       "epoch": 0.7643312101910829,
-      "grad_norm": 0.0006463359840785572,
+      "grad_norm": 10.2454630213605,
       "learning_rate": 1.3120567375886523e-07,
-      "logits/generated": -2.7338597774505615,
-      "logits/real": -2.3714773654937744,
-      "logps/generated": -498.8688049316406,
-      "logps/real": -0.0008279000176116824,
-      "loss": 0.0,
-      "rewards/accuracies": 1.0,
-      "rewards/generated": -6.969751834869385,
-      "rewards/margins": 41.174774169921875,
-      "rewards/real": 34.205020904541016,
+      "logits/generated": -2.6291651725769043,
+      "logits/real": -2.6084656715393066,
+      "logps/generated": -357.1519470214844,
+      "logps/real": -194.95156860351562,
+      "loss": 0.2818,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -6.045389175415039,
+      "rewards/margins": 10.61597728729248,
+      "rewards/real": 4.570586681365967,
       "step": 120
     },
     {
       "epoch": 0.8280254777070064,
-      "grad_norm": 0.0006205498614208318,
+      "grad_norm": 30.968094217096088,
       "learning_rate": 9.574468085106382e-08,
-      "logits/generated": -2.7348275184631348,
-      "logits/real": -2.3729214668273926,
-      "logps/generated": -498.8643493652344,
-      "logps/real": -0.0007825180655345321,
-      "loss": 0.0,
+      "logits/generated": -2.7041964530944824,
+      "logits/real": -2.6806530952453613,
+      "logps/generated": -349.06878662109375,
+      "logps/real": -164.90850830078125,
+      "loss": 0.1784,
       "rewards/accuracies": 1.0,
-      "rewards/generated": -6.969304084777832,
-      "rewards/margins": 41.17433547973633,
-      "rewards/real": 34.20502471923828,
+      "rewards/generated": -5.374934196472168,
+      "rewards/margins": 13.002920150756836,
+      "rewards/real": 7.627985954284668,
       "step": 130
     },
     {
       "epoch": 0.89171974522293,
-      "grad_norm": 0.0005942730744268325,
+      "grad_norm": 39.77463051390536,
       "learning_rate": 6.02836879432624e-08,
-      "logits/generated": -2.7360432147979736,
-      "logits/real": -2.374084949493408,
-      "logps/generated": -499.4146423339844,
-      "logps/real": -0.0007567574502900243,
-      "loss": 0.0,
-      "rewards/accuracies": 1.0,
-      "rewards/generated": -7.024338722229004,
-      "rewards/margins": 41.229366302490234,
-      "rewards/real": 34.20502471923828,
+      "logits/generated": -2.705962896347046,
+      "logits/real": -2.6498522758483887,
+      "logps/generated": -338.68121337890625,
+      "logps/real": -155.6457061767578,
+      "loss": 0.1802,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -4.318178176879883,
+      "rewards/margins": 11.076618194580078,
+      "rewards/real": 6.758440971374512,
       "step": 140
     },
     {
       "epoch": 0.9554140127388535,
-      "grad_norm": 0.0005956400007297931,
+      "grad_norm": 37.53390149893301,
       "learning_rate": 2.4822695035460993e-08,
-      "logits/generated": -2.73626708984375,
-      "logits/real": -2.374666690826416,
-      "logps/generated": -499.5492248535156,
-      "logps/real": -0.0007398539455607533,
-      "loss": 0.0,
+      "logits/generated": -2.6534829139709473,
+      "logits/real": -2.6331899166107178,
+      "logps/generated": -302.69781494140625,
+      "logps/real": -132.01724243164062,
+      "loss": 0.1803,
       "rewards/accuracies": 1.0,
-      "rewards/generated": -7.0377936363220215,
-      "rewards/margins": 41.242820739746094,
-      "rewards/real": 34.20502471923828,
+      "rewards/generated": -3.575101375579834,
+      "rewards/margins": 9.780950546264648,
+      "rewards/real": 6.20584774017334,
       "step": 150
     },
     {
       "epoch": 1.0,
       "step": 157,
       "total_flos": 0.0,
-      "train_loss": 0.03164779691052703,
-      "train_runtime": 1178.5993,
-      "train_samples_per_second": 4.242,
-      "train_steps_per_second": 0.133
+      "train_loss": 0.23225101619769054,
+      "train_runtime": 1278.722,
+      "train_samples_per_second": 3.91,
+      "train_steps_per_second": 0.123
     }
   ],
   "logging_steps": 10,