Model save

README.md

@@ -0,0 +1,74 @@
+---
+license: apache-2.0
+library_name: peft
+tags:
+- trl
+- dpo
+- generated_from_trainer
+base_model: alignment-handbook/zephyr-7b-sft-full
+model-index:
+- name: zephyr-dpo-timedial
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-dpo-timedial
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.2236
+- Rewards/chosen: 0.2987
+- Rewards/rejected: -1.0958
+- Rewards/accuracies: 1.0
+- Rewards/margins: 1.3944
+- Logps/rejected: -154.5925
+- Logps/chosen: -0.6286
+- Logits/rejected: -2.7419
+- Logits/chosen: -2.7480
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- distributed_type: multi-GPU
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 4
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.3995        | 0.35  | 100  | 0.3705          | 0.2856         | -0.5202          | 1.0                | 0.8058          | -97.0354       | -1.9368      | -2.7815         | -2.7803       |
+| 0.2236        | 0.69  | 200  | 0.2236          | 0.2987         | -1.0958          | 1.0                | 1.3944          | -154.5925      | -0.6286      | -2.7419         | -2.7480       |
+
+
+### Framework versions
+
+- PEFT 0.7.1
+- Transformers 4.39.0.dev0
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:913801ea55b598f7b58aaa007bb675321dd1e837e4dfa66a6569925581eb971d
+oid sha256:3f036b11d8bb4ff3055489120cb1ebff9118f7b36514d8ba27aecd6b2e1c3f6d
 size 83946192

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.36633619635162884,
+    "train_runtime": 628.6241,
+    "train_samples": 1157,
+    "train_samples_per_second": 1.841,
+    "train_steps_per_second": 0.46
+}

runs/Apr11_12-13-45_612e66badb5c/events.out.tfevents.1712837726.612e66badb5c.4808.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c958333c0e528a248dbd3c6fc69cdd9616580f3bae448122dcd2d1478c1bb160
-size 20525
+oid sha256:80644563065d10ab3cd240ce2956e96847e989a3ae0f7e0ab142e5d5e2e733c1
+size 26383

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.36633619635162884,
+    "train_runtime": 628.6241,
+    "train_samples": 1157,
+    "train_samples_per_second": 1.841,
+    "train_steps_per_second": 0.46
+}

trainer_state.json

@@ -0,0 +1,497 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.998272884283247,
+  "eval_steps": 100,
+  "global_step": 289,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 0.314453125,
+      "learning_rate": 1.7241379310344828e-07,
+      "logits/chosen": -2.648426055908203,
+      "logits/rejected": -2.6488823890686035,
+      "logps/chosen": -29.924301147460938,
+      "logps/rejected": -43.789119720458984,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 0.294921875,
+      "learning_rate": 1.724137931034483e-06,
+      "logits/chosen": -2.791017532348633,
+      "logits/rejected": -2.792222023010254,
+      "logps/chosen": -30.778282165527344,
+      "logps/rejected": -44.59426498413086,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0004109032452106476,
+      "rewards/margins": 0.0004249625199008733,
+      "rewards/rejected": -1.4059260138310492e-05,
+      "step": 10
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 0.3203125,
+      "learning_rate": 3.448275862068966e-06,
+      "logits/chosen": -2.8013782501220703,
+      "logits/rejected": -2.80298113822937,
+      "logps/chosen": -30.407978057861328,
+      "logps/rejected": -46.99640655517578,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.0021479069255292416,
+      "rewards/margins": 0.0025836334098130465,
+      "rewards/rejected": -0.00043572625145316124,
+      "step": 20
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 0.30078125,
+      "learning_rate": 4.999817502139027e-06,
+      "logits/chosen": -2.8514046669006348,
+      "logits/rejected": -2.851513385772705,
+      "logps/chosen": -29.340972900390625,
+      "logps/rejected": -46.38640594482422,
+      "loss": 0.6853,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.016690988093614578,
+      "rewards/margins": 0.01585637591779232,
+      "rewards/rejected": 0.0008346118847839534,
+      "step": 30
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 0.34765625,
+      "learning_rate": 4.977949980164773e-06,
+      "logits/chosen": -2.77081036567688,
+      "logits/rejected": -2.7683987617492676,
+      "logps/chosen": -24.99424934387207,
+      "logps/rejected": -43.76649475097656,
+      "loss": 0.6703,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.05648866295814514,
+      "rewards/margins": 0.04632381349802017,
+      "rewards/rejected": 0.010164851322770119,
+      "step": 40
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 0.53125,
+      "learning_rate": 4.919948367622307e-06,
+      "logits/chosen": -2.826796054840088,
+      "logits/rejected": -2.8314404487609863,
+      "logps/chosen": -20.64249610900879,
+      "logps/rejected": -46.383785247802734,
+      "loss": 0.6438,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.09788469970226288,
+      "rewards/margins": 0.10135942697525024,
+      "rewards/rejected": -0.003474730532616377,
+      "step": 50
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 0.57421875,
+      "learning_rate": 4.8266584586307555e-06,
+      "logits/chosen": -2.835439682006836,
+      "logits/rejected": -2.8375213146209717,
+      "logps/chosen": -19.745527267456055,
+      "logps/rejected": -48.08259582519531,
+      "loss": 0.6272,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.10895518958568573,
+      "rewards/margins": 0.13693755865097046,
+      "rewards/rejected": -0.02798236906528473,
+      "step": 60
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 0.80859375,
+      "learning_rate": 4.699440630133794e-06,
+      "logits/chosen": -2.8537039756774902,
+      "logits/rejected": -2.8549795150756836,
+      "logps/chosen": -16.767593383789062,
+      "logps/rejected": -53.961273193359375,
+      "loss": 0.5912,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.13634786009788513,
+      "rewards/margins": 0.2162630558013916,
+      "rewards/rejected": -0.07991518080234528,
+      "step": 70
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 0.7109375,
+      "learning_rate": 4.5401500045405126e-06,
+      "logits/chosen": -2.7856647968292236,
+      "logits/rejected": -2.7866556644439697,
+      "logps/chosen": -10.72642993927002,
+      "logps/rejected": -63.57000732421875,
+      "loss": 0.522,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.20102627575397491,
+      "rewards/margins": 0.37968239188194275,
+      "rewards/rejected": -0.17865613102912903,
+      "step": 80
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 0.73828125,
+      "learning_rate": 4.351109397863309e-06,
+      "logits/chosen": -2.784008026123047,
+      "logits/rejected": -2.7872986793518066,
+      "logps/chosen": -4.303569793701172,
+      "logps/rejected": -73.54255676269531,
+      "loss": 0.4602,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.26261404156684875,
+      "rewards/margins": 0.5405046343803406,
+      "rewards/rejected": -0.277890682220459,
+      "step": 90
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 0.671875,
+      "learning_rate": 4.135075447829912e-06,
+      "logits/chosen": -2.7918694019317627,
+      "logits/rejected": -2.797271490097046,
+      "logps/chosen": -2.3328211307525635,
+      "logps/rejected": -88.87330627441406,
+      "loss": 0.3995,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2833644449710846,
+      "rewards/margins": 0.7134484052658081,
+      "rewards/rejected": -0.4300839304924011,
+      "step": 100
+    },
+    {
+      "epoch": 0.35,
+      "eval_logits/chosen": -2.780285358428955,
+      "eval_logits/rejected": -2.7814674377441406,
+      "eval_logps/chosen": -1.9368014335632324,
+      "eval_logps/rejected": -97.03541564941406,
+      "eval_loss": 0.3704952299594879,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.28558260202407837,
+      "eval_rewards/margins": 0.8057959675788879,
+      "eval_rewards/rejected": -0.5202134251594543,
+      "eval_runtime": 59.0118,
+      "eval_samples_per_second": 4.897,
+      "eval_steps_per_second": 2.457,
+      "step": 100
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 0.87890625,
+      "learning_rate": 3.895198415897896e-06,
+      "logits/chosen": -2.757932186126709,
+      "logits/rejected": -2.75920033454895,
+      "logps/chosen": -1.1944591999053955,
+      "logps/rejected": -102.75828552246094,
+      "loss": 0.3536,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29440590739250183,
+      "rewards/margins": 0.8625706434249878,
+      "rewards/rejected": -0.5681648254394531,
+      "step": 110
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 0.5546875,
+      "learning_rate": 3.634976249348867e-06,
+      "logits/chosen": -2.7818870544433594,
+      "logits/rejected": -2.7803432941436768,
+      "logps/chosen": -1.4344559907913208,
+      "logps/rejected": -116.496337890625,
+      "loss": 0.3157,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2936590015888214,
+      "rewards/margins": 0.9929488897323608,
+      "rewards/rejected": -0.6992899179458618,
+      "step": 120
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 0.52734375,
+      "learning_rate": 3.3582035733403963e-06,
+      "logits/chosen": -2.784778594970703,
+      "logits/rejected": -2.78275465965271,
+      "logps/chosen": -1.0116920471191406,
+      "logps/rejected": -123.45560455322266,
+      "loss": 0.2951,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2951065003871918,
+      "rewards/margins": 1.0736355781555176,
+      "rewards/rejected": -0.7785290479660034,
+      "step": 130
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 0.47265625,
+      "learning_rate": 3.068916356726475e-06,
+      "logits/chosen": -2.732774019241333,
+      "logits/rejected": -2.730987071990967,
+      "logps/chosen": -0.8254967927932739,
+      "logps/rejected": -134.45706176757812,
+      "loss": 0.2701,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29469770193099976,
+      "rewards/margins": 1.1738824844360352,
+      "rewards/rejected": -0.8791847229003906,
+      "step": 140
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 0.451171875,
+      "learning_rate": 2.771333058543416e-06,
+      "logits/chosen": -2.7359633445739746,
+      "logits/rejected": -2.7314648628234863,
+      "logps/chosen": -0.7030662894248962,
+      "logps/rejected": -138.85061645507812,
+      "loss": 0.2582,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29896482825279236,
+      "rewards/margins": 1.2272465229034424,
+      "rewards/rejected": -0.9282817840576172,
+      "step": 150
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 0.37109375,
+      "learning_rate": 2.4697931133779566e-06,
+      "logits/chosen": -2.760554790496826,
+      "logits/rejected": -2.754014492034912,
+      "logps/chosen": -4.322839736938477,
+      "logps/rejected": -146.63290405273438,
+      "loss": 0.2537,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.26600345969200134,
+      "rewards/margins": 1.265166163444519,
+      "rewards/rejected": -0.9991627931594849,
+      "step": 160
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 0.5,
+      "learning_rate": 2.168693652639432e-06,
+      "logits/chosen": -2.749978542327881,
+      "logits/rejected": -2.7451446056365967,
+      "logps/chosen": -0.6626571416854858,
+      "logps/rejected": -147.86549377441406,
+      "loss": 0.2367,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3002447485923767,
+      "rewards/margins": 1.324704647064209,
+      "rewards/rejected": -1.0244598388671875,
+      "step": 170
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 0.50390625,
+      "learning_rate": 1.872425384482346e-06,
+      "logits/chosen": -2.7601447105407715,
+      "logits/rejected": -2.7531185150146484,
+      "logps/chosen": -0.5638567209243774,
+      "logps/rejected": -151.16998291015625,
+      "loss": 0.2307,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.30341148376464844,
+      "rewards/margins": 1.3561573028564453,
+      "rewards/rejected": -1.0527456998825073,
+      "step": 180
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 3.484375,
+      "learning_rate": 1.5853085673944695e-06,
+      "logits/chosen": -2.7549314498901367,
+      "logits/rejected": -2.748093366622925,
+      "logps/chosen": -1.0212732553482056,
+      "logps/rejected": -151.91207885742188,
+      "loss": 0.2298,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2937738597393036,
+      "rewards/margins": 1.36028254032135,
+      "rewards/rejected": -1.0665085315704346,
+      "step": 190
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 0.37109375,
+      "learning_rate": 1.3115300110997097e-06,
+      "logits/chosen": -2.73872447013855,
+      "logits/rejected": -2.734360933303833,
+      "logps/chosen": -0.8876104354858398,
+      "logps/rejected": -154.93753051757812,
+      "loss": 0.2236,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2938212454319,
+      "rewards/margins": 1.3927398920059204,
+      "rewards/rejected": -1.0989186763763428,
+      "step": 200
+    },
+    {
+      "epoch": 0.69,
+      "eval_logits/chosen": -2.7480151653289795,
+      "eval_logits/rejected": -2.741860866546631,
+      "eval_logps/chosen": -0.6285810470581055,
+      "eval_logps/rejected": -154.59251403808594,
+      "eval_loss": 0.22360171377658844,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.29866480827331543,
+      "eval_rewards/margins": 1.3944491147994995,
+      "eval_rewards/rejected": -1.095784306526184,
+      "eval_runtime": 58.9766,
+      "eval_samples_per_second": 4.9,
+      "eval_steps_per_second": 2.459,
+      "step": 200
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 0.56640625,
+      "learning_rate": 1.0550820234444627e-06,
+      "logits/chosen": -2.7622146606445312,
+      "logits/rejected": -2.7575910091400146,
+      "logps/chosen": -0.2955681085586548,
+      "logps/rejected": -156.5863494873047,
+      "loss": 0.2204,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3056582808494568,
+      "rewards/margins": 1.4064788818359375,
+      "rewards/rejected": -1.1008204221725464,
+      "step": 210
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 1.3125,
+      "learning_rate": 8.197041935593181e-07,
+      "logits/chosen": -2.7583320140838623,
+      "logits/rejected": -2.7535462379455566,
+      "logps/chosen": -0.26544955372810364,
+      "logps/rejected": -156.95286560058594,
+      "loss": 0.2193,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.30188480019569397,
+      "rewards/margins": 1.4142131805419922,
+      "rewards/rejected": -1.112328290939331,
+      "step": 220
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 0.373046875,
+      "learning_rate": 6.088288602287159e-07,
+      "logits/chosen": -2.723353385925293,
+      "logits/rejected": -2.71772837638855,
+      "logps/chosen": -0.17958000302314758,
+      "logps/rejected": -157.92813110351562,
+      "loss": 0.2166,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3015274405479431,
+      "rewards/margins": 1.4261300563812256,
+      "rewards/rejected": -1.1246027946472168,
+      "step": 230
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 0.6328125,
+      "learning_rate": 4.255310606625124e-07,
+      "logits/chosen": -2.762199640274048,
+      "logits/rejected": -2.758517265319824,
+      "logps/chosen": -0.2033136636018753,
+      "logps/rejected": -158.05380249023438,
+      "loss": 0.2164,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3013545870780945,
+      "rewards/margins": 1.4260454177856445,
+      "rewards/rejected": -1.1246907711029053,
+      "step": 240
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 0.478515625,
+      "learning_rate": 2.7248368952908055e-07,
+      "logits/chosen": -2.7318742275238037,
+      "logits/rejected": -2.724575996398926,
+      "logps/chosen": -0.2196466624736786,
+      "logps/rejected": -155.65301513671875,
+      "loss": 0.2205,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3047715723514557,
+      "rewards/margins": 1.406855583190918,
+      "rewards/rejected": -1.1020839214324951,
+      "step": 250
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 0.4453125,
+      "learning_rate": 1.5191852213221198e-07,
+      "logits/chosen": -2.7768027782440186,
+      "logits/rejected": -2.7685720920562744,
+      "logps/chosen": -1.5333685874938965,
+      "logps/rejected": -156.28958129882812,
+      "loss": 0.2237,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.29162219166755676,
+      "rewards/margins": 1.3938542604446411,
+      "rewards/rejected": -1.1022322177886963,
+      "step": 260
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 0.345703125,
+      "learning_rate": 6.559367010166629e-08,
+      "logits/chosen": -2.699760675430298,
+      "logits/rejected": -2.696230411529541,
+      "logps/chosen": -0.23872259259223938,
+      "logps/rejected": -159.244140625,
+      "loss": 0.2139,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3005048334598541,
+      "rewards/margins": 1.4419862031936646,
+      "rewards/rejected": -1.1414815187454224,
+      "step": 270
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 0.400390625,
+      "learning_rate": 1.4767944166687032e-08,
+      "logits/chosen": -2.699716091156006,
+      "logits/rejected": -2.6911072731018066,
+      "logps/chosen": -0.23203042149543762,
+      "logps/rejected": -159.73312377929688,
+      "loss": 0.2128,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.3060615360736847,
+      "rewards/margins": 1.44607675075531,
+      "rewards/rejected": -1.1400153636932373,
+      "step": 280
+    },
+    {
+      "epoch": 1.0,
+      "step": 289,
+      "total_flos": 0.0,
+      "train_loss": 0.36633619635162884,
+      "train_runtime": 628.6241,
+      "train_samples_per_second": 1.841,
+      "train_steps_per_second": 0.46
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 289,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}