Model save

README.md

@@ -0,0 +1,59 @@
+---
+base_model: mistralai/Mistral-7B-Instruct-v0.2
+library_name: peft
+license: apache-2.0
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: Mistral-7B-Instruct-v0.2-mergedORPO-r8
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# Mistral-7B-Instruct-v0.2-mergedORPO-r8
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 1
+- eval_batch_size: 1
+- seed: 42
+- gradient_accumulation_steps: 64
+- total_train_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 10
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.12.0
+- Transformers 4.44.2
+- Pytorch 2.4.0+cu121
+- Datasets 2.21.0
+- Tokenizers 0.19.1

last-checkpoint/adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
+    "up_proj",
     "q_proj",
     "o_proj",
+    "v_proj",
     "gate_proj",
     "k_proj",
-    "v_proj",
-    "up_proj"
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2a1e4c3b90d664cf744230b1635c69595b3e6feb863b87b843e43e3d3a7ae1d5
+oid sha256:349c41609a51e6cfc6896a2a5d1a67153c68f9e2b7976427d560cec1ef2695d0
 size 83945296

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d2a154beda3c570368f5acddceb38cab4dc0cd4ce9afacb1a4b5ede7235e6a54
+oid sha256:66a5843d74545eb236796350e0ece8c200e91256593c92b31b5d53bd3686a8f4
 size 43706258

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:25704ee5478cd4521ee5db3e9e14050e9c72edebe1088acf05e49ba2bb8ed7fb
+oid sha256:5f6daa0fed03c921127a64c255f462e66919bc76a6e6ba1e9093cf71d1068c6f
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9990442816183498,
+  "epoch": 0.897100987575661,
   "eval_steps": 500,
-  "global_step": 49,
+  "global_step": 44,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -799,96 +799,6 @@
       "rewards/margins": 0.0255814790725708,
       "rewards/rejected": -0.08916367590427399,
       "step": 44
-    },
-    {
-      "epoch": 0.9174896463841988,
-      "grad_norm": 1.5401713848114014,
-      "learning_rate": 1.0256410256410256e-05,
-      "log_odds_chosen": 0.5436590313911438,
-      "log_odds_ratio": -0.49072104692459106,
-      "logits/chosen": -2.357189655303955,
-      "logits/rejected": -2.368823528289795,
-      "logps/chosen": -0.6493522524833679,
-      "logps/rejected": -0.9346315860748291,
-      "loss": 0.9199,
-      "nll_loss": 0.8708474040031433,
-      "rewards/accuracies": 0.859375,
-      "rewards/chosen": -0.06493522971868515,
-      "rewards/margins": 0.02852793037891388,
-      "rewards/rejected": -0.09346316009759903,
-      "step": 45
-    },
-    {
-      "epoch": 0.9378783051927365,
-      "grad_norm": 1.42889404296875,
-      "learning_rate": 7.692307692307694e-06,
-      "log_odds_chosen": 0.49409711360931396,
-      "log_odds_ratio": -0.506333589553833,
-      "logits/chosen": -2.390031576156616,
-      "logits/rejected": -2.3763091564178467,
-      "logps/chosen": -0.6777428984642029,
-      "logps/rejected": -0.9434182643890381,
-      "loss": 0.9246,
-      "nll_loss": 0.8739202618598938,
-      "rewards/accuracies": 0.828125,
-      "rewards/chosen": -0.06777428835630417,
-      "rewards/margins": 0.02656753547489643,
-      "rewards/rejected": -0.09434183686971664,
-      "step": 46
-    },
-    {
-      "epoch": 0.9582669640012743,
-      "grad_norm": 1.4630545377731323,
-      "learning_rate": 5.128205128205128e-06,
-      "log_odds_chosen": 0.5089167356491089,
-      "log_odds_ratio": -0.49921250343322754,
-      "logits/chosen": -2.377633810043335,
-      "logits/rejected": -2.36307954788208,
-      "logps/chosen": -0.6796414256095886,
-      "logps/rejected": -0.9439955353736877,
-      "loss": 0.969,
-      "nll_loss": 0.9190601110458374,
-      "rewards/accuracies": 0.828125,
-      "rewards/chosen": -0.06796413660049438,
-      "rewards/margins": 0.02643541246652603,
-      "rewards/rejected": -0.09439954906702042,
-      "step": 47
-    },
-    {
-      "epoch": 0.978655622809812,
-      "grad_norm": 1.4280872344970703,
-      "learning_rate": 2.564102564102564e-06,
-      "log_odds_chosen": 0.5206915736198425,
-      "log_odds_ratio": -0.4926137626171112,
-      "logits/chosen": -2.381681442260742,
-      "logits/rejected": -2.3803200721740723,
-      "logps/chosen": -0.6376308798789978,
-      "logps/rejected": -0.9052294492721558,
-      "loss": 0.894,
-      "nll_loss": 0.8447284698486328,
-      "rewards/accuracies": 0.890625,
-      "rewards/chosen": -0.0637630894780159,
-      "rewards/margins": 0.026759855449199677,
-      "rewards/rejected": -0.09052293747663498,
-      "step": 48
-    },
-    {
-      "epoch": 0.9990442816183498,
-      "grad_norm": 1.549211859703064,
-      "learning_rate": 0.0,
-      "log_odds_chosen": 0.5898391008377075,
-      "log_odds_ratio": -0.47122782468795776,
-      "logits/chosen": -2.377497911453247,
-      "logits/rejected": -2.376384735107422,
-      "logps/chosen": -0.6647161245346069,
-      "logps/rejected": -0.974918007850647,
-      "loss": 0.9571,
-      "nll_loss": 0.9099375605583191,
-      "rewards/accuracies": 0.875,
-      "rewards/chosen": -0.06647161394357681,
-      "rewards/margins": 0.031020179390907288,
-      "rewards/rejected": -0.0974918082356453,
-      "step": 49
     }
   ],
   "logging_steps": 1,
@@ -903,7 +813,7 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:128f0cf3da8b1abd86530282a4bc926272323030313ebb6ed3dbec18e05701bf
+oid sha256:c6bcc3f11131b07d173fb79ad8c31d1a50ed3843eadae9ec2e4213021d99bed1
 size 5560