Upload folder using huggingface_hub

README.md

@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: EleutherAI/gpt-neo-1.3B
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

adapter_config.json

@@ -0,0 +1,30 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "EleutherAI/gpt-neo-1.3B",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj",
+    "k_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:feebdf15b898053d608d81daa4cd6dd099ba6cf7071641ae91ca882a1baebaf4
+size 9457000

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2641f2cb2b4fdeee3f9a4ee5a13215510784a49436a654bc8c3ccb112504636d
+size 18959674

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:901454eae3a785b11565176eda263a4901a5e801f61aaac1a63fa07ac7277b3e
+size 14180

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2754bf07d01336164e4bfa949e826e310b35cd5348cd170cf25a5bcbcf51c8d0
+size 1064

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,22 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 2048,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

trainer_state.json

@@ -0,0 +1,1355 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.260327357755261,
+  "eval_steps": 200,
+  "global_step": 5800,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.03897116134060795,
+      "grad_norm": 0.21226342022418976,
+      "learning_rate": 0.00025,
+      "logits/chosen": -19.687898635864258,
+      "logits/rejected": -18.633106231689453,
+      "logps/chosen": -343.7872619628906,
+      "logps/rejected": -250.41775512695312,
+      "loss": 0.4838,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 0.6970043182373047,
+      "rewards/margins": 0.9049937725067139,
+      "rewards/rejected": -0.2079893946647644,
+      "step": 100
+    },
+    {
+      "epoch": 0.0779423226812159,
+      "grad_norm": 1.92564857006073,
+      "learning_rate": 0.0005,
+      "logits/chosen": -19.59960174560547,
+      "logits/rejected": -18.517173767089844,
+      "logps/chosen": -349.5354309082031,
+      "logps/rejected": -277.8677062988281,
+      "loss": 0.2445,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 1.4490649700164795,
+      "rewards/margins": 3.1850626468658447,
+      "rewards/rejected": -1.7359976768493652,
+      "step": 200
+    },
+    {
+      "epoch": 0.0779423226812159,
+      "eval_logits/chosen": -16.551204681396484,
+      "eval_logits/rejected": -16.51001739501953,
+      "eval_logps/chosen": -340.37274169921875,
+      "eval_logps/rejected": -285.42059326171875,
+      "eval_loss": 0.19902318716049194,
+      "eval_rewards/accuracies": 0.9711538553237915,
+      "eval_rewards/chosen": 0.12859317660331726,
+      "eval_rewards/margins": 1.9796644449234009,
+      "eval_rewards/rejected": -1.8510712385177612,
+      "eval_runtime": 183.9267,
+      "eval_samples_per_second": 1.131,
+      "eval_steps_per_second": 0.565,
+      "step": 200
+    },
+    {
+      "epoch": 0.11691348402182385,
+      "grad_norm": 1.4603872299194336,
+      "learning_rate": 0.0004997805905390462,
+      "logits/chosen": -19.685632705688477,
+      "logits/rejected": -18.630176544189453,
+      "logps/chosen": -347.72479248046875,
+      "logps/rejected": -297.4257507324219,
+      "loss": 0.195,
+      "rewards/accuracies": 0.8799999952316284,
+      "rewards/chosen": 0.857117235660553,
+      "rewards/margins": 4.945696830749512,
+      "rewards/rejected": -4.088580131530762,
+      "step": 300
+    },
+    {
+      "epoch": 0.1558846453624318,
+      "grad_norm": 2.9201486110687256,
+      "learning_rate": 0.0004991227472802768,
+      "logits/chosen": -19.715879440307617,
+      "logits/rejected": -18.654006958007812,
+      "logps/chosen": -347.0750427246094,
+      "logps/rejected": -297.565185546875,
+      "loss": 0.1965,
+      "rewards/accuracies": 0.8949999809265137,
+      "rewards/chosen": 0.27759745717048645,
+      "rewards/margins": 5.794647693634033,
+      "rewards/rejected": -5.517050266265869,
+      "step": 400
+    },
+    {
+      "epoch": 0.1558846453624318,
+      "eval_logits/chosen": -16.683916091918945,
+      "eval_logits/rejected": -16.602561950683594,
+      "eval_logps/chosen": -343.2335510253906,
+      "eval_logps/rejected": -309.0007629394531,
+      "eval_loss": 0.04625505581498146,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.15748834609985352,
+      "eval_rewards/margins": 4.051597595214844,
+      "eval_rewards/rejected": -4.2090864181518555,
+      "eval_runtime": 183.7705,
+      "eval_samples_per_second": 1.132,
+      "eval_steps_per_second": 0.566,
+      "step": 400
+    },
+    {
+      "epoch": 0.19485580670303976,
+      "grad_norm": 0.7862759232521057,
+      "learning_rate": 0.0004980276249199705,
+      "logits/chosen": -19.461082458496094,
+      "logits/rejected": -18.434755325317383,
+      "logps/chosen": -370.48394775390625,
+      "logps/rejected": -314.01043701171875,
+      "loss": 0.1873,
+      "rewards/accuracies": 0.8974999785423279,
+      "rewards/chosen": 0.1783198118209839,
+      "rewards/margins": 5.953300952911377,
+      "rewards/rejected": -5.774980545043945,
+      "step": 500
+    },
+    {
+      "epoch": 0.2338269680436477,
+      "grad_norm": 0.5820030570030212,
+      "learning_rate": 0.0004964971456997812,
+      "logits/chosen": -19.28722381591797,
+      "logits/rejected": -18.402751922607422,
+      "logps/chosen": -369.10980224609375,
+      "logps/rejected": -328.44403076171875,
+      "loss": 0.19,
+      "rewards/accuracies": 0.9024999737739563,
+      "rewards/chosen": -0.12349352240562439,
+      "rewards/margins": 6.251636505126953,
+      "rewards/rejected": -6.375130653381348,
+      "step": 600
+    },
+    {
+      "epoch": 0.2338269680436477,
+      "eval_logits/chosen": -16.357208251953125,
+      "eval_logits/rejected": -16.30510139465332,
+      "eval_logps/chosen": -336.88409423828125,
+      "eval_logps/rejected": -304.0970764160156,
+      "eval_loss": 0.036708563566207886,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.47745734453201294,
+      "eval_rewards/margins": 4.196179389953613,
+      "eval_rewards/rejected": -3.718721866607666,
+      "eval_runtime": 183.7701,
+      "eval_samples_per_second": 1.132,
+      "eval_steps_per_second": 0.566,
+      "step": 600
+    },
+    {
+      "epoch": 0.2727981293842556,
+      "grad_norm": 0.3981655538082123,
+      "learning_rate": 0.0004945339960326746,
+      "logits/chosen": -19.472816467285156,
+      "logits/rejected": -18.500709533691406,
+      "logps/chosen": -368.1885681152344,
+      "logps/rejected": -322.74298095703125,
+      "loss": 0.171,
+      "rewards/accuracies": 0.9012500047683716,
+      "rewards/chosen": -0.00018103599722962826,
+      "rewards/margins": 6.297857761383057,
+      "rewards/rejected": -6.298038959503174,
+      "step": 700
+    },
+    {
+      "epoch": 0.3117692907248636,
+      "grad_norm": 0.429470956325531,
+      "learning_rate": 0.0004921416217875326,
+      "logits/chosen": -19.675304412841797,
+      "logits/rejected": -18.701597213745117,
+      "logps/chosen": -354.1678771972656,
+      "logps/rejected": -312.2345886230469,
+      "loss": 0.1742,
+      "rewards/accuracies": 0.8912500143051147,
+      "rewards/chosen": 0.06210971996188164,
+      "rewards/margins": 6.645627498626709,
+      "rewards/rejected": -6.583518028259277,
+      "step": 800
+    },
+    {
+      "epoch": 0.3117692907248636,
+      "eval_logits/chosen": -16.51410484313965,
+      "eval_logits/rejected": -16.480632781982422,
+      "eval_logps/chosen": -338.15936279296875,
+      "eval_logps/rejected": -315.7767333984375,
+      "eval_loss": 0.023502787575125694,
+      "eval_rewards/accuracies": 0.995192289352417,
+      "eval_rewards/chosen": 0.34993284940719604,
+      "eval_rewards/margins": 5.236622333526611,
+      "eval_rewards/rejected": -4.886689186096191,
+      "eval_runtime": 183.2137,
+      "eval_samples_per_second": 1.135,
+      "eval_steps_per_second": 0.568,
+      "step": 800
+    },
+    {
+      "epoch": 0.35074045206547155,
+      "grad_norm": 0.3401203453540802,
+      "learning_rate": 0.0004893242222407031,
+      "logits/chosen": -19.789016723632812,
+      "logits/rejected": -18.94336700439453,
+      "logps/chosen": -343.6241760253906,
+      "logps/rejected": -323.7110595703125,
+      "loss": 0.1681,
+      "rewards/accuracies": 0.9087499976158142,
+      "rewards/chosen": 0.2733341455459595,
+      "rewards/margins": 6.843422889709473,
+      "rewards/rejected": -6.5700883865356445,
+      "step": 900
+    },
+    {
+      "epoch": 0.3897116134060795,
+      "grad_norm": 1.2149152755737305,
+      "learning_rate": 0.00048608674270511344,
+      "logits/chosen": -19.847705841064453,
+      "logits/rejected": -18.942218780517578,
+      "logps/chosen": -359.46551513671875,
+      "logps/rejected": -334.84136962890625,
+      "loss": 0.1605,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 0.1338280439376831,
+      "rewards/margins": 7.973731517791748,
+      "rewards/rejected": -7.839903354644775,
+      "step": 1000
+    },
+    {
+      "epoch": 0.3897116134060795,
+      "eval_logits/chosen": -16.609466552734375,
+      "eval_logits/rejected": -16.567523956298828,
+      "eval_logps/chosen": -334.4723205566406,
+      "eval_logps/rejected": -314.89141845703125,
+      "eval_loss": 0.010429470799863338,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.7186338901519775,
+      "eval_rewards/margins": 5.516786575317383,
+      "eval_rewards/rejected": -4.798152446746826,
+      "eval_runtime": 183.9441,
+      "eval_samples_per_second": 1.131,
+      "eval_steps_per_second": 0.565,
+      "step": 1000
+    },
+    {
+      "epoch": 0.4286827747466875,
+      "grad_norm": 15.479838371276855,
+      "learning_rate": 0.00048243486584988133,
+      "logits/chosen": -19.61993408203125,
+      "logits/rejected": -18.709421157836914,
+      "logps/chosen": -361.1015625,
+      "logps/rejected": -337.32342529296875,
+      "loss": 0.173,
+      "rewards/accuracies": 0.9037500023841858,
+      "rewards/chosen": -0.6482878923416138,
+      "rewards/margins": 7.439374923706055,
+      "rewards/rejected": -8.087662696838379,
+      "step": 1100
+    },
+    {
+      "epoch": 0.4676539360872954,
+      "grad_norm": 2.251624822616577,
+      "learning_rate": 0.00047837500172566503,
+      "logits/chosen": -19.768835067749023,
+      "logits/rejected": -18.762409210205078,
+      "logps/chosen": -365.1903076171875,
+      "logps/rejected": -328.1022644042969,
+      "loss": 0.19,
+      "rewards/accuracies": 0.9200000166893005,
+      "rewards/chosen": -0.6630336046218872,
+      "rewards/margins": 6.923609733581543,
+      "rewards/rejected": -7.586642265319824,
+      "step": 1200
+    },
+    {
+      "epoch": 0.4676539360872954,
+      "eval_logits/chosen": -16.668439865112305,
+      "eval_logits/rejected": -16.63960075378418,
+      "eval_logps/chosen": -348.08587646484375,
+      "eval_logps/rejected": -331.30828857421875,
+      "eval_loss": 0.013935078866779804,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.6427211761474609,
+      "eval_rewards/margins": 5.797122478485107,
+      "eval_rewards/rejected": -6.43984317779541,
+      "eval_runtime": 183.93,
+      "eval_samples_per_second": 1.131,
+      "eval_steps_per_second": 0.565,
+      "step": 1200
+    },
+    {
+      "epoch": 0.5066250974279034,
+      "grad_norm": 2.569606304168701,
+      "learning_rate": 0.00047391427651325704,
+      "logits/chosen": -20.112619400024414,
+      "logits/rejected": -19.097084045410156,
+      "logps/chosen": -361.98895263671875,
+      "logps/rejected": -335.0581970214844,
+      "loss": 0.1348,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -0.5764763355255127,
+      "rewards/margins": 7.914137363433838,
+      "rewards/rejected": -8.49061393737793,
+      "step": 1300
+    },
+    {
+      "epoch": 0.5455962587685113,
+      "grad_norm": 1.0773099660873413,
+      "learning_rate": 0.00046906052001517164,
+      "logits/chosen": -20.002775192260742,
+      "logits/rejected": -19.024526596069336,
+      "logps/chosen": -369.9784851074219,
+      "logps/rejected": -346.8582763671875,
+      "loss": 0.1693,
+      "rewards/accuracies": 0.9237499833106995,
+      "rewards/chosen": -1.2993569374084473,
+      "rewards/margins": 8.027504920959473,
+      "rewards/rejected": -9.326862335205078,
+      "step": 1400
+    },
+    {
+      "epoch": 0.5455962587685113,
+      "eval_logits/chosen": -16.792936325073242,
+      "eval_logits/rejected": -16.73811912536621,
+      "eval_logps/chosen": -335.2118835449219,
+      "eval_logps/rejected": -317.7545471191406,
+      "eval_loss": 0.019140072166919708,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.6446801424026489,
+      "eval_rewards/margins": 5.729149341583252,
+      "eval_rewards/rejected": -5.084469318389893,
+      "eval_runtime": 183.9981,
+      "eval_samples_per_second": 1.13,
+      "eval_steps_per_second": 0.565,
+      "step": 1400
+    },
+    {
+      "epoch": 0.5845674201091192,
+      "grad_norm": 3.4432373046875,
+      "learning_rate": 0.00046382225191218373,
+      "logits/chosen": -20.071659088134766,
+      "logits/rejected": -19.052783966064453,
+      "logps/chosen": -363.5613708496094,
+      "logps/rejected": -345.07879638671875,
+      "loss": 0.1425,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -0.769163966178894,
+      "rewards/margins": 8.191140174865723,
+      "rewards/rejected": -8.96030330657959,
+      "step": 1500
+    },
+    {
+      "epoch": 0.6235385814497272,
+      "grad_norm": 1.9013152122497559,
+      "learning_rate": 0.0004582086668089399,
+      "logits/chosen": -19.665212631225586,
+      "logits/rejected": -18.75629234313965,
+      "logps/chosen": -367.5525817871094,
+      "logps/rejected": -342.85516357421875,
+      "loss": 0.1263,
+      "rewards/accuracies": 0.9325000047683716,
+      "rewards/chosen": -0.8581268191337585,
+      "rewards/margins": 8.124137878417969,
+      "rewards/rejected": -8.982264518737793,
+      "step": 1600
+    },
+    {
+      "epoch": 0.6235385814497272,
+      "eval_logits/chosen": -16.445354461669922,
+      "eval_logits/rejected": -16.387128829956055,
+      "eval_logps/chosen": -327.33563232421875,
+      "eval_logps/rejected": -312.2492980957031,
+      "eval_loss": 0.0060209427028894424,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 1.4323018789291382,
+      "eval_rewards/margins": 5.966245174407959,
+      "eval_rewards/rejected": -4.5339436531066895,
+      "eval_runtime": 184.0733,
+      "eval_samples_per_second": 1.13,
+      "eval_steps_per_second": 0.565,
+      "step": 1600
+    },
+    {
+      "epoch": 0.6625097427903351,
+      "grad_norm": 0.8941299915313721,
+      "learning_rate": 0.0004522296180948922,
+      "logits/chosen": -19.424589157104492,
+      "logits/rejected": -18.495784759521484,
+      "logps/chosen": -374.550537109375,
+      "logps/rejected": -353.1033630371094,
+      "loss": 0.125,
+      "rewards/accuracies": 0.9387500286102295,
+      "rewards/chosen": -0.5644223690032959,
+      "rewards/margins": 8.560413360595703,
+      "rewards/rejected": -9.124835014343262,
+      "step": 1700
+    },
+    {
+      "epoch": 0.7014809041309431,
+      "grad_norm": 1.1238608360290527,
+      "learning_rate": 0.00044589560064888347,
+      "logits/chosen": -19.239290237426758,
+      "logits/rejected": -18.375873565673828,
+      "logps/chosen": -369.72027587890625,
+      "logps/rejected": -349.2558898925781,
+      "loss": 0.1232,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.9773778319358826,
+      "rewards/margins": 8.248481750488281,
+      "rewards/rejected": -9.225860595703125,
+      "step": 1800
+    },
+    {
+      "epoch": 0.7014809041309431,
+      "eval_logits/chosen": -16.073932647705078,
+      "eval_logits/rejected": -16.034406661987305,
+      "eval_logps/chosen": -343.9805908203125,
+      "eval_logps/rejected": -342.4121398925781,
+      "eval_loss": 0.005078889429569244,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.23218610882759094,
+      "eval_rewards/margins": 7.318041801452637,
+      "eval_rewards/rejected": -7.550227642059326,
+      "eval_runtime": 183.2955,
+      "eval_samples_per_second": 1.135,
+      "eval_steps_per_second": 0.567,
+      "step": 1800
+    },
+    {
+      "epoch": 0.7404520654715511,
+      "grad_norm": 3.126462697982788,
+      "learning_rate": 0.00043921773241774185,
+      "logits/chosen": -19.31768226623535,
+      "logits/rejected": -18.37215232849121,
+      "logps/chosen": -348.50390625,
+      "logps/rejected": -336.8278503417969,
+      "loss": 0.0923,
+      "rewards/accuracies": 0.9587500095367432,
+      "rewards/chosen": -0.21741238236427307,
+      "rewards/margins": 8.6088228225708,
+      "rewards/rejected": -8.826234817504883,
+      "step": 1900
+    },
+    {
+      "epoch": 0.779423226812159,
+      "grad_norm": 0.6362993121147156,
+      "learning_rate": 0.0004322077349012186,
+      "logits/chosen": -19.6059513092041,
+      "logits/rejected": -18.7484073638916,
+      "logps/chosen": -359.93206787109375,
+      "logps/rejected": -353.2133483886719,
+      "loss": 0.0983,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -1.2052465677261353,
+      "rewards/margins": 8.998382568359375,
+      "rewards/rejected": -10.203628540039062,
+      "step": 2000
+    },
+    {
+      "epoch": 0.779423226812159,
+      "eval_logits/chosen": -16.276912689208984,
+      "eval_logits/rejected": -16.23944854736328,
+      "eval_logps/chosen": -340.11883544921875,
+      "eval_logps/rejected": -336.9291687011719,
+      "eval_loss": 0.003970090765506029,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.15398484468460083,
+      "eval_rewards/margins": 7.155914783477783,
+      "eval_rewards/rejected": -7.001929759979248,
+      "eval_runtime": 184.065,
+      "eval_samples_per_second": 1.13,
+      "eval_steps_per_second": 0.565,
+      "step": 2000
+    },
+    {
+      "epoch": 0.818394388152767,
+      "grad_norm": 3.1369752883911133,
+      "learning_rate": 0.00042487791257752493,
+      "logits/chosen": -19.724271774291992,
+      "logits/rejected": -18.804405212402344,
+      "logps/chosen": -371.5325012207031,
+      "logps/rejected": -357.1531677246094,
+      "loss": 0.1049,
+      "rewards/accuracies": 0.9512500166893005,
+      "rewards/chosen": -1.3020814657211304,
+      "rewards/margins": 8.643760681152344,
+      "rewards/rejected": -9.945842742919922,
+      "step": 2100
+    },
+    {
+      "epoch": 0.857365549493375,
+      "grad_norm": 1.0814727544784546,
+      "learning_rate": 0.0004172411313055802,
+      "logits/chosen": -19.90325164794922,
+      "logits/rejected": -19.001785278320312,
+      "logps/chosen": -376.0804748535156,
+      "logps/rejected": -359.4775390625,
+      "loss": 0.0924,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.4394536316394806,
+      "rewards/margins": 8.659097671508789,
+      "rewards/rejected": -9.098551750183105,
+      "step": 2200
+    },
+    {
+      "epoch": 0.857365549493375,
+      "eval_logits/chosen": -16.71300506591797,
+      "eval_logits/rejected": -16.685504913330078,
+      "eval_logps/chosen": -334.0523986816406,
+      "eval_logps/rejected": -324.8212890625,
+      "eval_loss": 0.00510050356388092,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.760625958442688,
+      "eval_rewards/margins": 6.551769733428955,
+      "eval_rewards/rejected": -5.791143417358398,
+      "eval_runtime": 183.3621,
+      "eval_samples_per_second": 1.134,
+      "eval_steps_per_second": 0.567,
+      "step": 2200
+    },
+    {
+      "epoch": 0.8963367108339828,
+      "grad_norm": 2.7771525382995605,
+      "learning_rate": 0.0004093107957418828,
+      "logits/chosen": -19.735408782958984,
+      "logits/rejected": -18.8337459564209,
+      "logps/chosen": -367.6742858886719,
+      "logps/rejected": -356.5105895996094,
+      "loss": 0.0789,
+      "rewards/accuracies": 0.9612500071525574,
+      "rewards/chosen": -1.1817975044250488,
+      "rewards/margins": 9.191315650939941,
+      "rewards/rejected": -10.373113632202148,
+      "step": 2300
+    },
+    {
+      "epoch": 0.9353078721745908,
+      "grad_norm": 0.0016880702460184693,
+      "learning_rate": 0.0004011008258116425,
+      "logits/chosen": -19.32187271118164,
+      "logits/rejected": -18.563722610473633,
+      "logps/chosen": -373.0425109863281,
+      "logps/rejected": -362.72589111328125,
+      "loss": 0.0848,
+      "rewards/accuracies": 0.9649999737739563,
+      "rewards/chosen": -2.054009437561035,
+      "rewards/margins": 9.20674991607666,
+      "rewards/rejected": -11.260760307312012,
+      "step": 2400
+    },
+    {
+      "epoch": 0.9353078721745908,
+      "eval_logits/chosen": -16.159223556518555,
+      "eval_logits/rejected": -16.161327362060547,
+      "eval_logps/chosen": -337.83880615234375,
+      "eval_logps/rejected": -339.8623046875,
+      "eval_loss": 0.0020727256778627634,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.38198867440223694,
+      "eval_rewards/margins": 7.677234172821045,
+      "eval_rewards/rejected": -7.295245170593262,
+      "eval_runtime": 183.8954,
+      "eval_samples_per_second": 1.131,
+      "eval_steps_per_second": 0.566,
+      "step": 2400
+    },
+    {
+      "epoch": 0.9742790335151987,
+      "grad_norm": 1.488077998161316,
+      "learning_rate": 0.00039262563227547396,
+      "logits/chosen": -19.280136108398438,
+      "logits/rejected": -18.451894760131836,
+      "logps/chosen": -376.1581115722656,
+      "logps/rejected": -366.6534118652344,
+      "loss": 0.0662,
+      "rewards/accuracies": 0.9725000262260437,
+      "rewards/chosen": -1.176061987876892,
+      "rewards/margins": 8.959639549255371,
+      "rewards/rejected": -10.135702133178711,
+      "step": 2500
+    },
+    {
+      "epoch": 1.0132501948558068,
+      "grad_norm": 2.0352509021759033,
+      "learning_rate": 0.0003839000914345393,
+      "logits/chosen": -19.330242156982422,
+      "logits/rejected": -18.416221618652344,
+      "logps/chosen": -373.55523681640625,
+      "logps/rejected": -357.8737487792969,
+      "loss": 0.0373,
+      "rewards/accuracies": 0.9837499856948853,
+      "rewards/chosen": -0.7969868183135986,
+      "rewards/margins": 9.221331596374512,
+      "rewards/rejected": -10.018318176269531,
+      "step": 2600
+    },
+    {
+      "epoch": 1.0132501948558068,
+      "eval_logits/chosen": -15.664155006408691,
+      "eval_logits/rejected": -15.684382438659668,
+      "eval_logps/chosen": -348.93939208984375,
+      "eval_logps/rejected": -357.6224365234375,
+      "eval_loss": 0.001914023538120091,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.7280683517456055,
+      "eval_rewards/margins": 8.34318733215332,
+      "eval_rewards/rejected": -9.071255683898926,
+      "eval_runtime": 183.3393,
+      "eval_samples_per_second": 1.135,
+      "eval_steps_per_second": 0.567,
+      "step": 2600
+    },
+    {
+      "epoch": 1.0522213561964147,
+      "grad_norm": 1.1277427673339844,
+      "learning_rate": 0.00037493951901853797,
+      "logits/chosen": -19.345949172973633,
+      "logits/rejected": -18.49242401123047,
+      "logps/chosen": -365.28350830078125,
+      "logps/rejected": -361.580810546875,
+      "loss": 0.021,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -0.9318392872810364,
+      "rewards/margins": 9.834463119506836,
+      "rewards/rejected": -10.766302108764648,
+      "step": 2700
+    },
+    {
+      "epoch": 1.0911925175370225,
+      "grad_norm": 0.07178483158349991,
+      "learning_rate": 0.00036575964330237904,
+      "logits/chosen": -19.510316848754883,
+      "logits/rejected": -18.55198860168457,
+      "logps/chosen": -367.44598388671875,
+      "logps/rejected": -363.74078369140625,
+      "loss": 0.0151,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -1.267324447631836,
+      "rewards/margins": 10.227116584777832,
+      "rewards/rejected": -11.494441032409668,
+      "step": 2800
+    },
+    {
+      "epoch": 1.0911925175370225,
+      "eval_logits/chosen": -15.850824356079102,
+      "eval_logits/rejected": -15.864118576049805,
+      "eval_logps/chosen": -359.7252502441406,
+      "eval_logps/rejected": -381.7374572753906,
+      "eval_loss": 0.0014377026818692684,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -1.8066591024398804,
+      "eval_rewards/margins": 9.676095962524414,
+      "eval_rewards/rejected": -11.482755661010742,
+      "eval_runtime": 183.4769,
+      "eval_samples_per_second": 1.134,
+      "eval_steps_per_second": 0.567,
+      "step": 2800
+    },
+    {
+      "epoch": 1.1301636788776306,
+      "grad_norm": 1.4340250492095947,
+      "learning_rate": 0.00035637657749872255,
+      "logits/chosen": -19.317781448364258,
+      "logits/rejected": -18.438331604003906,
+      "logps/chosen": -375.9151306152344,
+      "logps/rejected": -366.2434387207031,
+      "loss": 0.0137,
+      "rewards/accuracies": 0.9962499737739563,
+      "rewards/chosen": -2.468919277191162,
+      "rewards/margins": 10.26855754852295,
+      "rewards/rejected": -12.737476348876953,
+      "step": 2900
+    },
+    {
+      "epoch": 1.1691348402182384,
+      "grad_norm": 0.2114144116640091,
+      "learning_rate": 0.00034680679147484916,
+      "logits/chosen": -19.166513442993164,
+      "logits/rejected": -18.25996208190918,
+      "logps/chosen": -371.7158508300781,
+      "logps/rejected": -375.24481201171875,
+      "loss": 0.0177,
+      "rewards/accuracies": 0.9925000071525574,
+      "rewards/chosen": -1.933343768119812,
+      "rewards/margins": 10.125692367553711,
+      "rewards/rejected": -12.059035301208496,
+      "step": 3000
+    },
+    {
+      "epoch": 1.1691348402182384,
+      "eval_logits/chosen": -16.009708404541016,
+      "eval_logits/rejected": -15.956798553466797,
+      "eval_logps/chosen": -332.91302490234375,
+      "eval_logps/rejected": -345.1927490234375,
+      "eval_loss": 0.0008918559760786593,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.874567449092865,
+      "eval_rewards/margins": 8.702858924865723,
+      "eval_rewards/rejected": -7.8282904624938965,
+      "eval_runtime": 184.0136,
+      "eval_samples_per_second": 1.13,
+      "eval_steps_per_second": 0.565,
+      "step": 3000
+    },
+    {
+      "epoch": 1.2081060015588465,
+      "grad_norm": 1.6861902475357056,
+      "learning_rate": 0.00033706708284350227,
+      "logits/chosen": -19.25012969970703,
+      "logits/rejected": -18.424623489379883,
+      "logps/chosen": -372.6192932128906,
+      "logps/rejected": -379.19549560546875,
+      "loss": 0.0209,
+      "rewards/accuracies": 0.9950000047683716,
+      "rewards/chosen": -1.3639909029006958,
+      "rewards/margins": 10.262293815612793,
+      "rewards/rejected": -11.626285552978516,
+      "step": 3100
+    },
+    {
+      "epoch": 1.2470771628994544,
+      "grad_norm": 0.16628094017505646,
+      "learning_rate": 0.00032717454747844735,
+      "logits/chosen": -19.219282150268555,
+      "logits/rejected": -18.238243103027344,
+      "logps/chosen": -365.1196594238281,
+      "logps/rejected": -364.3663330078125,
+      "loss": 0.0168,
+      "rewards/accuracies": 0.9912499785423279,
+      "rewards/chosen": -1.1836423873901367,
+      "rewards/margins": 10.353758811950684,
+      "rewards/rejected": -11.53740119934082,
+      "step": 3200
+    },
+    {
+      "epoch": 1.2470771628994544,
+      "eval_logits/chosen": -15.836195945739746,
+      "eval_logits/rejected": -15.786764144897461,
+      "eval_logps/chosen": -340.96075439453125,
+      "eval_logps/rejected": -357.6408996582031,
+      "eval_loss": 0.0008726614178158343,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.06979300826787949,
+      "eval_rewards/margins": 9.142892837524414,
+      "eval_rewards/rejected": -9.073100090026855,
+      "eval_runtime": 183.8457,
+      "eval_samples_per_second": 1.131,
+      "eval_steps_per_second": 0.566,
+      "step": 3200
+    },
+    {
+      "epoch": 1.2860483242400624,
+      "grad_norm": 0.0003638103371486068,
+      "learning_rate": 0.00031714654950649947,
+      "logits/chosen": -19.43454933166504,
+      "logits/rejected": -18.49201011657715,
+      "logps/chosen": -371.1318664550781,
+      "logps/rejected": -372.17938232421875,
+      "loss": 0.0163,
+      "rewards/accuracies": 0.9912499785423279,
+      "rewards/chosen": -1.4684104919433594,
+      "rewards/margins": 10.204544067382812,
+      "rewards/rejected": -11.672954559326172,
+      "step": 3300
+    },
+    {
+      "epoch": 1.3250194855806703,
+      "grad_norm": 0.0004742901655845344,
+      "learning_rate": 0.0003070006908286945,
+      "logits/chosen": -19.53050994873047,
+      "logits/rejected": -18.605304718017578,
+      "logps/chosen": -383.9638671875,
+      "logps/rejected": -382.642333984375,
+      "loss": 0.0159,
+      "rewards/accuracies": 0.9925000071525574,
+      "rewards/chosen": -1.5249592065811157,
+      "rewards/margins": 10.782021522521973,
+      "rewards/rejected": -12.306981086730957,
+      "step": 3400
+    },
+    {
+      "epoch": 1.3250194855806703,
+      "eval_logits/chosen": -15.98876953125,
+      "eval_logits/rejected": -15.942709922790527,
+      "eval_logps/chosen": -337.0641174316406,
+      "eval_logps/rejected": -357.69342041015625,
+      "eval_loss": 0.0007918892079032958,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.4594591557979584,
+      "eval_rewards/margins": 9.53781795501709,
+      "eval_rewards/rejected": -9.078359603881836,
+      "eval_runtime": 183.8285,
+      "eval_samples_per_second": 1.131,
+      "eval_steps_per_second": 0.566,
+      "step": 3400
+    },
+    {
+      "epoch": 1.3639906469212781,
+      "grad_norm": 3.363085985183716,
+      "learning_rate": 0.0002967547802240997,
+      "logits/chosen": -19.533775329589844,
+      "logits/rejected": -18.48381233215332,
+      "logps/chosen": -369.41827392578125,
+      "logps/rejected": -367.9228820800781,
+      "loss": 0.0152,
+      "rewards/accuracies": 0.9950000047683716,
+      "rewards/chosen": -0.9302732944488525,
+      "rewards/margins": 10.337385177612305,
+      "rewards/rejected": -11.267658233642578,
+      "step": 3500
+    },
+    {
+      "epoch": 1.4029618082618862,
+      "grad_norm": 0.014138066209852695,
+      "learning_rate": 0.00028642680209049715,
+      "logits/chosen": -19.569242477416992,
+      "logits/rejected": -18.72142791748047,
+      "logps/chosen": -355.16339111328125,
+      "logps/rejected": -369.4004211425781,
+      "loss": 0.0106,
+      "rewards/accuracies": 0.9987499713897705,
+      "rewards/chosen": -0.6573285460472107,
+      "rewards/margins": 10.500960350036621,
+      "rewards/rejected": -11.15829086303711,
+      "step": 3600
+    },
+    {
+      "epoch": 1.4029618082618862,
+      "eval_logits/chosen": -16.05227279663086,
+      "eval_logits/rejected": -16.000469207763672,
+      "eval_logps/chosen": -334.4986572265625,
+      "eval_logps/rejected": -352.8683166503906,
+      "eval_loss": 0.0009424146264791489,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.7160030603408813,
+      "eval_rewards/margins": 9.311846733093262,
+      "eval_rewards/rejected": -8.595843315124512,
+      "eval_runtime": 183.6848,
+      "eval_samples_per_second": 1.132,
+      "eval_steps_per_second": 0.566,
+      "step": 3600
+    },
+    {
+      "epoch": 1.4419329696024943,
+      "grad_norm": 0.009088713675737381,
+      "learning_rate": 0.00027603488487680684,
+      "logits/chosen": -19.91989517211914,
+      "logits/rejected": -18.872535705566406,
+      "logps/chosen": -372.661865234375,
+      "logps/rejected": -367.7705078125,
+      "loss": 0.0184,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -1.1601972579956055,
+      "rewards/margins": 10.311330795288086,
+      "rewards/rejected": -11.471526145935059,
+      "step": 3700
+    },
+    {
+      "epoch": 1.4809041309431021,
+      "grad_norm": 0.010365006513893604,
+      "learning_rate": 0.00026559726926266204,
+      "logits/chosen": -19.595075607299805,
+      "logits/rejected": -18.658893585205078,
+      "logps/chosen": -390.93206787109375,
+      "logps/rejected": -393.7774963378906,
+      "loss": 0.0095,
+      "rewards/accuracies": 0.9950000047683716,
+      "rewards/chosen": -2.7839019298553467,
+      "rewards/margins": 10.808964729309082,
+      "rewards/rejected": -13.592867851257324,
+      "step": 3800
+    },
+    {
+      "epoch": 1.4809041309431021,
+      "eval_logits/chosen": -15.648859977722168,
+      "eval_logits/rejected": -15.611715316772461,
+      "eval_logps/chosen": -346.69561767578125,
+      "eval_logps/rejected": -376.822265625,
+      "eval_loss": 0.0005593308596871793,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.5036950707435608,
+      "eval_rewards/margins": 10.487546920776367,
+      "eval_rewards/rejected": -10.991241455078125,
+      "eval_runtime": 184.1015,
+      "eval_samples_per_second": 1.13,
+      "eval_steps_per_second": 0.565,
+      "step": 3800
+    },
+    {
+      "epoch": 1.51987529228371,
+      "grad_norm": 0.12639912962913513,
+      "learning_rate": 0.00025513227614098707,
+      "logits/chosen": -19.499780654907227,
+      "logits/rejected": -18.518644332885742,
+      "logps/chosen": -366.645263671875,
+      "logps/rejected": -380.7431640625,
+      "loss": 0.0067,
+      "rewards/accuracies": 0.9975000023841858,
+      "rewards/chosen": -1.6605476140975952,
+      "rewards/margins": 10.881020545959473,
+      "rewards/rejected": -12.541565895080566,
+      "step": 3900
+    },
+    {
+      "epoch": 1.558846453624318,
+      "grad_norm": 0.32782620191574097,
+      "learning_rate": 0.00024465827445977964,
+      "logits/chosen": -19.350522994995117,
+      "logits/rejected": -18.31049346923828,
+      "logps/chosen": -380.0142822265625,
+      "logps/rejected": -377.0948791503906,
+      "loss": 0.0107,
+      "rewards/accuracies": 0.9962499737739563,
+      "rewards/chosen": -1.4461276531219482,
+      "rewards/margins": 10.850979804992676,
+      "rewards/rejected": -12.297107696533203,
+      "step": 4000
+    },
+    {
+      "epoch": 1.558846453624318,
+      "eval_logits/chosen": -15.838950157165527,
+      "eval_logits/rejected": -15.806854248046875,
+      "eval_logps/chosen": -336.4289245605469,
+      "eval_logps/rejected": -364.8307189941406,
+      "eval_loss": 0.00040981665370054543,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 0.5229762196540833,
+      "eval_rewards/margins": 10.315059661865234,
+      "eval_rewards/rejected": -9.792083740234375,
+      "eval_runtime": 184.0487,
+      "eval_samples_per_second": 1.13,
+      "eval_steps_per_second": 0.565,
+      "step": 4000
+    },
+    {
+      "epoch": 1.597817614964926,
+      "grad_norm": 1.2895886898040771,
+      "learning_rate": 0.00023419364897954285,
+      "logits/chosen": -19.494075775146484,
+      "logits/rejected": -18.560813903808594,
+      "logps/chosen": -361.8075256347656,
+      "logps/rejected": -380.0975646972656,
+      "loss": 0.0108,
+      "rewards/accuracies": 0.9950000047683716,
+      "rewards/chosen": -1.1338391304016113,
+      "rewards/margins": 11.00627326965332,
+      "rewards/rejected": -12.140111923217773,
+      "step": 4100
+    },
+    {
+      "epoch": 1.6367887763055338,
+      "grad_norm": 0.00033555322443135083,
+      "learning_rate": 0.00022375676800296247,
+      "logits/chosen": -19.587282180786133,
+      "logits/rejected": -18.464643478393555,
+      "logps/chosen": -379.6680603027344,
+      "logps/rejected": -372.1777648925781,
+      "loss": 0.0124,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -1.0544594526290894,
+      "rewards/margins": 11.037619590759277,
+      "rewards/rejected": -12.092079162597656,
+      "step": 4200
+    },
+    {
+      "epoch": 1.6367887763055338,
+      "eval_logits/chosen": -15.83126163482666,
+      "eval_logits/rejected": -15.794379234313965,
+      "eval_logps/chosen": -330.6876525878906,
+      "eval_logps/rejected": -356.1661682128906,
+      "eval_loss": 0.0004220473056193441,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": 1.0971003770828247,
+      "eval_rewards/margins": 10.022727012634277,
+      "eval_rewards/rejected": -8.925626754760742,
+      "eval_runtime": 183.9983,
+      "eval_samples_per_second": 1.13,
+      "eval_steps_per_second": 0.565,
+      "step": 4200
+    },
+    {
+      "epoch": 1.6757599376461418,
+      "grad_norm": 1.2421733140945435,
+      "learning_rate": 0.00021336595113347144,
+      "logits/chosen": -19.402978897094727,
+      "logits/rejected": -18.351083755493164,
+      "logps/chosen": -370.1186828613281,
+      "logps/rejected": -374.5020446777344,
+      "loss": 0.009,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -1.1320860385894775,
+      "rewards/margins": 10.839487075805664,
+      "rewards/rejected": -11.971571922302246,
+      "step": 4300
+    },
+    {
+      "epoch": 1.71473109898675,
+      "grad_norm": 0.16441740095615387,
+      "learning_rate": 0.00020303943711929526,
+      "logits/chosen": -19.135257720947266,
+      "logits/rejected": -18.16560173034668,
+      "logps/chosen": -375.6964111328125,
+      "logps/rejected": -390.9460144042969,
+      "loss": 0.021,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -2.0195655822753906,
+      "rewards/margins": 11.327723503112793,
+      "rewards/rejected": -13.3472900390625,
+      "step": 4400
+    },
+    {
+      "epoch": 1.71473109898675,
+      "eval_logits/chosen": -15.488088607788086,
+      "eval_logits/rejected": -15.465119361877441,
+      "eval_logps/chosen": -344.2342529296875,
+      "eval_logps/rejected": -376.2817687988281,
+      "eval_loss": 0.0005891394102945924,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.25755709409713745,
+      "eval_rewards/margins": 10.679630279541016,
+      "eval_rewards/rejected": -10.937186241149902,
+      "eval_runtime": 184.1206,
+      "eval_samples_per_second": 1.13,
+      "eval_steps_per_second": 0.565,
+      "step": 4400
+    },
+    {
+      "epoch": 1.7537022603273578,
+      "grad_norm": 0.02925615757703781,
+      "learning_rate": 0.00019279535183942101,
+      "logits/chosen": -19.17304039001465,
+      "logits/rejected": -18.184049606323242,
+      "logps/chosen": -385.8622131347656,
+      "logps/rejected": -391.7120361328125,
+      "loss": 0.0083,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/chosen": -1.9831608533859253,
+      "rewards/margins": 11.25387954711914,
+      "rewards/rejected": -13.237038612365723,
+      "step": 4500
+    },
+    {
+      "epoch": 1.7926734216679656,
+      "grad_norm": 0.05078015476465225,
+      "learning_rate": 0.00018265167648768259,
+      "logits/chosen": -19.064685821533203,
+      "logits/rejected": -18.237638473510742,
+      "logps/chosen": -376.9311218261719,
+      "logps/rejected": -387.96905517578125,
+      "loss": 0.0061,
+      "rewards/accuracies": 0.9950000047683716,
+      "rewards/chosen": -2.660050392150879,
+      "rewards/margins": 11.354269027709961,
+      "rewards/rejected": -14.014320373535156,
+      "step": 4600
+    },
+    {
+      "epoch": 1.7926734216679656,
+      "eval_logits/chosen": -15.26915454864502,
+      "eval_logits/rejected": -15.260515213012695,
+      "eval_logps/chosen": -355.8375549316406,
+      "eval_logps/rejected": -390.41119384765625,
+      "eval_loss": 0.0009181927889585495,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -1.417887568473816,
+      "eval_rewards/margins": 10.932249069213867,
+      "eval_rewards/rejected": -12.350136756896973,
+      "eval_runtime": 183.689,
+      "eval_samples_per_second": 1.132,
+      "eval_steps_per_second": 0.566,
+      "step": 4600
+    },
+    {
+      "epoch": 1.8316445830085737,
+      "grad_norm": 0.2935536205768585,
+      "learning_rate": 0.00017262621601080811,
+      "logits/chosen": -19.163440704345703,
+      "logits/rejected": -18.167402267456055,
+      "logps/chosen": -379.1352233886719,
+      "logps/rejected": -383.6081237792969,
+      "loss": 0.0126,
+      "rewards/accuracies": 0.9950000047683716,
+      "rewards/chosen": -2.3724796772003174,
+      "rewards/margins": 11.351487159729004,
+      "rewards/rejected": -13.723965644836426,
+      "step": 4700
+    },
+    {
+      "epoch": 1.8706157443491818,
+      "grad_norm": 0.02069302648305893,
+      "learning_rate": 0.00016273656785582984,
+      "logits/chosen": -18.975814819335938,
+      "logits/rejected": -18.142276763916016,
+      "logps/chosen": -383.4511413574219,
+      "logps/rejected": -393.5997009277344,
+      "loss": 0.0102,
+      "rewards/accuracies": 0.9950000047683716,
+      "rewards/chosen": -2.359323024749756,
+      "rewards/margins": 11.0841064453125,
+      "rewards/rejected": -13.443428993225098,
+      "step": 4800
+    },
+    {
+      "epoch": 1.8706157443491818,
+      "eval_logits/chosen": -15.47636604309082,
+      "eval_logits/rejected": -15.429464340209961,
+      "eval_logps/chosen": -341.96356201171875,
+      "eval_logps/rejected": -369.5177307128906,
+      "eval_loss": 0.0004053489537909627,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.030484315007925034,
+      "eval_rewards/margins": 10.230300903320312,
+      "eval_rewards/rejected": -10.260786056518555,
+      "eval_runtime": 183.2032,
+      "eval_samples_per_second": 1.135,
+      "eval_steps_per_second": 0.568,
+      "step": 4800
+    },
+    {
+      "epoch": 1.9095869056897894,
+      "grad_norm": 0.06283093243837357,
+      "learning_rate": 0.00015300009108171347,
+      "logits/chosen": -19.286584854125977,
+      "logits/rejected": -18.341442108154297,
+      "logps/chosen": -376.3005065917969,
+      "logps/rejected": -383.4756774902344,
+      "loss": 0.0044,
+      "rewards/accuracies": 0.9987499713897705,
+      "rewards/chosen": -1.3465908765792847,
+      "rewards/margins": 11.345978736877441,
+      "rewards/rejected": -12.692569732666016,
+      "step": 4900
+    },
+    {
+      "epoch": 1.9485580670303975,
+      "grad_norm": 0.04220004007220268,
+      "learning_rate": 0.00014343387588942392,
+      "logits/chosen": -19.211626052856445,
+      "logits/rejected": -18.220531463623047,
+      "logps/chosen": -389.9902648925781,
+      "logps/rejected": -397.5294494628906,
+      "loss": 0.0057,
+      "rewards/accuracies": 0.9962499737739563,
+      "rewards/chosen": -1.984826922416687,
+      "rewards/margins": 11.562482833862305,
+      "rewards/rejected": -13.547309875488281,
+      "step": 5000
+    },
+    {
+      "epoch": 1.9485580670303975,
+      "eval_logits/chosen": -15.311283111572266,
+      "eval_logits/rejected": -15.284743309020996,
+      "eval_logps/chosen": -350.1474304199219,
+      "eval_logps/rejected": -386.7135314941406,
+      "eval_loss": 0.00032912034657783806,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.8488726019859314,
+      "eval_rewards/margins": 11.131490707397461,
+      "eval_rewards/rejected": -11.980363845825195,
+      "eval_runtime": 183.6747,
+      "eval_samples_per_second": 1.132,
+      "eval_steps_per_second": 0.566,
+      "step": 5000
+    },
+    {
+      "epoch": 1.9875292283710055,
+      "grad_norm": 0.05133282020688057,
+      "learning_rate": 0.00013405471362391068,
+      "logits/chosen": -19.146968841552734,
+      "logits/rejected": -18.36100196838379,
+      "logps/chosen": -375.5859680175781,
+      "logps/rejected": -405.4987487792969,
+      "loss": 0.0055,
+      "rewards/accuracies": 0.9962499737739563,
+      "rewards/chosen": -2.0062613487243652,
+      "rewards/margins": 11.725434303283691,
+      "rewards/rejected": -13.731695175170898,
+      "step": 5100
+    },
+    {
+      "epoch": 2.0265003897116136,
+      "grad_norm": 0.003434022655710578,
+      "learning_rate": 0.00012487906730066888,
+      "logits/chosen": -19.30167579650879,
+      "logits/rejected": -18.223770141601562,
+      "logps/chosen": -376.9130859375,
+      "logps/rejected": -380.2550354003906,
+      "loss": 0.0025,
+      "rewards/accuracies": 0.9975000023841858,
+      "rewards/chosen": -1.750290870666504,
+      "rewards/margins": 11.900991439819336,
+      "rewards/rejected": -13.651283264160156,
+      "step": 5200
+    },
+    {
+      "epoch": 2.0265003897116136,
+      "eval_logits/chosen": -15.498884201049805,
+      "eval_logits/rejected": -15.481465339660645,
+      "eval_logps/chosen": -349.6870422363281,
+      "eval_logps/rejected": -385.3147888183594,
+      "eval_loss": 0.0003257237549405545,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.8028372526168823,
+      "eval_rewards/margins": 11.037657737731934,
+      "eval_rewards/rejected": -11.840494155883789,
+      "eval_runtime": 183.7823,
+      "eval_samples_per_second": 1.132,
+      "eval_steps_per_second": 0.566,
+      "step": 5200
+    },
+    {
+      "epoch": 2.0654715510522212,
+      "grad_norm": 0.026280131191015244,
+      "learning_rate": 0.00011592304270860795,
+      "logits/chosen": -19.42230796813965,
+      "logits/rejected": -18.394390106201172,
+      "logps/chosen": -400.98980712890625,
+      "logps/rejected": -409.95098876953125,
+      "loss": 0.005,
+      "rewards/accuracies": 0.9962499737739563,
+      "rewards/chosen": -2.090318202972412,
+      "rewards/margins": 11.665027618408203,
+      "rewards/rejected": -13.755346298217773,
+      "step": 5300
+    },
+    {
+      "epoch": 2.1044427123928293,
+      "grad_norm": 0.00010202277189819142,
+      "learning_rate": 0.00010720236013995221,
+      "logits/chosen": -19.477603912353516,
+      "logits/rejected": -18.485740661621094,
+      "logps/chosen": -365.8956298828125,
+      "logps/rejected": -397.5658264160156,
+      "loss": 0.0015,
+      "rewards/accuracies": 0.9987499713897705,
+      "rewards/chosen": -1.8521977663040161,
+      "rewards/margins": 12.37298583984375,
+      "rewards/rejected": -14.225183486938477,
+      "step": 5400
+    },
+    {
+      "epoch": 2.1044427123928293,
+      "eval_logits/chosen": -15.460063934326172,
+      "eval_logits/rejected": -15.440217018127441,
+      "eval_logps/chosen": -351.2416687011719,
+      "eval_logps/rejected": -389.2104797363281,
+      "eval_loss": 0.00034331483766436577,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.958295464515686,
+      "eval_rewards/margins": 11.27176570892334,
+      "eval_rewards/rejected": -12.230061531066895,
+      "eval_runtime": 183.6747,
+      "eval_samples_per_second": 1.132,
+      "eval_steps_per_second": 0.566,
+      "step": 5400
+    },
+    {
+      "epoch": 2.1434138737334374,
+      "grad_norm": 0.005236598663032055,
+      "learning_rate": 9.873232679679392e-05,
+      "logits/chosen": -19.37502670288086,
+      "logits/rejected": -18.3719482421875,
+      "logps/chosen": -381.2720947265625,
+      "logps/rejected": -403.01593017578125,
+      "loss": 0.0024,
+      "rewards/accuracies": 0.9975000023841858,
+      "rewards/chosen": -2.047292709350586,
+      "rewards/margins": 12.199724197387695,
+      "rewards/rejected": -14.247018814086914,
+      "step": 5500
+    },
+    {
+      "epoch": 2.182385035074045,
+      "grad_norm": 0.029128100723028183,
+      "learning_rate": 9.052780992273379e-05,
+      "logits/chosen": -19.223974227905273,
+      "logits/rejected": -18.315275192260742,
+      "logps/chosen": -378.5389404296875,
+      "logps/rejected": -399.8840637207031,
+      "loss": 0.0021,
+      "rewards/accuracies": 0.9975000023841858,
+      "rewards/chosen": -1.8436551094055176,
+      "rewards/margins": 12.235860824584961,
+      "rewards/rejected": -14.07951545715332,
+      "step": 5600
+    },
+    {
+      "epoch": 2.182385035074045,
+      "eval_logits/chosen": -15.46290111541748,
+      "eval_logits/rejected": -15.436098098754883,
+      "eval_logps/chosen": -349.15423583984375,
+      "eval_logps/rejected": -387.6983947753906,
+      "eval_loss": 0.00030186420190148056,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.7495561242103577,
+      "eval_rewards/margins": 11.329290390014648,
+      "eval_rewards/rejected": -12.078847885131836,
+      "eval_runtime": 183.2993,
+      "eval_samples_per_second": 1.135,
+      "eval_steps_per_second": 0.567,
+      "step": 5600
+    },
+    {
+      "epoch": 2.221356196414653,
+      "grad_norm": 0.011133521795272827,
+      "learning_rate": 8.26032107067696e-05,
+      "logits/chosen": -19.451305389404297,
+      "logits/rejected": -18.38578224182129,
+      "logps/chosen": -374.8850402832031,
+      "logps/rejected": -388.83575439453125,
+      "loss": 0.0014,
+      "rewards/accuracies": 0.9987499713897705,
+      "rewards/chosen": -1.683659315109253,
+      "rewards/margins": 12.163559913635254,
+      "rewards/rejected": -13.847216606140137,
+      "step": 5700
+    },
+    {
+      "epoch": 2.260327357755261,
+      "grad_norm": 7.387495134025812e-05,
+      "learning_rate": 7.497243900523937e-05,
+      "logits/chosen": -19.474388122558594,
+      "logits/rejected": -18.39251708984375,
+      "logps/chosen": -382.3981628417969,
+      "logps/rejected": -390.441162109375,
+      "loss": 0.0017,
+      "rewards/accuracies": 0.9987499713897705,
+      "rewards/chosen": -1.4145100116729736,
+      "rewards/margins": 11.843048095703125,
+      "rewards/rejected": -13.257555961608887,
+      "step": 5800
+    },
+    {
+      "epoch": 2.260327357755261,
+      "eval_logits/chosen": -15.475095748901367,
+      "eval_logits/rejected": -15.438820838928223,
+      "eval_logps/chosen": -347.4687805175781,
+      "eval_logps/rejected": -385.78497314453125,
+      "eval_loss": 0.0002713102730922401,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/chosen": -0.5810099244117737,
+      "eval_rewards/margins": 11.306499481201172,
+      "eval_rewards/rejected": -11.887508392333984,
+      "eval_runtime": 183.4323,
+      "eval_samples_per_second": 1.134,
+      "eval_steps_per_second": 0.567,
+      "step": 5800
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 7698,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 200,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff9a69f1507360b7e3abf8d20008aa5952976ad08fbbe6661f13498456902a1b
+size 5560