Model save

README.md

@@ -0,0 +1,72 @@
+---
+license: mit
+base_model: microsoft/Phi-3-small-8k-instruct
+tags:
+- generated_from_trainer
+model-index:
+- name: phi3-spin-phi3-data
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# phi3-spin-phi3-data
+
+This model is a fine-tuned version of [microsoft/Phi-3-small-8k-instruct](https://huggingface.co/microsoft/Phi-3-small-8k-instruct) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0000
+- Rewards/real: -20.4314
+- Rewards/generated: -59.7851
+- Rewards/accuracies: 1.0
+- Rewards/margins: 39.3538
+- Logps/generated: -1415.6521
+- Logps/real: -459.2887
+- Logits/generated: -inf
+- Logits/real: -inf
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/real | Rewards/generated | Rewards/accuracies | Rewards/margins | Logps/generated | Logps/real | Logits/generated | Logits/real |
+|:-------------:|:-----:|:----:|:---------------:|:------------:|:-----------------:|:------------------:|:---------------:|:---------------:|:----------:|:----------------:|:-----------:|
+| 0.0013        | 0.32  | 500  | 0.0028          | -15.0276     | -43.9108          | 1.0                | 28.8832         | -1256.9089      | -405.2515  | -inf             | -inf        |
+| 0.0007        | 0.64  | 1000 | 0.0001          | -20.5002     | -56.1393          | 1.0                | 35.6391         | -1379.1938      | -459.9772  | -inf             | -inf        |
+| 0.0081        | 0.96  | 1500 | 0.0000          | -20.4314     | -59.7851          | 1.0                | 39.3538         | -1415.6521      | -459.2887  | -inf             | -inf        |
+
+
+### Framework versions
+
+- Transformers 4.37.0
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.22308072375661545,
+    "train_runtime": 12352.7864,
+    "train_samples": 50000,
+    "train_samples_per_second": 4.048,
+    "train_steps_per_second": 0.127
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257,
+    100266
+  ],
+  "transformers_version": "4.37.0"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2422ed114eb44cac8237dcb4c48ee405f736391db0ad06b7ad4f15c494c6c266
+size 4832943104

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa593f4f4e07a3f5466aa11dee4f040aaa6e53f0edc488e705c1178320d8a3b3
+size 4799608224

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bac659d89668549f53193d3cf98590858ae695844d44a8e59687b50dcd38db9e
+size 4799608240

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5cf501b90bc59b0c8fcedc155cb8e0b3d9ba5a9fef4b21ef5e5febefa726cd42
+size 352437304

model.safetensors.index.json

@@ -0,0 +1,426 @@
+{
+  "metadata": {
+    "total_size": 14784548864
+  },
+  "weight_map": {
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.final_layernorm.bias": "model-00004-of-00004.safetensors",
+    "model.final_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.0.input_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.dense.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.dense.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.query_key_value.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.query_key_value.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.rotary_emb.inv_freq": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.dense.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.dense.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.query_key_value.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.query_key_value.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.rotary_emb.inv_freq": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.dense.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.dense.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.query_key_value.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.query_key_value.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.rotary_emb.inv_freq": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.dense.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.dense.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.query_key_value.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.query_key_value.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.rotary_emb.inv_freq": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.dense.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.dense.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.query_key_value.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.query_key_value.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.rotary_emb.inv_freq": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.dense.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.dense.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.query_key_value.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.query_key_value.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.rotary_emb.inv_freq": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.dense.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.dense.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.query_key_value.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.query_key_value.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.rotary_emb.inv_freq": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.dense.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.dense.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.query_key_value.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.query_key_value.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.rotary_emb.inv_freq": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.dense.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.dense.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.query_key_value.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.query_key_value.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.rotary_emb.inv_freq": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.dense.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.dense.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.query_key_value.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.query_key_value.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.rotary_emb.inv_freq": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.dense.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.dense.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.query_key_value.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.query_key_value.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.rotary_emb.inv_freq": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.dense.bias": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.dense.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.query_key_value.bias": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.query_key_value.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.rotary_emb.inv_freq": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.dense.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.dense.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.query_key_value.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.query_key_value.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.rotary_emb.inv_freq": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.dense.bias": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.dense.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.query_key_value.bias": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.query_key_value.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.rotary_emb.inv_freq": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.dense.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.dense.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.query_key_value.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.query_key_value.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.rotary_emb.inv_freq": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.dense.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.dense.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.query_key_value.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.query_key_value.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.rotary_emb.inv_freq": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.dense.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.dense.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.query_key_value.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.query_key_value.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.rotary_emb.inv_freq": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.dense.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.dense.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.query_key_value.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.query_key_value.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.rotary_emb.inv_freq": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.dense.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.dense.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.query_key_value.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.query_key_value.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.rotary_emb.inv_freq": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.dense.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.dense.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.query_key_value.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.query_key_value.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.rotary_emb.inv_freq": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.dense.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.dense.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.query_key_value.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.query_key_value.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.rotary_emb.inv_freq": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.dense.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.dense.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.query_key_value.bias": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.query_key_value.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.rotary_emb.inv_freq": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.dense.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.dense.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.query_key_value.bias": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.query_key_value.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.rotary_emb.inv_freq": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.dense.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.dense.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.query_key_value.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.query_key_value.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.rotary_emb.inv_freq": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.bias": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.dense.bias": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.dense.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.query_key_value.bias": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.query_key_value.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.rotary_emb.inv_freq": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.bias": "model-00004-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.bias": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.bias": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.bias": "model-00004-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.dense.bias": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.dense.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.query_key_value.bias": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.query_key_value.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.rotary_emb.inv_freq": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.dense.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.dense.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.query_key_value.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.query_key_value.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.rotary_emb.inv_freq": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.dense.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.dense.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.query_key_value.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.query_key_value.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.rotary_emb.inv_freq": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.dense.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.dense.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.query_key_value.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.query_key_value.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.rotary_emb.inv_freq": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.dense.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.dense.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.query_key_value.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.query_key_value.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.rotary_emb.inv_freq": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.dense.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.dense.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.query_key_value.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.query_key_value.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.rotary_emb.inv_freq": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.dense.bias": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.dense.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.query_key_value.bias": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.query_key_value.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.self_attn.rotary_emb.inv_freq": "model-00001-of-00004.safetensors"
+  }
+}

runs/Jul22_04-01-30_gilbreth-j001.rcac.purdue.edu/events.out.tfevents.1721635450.gilbreth-j001.rcac.purdue.edu.78217.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:36fd560dfdb7af3d22a16c5de2bf4d2858edaaf36e0195ff25e6afd41aab0a04
-size 104153
+oid sha256:4c32ec8a590dfe8bdd9f4ff4fbb67d110ff793ea9b05ad86011c930b2db30d1b
+size 108293

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.22308072375661545,
+    "train_runtime": 12352.7864,
+    "train_samples": 50000,
+    "train_samples_per_second": 4.048,
+    "train_steps_per_second": 0.127
+}

trainer_state.json

@@ -0,0 +1,2276 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1563,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 3.1847133757961784e-09,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -966.953369140625,
+      "logps/real": -375.4235534667969,
+      "loss": 2.3326,
+      "rewards/accuracies": 0.5,
+      "rewards/generated": -24.445566177368164,
+      "rewards/margins": 7.51291561126709,
+      "rewards/real": -16.932649612426758,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.184713375796178e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1466.3399658203125,
+      "logps/real": -367.64337158203125,
+      "loss": 3.2315,
+      "rewards/accuracies": 0.7916666865348816,
+      "rewards/generated": -59.35597610473633,
+      "rewards/margins": 47.49872970581055,
+      "rewards/real": -11.857250213623047,
+      "step": 10
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.369426751592356e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1373.967041015625,
+      "logps/real": -417.777099609375,
+      "loss": 3.2448,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/generated": -58.472434997558594,
+      "rewards/margins": 41.433250427246094,
+      "rewards/real": -17.039188385009766,
+      "step": 20
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.554140127388536e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1141.670654296875,
+      "logps/real": -389.3248291015625,
+      "loss": 3.3741,
+      "rewards/accuracies": 0.6875,
+      "rewards/generated": -40.76832962036133,
+      "rewards/margins": 27.845264434814453,
+      "rewards/real": -12.923068046569824,
+      "step": 30
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.2738853503184713e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1452.003173828125,
+      "logps/real": -380.2642822265625,
+      "loss": 2.4483,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/generated": -59.415374755859375,
+      "rewards/margins": 47.07177734375,
+      "rewards/real": -12.343605041503906,
+      "step": 40
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.592356687898089e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1416.5321044921875,
+      "logps/real": -375.83770751953125,
+      "loss": 2.6791,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/generated": -57.3775520324707,
+      "rewards/margins": 43.96550369262695,
+      "rewards/real": -13.4120454788208,
+      "step": 50
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.9108280254777072e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1459.74658203125,
+      "logps/real": -403.6402893066406,
+      "loss": 2.7707,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/generated": -65.47557830810547,
+      "rewards/margins": 50.47956085205078,
+      "rewards/real": -14.996012687683105,
+      "step": 60
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.2292993630573247e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1399.264892578125,
+      "logps/real": -417.351318359375,
+      "loss": 2.7387,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/generated": -58.04237747192383,
+      "rewards/margins": 42.52684020996094,
+      "rewards/real": -15.515554428100586,
+      "step": 70
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.5477707006369425e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1565.3223876953125,
+      "logps/real": -387.29339599609375,
+      "loss": 1.8942,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/generated": -69.57615661621094,
+      "rewards/margins": 55.375877380371094,
+      "rewards/real": -14.200276374816895,
+      "step": 80
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.86624203821656e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1446.0582275390625,
+      "logps/real": -406.51068115234375,
+      "loss": 1.4942,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/generated": -63.372100830078125,
+      "rewards/margins": 48.403770446777344,
+      "rewards/real": -14.968330383300781,
+      "step": 90
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.184713375796178e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1510.2962646484375,
+      "logps/real": -393.2122802734375,
+      "loss": 1.0954,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/generated": -73.94338989257812,
+      "rewards/margins": 58.87176513671875,
+      "rewards/real": -15.071626663208008,
+      "step": 100
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.5031847133757957e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1613.8096923828125,
+      "logps/real": -408.3921203613281,
+      "loss": 1.06,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/generated": -75.48204040527344,
+      "rewards/margins": 61.25355911254883,
+      "rewards/real": -14.228479385375977,
+      "step": 110
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.8216560509554143e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1446.879150390625,
+      "logps/real": -389.1916809082031,
+      "loss": 0.3895,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -68.02391052246094,
+      "rewards/margins": 54.14775466918945,
+      "rewards/real": -13.876157760620117,
+      "step": 120
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.140127388535032e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1689.6607666015625,
+      "logps/real": -398.95330810546875,
+      "loss": 0.6185,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/generated": -84.8166275024414,
+      "rewards/margins": 71.48504638671875,
+      "rewards/real": -13.331570625305176,
+      "step": 130
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.4585987261146494e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1808.442138671875,
+      "logps/real": -405.91876220703125,
+      "loss": 0.4881,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/generated": -96.70143127441406,
+      "rewards/margins": 80.24392700195312,
+      "rewards/real": -16.457504272460938,
+      "step": 140
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.777070063694267e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1815.619140625,
+      "logps/real": -398.8287353515625,
+      "loss": 0.1995,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/generated": -97.90288543701172,
+      "rewards/margins": 83.45794677734375,
+      "rewards/real": -14.444938659667969,
+      "step": 150
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.989331436699858e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1700.7669677734375,
+      "logps/real": -392.6664733886719,
+      "loss": 0.2746,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/generated": -88.56529998779297,
+      "rewards/margins": 74.306640625,
+      "rewards/real": -14.25866985321045,
+      "step": 160
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.953769559032717e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1837.351806640625,
+      "logps/real": -367.84075927734375,
+      "loss": 0.2882,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/generated": -99.11883544921875,
+      "rewards/margins": 85.6162109375,
+      "rewards/real": -13.5026273727417,
+      "step": 170
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.918207681365576e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1801.8463134765625,
+      "logps/real": -429.1656799316406,
+      "loss": 0.3965,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -92.02323913574219,
+      "rewards/margins": 77.11473083496094,
+      "rewards/real": -14.90850830078125,
+      "step": 180
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.882645803698435e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1854.0181884765625,
+      "logps/real": -426.25140380859375,
+      "loss": 0.3915,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/generated": -101.48790740966797,
+      "rewards/margins": 85.09318542480469,
+      "rewards/real": -16.39472770690918,
+      "step": 190
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.847083926031294e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1771.7728271484375,
+      "logps/real": -393.14154052734375,
+      "loss": 0.3517,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -89.36502075195312,
+      "rewards/margins": 74.54100036621094,
+      "rewards/real": -14.824020385742188,
+      "step": 200
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.811522048364154e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1783.140869140625,
+      "logps/real": -359.3192443847656,
+      "loss": 0.215,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/generated": -92.24617004394531,
+      "rewards/margins": 79.44178771972656,
+      "rewards/real": -12.8043851852417,
+      "step": 210
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.775960170697012e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2027.943115234375,
+      "logps/real": -366.04815673828125,
+      "loss": 0.1675,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -115.5037841796875,
+      "rewards/margins": 101.34663391113281,
+      "rewards/real": -14.157157897949219,
+      "step": 220
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.7403982930298717e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2005.3988037109375,
+      "logps/real": -382.80316162109375,
+      "loss": 0.1831,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -116.76234436035156,
+      "rewards/margins": 102.41606140136719,
+      "rewards/real": -14.346267700195312,
+      "step": 230
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.7048364153627306e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1924.673828125,
+      "logps/real": -451.62957763671875,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -112.26811218261719,
+      "rewards/margins": 95.80821228027344,
+      "rewards/real": -16.459903717041016,
+      "step": 240
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.66927453769559e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2092.075439453125,
+      "logps/real": -376.3299560546875,
+      "loss": 0.0755,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -122.57212829589844,
+      "rewards/margins": 108.68547058105469,
+      "rewards/real": -13.886662483215332,
+      "step": 250
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.633712660028449e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1976.077392578125,
+      "logps/real": -378.0892028808594,
+      "loss": 0.078,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/generated": -113.3365249633789,
+      "rewards/margins": 98.88800811767578,
+      "rewards/real": -14.448512077331543,
+      "step": 260
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.5981507823613085e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1872.919189453125,
+      "logps/real": -390.10089111328125,
+      "loss": 0.2767,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -107.5772933959961,
+      "rewards/margins": 92.26273345947266,
+      "rewards/real": -15.314567565917969,
+      "step": 270
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.562588904694168e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2175.498779296875,
+      "logps/real": -416.78863525390625,
+      "loss": 0.0438,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -131.2454376220703,
+      "rewards/margins": 116.00118255615234,
+      "rewards/real": -15.244237899780273,
+      "step": 280
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.5270270270270264e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2091.298583984375,
+      "logps/real": -409.22003173828125,
+      "loss": 0.1056,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/generated": -125.65919494628906,
+      "rewards/margins": 110.78924560546875,
+      "rewards/real": -14.869955062866211,
+      "step": 290
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.491465149359886e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2561.4248046875,
+      "logps/real": -387.7209167480469,
+      "loss": 0.1194,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -162.672607421875,
+      "rewards/margins": 147.49693298339844,
+      "rewards/real": -15.175674438476562,
+      "step": 300
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.4559032716927454e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1870.7001953125,
+      "logps/real": -414.51519775390625,
+      "loss": 0.0126,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -106.3661880493164,
+      "rewards/margins": 92.60579681396484,
+      "rewards/real": -13.760396957397461,
+      "step": 310
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.420341394025605e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2169.3603515625,
+      "logps/real": -430.83123779296875,
+      "loss": 0.1027,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -134.11862182617188,
+      "rewards/margins": 116.77679443359375,
+      "rewards/real": -17.341833114624023,
+      "step": 320
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.384779516358463e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1846.721435546875,
+      "logps/real": -438.3114318847656,
+      "loss": 0.0967,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/generated": -102.0712661743164,
+      "rewards/margins": 84.29792785644531,
+      "rewards/real": -17.773338317871094,
+      "step": 330
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.3492176386913227e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1876.280517578125,
+      "logps/real": -434.6044921875,
+      "loss": 0.0503,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -112.82167053222656,
+      "rewards/margins": 97.01448059082031,
+      "rewards/real": -15.807202339172363,
+      "step": 340
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.313655761024182e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2239.822998046875,
+      "logps/real": -415.4873046875,
+      "loss": 0.0054,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -141.17965698242188,
+      "rewards/margins": 125.756103515625,
+      "rewards/real": -15.423541069030762,
+      "step": 350
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.278093883357041e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2111.564697265625,
+      "logps/real": -414.2516174316406,
+      "loss": 0.0731,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -130.3516082763672,
+      "rewards/margins": 114.76496887207031,
+      "rewards/real": -15.586637496948242,
+      "step": 360
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.2425320056899e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1973.482666015625,
+      "logps/real": -460.66253662109375,
+      "loss": 0.0208,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -122.2149429321289,
+      "rewards/margins": 103.8694076538086,
+      "rewards/real": -18.345508575439453,
+      "step": 370
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.2069701280227595e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2023.71484375,
+      "logps/real": -423.6307678222656,
+      "loss": 0.0845,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -119.55964660644531,
+      "rewards/margins": 103.66563415527344,
+      "rewards/real": -15.89399528503418,
+      "step": 380
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.1714082503556185e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2193.784423828125,
+      "logps/real": -412.4560546875,
+      "loss": 0.0184,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -129.15220642089844,
+      "rewards/margins": 112.60185241699219,
+      "rewards/real": -16.550355911254883,
+      "step": 390
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.135846372688478e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1906.6185302734375,
+      "logps/real": -447.01031494140625,
+      "loss": 0.1088,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -116.6185302734375,
+      "rewards/margins": 98.093017578125,
+      "rewards/real": -18.525516510009766,
+      "step": 400
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.100284495021337e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1921.3818359375,
+      "logps/real": -405.3843688964844,
+      "loss": 0.1989,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -116.56185150146484,
+      "rewards/margins": 100.868408203125,
+      "rewards/real": -15.693446159362793,
+      "step": 410
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.064722617354196e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2570.93408203125,
+      "logps/real": -405.2688903808594,
+      "loss": 0.1152,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -167.54176330566406,
+      "rewards/margins": 153.5286102294922,
+      "rewards/real": -14.013137817382812,
+      "step": 420
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.0291607396870553e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2243.0859375,
+      "logps/real": -415.53009033203125,
+      "loss": 0.035,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -138.50875854492188,
+      "rewards/margins": 121.80815124511719,
+      "rewards/real": -16.700613021850586,
+      "step": 430
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 3.993598862019915e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2233.75244140625,
+      "logps/real": -409.6092834472656,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -143.94195556640625,
+      "rewards/margins": 128.61239624023438,
+      "rewards/real": -15.32953929901123,
+      "step": 440
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 3.9580369843527737e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2093.072998046875,
+      "logps/real": -405.0209045410156,
+      "loss": 0.0541,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -124.28369140625,
+      "rewards/margins": 108.1083755493164,
+      "rewards/real": -16.17531394958496,
+      "step": 450
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 3.9224751066856327e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2211.914794921875,
+      "logps/real": -460.36224365234375,
+      "loss": 0.0491,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -143.0950164794922,
+      "rewards/margins": 124.48175048828125,
+      "rewards/real": -18.613265991210938,
+      "step": 460
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 3.886913229018492e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2252.41162109375,
+      "logps/real": -433.734375,
+      "loss": 0.1377,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/generated": -145.9087677001953,
+      "rewards/margins": 127.59245300292969,
+      "rewards/real": -18.316326141357422,
+      "step": 470
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 3.851351351351351e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2520.988525390625,
+      "logps/real": -441.86834716796875,
+      "loss": 0.0248,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -159.65313720703125,
+      "rewards/margins": 141.00930786132812,
+      "rewards/real": -18.64383888244629,
+      "step": 480
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 3.8157894736842105e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2456.33642578125,
+      "logps/real": -404.4278259277344,
+      "loss": 0.0541,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -158.11917114257812,
+      "rewards/margins": 140.7222137451172,
+      "rewards/real": -17.396974563598633,
+      "step": 490
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 3.7802275960170695e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2015.9263916015625,
+      "logps/real": -430.71563720703125,
+      "loss": 0.0013,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -123.14335632324219,
+      "rewards/margins": 104.85958099365234,
+      "rewards/real": -18.283767700195312,
+      "step": 500
+    },
+    {
+      "epoch": 0.32,
+      "eval_logits/generated": -Infinity,
+      "eval_logits/real": -Infinity,
+      "eval_logps/generated": -1256.908935546875,
+      "eval_logps/real": -405.2514953613281,
+      "eval_loss": 0.0027930852957069874,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/generated": -43.91083526611328,
+      "eval_rewards/margins": 28.883193969726562,
+      "eval_rewards/real": -15.027642250061035,
+      "eval_runtime": 58.2391,
+      "eval_samples_per_second": 8.585,
+      "eval_steps_per_second": 0.275,
+      "step": 500
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 3.7446657183499284e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1968.699951171875,
+      "logps/real": -415.49310302734375,
+      "loss": 0.0459,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -120.45564270019531,
+      "rewards/margins": 103.67842864990234,
+      "rewards/real": -16.7772159576416,
+      "step": 510
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 3.709103840682788e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2040.2958984375,
+      "logps/real": -397.3074951171875,
+      "loss": 0.0661,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -124.30355072021484,
+      "rewards/margins": 107.7059326171875,
+      "rewards/real": -16.59762191772461,
+      "step": 520
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 3.6735419630156474e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2419.407470703125,
+      "logps/real": -390.42291259765625,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -149.4462890625,
+      "rewards/margins": 132.99026489257812,
+      "rewards/real": -16.45602798461914,
+      "step": 530
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 3.637980085348506e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1950.378662109375,
+      "logps/real": -427.7591857910156,
+      "loss": 0.0007,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -117.6002426147461,
+      "rewards/margins": 100.45401000976562,
+      "rewards/real": -17.14624786376953,
+      "step": 540
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 3.602418207681365e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2320.71337890625,
+      "logps/real": -444.1558532714844,
+      "loss": 0.0102,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -151.31004333496094,
+      "rewards/margins": 133.40000915527344,
+      "rewards/real": -17.91002655029297,
+      "step": 550
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 3.5668563300142247e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2030.7210693359375,
+      "logps/real": -419.117919921875,
+      "loss": 0.0758,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -125.63960266113281,
+      "rewards/margins": 108.42061614990234,
+      "rewards/real": -17.218996047973633,
+      "step": 560
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 3.5312944523470837e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2275.38525390625,
+      "logps/real": -426.5855407714844,
+      "loss": 0.0114,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -142.7791748046875,
+      "rewards/margins": 124.15544128417969,
+      "rewards/real": -18.62372589111328,
+      "step": 570
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.495732574679943e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2312.51416015625,
+      "logps/real": -432.1441345214844,
+      "loss": 0.0364,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/generated": -146.12319946289062,
+      "rewards/margins": 127.83203125,
+      "rewards/real": -18.291170120239258,
+      "step": 580
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.460170697012802e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2241.414306640625,
+      "logps/real": -407.19952392578125,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -140.54261779785156,
+      "rewards/margins": 124.97169494628906,
+      "rewards/real": -15.570927619934082,
+      "step": 590
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.424608819345661e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2033.769775390625,
+      "logps/real": -422.98358154296875,
+      "loss": 0.059,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -127.4608383178711,
+      "rewards/margins": 108.93534851074219,
+      "rewards/real": -18.525497436523438,
+      "step": 600
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.3890469416785205e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2355.256103515625,
+      "logps/real": -440.0362243652344,
+      "loss": 0.001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -151.5084228515625,
+      "rewards/margins": 132.78128051757812,
+      "rewards/real": -18.727153778076172,
+      "step": 610
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.35348506401138e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2205.610595703125,
+      "logps/real": -405.8412780761719,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -141.056396484375,
+      "rewards/margins": 123.6712875366211,
+      "rewards/real": -17.38511848449707,
+      "step": 620
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.3179231863442384e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2315.1484375,
+      "logps/real": -461.20794677734375,
+      "loss": 0.0428,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -147.27651977539062,
+      "rewards/margins": 126.5012435913086,
+      "rewards/real": -20.775270462036133,
+      "step": 630
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.282361308677098e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2220.00439453125,
+      "logps/real": -438.3214416503906,
+      "loss": 0.0645,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -142.0787353515625,
+      "rewards/margins": 123.02635192871094,
+      "rewards/real": -19.052383422851562,
+      "step": 640
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.2467994310099573e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2540.353759765625,
+      "logps/real": -473.17254638671875,
+      "loss": 0.0579,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -170.84814453125,
+      "rewards/margins": 150.62413024902344,
+      "rewards/real": -20.223987579345703,
+      "step": 650
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.211237553342817e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2051.99853515625,
+      "logps/real": -422.4043884277344,
+      "loss": 0.0003,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -129.46347045898438,
+      "rewards/margins": 111.48968505859375,
+      "rewards/real": -17.973770141601562,
+      "step": 660
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.175675675675675e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2427.76513671875,
+      "logps/real": -401.15203857421875,
+      "loss": 0.0062,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -156.4727020263672,
+      "rewards/margins": 139.4108123779297,
+      "rewards/real": -17.061885833740234,
+      "step": 670
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.1401137980085347e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2219.527099609375,
+      "logps/real": -413.57635498046875,
+      "loss": 0.0381,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -136.24771118164062,
+      "rewards/margins": 119.32716369628906,
+      "rewards/real": -16.92055320739746,
+      "step": 680
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.104551920341394e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2171.81005859375,
+      "logps/real": -436.612548828125,
+      "loss": 0.1275,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -136.0384063720703,
+      "rewards/margins": 117.75749206542969,
+      "rewards/real": -18.28091049194336,
+      "step": 690
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.068990042674253e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2208.45556640625,
+      "logps/real": -398.7838439941406,
+      "loss": 0.0285,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -140.2645721435547,
+      "rewards/margins": 122.7247314453125,
+      "rewards/real": -17.539838790893555,
+      "step": 700
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.033428165007112e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2175.37353515625,
+      "logps/real": -441.8241271972656,
+      "loss": 0.0224,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -130.23345947265625,
+      "rewards/margins": 110.2350082397461,
+      "rewards/real": -19.998470306396484,
+      "step": 710
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 2.9978662873399715e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2355.478515625,
+      "logps/real": -450.7440490722656,
+      "loss": 0.0125,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -152.7926025390625,
+      "rewards/margins": 132.38949584960938,
+      "rewards/real": -20.403091430664062,
+      "step": 720
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 2.9623044096728305e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2129.60009765625,
+      "logps/real": -481.05889892578125,
+      "loss": 0.0169,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -131.75416564941406,
+      "rewards/margins": 110.94267272949219,
+      "rewards/real": -20.811492919921875,
+      "step": 730
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 2.92674253200569e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2132.948486328125,
+      "logps/real": -449.02337646484375,
+      "loss": 0.0687,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/generated": -137.2069091796875,
+      "rewards/margins": 116.990966796875,
+      "rewards/real": -20.2159366607666,
+      "step": 740
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 2.8911806543385494e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2116.66748046875,
+      "logps/real": -480.8214416503906,
+      "loss": 0.0002,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -130.63247680664062,
+      "rewards/margins": 111.26945495605469,
+      "rewards/real": -19.363012313842773,
+      "step": 750
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.855618776671408e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2416.93408203125,
+      "logps/real": -500.4256896972656,
+      "loss": 0.0005,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -158.8488311767578,
+      "rewards/margins": 137.0421905517578,
+      "rewards/real": -21.806636810302734,
+      "step": 760
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.8200568990042673e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2503.43017578125,
+      "logps/real": -443.42889404296875,
+      "loss": 0.0008,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -162.8308868408203,
+      "rewards/margins": 143.54254150390625,
+      "rewards/real": -19.288341522216797,
+      "step": 770
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.784495021337127e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2201.969482421875,
+      "logps/real": -411.990478515625,
+      "loss": 0.0124,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -137.68695068359375,
+      "rewards/margins": 119.24308776855469,
+      "rewards/real": -18.443859100341797,
+      "step": 780
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.7489331436699857e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2512.57861328125,
+      "logps/real": -472.49212646484375,
+      "loss": 0.0512,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -168.89157104492188,
+      "rewards/margins": 148.9585418701172,
+      "rewards/real": -19.933027267456055,
+      "step": 790
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.7133712660028446e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2306.381103515625,
+      "logps/real": -424.8629455566406,
+      "loss": 0.0048,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -145.5426025390625,
+      "rewards/margins": 127.3676986694336,
+      "rewards/real": -18.174896240234375,
+      "step": 800
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.677809388335704e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2429.31591796875,
+      "logps/real": -437.7942810058594,
+      "loss": 0.018,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -154.94802856445312,
+      "rewards/margins": 134.9539794921875,
+      "rewards/real": -19.994056701660156,
+      "step": 810
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.642247510668563e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2294.972900390625,
+      "logps/real": -437.0038146972656,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -147.67648315429688,
+      "rewards/margins": 128.96316528320312,
+      "rewards/real": -18.713321685791016,
+      "step": 820
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6066856330014225e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2357.01025390625,
+      "logps/real": -464.3094177246094,
+      "loss": 0.0305,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -147.08970642089844,
+      "rewards/margins": 126.06890869140625,
+      "rewards/real": -21.020793914794922,
+      "step": 830
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5711237553342815e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2230.1494140625,
+      "logps/real": -490.10662841796875,
+      "loss": 0.022,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -133.9390411376953,
+      "rewards/margins": 112.09165954589844,
+      "rewards/real": -21.847393035888672,
+      "step": 840
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5355618776671404e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2354.697265625,
+      "logps/real": -444.6630859375,
+      "loss": 0.0324,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -150.9205322265625,
+      "rewards/margins": 132.67068481445312,
+      "rewards/real": -18.24983787536621,
+      "step": 850
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2163.265869140625,
+      "logps/real": -441.83087158203125,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -132.36106872558594,
+      "rewards/margins": 112.86710357666016,
+      "rewards/real": -19.493968963623047,
+      "step": 860
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4644381223328594e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2413.11376953125,
+      "logps/real": -477.031494140625,
+      "loss": 0.032,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -166.50555419921875,
+      "rewards/margins": 144.79429626464844,
+      "rewards/real": -21.711240768432617,
+      "step": 870
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4288762446657183e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2381.630615234375,
+      "logps/real": -464.4560546875,
+      "loss": 0.0114,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -149.9140625,
+      "rewards/margins": 129.8895721435547,
+      "rewards/real": -20.02446746826172,
+      "step": 880
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.393314366998578e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2167.195556640625,
+      "logps/real": -447.3040466308594,
+      "loss": 0.0515,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -134.86166381835938,
+      "rewards/margins": 115.23634338378906,
+      "rewards/real": -19.625324249267578,
+      "step": 890
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.3577524893314365e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2326.58447265625,
+      "logps/real": -426.9696350097656,
+      "loss": 0.0547,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -145.77752685546875,
+      "rewards/margins": 127.47369384765625,
+      "rewards/real": -18.303844451904297,
+      "step": 900
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.322190611664296e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2220.4169921875,
+      "logps/real": -489.0972595214844,
+      "loss": 0.0636,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -143.51425170898438,
+      "rewards/margins": 121.31208801269531,
+      "rewards/real": -22.202152252197266,
+      "step": 910
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.2866287339971549e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2354.84912109375,
+      "logps/real": -450.4208068847656,
+      "loss": 0.0845,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -153.4873809814453,
+      "rewards/margins": 133.9937744140625,
+      "rewards/real": -19.49359130859375,
+      "step": 920
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.251066856330014e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2735.659423828125,
+      "logps/real": -416.12054443359375,
+      "loss": 0.0213,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -183.10470581054688,
+      "rewards/margins": 165.31044006347656,
+      "rewards/real": -17.794281005859375,
+      "step": 930
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.2155049786628733e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2315.6005859375,
+      "logps/real": -463.85919189453125,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -149.17642211914062,
+      "rewards/margins": 129.64544677734375,
+      "rewards/real": -19.530961990356445,
+      "step": 940
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.1799431009957325e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2543.94580078125,
+      "logps/real": -417.3636779785156,
+      "loss": 0.0938,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -164.8701629638672,
+      "rewards/margins": 146.69618225097656,
+      "rewards/real": -18.173999786376953,
+      "step": 950
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.1443812233285914e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2010.436279296875,
+      "logps/real": -479.859130859375,
+      "loss": 0.0274,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -120.16337585449219,
+      "rewards/margins": 98.71519470214844,
+      "rewards/real": -21.44818115234375,
+      "step": 960
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.108819345661451e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2313.285888671875,
+      "logps/real": -467.345703125,
+      "loss": 0.0361,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -150.2023162841797,
+      "rewards/margins": 128.9005126953125,
+      "rewards/real": -21.301822662353516,
+      "step": 970
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 2.0732574679943098e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2233.77978515625,
+      "logps/real": -422.1748046875,
+      "loss": 0.0049,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -142.33438110351562,
+      "rewards/margins": 122.62577819824219,
+      "rewards/real": -19.70859146118164,
+      "step": 980
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 2.0376955903271693e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -1923.1666259765625,
+      "logps/real": -499.18792724609375,
+      "loss": 0.0308,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -117.52037048339844,
+      "rewards/margins": 95.38182067871094,
+      "rewards/real": -22.138551712036133,
+      "step": 990
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 2.0021337126600283e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2139.812255859375,
+      "logps/real": -437.359375,
+      "loss": 0.0007,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -138.1731719970703,
+      "rewards/margins": 117.10591888427734,
+      "rewards/real": -21.067237854003906,
+      "step": 1000
+    },
+    {
+      "epoch": 0.64,
+      "eval_logits/generated": -Infinity,
+      "eval_logits/real": -Infinity,
+      "eval_logps/generated": -1379.19384765625,
+      "eval_logps/real": -459.97723388671875,
+      "eval_loss": 0.00011698609159793705,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/generated": -56.13932800292969,
+      "eval_rewards/margins": 35.63911437988281,
+      "eval_rewards/real": -20.50020980834961,
+      "eval_runtime": 57.3321,
+      "eval_samples_per_second": 8.721,
+      "eval_steps_per_second": 0.279,
+      "step": 1000
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.9665718349928875e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2620.46630859375,
+      "logps/real": -439.03289794921875,
+      "loss": 0.0023,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -172.01492309570312,
+      "rewards/margins": 151.74253845214844,
+      "rewards/real": -20.27237892150879,
+      "step": 1010
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.931009957325747e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2179.395263671875,
+      "logps/real": -436.818359375,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -134.63832092285156,
+      "rewards/margins": 113.6363296508789,
+      "rewards/real": -21.00200843811035,
+      "step": 1020
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.895448079658606e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2512.95654296875,
+      "logps/real": -476.609375,
+      "loss": 0.003,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -164.37026977539062,
+      "rewards/margins": 142.48654174804688,
+      "rewards/real": -21.883718490600586,
+      "step": 1030
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.859886201991465e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2187.837890625,
+      "logps/real": -466.7958984375,
+      "loss": 0.0237,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -146.79244995117188,
+      "rewards/margins": 126.20684814453125,
+      "rewards/real": -20.58557891845703,
+      "step": 1040
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.8243243243243243e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2336.830078125,
+      "logps/real": -449.9044494628906,
+      "loss": 0.0288,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -152.05953979492188,
+      "rewards/margins": 132.2141876220703,
+      "rewards/real": -19.845373153686523,
+      "step": 1050
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.7887624466571835e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2642.74560546875,
+      "logps/real": -453.025634765625,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -175.72708129882812,
+      "rewards/margins": 155.6625518798828,
+      "rewards/real": -20.06453514099121,
+      "step": 1060
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.7532005689900424e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2573.627685546875,
+      "logps/real": -460.2877502441406,
+      "loss": 0.1833,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -175.51333618164062,
+      "rewards/margins": 154.97291564941406,
+      "rewards/real": -20.540424346923828,
+      "step": 1070
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.717638691322902e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2539.660400390625,
+      "logps/real": -456.41156005859375,
+      "loss": 0.0012,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -176.18435668945312,
+      "rewards/margins": 155.01394653320312,
+      "rewards/real": -21.170413970947266,
+      "step": 1080
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.6820768136557609e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2953.92626953125,
+      "logps/real": -455.2708435058594,
+      "loss": 0.0083,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -206.5038604736328,
+      "rewards/margins": 186.19964599609375,
+      "rewards/real": -20.304224014282227,
+      "step": 1090
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.64651493598862e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2637.98876953125,
+      "logps/real": -484.7318420410156,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -176.80865478515625,
+      "rewards/margins": 154.76345825195312,
+      "rewards/real": -22.04518699645996,
+      "step": 1100
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.6109530583214793e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2345.23046875,
+      "logps/real": -442.49407958984375,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -145.41603088378906,
+      "rewards/margins": 124.95213317871094,
+      "rewards/real": -20.463890075683594,
+      "step": 1110
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.5753911806543385e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2436.03466796875,
+      "logps/real": -454.4659729003906,
+      "loss": 0.0775,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -157.82691955566406,
+      "rewards/margins": 137.90733337402344,
+      "rewards/real": -19.919586181640625,
+      "step": 1120
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.5398293029871974e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2562.81689453125,
+      "logps/real": -479.8397521972656,
+      "loss": 0.0046,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -169.16990661621094,
+      "rewards/margins": 147.8955535888672,
+      "rewards/real": -21.274364471435547,
+      "step": 1130
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.504267425320057e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2312.70654296875,
+      "logps/real": -442.69287109375,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -152.26504516601562,
+      "rewards/margins": 131.82748413085938,
+      "rewards/real": -20.437549591064453,
+      "step": 1140
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 1.4687055476529158e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2099.454345703125,
+      "logps/real": -452.2735900878906,
+      "loss": 0.0276,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -133.490478515625,
+      "rewards/margins": 114.50862121582031,
+      "rewards/real": -18.981868743896484,
+      "step": 1150
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 1.4331436699857753e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2273.83154296875,
+      "logps/real": -468.0013732910156,
+      "loss": 0.0002,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -143.70883178710938,
+      "rewards/margins": 122.0762939453125,
+      "rewards/real": -21.63254737854004,
+      "step": 1160
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 1.3975817923186345e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2436.0341796875,
+      "logps/real": -490.1158752441406,
+      "loss": 0.0646,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/generated": -164.6710662841797,
+      "rewards/margins": 143.2229766845703,
+      "rewards/real": -21.44809913635254,
+      "step": 1170
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 1.3620199146514935e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2336.074951171875,
+      "logps/real": -396.33245849609375,
+      "loss": 0.0385,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -148.41668701171875,
+      "rewards/margins": 130.65158081054688,
+      "rewards/real": -17.765100479125977,
+      "step": 1180
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 1.326458036984353e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2272.589599609375,
+      "logps/real": -452.4452209472656,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -145.31480407714844,
+      "rewards/margins": 124.13655090332031,
+      "rewards/real": -21.178268432617188,
+      "step": 1190
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.290896159317212e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2375.04833984375,
+      "logps/real": -424.8208923339844,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -156.83316040039062,
+      "rewards/margins": 138.62127685546875,
+      "rewards/real": -18.211872100830078,
+      "step": 1200
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.255334281650071e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2136.970458984375,
+      "logps/real": -470.10577392578125,
+      "loss": 0.008,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -137.5404052734375,
+      "rewards/margins": 116.5934829711914,
+      "rewards/real": -20.946937561035156,
+      "step": 1210
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 1.2197724039829303e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2407.7744140625,
+      "logps/real": -444.0035705566406,
+      "loss": 0.0677,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -150.90780639648438,
+      "rewards/margins": 130.57858276367188,
+      "rewards/real": -20.329204559326172,
+      "step": 1220
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 1.1842105263157894e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2083.391357421875,
+      "logps/real": -432.478515625,
+      "loss": 0.0868,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -131.513427734375,
+      "rewards/margins": 110.8812484741211,
+      "rewards/real": -20.632179260253906,
+      "step": 1230
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 1.1486486486486487e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2315.928466796875,
+      "logps/real": -454.92041015625,
+      "loss": 0.1006,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -143.36203002929688,
+      "rewards/margins": 123.34745025634766,
+      "rewards/real": -20.014583587646484,
+      "step": 1240
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.1130867709815078e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2121.920654296875,
+      "logps/real": -444.47784423828125,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -133.88516235351562,
+      "rewards/margins": 113.72652435302734,
+      "rewards/real": -20.158626556396484,
+      "step": 1250
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.077524893314367e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2223.19091796875,
+      "logps/real": -423.525146484375,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -137.70986938476562,
+      "rewards/margins": 117.95418548583984,
+      "rewards/real": -19.75567054748535,
+      "step": 1260
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.0419630156472262e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2461.502685546875,
+      "logps/real": -495.8019104003906,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -160.56712341308594,
+      "rewards/margins": 138.31765747070312,
+      "rewards/real": -22.249460220336914,
+      "step": 1270
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.0064011379800854e-07,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2393.97216796875,
+      "logps/real": -476.8765563964844,
+      "loss": 0.0052,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -154.5055694580078,
+      "rewards/margins": 133.27890014648438,
+      "rewards/real": -21.226654052734375,
+      "step": 1280
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 9.708392603129445e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2168.818603515625,
+      "logps/real": -469.14227294921875,
+      "loss": 0.1099,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -134.32223510742188,
+      "rewards/margins": 112.4707260131836,
+      "rewards/real": -21.851505279541016,
+      "step": 1290
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 9.352773826458037e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2151.55419921875,
+      "logps/real": -462.02569580078125,
+      "loss": 0.0003,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -138.9175567626953,
+      "rewards/margins": 118.1842269897461,
+      "rewards/real": -20.733333587646484,
+      "step": 1300
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.997155049786629e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2486.102294921875,
+      "logps/real": -419.736083984375,
+      "loss": 0.0031,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -167.7966766357422,
+      "rewards/margins": 149.419677734375,
+      "rewards/real": -18.376995086669922,
+      "step": 1310
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.64153627311522e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2487.68408203125,
+      "logps/real": -441.118896484375,
+      "loss": 0.0001,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -164.9488983154297,
+      "rewards/margins": 145.40994262695312,
+      "rewards/real": -19.538982391357422,
+      "step": 1320
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 8.285917496443812e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2352.103271484375,
+      "logps/real": -464.57684326171875,
+      "loss": 0.0255,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -153.78616333007812,
+      "rewards/margins": 132.96963500976562,
+      "rewards/real": -20.816532135009766,
+      "step": 1330
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.930298719772404e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2409.52734375,
+      "logps/real": -470.6730041503906,
+      "loss": 0.0236,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -150.34481811523438,
+      "rewards/margins": 130.0278778076172,
+      "rewards/real": -20.31694984436035,
+      "step": 1340
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.574679943100994e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2062.15576171875,
+      "logps/real": -416.6309509277344,
+      "loss": 0.0741,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -132.1886444091797,
+      "rewards/margins": 113.09898376464844,
+      "rewards/real": -19.08967399597168,
+      "step": 1350
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 7.219061166429587e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2337.429443359375,
+      "logps/real": -436.75006103515625,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -150.75607299804688,
+      "rewards/margins": 132.02603149414062,
+      "rewards/real": -18.730056762695312,
+      "step": 1360
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.863442389758179e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2418.18896484375,
+      "logps/real": -488.41986083984375,
+      "loss": 0.0339,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -164.10385131835938,
+      "rewards/margins": 142.82009887695312,
+      "rewards/real": -21.283767700195312,
+      "step": 1370
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.507823613086771e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2458.375732421875,
+      "logps/real": -475.74603271484375,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -164.42733764648438,
+      "rewards/margins": 143.57705688476562,
+      "rewards/real": -20.85028648376465,
+      "step": 1380
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 6.152204836415363e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2469.349609375,
+      "logps/real": -476.27166748046875,
+      "loss": 0.0519,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -164.94485473632812,
+      "rewards/margins": 143.2485809326172,
+      "rewards/real": -21.696279525756836,
+      "step": 1390
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.796586059743954e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2470.31005859375,
+      "logps/real": -454.34576416015625,
+      "loss": 0.0016,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -157.91641235351562,
+      "rewards/margins": 136.79840087890625,
+      "rewards/real": -21.11802101135254,
+      "step": 1400
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.4409672830725456e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2465.37451171875,
+      "logps/real": -449.94720458984375,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -155.06021118164062,
+      "rewards/margins": 134.5743408203125,
+      "rewards/real": -20.485883712768555,
+      "step": 1410
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 5.0853485064011376e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2366.994140625,
+      "logps/real": -421.33599853515625,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -157.2525634765625,
+      "rewards/margins": 138.6487579345703,
+      "rewards/real": -18.603818893432617,
+      "step": 1420
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.72972972972973e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2792.276611328125,
+      "logps/real": -460.935302734375,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -190.84226989746094,
+      "rewards/margins": 169.98809814453125,
+      "rewards/real": -20.85416603088379,
+      "step": 1430
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.374110953058322e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2973.617431640625,
+      "logps/real": -465.82794189453125,
+      "loss": 0.0383,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -210.22744750976562,
+      "rewards/margins": 188.23648071289062,
+      "rewards/real": -21.990949630737305,
+      "step": 1440
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 4.018492176386913e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2328.478271484375,
+      "logps/real": -446.6985778808594,
+      "loss": 0.053,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -153.81591796875,
+      "rewards/margins": 134.17135620117188,
+      "rewards/real": -19.644563674926758,
+      "step": 1450
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.6628733997155046e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2488.30126953125,
+      "logps/real": -408.4414978027344,
+      "loss": 0.0338,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -165.46292114257812,
+      "rewards/margins": 145.89773559570312,
+      "rewards/real": -19.5651798248291,
+      "step": 1460
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.3072546230440967e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2582.849609375,
+      "logps/real": -454.8753356933594,
+      "loss": 0.0158,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -177.5186309814453,
+      "rewards/margins": 156.65342712402344,
+      "rewards/real": -20.865222930908203,
+      "step": 1470
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.9516358463726884e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2417.738037109375,
+      "logps/real": -467.0601501464844,
+      "loss": 0.0035,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -158.90713500976562,
+      "rewards/margins": 138.79782104492188,
+      "rewards/real": -20.109325408935547,
+      "step": 1480
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 2.59601706970128e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2566.396484375,
+      "logps/real": -479.7379455566406,
+      "loss": 0.0403,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -173.2800750732422,
+      "rewards/margins": 153.24374389648438,
+      "rewards/real": -20.036338806152344,
+      "step": 1490
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.240398293029872e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2365.49560546875,
+      "logps/real": -448.97821044921875,
+      "loss": 0.0081,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -151.83856201171875,
+      "rewards/margins": 132.27145385742188,
+      "rewards/real": -19.567106246948242,
+      "step": 1500
+    },
+    {
+      "epoch": 0.96,
+      "eval_logits/generated": -Infinity,
+      "eval_logits/real": -Infinity,
+      "eval_logps/generated": -1415.652099609375,
+      "eval_logps/real": -459.2886962890625,
+      "eval_loss": 3.275389826740138e-05,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/generated": -59.78514099121094,
+      "eval_rewards/margins": 39.35377883911133,
+      "eval_rewards/real": -20.431360244750977,
+      "eval_runtime": 81.8497,
+      "eval_samples_per_second": 6.109,
+      "eval_steps_per_second": 0.195,
+      "step": 1500
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.8847795163584636e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2331.526123046875,
+      "logps/real": -453.56903076171875,
+      "loss": 0.0128,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -153.5113525390625,
+      "rewards/margins": 133.09779357910156,
+      "rewards/real": -20.413562774658203,
+      "step": 1510
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.5291607396870554e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2519.184814453125,
+      "logps/real": -455.0816955566406,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -166.50537109375,
+      "rewards/margins": 145.3622589111328,
+      "rewards/real": -21.14311408996582,
+      "step": 1520
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.1735419630156473e-08,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2311.308349609375,
+      "logps/real": -446.6311950683594,
+      "loss": 0.0639,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -147.44296264648438,
+      "rewards/margins": 127.8827133178711,
+      "rewards/real": -19.560251235961914,
+      "step": 1530
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.179231863442388e-09,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2288.357666015625,
+      "logps/real": -467.3319396972656,
+      "loss": 0.0003,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -145.20071411132812,
+      "rewards/margins": 123.99946594238281,
+      "rewards/real": -21.201269149780273,
+      "step": 1540
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.623044096728307e-09,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2068.825439453125,
+      "logps/real": -489.23681640625,
+      "loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -129.2821044921875,
+      "rewards/margins": 106.22123718261719,
+      "rewards/real": -23.060855865478516,
+      "step": 1550
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.0668563300142248e-09,
+      "logits/generated": -Infinity,
+      "logits/real": -Infinity,
+      "logps/generated": -2547.381591796875,
+      "logps/real": -438.65875244140625,
+      "loss": 0.0662,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -166.4281463623047,
+      "rewards/margins": 146.95614624023438,
+      "rewards/real": -19.471982955932617,
+      "step": 1560
+    },
+    {
+      "epoch": 1.0,
+      "step": 1563,
+      "total_flos": 0.0,
+      "train_loss": 0.22308072375661545,
+      "train_runtime": 12352.7864,
+      "train_samples_per_second": 4.048,
+      "train_steps_per_second": 0.127
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1563,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}