Training in progress, epoch 59, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b392aff3a6ab84e321c0da912a52394cdf80c96d908bc82cd18229c180b3a16a
+oid sha256:79b71bf8526feaf3fbe640362c921c3bd5d9b567af82dd4997d42973aa321991
 size 166496880

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd9a916036f1c4feb33e98f71780fdaa5eecd9c63d4096c79d97b0f84384125c
+oid sha256:c277767943f1e035c0d73c794bfd2e290cf536330fec91bf85bb7c955776135f
 size 330495866

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a728912d3338c692b48d80b79e8160131e5fa9d986b207c5ffb9444c99eab6bc
+oid sha256:7021b7f290ce2ecee8960d7d8842b39007015ec1c207fbea7eb79b547a00e6ea
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d91fda77405c63be9ba69cfebcac04e28133f651f301ae840d5c36ddddb1529b
+oid sha256:cbb398868d9495d1d98703c67e6d962238241ad82fc84d6f95dd75ff664b33ec
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.22530820965766907,
   "best_model_checkpoint": "joe611/chickens-60-epoch-1000-images-aug/checkpoint-25000",
-  "epoch": 58.0,
+  "epoch": 59.0,
   "eval_steps": 500,
-  "global_step": 29000,
+  "global_step": 29500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -8277,6 +8277,151 @@
       "eval_samples_per_second": 10.168,
       "eval_steps_per_second": 1.322,
       "step": 29000
+    },
+    {
+      "epoch": 58.02,
+      "grad_norm": 31.76934242248535,
+      "learning_rate": 2.684594004283836e-08,
+      "loss": 0.2661,
+      "step": 29010
+    },
+    {
+      "epoch": 58.08,
+      "grad_norm": 33.10856628417969,
+      "learning_rate": 2.5244915093499134e-08,
+      "loss": 0.2613,
+      "step": 29040
+    },
+    {
+      "epoch": 58.14,
+      "grad_norm": 54.95156478881836,
+      "learning_rate": 2.3692988968458398e-08,
+      "loss": 0.3306,
+      "step": 29070
+    },
+    {
+      "epoch": 58.2,
+      "grad_norm": 50.21295928955078,
+      "learning_rate": 2.219017698460002e-08,
+      "loss": 0.2639,
+      "step": 29100
+    },
+    {
+      "epoch": 58.26,
+      "grad_norm": 34.72473907470703,
+      "learning_rate": 2.0736493974071736e-08,
+      "loss": 0.2776,
+      "step": 29130
+    },
+    {
+      "epoch": 58.32,
+      "grad_norm": 47.885765075683594,
+      "learning_rate": 1.9331954284137476e-08,
+      "loss": 0.2702,
+      "step": 29160
+    },
+    {
+      "epoch": 58.38,
+      "grad_norm": 48.69330978393555,
+      "learning_rate": 1.7976571777038044e-08,
+      "loss": 0.2764,
+      "step": 29190
+    },
+    {
+      "epoch": 58.44,
+      "grad_norm": 48.01133728027344,
+      "learning_rate": 1.6670359829850657e-08,
+      "loss": 0.2787,
+      "step": 29220
+    },
+    {
+      "epoch": 58.5,
+      "grad_norm": 38.076595306396484,
+      "learning_rate": 1.541333133436018e-08,
+      "loss": 0.3156,
+      "step": 29250
+    },
+    {
+      "epoch": 58.56,
+      "grad_norm": 77.71937561035156,
+      "learning_rate": 1.4205498696930332e-08,
+      "loss": 0.2821,
+      "step": 29280
+    },
+    {
+      "epoch": 58.62,
+      "grad_norm": 69.0176010131836,
+      "learning_rate": 1.3046873838381546e-08,
+      "loss": 0.2954,
+      "step": 29310
+    },
+    {
+      "epoch": 58.68,
+      "grad_norm": 43.61737823486328,
+      "learning_rate": 1.1937468193873869e-08,
+      "loss": 0.2688,
+      "step": 29340
+    },
+    {
+      "epoch": 58.74,
+      "grad_norm": 48.39431381225586,
+      "learning_rate": 1.0877292712792586e-08,
+      "loss": 0.2566,
+      "step": 29370
+    },
+    {
+      "epoch": 58.8,
+      "grad_norm": 66.10430908203125,
+      "learning_rate": 9.866357858642206e-09,
+      "loss": 0.2634,
+      "step": 29400
+    },
+    {
+      "epoch": 58.86,
+      "grad_norm": 147.0448760986328,
+      "learning_rate": 8.904673608940983e-09,
+      "loss": 0.3025,
+      "step": 29430
+    },
+    {
+      "epoch": 58.92,
+      "grad_norm": 52.136348724365234,
+      "learning_rate": 7.992249455124889e-09,
+      "loss": 0.3041,
+      "step": 29460
+    },
+    {
+      "epoch": 58.98,
+      "grad_norm": 63.46728515625,
+      "learning_rate": 7.129094402451575e-09,
+      "loss": 0.2693,
+      "step": 29490
+    },
+    {
+      "epoch": 59.0,
+      "eval_loss": 0.2272312492132187,
+      "eval_map": 0.8302,
+      "eval_map_50": 0.9765,
+      "eval_map_75": 0.9394,
+      "eval_map_chicken": 0.8148,
+      "eval_map_duck": 0.8005,
+      "eval_map_large": 0.9097,
+      "eval_map_medium": 0.8372,
+      "eval_map_plant": 0.8752,
+      "eval_map_small": 0.3768,
+      "eval_mar_1": 0.3092,
+      "eval_mar_10": 0.8633,
+      "eval_mar_100": 0.8673,
+      "eval_mar_100_chicken": 0.852,
+      "eval_mar_100_duck": 0.832,
+      "eval_mar_100_plant": 0.9179,
+      "eval_mar_large": 0.9469,
+      "eval_mar_medium": 0.8825,
+      "eval_mar_small": 0.4523,
+      "eval_runtime": 10.2822,
+      "eval_samples_per_second": 9.726,
+      "eval_steps_per_second": 1.264,
+      "step": 29500
     }
   ],
   "logging_steps": 30,
@@ -8296,7 +8441,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 9.9766702743552e+18,
+  "total_flos": 1.01486818308096e+19,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null