Training in progress, step 12000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b15938f7058276e6bdfc2fddcc7a120da5e6a0c7c9c19c71e16cb873fc6886b3
+oid sha256:178b5b502a485d7c2fe861b88b73958c4bdd88fbe0bd804aaac864b4c3d58b52
 size 12347192855

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6bdd501f6153777369e4dc961faa1bf60f1f7535ded5d73c87f2ff06b5aa8064
+oid sha256:d40f3344c48b8c1de58d7e8467befdcdcde144b9ff8ed4465cd30973a433d738
 size 6173655480

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:069bc79c0b97b8bebabb6777e867a3617901ff0e79b431f2ba4a791a90815e7f
-size 14639
+oid sha256:35e348aa5f9f6cf6306d6ba94fb59dc91d14cd792144b93c44e821af2cb774ce
+size 14575

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b4b38bb567a4c3bd8aafad6c492b89fde494be62ffddc238da23107b19a9945
+oid sha256:a3ab4d8f5a2206989f1574e42f0b2264f32c39696956f3c8247ab96f9f887022
 size 557

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e9a74567933ed8c342d4d89869156924da4e6db37efcc7105e4c34e754e42a4
+oid sha256:e22e62dec3718a0ec90f6becc88cb24f88c5c482567956f03f4b33d800c0df20
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": 0.11536111854119403,
   "best_model_checkpoint": "/storage/elsayedissa/whisper-large-v2-spanish/checkpoint-1000",
-  "epoch": 0.3818251240931653,
-  "global_step": 11000,
+  "epoch": 0.4165364990107258,
+  "global_step": 12000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2745,11 +2745,260 @@
       "eval_steps_per_second": 0.062,
       "eval_wer": 0.09984380314573199,
       "step": 11000
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 5.7073469387755105e-06,
+      "loss": 0.1738,
+      "step": 11025
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 5.697142857142858e-06,
+      "loss": 0.156,
+      "step": 11050
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 5.686938775510205e-06,
+      "loss": 0.1786,
+      "step": 11075
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 5.676734693877551e-06,
+      "loss": 0.1681,
+      "step": 11100
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 5.666530612244898e-06,
+      "loss": 0.1678,
+      "step": 11125
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 5.656326530612245e-06,
+      "loss": 0.1696,
+      "step": 11150
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 5.646122448979592e-06,
+      "loss": 0.1693,
+      "step": 11175
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 5.6359183673469395e-06,
+      "loss": 0.1342,
+      "step": 11200
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 5.625714285714286e-06,
+      "loss": 0.2051,
+      "step": 11225
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 5.615510204081634e-06,
+      "loss": 0.1534,
+      "step": 11250
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 5.60530612244898e-06,
+      "loss": 0.1683,
+      "step": 11275
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 5.595102040816327e-06,
+      "loss": 0.1524,
+      "step": 11300
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 5.584897959183674e-06,
+      "loss": 0.184,
+      "step": 11325
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 5.574693877551021e-06,
+      "loss": 0.1486,
+      "step": 11350
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 5.564489795918368e-06,
+      "loss": 0.1685,
+      "step": 11375
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 5.554285714285714e-06,
+      "loss": 0.1645,
+      "step": 11400
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 5.544081632653062e-06,
+      "loss": 0.1743,
+      "step": 11425
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 5.533877551020409e-06,
+      "loss": 0.1377,
+      "step": 11450
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 5.524489795918367e-06,
+      "loss": 0.1776,
+      "step": 11475
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 5.514285714285714e-06,
+      "loss": 0.1365,
+      "step": 11500
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 5.504081632653062e-06,
+      "loss": 0.1777,
+      "step": 11525
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 5.493877551020409e-06,
+      "loss": 0.149,
+      "step": 11550
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 5.4836734693877555e-06,
+      "loss": 0.1855,
+      "step": 11575
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 5.473469387755103e-06,
+      "loss": 0.1543,
+      "step": 11600
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 5.46326530612245e-06,
+      "loss": 0.1817,
+      "step": 11625
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 5.453061224489796e-06,
+      "loss": 0.1614,
+      "step": 11650
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.442857142857143e-06,
+      "loss": 0.1544,
+      "step": 11675
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.4326530612244895e-06,
+      "loss": 0.1592,
+      "step": 11700
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.422448979591837e-06,
+      "loss": 0.1658,
+      "step": 11725
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.4122448979591845e-06,
+      "loss": 0.1596,
+      "step": 11750
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.402040816326531e-06,
+      "loss": 0.1707,
+      "step": 11775
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.391836734693879e-06,
+      "loss": 0.1383,
+      "step": 11800
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.381632653061225e-06,
+      "loss": 0.175,
+      "step": 11825
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.371428571428572e-06,
+      "loss": 0.1658,
+      "step": 11850
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.3612244897959186e-06,
+      "loss": 0.1864,
+      "step": 11875
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.351020408163265e-06,
+      "loss": 0.1441,
+      "step": 11900
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.340816326530613e-06,
+      "loss": 0.1559,
+      "step": 11925
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 5.330612244897959e-06,
+      "loss": 0.1559,
+      "step": 11950
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.320408163265307e-06,
+      "loss": 0.1786,
+      "step": 11975
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.310204081632654e-06,
+      "loss": 0.169,
+      "step": 12000
+    },
+    {
+      "epoch": 0.42,
+      "eval_loss": 0.17712195217609406,
+      "eval_runtime": 30926.8901,
+      "eval_samples_per_second": 1.004,
+      "eval_steps_per_second": 0.063,
+      "eval_wer": 0.09824881201229761,
+      "step": 12000
     }
   ],
   "max_steps": 25000,
   "num_train_epochs": 1,
-  "total_flos": 3.736780996608e+20,
+  "total_flos": 4.076488359936e+20,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6bdd501f6153777369e4dc961faa1bf60f1f7535ded5d73c87f2ff06b5aa8064
+oid sha256:d40f3344c48b8c1de58d7e8467befdcdcde144b9ff8ed4465cd30973a433d738
 size 6173655480

runs/Dec26_19-57-19_gpu07.cyverse.org/events.out.tfevents.1672109970.gpu07.cyverse.org.126369.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9c954f05ee34517b3eb4c4f0da7bbc163f490b1b0ac3836bb3e4f0baf08b45f5
-size 76967
+oid sha256:dc505845f9a549bc125c53e3e46d54bfa0b789b17c0fc518106e8b0e000bbc35
+size 83565