Training in progress, step 8000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6193d4dcef6d888921a1166b7db92e1165f3686f778873a319aa454e17ea622
+oid sha256:63d424da125909272adbcd6febb158fb464d384fcc7052b7279506ac6f5969ac
 size 12347192855

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e91ae1bbae769e9ce17ed73669048b57cd296ecd5ff7d7b0071e74ca1c28a4f
+oid sha256:00f04d4ca24866b389dc2df4457f672ba5912421795c884ce9a09b3caf5317e1
 size 6173655480

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49ab9d2b8499c20583cc80db00868f9477a33e6dd8363c0d4ff3b9c6bb471b6b
+oid sha256:fbfadcfc0728788eb8464114731524e59e0eeea3de78fb600f90df25108582a6
 size 14575

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cafc80e01cebde114db040b18403e6c790dd6f4096e4412ec1d94153944f97a8
+oid sha256:2033db9ad36b6f8a091d260383726ce676844e1562b9251b94e781e66324da99
 size 557

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d0b2313199b9b45d740a2bffd20986f78b2b72606b98ad7849fa54f34c2f5de
+oid sha256:72ca6c784ba72f382b1bcb047ab8308017dc36fd5ba0484d4a1d02017086a52b
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": 0.11536111854119403,
   "best_model_checkpoint": "/storage/elsayedissa/whisper-large-v2-spanish/checkpoint-1000",
-  "epoch": 0.2429796244229234,
-  "global_step": 7000,
+  "epoch": 0.27769099934048386,
+  "global_step": 8000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1749,11 +1749,260 @@
       "eval_steps_per_second": 0.063,
       "eval_wer": 0.10483351660871203,
       "step": 7000
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.339591836734694e-06,
+      "loss": 0.2032,
+      "step": 7025
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 7.329387755102041e-06,
+      "loss": 0.1564,
+      "step": 7050
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.319183673469389e-06,
+      "loss": 0.1912,
+      "step": 7075
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.309387755102041e-06,
+      "loss": 0.2008,
+      "step": 7100
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.299183673469389e-06,
+      "loss": 0.1955,
+      "step": 7125
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.288979591836736e-06,
+      "loss": 0.1671,
+      "step": 7150
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.278775510204082e-06,
+      "loss": 0.178,
+      "step": 7175
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.268571428571429e-06,
+      "loss": 0.1683,
+      "step": 7200
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.258367346938776e-06,
+      "loss": 0.1977,
+      "step": 7225
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.248163265306123e-06,
+      "loss": 0.159,
+      "step": 7250
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.23795918367347e-06,
+      "loss": 0.205,
+      "step": 7275
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.2277551020408164e-06,
+      "loss": 0.1717,
+      "step": 7300
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 7.217551020408163e-06,
+      "loss": 0.1921,
+      "step": 7325
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.207346938775511e-06,
+      "loss": 0.1541,
+      "step": 7350
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.197142857142858e-06,
+      "loss": 0.189,
+      "step": 7375
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.186938775510205e-06,
+      "loss": 0.1533,
+      "step": 7400
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.176734693877551e-06,
+      "loss": 0.2029,
+      "step": 7425
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.166530612244899e-06,
+      "loss": 0.1711,
+      "step": 7450
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.1563265306122455e-06,
+      "loss": 0.1906,
+      "step": 7475
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.146122448979592e-06,
+      "loss": 0.1598,
+      "step": 7500
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.135918367346939e-06,
+      "loss": 0.1761,
+      "step": 7525
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.125714285714286e-06,
+      "loss": 0.1641,
+      "step": 7550
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.115510204081634e-06,
+      "loss": 0.1971,
+      "step": 7575
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.10530612244898e-06,
+      "loss": 0.138,
+      "step": 7600
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 7.095102040816327e-06,
+      "loss": 0.1919,
+      "step": 7625
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.0848979591836745e-06,
+      "loss": 0.1698,
+      "step": 7650
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.074693877551021e-06,
+      "loss": 0.1832,
+      "step": 7675
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.064489795918368e-06,
+      "loss": 0.1643,
+      "step": 7700
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.054285714285714e-06,
+      "loss": 0.1919,
+      "step": 7725
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.044081632653061e-06,
+      "loss": 0.1675,
+      "step": 7750
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.0338775510204085e-06,
+      "loss": 0.1848,
+      "step": 7775
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.023673469387756e-06,
+      "loss": 0.1878,
+      "step": 7800
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.013469387755103e-06,
+      "loss": 0.2086,
+      "step": 7825
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 7.00326530612245e-06,
+      "loss": 0.1665,
+      "step": 7850
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 6.993061224489797e-06,
+      "loss": 0.2067,
+      "step": 7875
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 6.982857142857143e-06,
+      "loss": 0.1687,
+      "step": 7900
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 6.97265306122449e-06,
+      "loss": 0.1953,
+      "step": 7925
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 6.962448979591837e-06,
+      "loss": 0.1844,
+      "step": 7950
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 6.952244897959184e-06,
+      "loss": 0.1728,
+      "step": 7975
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 6.942040816326531e-06,
+      "loss": 0.1722,
+      "step": 8000
+    },
+    {
+      "epoch": 0.28,
+      "eval_loss": 0.18894970417022705,
+      "eval_runtime": 31292.0854,
+      "eval_samples_per_second": 0.992,
+      "eval_steps_per_second": 0.062,
+      "eval_wer": 0.10428864386126549,
+      "step": 8000
     }
   ],
   "max_steps": 25000,
   "num_train_epochs": 1,
-  "total_flos": 2.377951543296e+20,
+  "total_flos": 2.717658906624e+20,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e91ae1bbae769e9ce17ed73669048b57cd296ecd5ff7d7b0071e74ca1c28a4f
+oid sha256:00f04d4ca24866b389dc2df4457f672ba5912421795c884ce9a09b3caf5317e1
 size 6173655480

runs/Dec26_19-57-19_gpu07.cyverse.org/events.out.tfevents.1672109970.gpu07.cyverse.org.126369.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e2bff7f5195a10fc570bcd185faa14302e6ca8f8cd84b4a9fa8001f74a0166e
-size 50575
+oid sha256:076eb2bada8a4a94fa9d189e543bfecad75db25a7e323324e16521b9cba64240
+size 57173