Training in progress, step 6000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae1044aaa1ff0119e85e5a0d94822e141e873a1f2c86ff72ffd19663c3603950
+oid sha256:f0460400a279e7766ba1a5c0ca78380731e0774cf4d291edfbc84654ee4a6aca
 size 12347192855

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:10329c118d1ff193245b323dfb24913485913d730509ac1e75a72918eceefd52
+oid sha256:5e0ff690a614ad4feb1584be1b22e3316419b9b45d45121399a24f7b7785e65c
 size 6173655480

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:51a6d2be8295de6c5e778332840e9e6dea5c8558aaf65c1f10a5f7fcbf6bc26c
-size 14575
+oid sha256:3788ae40196f57221f45ac4d182ab0f36e1ff61517502c4d51687db9b9b71e1f
+size 14511

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b99c6a597945a355f19ec7e8be26c1ddbe903987bb1e029b16374ec80b93d036
+oid sha256:da3defc4121d1071efa04d9a3c24a084fc57cd835b7f2d21ed3bc283ad291901
 size 557

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:15ee7b0a16db565d640aec25ccf9707932c3a303f7fa345a261173fe1e99fbfc
+oid sha256:6c16fd3410fdd3660983e7e486d0b41995f7675ed13e0a940e3532e9d1a7db51
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": 0.11536111854119403,
   "best_model_checkpoint": "/storage/elsayedissa/whisper-large-v2-spanish/checkpoint-1000",
-  "epoch": 0.17355687458780242,
-  "global_step": 5000,
+  "epoch": 0.2082682495053629,
+  "global_step": 6000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1251,11 +1251,260 @@
       "eval_steps_per_second": 0.063,
       "eval_wer": 0.10732672221066432,
       "step": 5000
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.155918367346939e-06,
+      "loss": 0.2001,
+      "step": 5025
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.145714285714287e-06,
+      "loss": 0.1835,
+      "step": 5050
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.135510204081634e-06,
+      "loss": 0.2016,
+      "step": 5075
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.12530612244898e-06,
+      "loss": 0.1907,
+      "step": 5100
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.115102040816327e-06,
+      "loss": 0.1955,
+      "step": 5125
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.104897959183675e-06,
+      "loss": 0.1904,
+      "step": 5150
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.09469387755102e-06,
+      "loss": 0.1986,
+      "step": 5175
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.084489795918368e-06,
+      "loss": 0.1499,
+      "step": 5200
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.074285714285714e-06,
+      "loss": 0.1963,
+      "step": 5225
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.064081632653063e-06,
+      "loss": 0.1821,
+      "step": 5250
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.053877551020409e-06,
+      "loss": 0.1766,
+      "step": 5275
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.043673469387756e-06,
+      "loss": 0.1827,
+      "step": 5300
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 8.033469387755102e-06,
+      "loss": 0.1911,
+      "step": 5325
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.02326530612245e-06,
+      "loss": 0.1809,
+      "step": 5350
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.013061224489797e-06,
+      "loss": 0.2068,
+      "step": 5375
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 8.002857142857143e-06,
+      "loss": 0.1841,
+      "step": 5400
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 7.99265306122449e-06,
+      "loss": 0.1834,
+      "step": 5425
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 7.982448979591836e-06,
+      "loss": 0.1848,
+      "step": 5450
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 7.972244897959184e-06,
+      "loss": 0.1938,
+      "step": 5475
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 7.962040816326531e-06,
+      "loss": 0.1747,
+      "step": 5500
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 7.951836734693879e-06,
+      "loss": 0.1929,
+      "step": 5525
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 7.941632653061226e-06,
+      "loss": 0.1785,
+      "step": 5550
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 7.931428571428572e-06,
+      "loss": 0.1894,
+      "step": 5575
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 7.92122448979592e-06,
+      "loss": 0.172,
+      "step": 5600
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 7.911020408163265e-06,
+      "loss": 0.2007,
+      "step": 5625
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 7.900816326530613e-06,
+      "loss": 0.1493,
+      "step": 5650
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 7.89061224489796e-06,
+      "loss": 0.2092,
+      "step": 5675
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 7.880408163265306e-06,
+      "loss": 0.1795,
+      "step": 5700
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 7.870204081632653e-06,
+      "loss": 0.1958,
+      "step": 5725
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 7.860000000000001e-06,
+      "loss": 0.165,
+      "step": 5750
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 7.849795918367348e-06,
+      "loss": 0.1826,
+      "step": 5775
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 7.839591836734694e-06,
+      "loss": 0.162,
+      "step": 5800
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 7.829387755102042e-06,
+      "loss": 0.1729,
+      "step": 5825
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 7.819183673469388e-06,
+      "loss": 0.1912,
+      "step": 5850
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 7.808979591836735e-06,
+      "loss": 0.181,
+      "step": 5875
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 7.798775510204083e-06,
+      "loss": 0.1547,
+      "step": 5900
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.788571428571428e-06,
+      "loss": 0.2268,
+      "step": 5925
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.778367346938776e-06,
+      "loss": 0.1674,
+      "step": 5950
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.768163265306123e-06,
+      "loss": 0.1934,
+      "step": 5975
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 7.75795918367347e-06,
+      "loss": 0.1535,
+      "step": 6000
+    },
+    {
+      "epoch": 0.21,
+      "eval_loss": 0.1971120834350586,
+      "eval_runtime": 31128.1173,
+      "eval_samples_per_second": 0.997,
+      "eval_steps_per_second": 0.062,
+      "eval_wer": 0.10863441680453598,
+      "step": 6000
     }
   ],
   "max_steps": 25000,
   "num_train_epochs": 1,
-  "total_flos": 1.69853681664e+20,
+  "total_flos": 2.038244179968e+20,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:10329c118d1ff193245b323dfb24913485913d730509ac1e75a72918eceefd52
+oid sha256:5e0ff690a614ad4feb1584be1b22e3316419b9b45d45121399a24f7b7785e65c
 size 6173655480

runs/Dec26_19-57-19_gpu07.cyverse.org/events.out.tfevents.1672109970.gpu07.cyverse.org.126369.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1de67b1e2d6be047867041a9be230f9e7c3ad6c837579ee7e86bfee1abf00cb1
-size 37379
+oid sha256:5c471ec8b7d00e44f926ec8e72e6951f01dad13c670f4fcd66f750928299dbb9
+size 43977