Training in progress, step 5000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:071c5f4b0d6b94f41e4ec3d1d9e0d746d356fee79d390bbf5696fdec12d09eee
+oid sha256:ae1044aaa1ff0119e85e5a0d94822e141e873a1f2c86ff72ffd19663c3603950
 size 12347192855

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d13c1515c5bec3b1b6df88233162654239b2f0a7c1119dddff788e8c045bd24
+oid sha256:10329c118d1ff193245b323dfb24913485913d730509ac1e75a72918eceefd52
 size 6173655480

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec70788b682d89359da85d225e858c6024d3fd2773541d2690bca2b322a42f06
+oid sha256:51a6d2be8295de6c5e778332840e9e6dea5c8558aaf65c1f10a5f7fcbf6bc26c
 size 14575

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d92186bd8b7a36cbab743326563ee77d960ade57f12c3c59d4d213b20af4bd62
+oid sha256:b99c6a597945a355f19ec7e8be26c1ddbe903987bb1e029b16374ec80b93d036
 size 557

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8e131db79df8f2c3d12190a618e4f984ac87c57b6930e6ec08f7edadfe13646
+oid sha256:15ee7b0a16db565d640aec25ccf9707932c3a303f7fa345a261173fe1e99fbfc
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": 0.11536111854119403,
   "best_model_checkpoint": "/storage/elsayedissa/whisper-large-v2-spanish/checkpoint-1000",
-  "epoch": 0.13884549967024193,
-  "global_step": 4000,
+  "epoch": 0.17355687458780242,
+  "global_step": 5000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1002,11 +1002,260 @@
       "eval_steps_per_second": 0.062,
       "eval_wer": 0.11463792380367409,
       "step": 4000
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.564081632653062e-06,
+      "loss": 0.2078,
+      "step": 4025
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.55387755102041e-06,
+      "loss": 0.1935,
+      "step": 4050
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.543673469387755e-06,
+      "loss": 0.1966,
+      "step": 4075
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.533469387755102e-06,
+      "loss": 0.2027,
+      "step": 4100
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.52326530612245e-06,
+      "loss": 0.1985,
+      "step": 4125
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.513061224489797e-06,
+      "loss": 0.1822,
+      "step": 4150
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 8.502857142857143e-06,
+      "loss": 0.2102,
+      "step": 4175
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.49265306122449e-06,
+      "loss": 0.2183,
+      "step": 4200
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.482448979591838e-06,
+      "loss": 0.2299,
+      "step": 4225
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.472244897959184e-06,
+      "loss": 0.1802,
+      "step": 4250
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.462040816326531e-06,
+      "loss": 0.1931,
+      "step": 4275
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.451836734693877e-06,
+      "loss": 0.1847,
+      "step": 4300
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.441632653061225e-06,
+      "loss": 0.221,
+      "step": 4325
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.431428571428572e-06,
+      "loss": 0.1894,
+      "step": 4350
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.42122448979592e-06,
+      "loss": 0.1987,
+      "step": 4375
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.411020408163266e-06,
+      "loss": 0.1714,
+      "step": 4400
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.400816326530613e-06,
+      "loss": 0.2167,
+      "step": 4425
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 8.39061224489796e-06,
+      "loss": 0.1895,
+      "step": 4450
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.380408163265306e-06,
+      "loss": 0.1955,
+      "step": 4475
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.370204081632654e-06,
+      "loss": 0.1764,
+      "step": 4500
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.36e-06,
+      "loss": 0.2209,
+      "step": 4525
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.349795918367347e-06,
+      "loss": 0.1681,
+      "step": 4550
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.339591836734695e-06,
+      "loss": 0.218,
+      "step": 4575
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.329387755102042e-06,
+      "loss": 0.1725,
+      "step": 4600
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.31918367346939e-06,
+      "loss": 0.1768,
+      "step": 4625
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.308979591836735e-06,
+      "loss": 0.1893,
+      "step": 4650
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.298775510204083e-06,
+      "loss": 0.2104,
+      "step": 4675
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.288571428571429e-06,
+      "loss": 0.1708,
+      "step": 4700
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.278367346938776e-06,
+      "loss": 0.2091,
+      "step": 4725
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 8.268163265306124e-06,
+      "loss": 0.2185,
+      "step": 4750
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.25795918367347e-06,
+      "loss": 0.2086,
+      "step": 4775
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.247755102040817e-06,
+      "loss": 0.1724,
+      "step": 4800
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.237551020408164e-06,
+      "loss": 0.2082,
+      "step": 4825
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.227346938775512e-06,
+      "loss": 0.1546,
+      "step": 4850
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.217142857142858e-06,
+      "loss": 0.2014,
+      "step": 4875
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.206938775510205e-06,
+      "loss": 0.1666,
+      "step": 4900
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.196734693877551e-06,
+      "loss": 0.1793,
+      "step": 4925
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.186530612244898e-06,
+      "loss": 0.1734,
+      "step": 4950
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.176326530612246e-06,
+      "loss": 0.2033,
+      "step": 4975
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 8.166122448979592e-06,
+      "loss": 0.1656,
+      "step": 5000
+    },
+    {
+      "epoch": 0.17,
+      "eval_loss": 0.20018751919269562,
+      "eval_runtime": 30831.2936,
+      "eval_samples_per_second": 1.007,
+      "eval_steps_per_second": 0.063,
+      "eval_wer": 0.10732672221066432,
+      "step": 5000
     }
   ],
   "max_steps": 25000,
   "num_train_epochs": 1,
-  "total_flos": 1.358829453312e+20,
+  "total_flos": 1.69853681664e+20,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d13c1515c5bec3b1b6df88233162654239b2f0a7c1119dddff788e8c045bd24
+oid sha256:10329c118d1ff193245b323dfb24913485913d730509ac1e75a72918eceefd52
 size 6173655480

runs/Dec26_19-57-19_gpu07.cyverse.org/events.out.tfevents.1672109970.gpu07.cyverse.org.126369.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3be876d39b9ffab345bd37299ad53f20801693f40e6411525ec37c8d2d55b4dd
-size 30781
+oid sha256:1de67b1e2d6be047867041a9be230f9e7c3ad6c837579ee7e86bfee1abf00cb1
+size 37379