Training in progress, step 24000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2daba987c3635c07dbbfcc0f0b00435568b055cd9e65badd42c48ee0e8cc6eb7
+oid sha256:bd7eb848685ef3f06accf08b0991ae5fb7aca4770e2f824f2a4e024e639e7fb2
 size 12347192855

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7110b14f56f7f4c10e6b71fbe6f5251480a4ae2ea07b69ac339d59349d6f9560
+oid sha256:8e71d25993f756e62aa28d38555706c7d09a30a1163fed42767f77816851100d
 size 6173655480

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ee749995a148c38ebbb92f25aca7282f391ce6f8f7323887d7232d39b5ea5d4
-size 14575
+oid sha256:9eaf0badf99eb2f5a4bcf7352b7e7e60bdb9fb3eb0aafd7a775251c7a5269cfd
+size 14511

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:13a7589714ca42744cc53ba86bd24fc9f77beb3a4a23c480fa17e3779ac95997
+oid sha256:cf5ce6709b8cf97417772fb30cceb44ce3363ceb800bc698892d9018ba93c60f
 size 557

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0261b0ef9ff8bcf7f335c8ff2f03f795976c908bbc906ae5bd82b9ccd45be639
+oid sha256:d0b3ee000403ded4a43ceaf77d66b9d28960f5cc5d0851756133984c89e3a2c4
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": 0.11536111854119403,
   "best_model_checkpoint": "/storage/elsayedissa/whisper-large-v2-spanish/checkpoint-1000",
-  "epoch": 0.7983616231038911,
-  "global_step": 23000,
+  "epoch": 0.8330729980214516,
+  "global_step": 24000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5733,11 +5733,260 @@
       "eval_steps_per_second": 0.062,
       "eval_wer": 0.08671732332088382,
       "step": 23000
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 8.110204081632655e-07,
+      "loss": 0.1585,
+      "step": 23025
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 8.008163265306124e-07,
+      "loss": 0.1183,
+      "step": 23050
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 7.906122448979592e-07,
+      "loss": 0.1367,
+      "step": 23075
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 7.804081632653061e-07,
+      "loss": 0.1175,
+      "step": 23100
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 7.702040816326531e-07,
+      "loss": 0.1487,
+      "step": 23125
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 7.6e-07,
+      "loss": 0.119,
+      "step": 23150
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 7.49795918367347e-07,
+      "loss": 0.1402,
+      "step": 23175
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 7.395918367346939e-07,
+      "loss": 0.1307,
+      "step": 23200
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 7.293877551020409e-07,
+      "loss": 0.1547,
+      "step": 23225
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 7.191836734693879e-07,
+      "loss": 0.1208,
+      "step": 23250
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 7.089795918367348e-07,
+      "loss": 0.1647,
+      "step": 23275
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 6.987755102040817e-07,
+      "loss": 0.1025,
+      "step": 23300
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 6.885714285714286e-07,
+      "loss": 0.1778,
+      "step": 23325
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 6.783673469387756e-07,
+      "loss": 0.1232,
+      "step": 23350
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 6.681632653061225e-07,
+      "loss": 0.1301,
+      "step": 23375
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 6.579591836734695e-07,
+      "loss": 0.1401,
+      "step": 23400
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 6.477551020408164e-07,
+      "loss": 0.1588,
+      "step": 23425
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 6.375510204081634e-07,
+      "loss": 0.1226,
+      "step": 23450
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 6.273469387755104e-07,
+      "loss": 0.1594,
+      "step": 23475
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 6.171428571428572e-07,
+      "loss": 0.1341,
+      "step": 23500
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 6.069387755102042e-07,
+      "loss": 0.1606,
+      "step": 23525
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 5.96734693877551e-07,
+      "loss": 0.1174,
+      "step": 23550
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 5.86530612244898e-07,
+      "loss": 0.156,
+      "step": 23575
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 5.763265306122449e-07,
+      "loss": 0.1323,
+      "step": 23600
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 5.661224489795919e-07,
+      "loss": 0.1532,
+      "step": 23625
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 5.559183673469388e-07,
+      "loss": 0.119,
+      "step": 23650
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 5.457142857142857e-07,
+      "loss": 0.1476,
+      "step": 23675
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 5.355102040816326e-07,
+      "loss": 0.1331,
+      "step": 23700
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 5.253061224489796e-07,
+      "loss": 0.1498,
+      "step": 23725
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 5.151020408163266e-07,
+      "loss": 0.1268,
+      "step": 23750
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 5.048979591836735e-07,
+      "loss": 0.1327,
+      "step": 23775
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.946938775510205e-07,
+      "loss": 0.1253,
+      "step": 23800
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.844897959183674e-07,
+      "loss": 0.1756,
+      "step": 23825
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.7428571428571434e-07,
+      "loss": 0.1487,
+      "step": 23850
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.6408163265306124e-07,
+      "loss": 0.1502,
+      "step": 23875
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.538775510204082e-07,
+      "loss": 0.1231,
+      "step": 23900
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.4367346938775515e-07,
+      "loss": 0.1352,
+      "step": 23925
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.334693877551021e-07,
+      "loss": 0.141,
+      "step": 23950
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.2326530612244896e-07,
+      "loss": 0.1539,
+      "step": 23975
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.130612244897959e-07,
+      "loss": 0.1337,
+      "step": 24000
+    },
+    {
+      "epoch": 0.83,
+      "eval_loss": 0.1472465544939041,
+      "eval_runtime": 31230.9152,
+      "eval_samples_per_second": 0.994,
+      "eval_steps_per_second": 0.062,
+      "eval_wer": 0.08535018806365434,
+      "step": 24000
     }
   ],
   "max_steps": 25000,
   "num_train_epochs": 1,
-  "total_flos": 7.813269356544e+20,
+  "total_flos": 8.152976719872e+20,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7110b14f56f7f4c10e6b71fbe6f5251480a4ae2ea07b69ac339d59349d6f9560
+oid sha256:8e71d25993f756e62aa28d38555706c7d09a30a1163fed42767f77816851100d
 size 6173655480

runs/Dec26_19-57-19_gpu07.cyverse.org/events.out.tfevents.1672109970.gpu07.cyverse.org.126369.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:40bb4a9db04f1abcaa80cc21618c523855a5b501d308ce0a0b153c875673d03d
-size 156980
+oid sha256:3be1bf9d6afc1185ac6ba7539df5045fb2a516aaf0676ce7cb1d8b30a8903c18
+size 163704