Training in progress, step 9000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63d424da125909272adbcd6febb158fb464d384fcc7052b7279506ac6f5969ac
+oid sha256:2f166b67a2e5e8da54bcd327bceca4b864da3d9470f1e27ae8c07237158f24f2
 size 12347192855

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:00f04d4ca24866b389dc2df4457f672ba5912421795c884ce9a09b3caf5317e1
+oid sha256:fd38db0ab531900a0edf77b4884776dbc30584f579ce891d6129463490ea14aa
 size 6173655480

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fbfadcfc0728788eb8464114731524e59e0eeea3de78fb600f90df25108582a6
+oid sha256:14d50af3ad80506d9b5eb2378638e4d6bd5b02f77a4e8ed4e65b44aee3f7410c
 size 14575

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2033db9ad36b6f8a091d260383726ce676844e1562b9251b94e781e66324da99
+oid sha256:4a05fc2aa63e1878582f11ce1a8a19360e9ade1f13595895a6cf908de48f3707
 size 557

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:72ca6c784ba72f382b1bcb047ab8308017dc36fd5ba0484d4a1d02017086a52b
+oid sha256:adf20911833533da369ecd232dea5963d4a78412ad9ec6b3923960496ee6c41e
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": 0.11536111854119403,
   "best_model_checkpoint": "/storage/elsayedissa/whisper-large-v2-spanish/checkpoint-1000",
-  "epoch": 0.27769099934048386,
-  "global_step": 8000,
+  "epoch": 0.31240237425804435,
+  "global_step": 9000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1998,11 +1998,260 @@
       "eval_steps_per_second": 0.062,
       "eval_wer": 0.10428864386126549,
       "step": 8000
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 6.931836734693878e-06,
+      "loss": 0.179,
+      "step": 8025
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 6.921632653061226e-06,
+      "loss": 0.1647,
+      "step": 8050
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 6.911428571428572e-06,
+      "loss": 0.1792,
+      "step": 8075
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 6.901224489795919e-06,
+      "loss": 0.1815,
+      "step": 8100
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 6.891020408163266e-06,
+      "loss": 0.1943,
+      "step": 8125
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 6.880816326530612e-06,
+      "loss": 0.1473,
+      "step": 8150
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 6.87061224489796e-06,
+      "loss": 0.196,
+      "step": 8175
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 6.8604081632653065e-06,
+      "loss": 0.1471,
+      "step": 8200
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 6.850204081632653e-06,
+      "loss": 0.1928,
+      "step": 8225
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 6.8400000000000014e-06,
+      "loss": 0.1693,
+      "step": 8250
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 6.829795918367348e-06,
+      "loss": 0.1941,
+      "step": 8275
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 6.819591836734695e-06,
+      "loss": 0.1627,
+      "step": 8300
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 6.809387755102041e-06,
+      "loss": 0.1591,
+      "step": 8325
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 6.799183673469388e-06,
+      "loss": 0.1641,
+      "step": 8350
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 6.7889795918367355e-06,
+      "loss": 0.1943,
+      "step": 8375
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 6.778775510204082e-06,
+      "loss": 0.1474,
+      "step": 8400
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 6.768571428571429e-06,
+      "loss": 0.2049,
+      "step": 8425
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 6.758367346938775e-06,
+      "loss": 0.1701,
+      "step": 8450
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 6.748163265306124e-06,
+      "loss": 0.1948,
+      "step": 8475
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 6.73795918367347e-06,
+      "loss": 0.176,
+      "step": 8500
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 6.727755102040817e-06,
+      "loss": 0.2003,
+      "step": 8525
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 6.717551020408164e-06,
+      "loss": 0.1687,
+      "step": 8550
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 6.707346938775511e-06,
+      "loss": 0.1838,
+      "step": 8575
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 6.697142857142858e-06,
+      "loss": 0.1492,
+      "step": 8600
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 6.686938775510204e-06,
+      "loss": 0.1886,
+      "step": 8625
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 6.676734693877551e-06,
+      "loss": 0.1604,
+      "step": 8650
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 6.666530612244898e-06,
+      "loss": 0.1935,
+      "step": 8675
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 6.656326530612246e-06,
+      "loss": 0.1571,
+      "step": 8700
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 6.646122448979593e-06,
+      "loss": 0.1697,
+      "step": 8725
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 6.635918367346939e-06,
+      "loss": 0.1644,
+      "step": 8750
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 6.625714285714287e-06,
+      "loss": 0.1977,
+      "step": 8775
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.615510204081633e-06,
+      "loss": 0.1566,
+      "step": 8800
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.60530612244898e-06,
+      "loss": 0.1676,
+      "step": 8825
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.595102040816327e-06,
+      "loss": 0.151,
+      "step": 8850
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.584897959183673e-06,
+      "loss": 0.1828,
+      "step": 8875
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.574693877551021e-06,
+      "loss": 0.1781,
+      "step": 8900
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.564489795918368e-06,
+      "loss": 0.1689,
+      "step": 8925
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.554285714285715e-06,
+      "loss": 0.1698,
+      "step": 8950
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.5440816326530624e-06,
+      "loss": 0.1893,
+      "step": 8975
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.533877551020409e-06,
+      "loss": 0.166,
+      "step": 9000
+    },
+    {
+      "epoch": 0.31,
+      "eval_loss": 0.18502512574195862,
+      "eval_runtime": 31506.0133,
+      "eval_samples_per_second": 0.985,
+      "eval_steps_per_second": 0.062,
+      "eval_wer": 0.10222473193911955,
+      "step": 9000
     }
   ],
   "max_steps": 25000,
   "num_train_epochs": 1,
-  "total_flos": 2.717658906624e+20,
+  "total_flos": 3.057366269952e+20,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:00f04d4ca24866b389dc2df4457f672ba5912421795c884ce9a09b3caf5317e1
+oid sha256:fd38db0ab531900a0edf77b4884776dbc30584f579ce891d6129463490ea14aa
 size 6173655480

runs/Dec26_19-57-19_gpu07.cyverse.org/events.out.tfevents.1672109970.gpu07.cyverse.org.126369.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:076eb2bada8a4a94fa9d189e543bfecad75db25a7e323324e16521b9cba64240
-size 57173
+oid sha256:3d63ccbe1acfa4c9224bc8fd7be72373fbc520dadad5e653bda1403d45e1622f
+size 63771