Training in progress, step 10000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f166b67a2e5e8da54bcd327bceca4b864da3d9470f1e27ae8c07237158f24f2
+oid sha256:666bd295da5110fc8ed72c097e043b17b68a4f46ec1f780894808ac7d810702a
 size 12347192855

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd38db0ab531900a0edf77b4884776dbc30584f579ce891d6129463490ea14aa
+oid sha256:492b321ad12ae8744ac23f006fa941f8bd7881d2b5e9aecb269dbdaf98b5ee88
 size 6173655480

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:14d50af3ad80506d9b5eb2378638e4d6bd5b02f77a4e8ed4e65b44aee3f7410c
+oid sha256:aad3fc937f2c6f56a2a6ddd2d655b320e20d5cf3c0672d854ed64624351b0488
 size 14575

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4a05fc2aa63e1878582f11ce1a8a19360e9ade1f13595895a6cf908de48f3707
+oid sha256:0ff19fa2317539f8bfb616fcdefc3caa2efd60d151a696a55bb38626512e9ba9
 size 557

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:adf20911833533da369ecd232dea5963d4a78412ad9ec6b3923960496ee6c41e
+oid sha256:1196c99a263cfa6a64b1d73b220e535cc282c309df0a28be120a2a37266e02ba
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": 0.11536111854119403,
   "best_model_checkpoint": "/storage/elsayedissa/whisper-large-v2-spanish/checkpoint-1000",
-  "epoch": 0.31240237425804435,
-  "global_step": 9000,
+  "epoch": 0.34711374917560484,
+  "global_step": 10000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2247,11 +2247,260 @@
       "eval_steps_per_second": 0.062,
       "eval_wer": 0.10222473193911955,
       "step": 9000
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.523673469387756e-06,
+      "loss": 0.1991,
+      "step": 9025
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 6.513469387755102e-06,
+      "loss": 0.1732,
+      "step": 9050
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.503265306122449e-06,
+      "loss": 0.1864,
+      "step": 9075
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.4930612244897965e-06,
+      "loss": 0.1535,
+      "step": 9100
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.482857142857143e-06,
+      "loss": 0.1749,
+      "step": 9125
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.47265306122449e-06,
+      "loss": 0.1707,
+      "step": 9150
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.462448979591838e-06,
+      "loss": 0.1855,
+      "step": 9175
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.452244897959185e-06,
+      "loss": 0.1667,
+      "step": 9200
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.442040816326531e-06,
+      "loss": 0.1997,
+      "step": 9225
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.431836734693878e-06,
+      "loss": 0.1476,
+      "step": 9250
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.421632653061225e-06,
+      "loss": 0.1955,
+      "step": 9275
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.411428571428572e-06,
+      "loss": 0.1701,
+      "step": 9300
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.401224489795919e-06,
+      "loss": 0.1738,
+      "step": 9325
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 6.391020408163265e-06,
+      "loss": 0.1659,
+      "step": 9350
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.380816326530612e-06,
+      "loss": 0.1665,
+      "step": 9375
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.37061224489796e-06,
+      "loss": 0.1668,
+      "step": 9400
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.360408163265307e-06,
+      "loss": 0.2077,
+      "step": 9425
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.350204081632654e-06,
+      "loss": 0.1615,
+      "step": 9450
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.34e-06,
+      "loss": 0.1833,
+      "step": 9475
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.329795918367348e-06,
+      "loss": 0.1696,
+      "step": 9500
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.319591836734694e-06,
+      "loss": 0.183,
+      "step": 9525
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.309387755102041e-06,
+      "loss": 0.1892,
+      "step": 9550
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.299183673469388e-06,
+      "loss": 0.168,
+      "step": 9575
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.288979591836734e-06,
+      "loss": 0.1645,
+      "step": 9600
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.278775510204083e-06,
+      "loss": 0.181,
+      "step": 9625
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 6.268571428571429e-06,
+      "loss": 0.1508,
+      "step": 9650
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.258367346938776e-06,
+      "loss": 0.1776,
+      "step": 9675
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.248163265306123e-06,
+      "loss": 0.161,
+      "step": 9700
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.23795918367347e-06,
+      "loss": 0.1873,
+      "step": 9725
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.227755102040817e-06,
+      "loss": 0.166,
+      "step": 9750
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.217551020408163e-06,
+      "loss": 0.1739,
+      "step": 9775
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.20734693877551e-06,
+      "loss": 0.1496,
+      "step": 9800
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.1971428571428575e-06,
+      "loss": 0.1924,
+      "step": 9825
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.186938775510205e-06,
+      "loss": 0.1453,
+      "step": 9850
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.176734693877552e-06,
+      "loss": 0.196,
+      "step": 9875
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.166530612244899e-06,
+      "loss": 0.1782,
+      "step": 9900
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 6.156326530612246e-06,
+      "loss": 0.1855,
+      "step": 9925
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.146122448979592e-06,
+      "loss": 0.1628,
+      "step": 9950
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.135918367346939e-06,
+      "loss": 0.1733,
+      "step": 9975
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 6.125714285714286e-06,
+      "loss": 0.1277,
+      "step": 10000
+    },
+    {
+      "epoch": 0.35,
+      "eval_loss": 0.1820017397403717,
+      "eval_runtime": 31277.5076,
+      "eval_samples_per_second": 0.992,
+      "eval_steps_per_second": 0.062,
+      "eval_wer": 0.10322861869805133,
+      "step": 10000
     }
   ],
   "max_steps": 25000,
   "num_train_epochs": 1,
-  "total_flos": 3.057366269952e+20,
+  "total_flos": 3.39707363328e+20,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd38db0ab531900a0edf77b4884776dbc30584f579ce891d6129463490ea14aa
+oid sha256:492b321ad12ae8744ac23f006fa941f8bd7881d2b5e9aecb269dbdaf98b5ee88
 size 6173655480

runs/Dec26_19-57-19_gpu07.cyverse.org/events.out.tfevents.1672109970.gpu07.cyverse.org.126369.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d63ccbe1acfa4c9224bc8fd7be72373fbc520dadad5e653bda1403d45e1622f
-size 63771
+oid sha256:3649b56169acd2abda9b0ee07a52cc5ff3d90c7e405842df1439d1a6f490683e
+size 70369