Training in progress, step 13000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:178b5b502a485d7c2fe861b88b73958c4bdd88fbe0bd804aaac864b4c3d58b52
+oid sha256:9b82e023bcdce3d9867de00994de7448663053d12658724461977f225ef150ba
 size 12347192855

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d40f3344c48b8c1de58d7e8467befdcdcde144b9ff8ed4465cd30973a433d738
+oid sha256:b873f46f3287d785fe9e0e65cbecdf06f727c84e188245367c48b687b9339708
 size 6173655480

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:35e348aa5f9f6cf6306d6ba94fb59dc91d14cd792144b93c44e821af2cb774ce
-size 14575
+oid sha256:c516e23ac00a4fed94805529a2f025da998c6366d20bf4032bb8f1a2f5ae0ce3
+size 14511

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a3ab4d8f5a2206989f1574e42f0b2264f32c39696956f3c8247ab96f9f887022
+oid sha256:c19297e7f4fe97d7584a052fffbfde0e921e6770e0bba0e78fc588359f317245
 size 557

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e22e62dec3718a0ec90f6becc88cb24f88c5c482567956f03f4b33d800c0df20
+oid sha256:52755b2d249438bf866dbd6a565d7c609e276075f6ad3cb63ef2a63e3670f1cd
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": 0.11536111854119403,
   "best_model_checkpoint": "/storage/elsayedissa/whisper-large-v2-spanish/checkpoint-1000",
-  "epoch": 0.4165364990107258,
-  "global_step": 12000,
+  "epoch": 0.4512478739282863,
+  "global_step": 13000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2994,11 +2994,260 @@
       "eval_steps_per_second": 0.063,
       "eval_wer": 0.09824881201229761,
       "step": 12000
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.300000000000001e-06,
+      "loss": 0.1843,
+      "step": 12025
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.2897959183673476e-06,
+      "loss": 0.155,
+      "step": 12050
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.279591836734694e-06,
+      "loss": 0.1799,
+      "step": 12075
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.269387755102041e-06,
+      "loss": 0.1632,
+      "step": 12100
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.259183673469388e-06,
+      "loss": 0.172,
+      "step": 12125
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.248979591836735e-06,
+      "loss": 0.17,
+      "step": 12150
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.238775510204082e-06,
+      "loss": 0.1863,
+      "step": 12175
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.22857142857143e-06,
+      "loss": 0.1611,
+      "step": 12200
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 5.218367346938777e-06,
+      "loss": 0.1737,
+      "step": 12225
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.208163265306123e-06,
+      "loss": 0.164,
+      "step": 12250
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.19795918367347e-06,
+      "loss": 0.1639,
+      "step": 12275
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.1877551020408165e-06,
+      "loss": 0.1428,
+      "step": 12300
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.177551020408164e-06,
+      "loss": 0.1688,
+      "step": 12325
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.167346938775511e-06,
+      "loss": 0.155,
+      "step": 12350
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.157142857142857e-06,
+      "loss": 0.1702,
+      "step": 12375
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.146938775510204e-06,
+      "loss": 0.1409,
+      "step": 12400
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.136734693877552e-06,
+      "loss": 0.1677,
+      "step": 12425
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.126530612244899e-06,
+      "loss": 0.139,
+      "step": 12450
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.1163265306122455e-06,
+      "loss": 0.184,
+      "step": 12475
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.106122448979592e-06,
+      "loss": 0.1452,
+      "step": 12500
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 5.09591836734694e-06,
+      "loss": 0.1651,
+      "step": 12525
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.085714285714286e-06,
+      "loss": 0.1628,
+      "step": 12550
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.075510204081633e-06,
+      "loss": 0.181,
+      "step": 12575
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.0653061224489795e-06,
+      "loss": 0.1515,
+      "step": 12600
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.055102040816326e-06,
+      "loss": 0.1748,
+      "step": 12625
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.0448979591836745e-06,
+      "loss": 0.1504,
+      "step": 12650
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.034693877551021e-06,
+      "loss": 0.1696,
+      "step": 12675
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.024489795918368e-06,
+      "loss": 0.1604,
+      "step": 12700
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.014285714285715e-06,
+      "loss": 0.1779,
+      "step": 12725
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 5.004081632653062e-06,
+      "loss": 0.1646,
+      "step": 12750
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 4.9938775510204086e-06,
+      "loss": 0.1556,
+      "step": 12775
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 4.983673469387755e-06,
+      "loss": 0.1498,
+      "step": 12800
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 4.973469387755103e-06,
+      "loss": 0.1715,
+      "step": 12825
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 4.963265306122449e-06,
+      "loss": 0.1219,
+      "step": 12850
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 4.953061224489796e-06,
+      "loss": 0.1666,
+      "step": 12875
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 4.9428571428571435e-06,
+      "loss": 0.1476,
+      "step": 12900
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 4.93265306122449e-06,
+      "loss": 0.1741,
+      "step": 12925
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 4.9224489795918376e-06,
+      "loss": 0.1579,
+      "step": 12950
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 4.912244897959184e-06,
+      "loss": 0.164,
+      "step": 12975
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 4.902040816326531e-06,
+      "loss": 0.1612,
+      "step": 13000
+    },
+    {
+      "epoch": 0.45,
+      "eval_loss": 0.17242176830768585,
+      "eval_runtime": 31404.3519,
+      "eval_samples_per_second": 0.988,
+      "eval_steps_per_second": 0.062,
+      "eval_wer": 0.09758505793813547,
+      "step": 13000
     }
   ],
   "max_steps": 25000,
   "num_train_epochs": 1,
-  "total_flos": 4.076488359936e+20,
+  "total_flos": 4.416195723264e+20,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d40f3344c48b8c1de58d7e8467befdcdcde144b9ff8ed4465cd30973a433d738
+oid sha256:b873f46f3287d785fe9e0e65cbecdf06f727c84e188245367c48b687b9339708
 size 6173655480

runs/Dec26_19-57-19_gpu07.cyverse.org/events.out.tfevents.1672109970.gpu07.cyverse.org.126369.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc505845f9a549bc125c53e3e46d54bfa0b789b17c0fc518106e8b0e000bbc35
-size 83565
+oid sha256:6a3ad336968cb511fe9ae780cd83e8101970e1c047eb76f8596c5e84f34771a2
+size 90163