Training in progress, step 16000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:36dbc50cf9140fb7f0d07a538501afa688944f7d4db346dd9f7ac42fb1be590a
+oid sha256:b2828bba9c0610d92bcb9ced115dba1825e24c111402867d65a37cba2fd4cdf3
 size 12347192855

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1327fc0bd4b84f3f75a5ae566a31c8d3866b9addc072e36ebcf7aa9bd4b067fa
+oid sha256:41ddc7f21fd5cb7cbeac420cbf26556fe05d0d0276c5d96c3c5d572508243cc6
 size 6173655480

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a48bbb35f10fc80761770aebcc3f7c466fe7a23a21b518a2ddab7c9475727995
+oid sha256:d6024b9a42abf2c5787dca8b8ac21caaa78ef8edc18fdd3f5727713a6bded2de
 size 14575

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:870e427bfebba6376a09e42afc7cf27343ead25cf07419d6ca98b32ac4723fc2
+oid sha256:8beddb254d719968614077f7da7885f0decec146856dee753939f87ce125fd6c
 size 557

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee2e34fac94bd8efbe6863121732ff5580c12a2335033285d23cf8ff6a7c8cb1
+oid sha256:0fd534b0cae4142fc61b98b6723afbc5b4255a5022df3ad8a29653d61c4f84f8
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": 0.11536111854119403,
   "best_model_checkpoint": "/storage/elsayedissa/whisper-large-v2-spanish/checkpoint-1000",
-  "epoch": 0.5206706237634072,
-  "global_step": 15000,
+  "epoch": 0.5553819986809677,
+  "global_step": 16000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3741,11 +3741,260 @@
       "eval_steps_per_second": 0.062,
       "eval_wer": 0.09415070849968464,
       "step": 15000
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.075918367346939e-06,
+      "loss": 0.1621,
+      "step": 15025
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.065714285714286e-06,
+      "loss": 0.1453,
+      "step": 15050
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.055510204081633e-06,
+      "loss": 0.1684,
+      "step": 15075
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 4.04530612244898e-06,
+      "loss": 0.1425,
+      "step": 15100
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.035102040816327e-06,
+      "loss": 0.173,
+      "step": 15125
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.024897959183674e-06,
+      "loss": 0.1417,
+      "step": 15150
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.014693877551021e-06,
+      "loss": 0.1659,
+      "step": 15175
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 4.004489795918367e-06,
+      "loss": 0.1472,
+      "step": 15200
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 3.994285714285714e-06,
+      "loss": 0.1528,
+      "step": 15225
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 3.9840816326530615e-06,
+      "loss": 0.1337,
+      "step": 15250
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 3.973877551020409e-06,
+      "loss": 0.1851,
+      "step": 15275
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 3.963673469387756e-06,
+      "loss": 0.1305,
+      "step": 15300
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 3.953469387755102e-06,
+      "loss": 0.161,
+      "step": 15325
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 3.943265306122449e-06,
+      "loss": 0.1332,
+      "step": 15350
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 3.933061224489796e-06,
+      "loss": 0.1638,
+      "step": 15375
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 3.922857142857143e-06,
+      "loss": 0.151,
+      "step": 15400
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 3.91265306122449e-06,
+      "loss": 0.1606,
+      "step": 15425
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 3.902448979591837e-06,
+      "loss": 0.1567,
+      "step": 15450
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 3.892244897959185e-06,
+      "loss": 0.1601,
+      "step": 15475
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 3.882040816326531e-06,
+      "loss": 0.1424,
+      "step": 15500
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 3.871836734693878e-06,
+      "loss": 0.1705,
+      "step": 15525
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 3.861632653061225e-06,
+      "loss": 0.1169,
+      "step": 15550
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 3.851428571428571e-06,
+      "loss": 0.176,
+      "step": 15575
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 3.841224489795919e-06,
+      "loss": 0.1359,
+      "step": 15600
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 3.831020408163265e-06,
+      "loss": 0.1479,
+      "step": 15625
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 3.820816326530613e-06,
+      "loss": 0.1404,
+      "step": 15650
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 3.8106122448979595e-06,
+      "loss": 0.1683,
+      "step": 15675
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 3.8004081632653065e-06,
+      "loss": 0.1315,
+      "step": 15700
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 3.7902040816326536e-06,
+      "loss": 0.1603,
+      "step": 15725
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 3.7800000000000002e-06,
+      "loss": 0.1294,
+      "step": 15750
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 3.7697959183673473e-06,
+      "loss": 0.1443,
+      "step": 15775
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 3.759591836734694e-06,
+      "loss": 0.1243,
+      "step": 15800
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 3.7493877551020414e-06,
+      "loss": 0.1358,
+      "step": 15825
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 3.739183673469388e-06,
+      "loss": 0.1554,
+      "step": 15850
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 3.728979591836735e-06,
+      "loss": 0.1695,
+      "step": 15875
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 3.7187755102040818e-06,
+      "loss": 0.1207,
+      "step": 15900
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 3.7085714285714284e-06,
+      "loss": 0.1588,
+      "step": 15925
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 3.698367346938776e-06,
+      "loss": 0.1268,
+      "step": 15950
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 3.688163265306123e-06,
+      "loss": 0.1597,
+      "step": 15975
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 3.6779591836734696e-06,
+      "loss": 0.1448,
+      "step": 16000
+    },
+    {
+      "epoch": 0.56,
+      "eval_loss": 0.1646193414926529,
+      "eval_runtime": 30931.6929,
+      "eval_samples_per_second": 1.003,
+      "eval_steps_per_second": 0.063,
+      "eval_wer": 0.09301142911866007,
+      "step": 16000
     }
   ],
   "max_steps": 25000,
   "num_train_epochs": 1,
-  "total_flos": 5.09561044992e+20,
+  "total_flos": 5.435317813248e+20,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1327fc0bd4b84f3f75a5ae566a31c8d3866b9addc072e36ebcf7aa9bd4b067fa
+oid sha256:41ddc7f21fd5cb7cbeac420cbf26556fe05d0d0276c5d96c3c5d572508243cc6
 size 6173655480

runs/Dec26_19-57-19_gpu07.cyverse.org/events.out.tfevents.1672109970.gpu07.cyverse.org.126369.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6358803618c5f009d0a7ba4001fd025bc6f24e6e4315f0b1153ccc109884fa00
-size 103359
+oid sha256:75cd60efea19799c78e56543455f4479b2c2960615ec3453f56a3b9470094bf2
+size 109957