Training in progress, step 18000

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0afdff77222472257de0ba34c36739c26555750dfdb27e7b6f2ffed753b60d4e
+oid sha256:a8a90d99fa5e493d103776713ee2ee6b9fca9948380c5ad509565e448a0d3c2f
 size 12347192855

last-checkpoint/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a46b6b71b47c9b3cdde4a1d39cea580e81a6bbf969d5738793f41c7f51939ad5
+oid sha256:2b690154ee77cdbea8a14f2965218b2b3790caf8955482e539a12fa93fdc53f8
 size 6173655480

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:009dbf954364aba5d9d07623c2bc3be59611e798b46569f55549a02c3eabd95e
+oid sha256:537f8d4bc472f85d8aaced00700f04502f899659cbfa067468feee29a0f0233a
 size 14575

last-checkpoint/scaler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b9006cc2f9bf0c15a1ea19a9fdf3bcc02dfdb969110e593a43ce6b82aac6cf3
+oid sha256:90be1018263c5a8b0b93a055ffb96e8625ade303fea979c01cbf658c8aed8c95
 size 557

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3675e25dffe1ac318cb230ca76cdb84aec69a7bf2cb79f6d7b5f6265ec4df7c8
+oid sha256:9205cfac3dc2d508e85efbdf6cfda796cb8dde46c8f1bd9e062253fea417d2b3
 size 627

last-checkpoint/trainer_state.json

@@ -1,8 +1,8 @@
 {
   "best_metric": 0.11536111854119403,
   "best_model_checkpoint": "/storage/elsayedissa/whisper-large-v2-spanish/checkpoint-1000",
-  "epoch": 0.5900933735985282,
-  "global_step": 17000,
+  "epoch": 0.6248047485160887,
+  "global_step": 18000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4239,11 +4239,260 @@
       "eval_steps_per_second": 0.062,
       "eval_wer": 0.09140322894892396,
       "step": 17000
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 3.259591836734694e-06,
+      "loss": 0.1475,
+      "step": 17025
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 3.2493877551020407e-06,
+      "loss": 0.1458,
+      "step": 17050
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 3.239183673469388e-06,
+      "loss": 0.155,
+      "step": 17075
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 3.2289795918367352e-06,
+      "loss": 0.1446,
+      "step": 17100
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 3.218775510204082e-06,
+      "loss": 0.1645,
+      "step": 17125
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.2085714285714285e-06,
+      "loss": 0.1445,
+      "step": 17150
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.198367346938776e-06,
+      "loss": 0.1561,
+      "step": 17175
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.1881632653061227e-06,
+      "loss": 0.1518,
+      "step": 17200
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.1779591836734697e-06,
+      "loss": 0.1521,
+      "step": 17225
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.1677551020408164e-06,
+      "loss": 0.1426,
+      "step": 17250
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.1575510204081634e-06,
+      "loss": 0.145,
+      "step": 17275
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.1473469387755105e-06,
+      "loss": 0.1287,
+      "step": 17300
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.1371428571428575e-06,
+      "loss": 0.1598,
+      "step": 17325
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.126938775510204e-06,
+      "loss": 0.1581,
+      "step": 17350
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.1167346938775512e-06,
+      "loss": 0.1566,
+      "step": 17375
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.106530612244898e-06,
+      "loss": 0.1291,
+      "step": 17400
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 3.0963265306122454e-06,
+      "loss": 0.1532,
+      "step": 17425
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.086122448979592e-06,
+      "loss": 0.146,
+      "step": 17450
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.075918367346939e-06,
+      "loss": 0.157,
+      "step": 17475
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.0657142857142857e-06,
+      "loss": 0.1252,
+      "step": 17500
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.055510204081633e-06,
+      "loss": 0.1701,
+      "step": 17525
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.04530612244898e-06,
+      "loss": 0.1354,
+      "step": 17550
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.035102040816327e-06,
+      "loss": 0.1687,
+      "step": 17575
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.0248979591836735e-06,
+      "loss": 0.1178,
+      "step": 17600
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.0146938775510206e-06,
+      "loss": 0.1564,
+      "step": 17625
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 3.0044897959183677e-06,
+      "loss": 0.1446,
+      "step": 17650
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.9942857142857147e-06,
+      "loss": 0.1463,
+      "step": 17675
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.9840816326530614e-06,
+      "loss": 0.1206,
+      "step": 17700
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.973877551020408e-06,
+      "loss": 0.1473,
+      "step": 17725
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.9636734693877555e-06,
+      "loss": 0.1327,
+      "step": 17750
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.9534693877551025e-06,
+      "loss": 0.1654,
+      "step": 17775
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.943265306122449e-06,
+      "loss": 0.1237,
+      "step": 17800
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.933061224489796e-06,
+      "loss": 0.1615,
+      "step": 17825
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.922857142857143e-06,
+      "loss": 0.142,
+      "step": 17850
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.9126530612244904e-06,
+      "loss": 0.1608,
+      "step": 17875
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.902448979591837e-06,
+      "loss": 0.1296,
+      "step": 17900
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.8922448979591836e-06,
+      "loss": 0.1569,
+      "step": 17925
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.8820408163265307e-06,
+      "loss": 0.1331,
+      "step": 17950
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.871836734693878e-06,
+      "loss": 0.1872,
+      "step": 17975
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.861632653061225e-06,
+      "loss": 0.1441,
+      "step": 18000
+    },
+    {
+      "epoch": 0.62,
+      "eval_loss": 0.15962253510951996,
+      "eval_runtime": 31151.8818,
+      "eval_samples_per_second": 0.996,
+      "eval_steps_per_second": 0.062,
+      "eval_wer": 0.08993702591943148,
+      "step": 18000
     }
   ],
   "max_steps": 25000,
   "num_train_epochs": 1,
-  "total_flos": 5.775025176576e+20,
+  "total_flos": 6.114732539904e+20,
   "trial_name": null,
   "trial_params": null
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a46b6b71b47c9b3cdde4a1d39cea580e81a6bbf969d5738793f41c7f51939ad5
+oid sha256:2b690154ee77cdbea8a14f2965218b2b3790caf8955482e539a12fa93fdc53f8
 size 6173655480

runs/Dec26_19-57-19_gpu07.cyverse.org/events.out.tfevents.1672109970.gpu07.cyverse.org.126369.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e78bc9cee84c391c8ae997316cec7200e593d160d602a2fd280af39d860d9f6a
-size 116636
+oid sha256:54f4f30667756976745aebfb4e84b0aa810e69c9ffaa1a0e78323a5e4c0623c1
+size 123360