huggingartists

README.md

@@ -45,15 +45,15 @@ from datasets import load_dataset
 dataset = load_dataset("huggingartists/lady-gaga")
 ```
 
-[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/s6109nue/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
+[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/17c0d4ej/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
 
 ## Training procedure
 
 The model is based on a pre-trained [GPT-2](https://huggingface.co/gpt2) which is fine-tuned on Lady Gaga's lyrics.
 
-Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/3g2fq2s0) for full transparency and reproducibility.
+Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/2j7yp9qd) for full transparency and reproducibility.
 
-At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/3g2fq2s0/artifacts) is logged and versioned.
+At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/2j7yp9qd/artifacts) is logged and versioned.
 
 ## How to use
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "gpt2",
+  "_name_or_path": "lady-gaga",
   "activation_function": "gelu_new",
   "architectures": [
     "GPT2LMHeadModel"
@@ -18,7 +18,9 @@
   "n_inner": null,
   "n_layer": 12,
   "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
   "scale_attn_weights": true,
   "summary_activation": null,
   "summary_first_dropout": 0.1,
@@ -35,7 +37,7 @@
     }
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.10.0",
+  "transformers_version": "4.17.0",
   "use_cache": true,
   "vocab_size": 50257
 }

evaluation.txt

@@ -1 +1 @@
-{"eval_loss": 1.7604948282241821, "eval_runtime": 8.1984, "eval_samples_per_second": 22.078, "eval_steps_per_second": 2.805, "epoch": 20.0}
+{"eval_loss": 1.1818922758102417, "eval_runtime": 4.5799, "eval_samples_per_second": 42.359, "eval_steps_per_second": 5.459, "epoch": 10.0}

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b7afa87885c4237efdbd54f14a90c394a7789cee5d9dc805cefa09922257eca9
+oid sha256:99dd1e512d6d5a8bb64ae4522bfb7b135369d388d25a4d9301825fcb20cba277
 size 497764120

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e975e394301ef8deef76b9ee8c42cf177aa52fc2de166eb69e8a52fa5f03089
+oid sha256:260937b68e76f08ab05abc4034e8a26d999dba3a07d5dd085f85eafddea8716b
 size 995604017

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:075245bdc5c5f7cd7a9cdac5907183e6b2f805ffe1d84da4d6b08b01b13cdde7
-size 510403817
+oid sha256:7b8b67c493edd3b41282c01b0aeeb6ec02fb37cddd445842e671281f8fe93481
+size 510404393

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03a2d5e765e5cf5ecb55bf1d575b6d6e34d62c85ee9fecda0b266d1418fa375b
+oid sha256:543fe6be835f78a08147e71f220ab369938164ecea602c709f497ffb01da0814
 size 14503

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e3679dd478a770c99d8cf2c99947149cd3de03554a50244d19b822d7b456455
+oid sha256:19b8ed0e0974b735985e6aeb8c7bbf0ef2968c7807776eb9b1a23504cdae1948
 size 623

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "gpt2", "tokenizer_class": "GPT2Tokenizer"}
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "huggingartists/lady-gaga", "tokenizer_class": "GPT2Tokenizer"}

trainer_state.json

@@ -1,8 +1,8 @@
 {
-  "best_metric": 1.7604948282241821,
-  "best_model_checkpoint": "output/lady-gaga/checkpoint-822",
-  "epoch": 6.0,
-  "global_step": 822,
+  "best_metric": 1.1818922758102417,
+  "best_model_checkpoint": "output/lady-gaga/checkpoint-945",
+  "epoch": 7.0,
+  "global_step": 945,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1038,11 +1038,169 @@
       "eval_samples_per_second": 22.462,
       "eval_steps_per_second": 2.854,
       "step": 822
+    },
+    {
+      "epoch": 6.11,
+      "learning_rate": 0.00013306291378591335,
+      "loss": 1.3683,
+      "step": 825
+    },
+    {
+      "epoch": 6.15,
+      "learning_rate": 0.00012990319912618617,
+      "loss": 1.3883,
+      "step": 830
+    },
+    {
+      "epoch": 6.19,
+      "learning_rate": 0.00012591446386292745,
+      "loss": 1.289,
+      "step": 835
+    },
+    {
+      "epoch": 6.22,
+      "learning_rate": 0.00012115064879796196,
+      "loss": 1.3834,
+      "step": 840
+    },
+    {
+      "epoch": 6.26,
+      "learning_rate": 0.00011567617635779509,
+      "loss": 1.2739,
+      "step": 845
+    },
+    {
+      "epoch": 6.3,
+      "learning_rate": 0.00010956507939081116,
+      "loss": 1.0868,
+      "step": 850
+    },
+    {
+      "epoch": 6.33,
+      "learning_rate": 0.00010290000000000012,
+      "loss": 1.3412,
+      "step": 855
+    },
+    {
+      "epoch": 6.37,
+      "learning_rate": 9.577107195028614e-05,
+      "loss": 1.2163,
+      "step": 860
+    },
+    {
+      "epoch": 6.41,
+      "learning_rate": 8.827470176398086e-05,
+      "loss": 1.1091,
+      "step": 865
+    },
+    {
+      "epoch": 6.44,
+      "learning_rate": 8.051226498795135e-05,
+      "loss": 1.2886,
+      "step": 870
+    },
+    {
+      "epoch": 6.48,
+      "learning_rate": 7.258873526325866e-05,
+      "loss": 1.5192,
+      "step": 875
+    },
+    {
+      "epoch": 6.52,
+      "learning_rate": 6.461126473674146e-05,
+      "loss": 1.3007,
+      "step": 880
+    },
+    {
+      "epoch": 6.56,
+      "learning_rate": 5.6687735012048764e-05,
+      "loss": 1.0916,
+      "step": 885
+    },
+    {
+      "epoch": 6.59,
+      "learning_rate": 4.892529823601925e-05,
+      "loss": 1.1481,
+      "step": 890
+    },
+    {
+      "epoch": 6.63,
+      "learning_rate": 4.1428928049713965e-05,
+      "loss": 1.2328,
+      "step": 895
+    },
+    {
+      "epoch": 6.67,
+      "learning_rate": 3.429999999999998e-05,
+      "loss": 1.0816,
+      "step": 900
+    },
+    {
+      "epoch": 6.7,
+      "learning_rate": 2.763492060918892e-05,
+      "loss": 1.2086,
+      "step": 905
+    },
+    {
+      "epoch": 6.74,
+      "learning_rate": 2.152382364220499e-05,
+      "loss": 0.9535,
+      "step": 910
+    },
+    {
+      "epoch": 6.78,
+      "learning_rate": 1.604935120203811e-05,
+      "loss": 1.3744,
+      "step": 915
+    },
+    {
+      "epoch": 6.81,
+      "learning_rate": 1.128553613707261e-05,
+      "loss": 1.2614,
+      "step": 920
+    },
+    {
+      "epoch": 6.85,
+      "learning_rate": 7.296800873813895e-06,
+      "loss": 1.2024,
+      "step": 925
+    },
+    {
+      "epoch": 6.89,
+      "learning_rate": 4.137086214086697e-06,
+      "loss": 1.2731,
+      "step": 930
+    },
+    {
+      "epoch": 6.93,
+      "learning_rate": 1.8491218782241098e-06,
+      "loss": 1.1451,
+      "step": 935
+    },
+    {
+      "epoch": 6.96,
+      "learning_rate": 4.638486589027391e-07,
+      "loss": 1.2302,
+      "step": 940
+    },
+    {
+      "epoch": 7.0,
+      "learning_rate": 0.0,
+      "loss": 1.2435,
+      "step": 945
+    },
+    {
+      "epoch": 7.0,
+      "eval_loss": 1.1818922758102417,
+      "eval_runtime": 4.3047,
+      "eval_samples_per_second": 45.067,
+      "eval_steps_per_second": 5.808,
+      "step": 945
     }
   ],
-  "max_steps": 2740,
-  "num_train_epochs": 20,
-  "total_flos": 855992696832000.0,
+  "max_steps": 1350,
+  "num_train_epochs": 10,
+  "total_flos": 984287084544000.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e887f9fc06cce3b63f585b7c5b8b706b54fff3e04c1880ad0fa1b0ea356cea0
-size 2671
+oid sha256:4756263e940a9553814d86ac90b5fae14148f687aa8b2a4a1fdc0adfe160f9a8
+size 3055