huggingartists

README.md

@@ -45,15 +45,15 @@ from datasets import load_dataset
 dataset = load_dataset("huggingartists/andre-3000")
 ```
 
-[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/2aj9iybn/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
+[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/2hnhboqf/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
 
 ## Training procedure
 
 The model is based on a pre-trained [GPT-2](https://huggingface.co/gpt2) which is fine-tuned on André 3000's lyrics.
 
-Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/3lbg2cit) for full transparency and reproducibility.
+Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/1mydp6nh) for full transparency and reproducibility.
 
-At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/3lbg2cit/artifacts) is logged and versioned.
+At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/1mydp6nh/artifacts) is logged and versioned.
 
 ## How to use
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "huggingartists/andre-3000",
+  "_name_or_path": "andre-3000",
   "activation_function": "gelu_new",
   "architectures": [
     "GPT2LMHeadModel"
@@ -17,7 +17,9 @@
   "n_inner": null,
   "n_layer": 12,
   "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
   "scale_attn_weights": true,
   "summary_activation": null,
   "summary_first_dropout": 0.1,
@@ -34,7 +36,7 @@
     }
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.11.3",
+  "transformers_version": "4.16.2",
   "use_cache": true,
   "vocab_size": 50257
 }

evaluation.txt

@@ -1 +1 @@
-{"eval_loss": 2.7857184410095215, "eval_runtime": 3.2483, "eval_samples_per_second": 43.715, "eval_steps_per_second": 5.541, "epoch": 105.0}
+{"eval_loss": 2.1131954193115234, "eval_runtime": 1.8568, "eval_samples_per_second": 75.401, "eval_steps_per_second": 9.694, "epoch": 13.0}

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ca2d40ae75a2d9347d2b27e73aece5aa55fffa7ae760d8a75b258044ec799c4
+oid sha256:989d280b8f58c2ff5da2508491e0ae6778349dc1b34a169c32da297305ff01d6
 size 497764120

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f6ec27a8db3a3d8e12f1403c8e1ddd7d9d8f7c369361110364668a7d3905bcc
+oid sha256:759bb1bfd5a6fd243d563dbd3b6c11508cb5d3cbc90e52a03119979a9d759d12
 size 995604017

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:644cac333a83af4607d4d90810953f70e30bdc7159f4a9dc40f8b242c88c2770
+oid sha256:1590cd45efa4fe8174b9c88c70b68bbd7cf75b266511a97a2900be237518f868
 size 510403817

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16e34fba951f86dbe52880106da1b406b7bf5d467c83625264b1d27faf3245c0
+oid sha256:c4b07ce20b7b785ae36b466b0be7ef39edcf348c4b3a68bf0463d5c30234dc25
 size 14503

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4215fb2fd58e7d9127b8dee8649641af0662fd1b11cda980f7e7dc26e6aa301f
+oid sha256:a6450797a427e797c153fb64efaf17b83fd9c09303bffab938f0a1b399542842
 size 623

trainer_state.json

@@ -1,8 +1,8 @@
 {
-  "best_metric": 2.7857184410095215,
-  "best_model_checkpoint": "output/andre-3000/checkpoint-990",
-  "epoch": 10.0,
-  "global_step": 990,
+  "best_metric": 2.1131954193115234,
+  "best_model_checkpoint": "output/andre-3000/checkpoint-1089",
+  "epoch": 11.0,
+  "global_step": 1089,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1274,11 +1274,133 @@
       "eval_samples_per_second": 43.32,
       "eval_steps_per_second": 5.491,
       "step": 990
+    },
+    {
+      "epoch": 10.05,
+      "learning_rate": 0.00013633830776320067,
+      "loss": 2.2465,
+      "step": 995
+    },
+    {
+      "epoch": 10.1,
+      "learning_rate": 0.00013377487867702895,
+      "loss": 2.0385,
+      "step": 1000
+    },
+    {
+      "epoch": 10.15,
+      "learning_rate": 0.0001295741117777277,
+      "loss": 2.0847,
+      "step": 1005
+    },
+    {
+      "epoch": 10.2,
+      "learning_rate": 0.00012384153966663063,
+      "loss": 2.0744,
+      "step": 1010
+    },
+    {
+      "epoch": 10.25,
+      "learning_rate": 0.00011672117729665371,
+      "loss": 2.2347,
+      "step": 1015
+    },
+    {
+      "epoch": 10.3,
+      "learning_rate": 0.00010839190399658435,
+      "loss": 2.3577,
+      "step": 1020
+    },
+    {
+      "epoch": 10.35,
+      "learning_rate": 9.906296962475639e-05,
+      "loss": 2.1928,
+      "step": 1025
+    },
+    {
+      "epoch": 10.4,
+      "learning_rate": 8.896873774751959e-05,
+      "loss": 2.3981,
+      "step": 1030
+    },
+    {
+      "epoch": 10.45,
+      "learning_rate": 7.836279790554739e-05,
+      "loss": 2.1466,
+      "step": 1035
+    },
+    {
+      "epoch": 10.51,
+      "learning_rate": 6.75115948809323e-05,
+      "loss": 2.0252,
+      "step": 1040
+    },
+    {
+      "epoch": 10.56,
+      "learning_rate": 5.66877350120488e-05,
+      "loss": 2.1059,
+      "step": 1045
+    },
+    {
+      "epoch": 10.61,
+      "learning_rate": 4.616313771642518e-05,
+      "loss": 2.251,
+      "step": 1050
+    },
+    {
+      "epoch": 10.66,
+      "learning_rate": 3.620220427059434e-05,
+      "loss": 2.332,
+      "step": 1055
+    },
+    {
+      "epoch": 10.71,
+      "learning_rate": 2.7055175462356102e-05,
+      "loss": 2.0934,
+      "step": 1060
+    },
+    {
+      "epoch": 10.76,
+      "learning_rate": 1.8951844985992284e-05,
+      "loss": 2.2116,
+      "step": 1065
+    },
+    {
+      "epoch": 10.81,
+      "learning_rate": 1.2095786513913607e-05,
+      "loss": 2.1227,
+      "step": 1070
+    },
+    {
+      "epoch": 10.86,
+      "learning_rate": 6.659239473537932e-06,
+      "loss": 2.1152,
+      "step": 1075
+    },
+    {
+      "epoch": 10.91,
+      "learning_rate": 2.7787820100456023e-06,
+      "loss": 1.9454,
+      "step": 1080
+    },
+    {
+      "epoch": 10.96,
+      "learning_rate": 5.518998398074473e-07,
+      "loss": 2.208,
+      "step": 1085
+    },
+    {
+      "epoch": 11.0,
+      "eval_loss": 2.1131954193115234,
+      "eval_runtime": 1.8527,
+      "eval_samples_per_second": 75.565,
+      "eval_steps_per_second": 9.716,
+      "step": 1089
     }
   ],
-  "max_steps": 10395,
-  "num_train_epochs": 105,
-  "total_flos": 1032103526400000.0,
+  "max_steps": 1287,
+  "num_train_epochs": 13,
+  "total_flos": 1135575171072000.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:87b339441b1f6019e8540c7c71ded5ff035d2f0e1f074c763f744b0da4bc0c37
-size 2863
+oid sha256:2eaf90804f86898f984a6fa365311f924317fbe67459b0c35ed5b4fc325b6372
+size 3055