End of training

README.md

@@ -4,6 +4,7 @@ license: llama3
 base_model: meta-llama/Meta-Llama-3-8B-Instruct
 tags:
 - llama-factory
+- prompt-tuning
 - generated_from_trainer
 model-index:
 - name: train_wsc_101112_1760351840
@@ -15,10 +16,10 @@ should probably proofread and complete it, then remove this comment. -->
 
 # train_wsc_101112_1760351840
 
-This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) on an unknown dataset.
+This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) on the wsc dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.3495
-- Num Input Tokens Seen: 469328
+- Loss: 0.3449
+- Num Input Tokens Seen: 488816
 
 ## Model description
 

all_results.json

@@ -0,0 +1,13 @@
+{
+    "epoch": 10.0,
+    "eval_loss": 0.3449070155620575,
+    "eval_runtime": 0.949,
+    "eval_samples_per_second": 59.012,
+    "eval_steps_per_second": 14.753,
+    "num_input_tokens_seen": 488816,
+    "total_flos": 2.201116355081011e+16,
+    "train_loss": 0.41895289707183836,
+    "train_runtime": 188.2345,
+    "train_samples_per_second": 26.456,
+    "train_steps_per_second": 6.641
+}

eval_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 10.0,
+    "eval_loss": 0.3449070155620575,
+    "eval_runtime": 0.949,
+    "eval_samples_per_second": 59.012,
+    "eval_steps_per_second": 14.753,
+    "num_input_tokens_seen": 488816
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 10.0,
+    "num_input_tokens_seen": 488816,
+    "total_flos": 2.201116355081011e+16,
+    "train_loss": 0.41895289707183836,
+    "train_runtime": 188.2345,
+    "train_samples_per_second": 26.456,
+    "train_steps_per_second": 6.641
+}

trainer_state.json

@@ -0,0 +1,2215 @@
+{
+  "best_global_step": 252,
+  "best_metric": 0.3449070155620575,
+  "best_model_checkpoint": "saves/prompt-tuning/llama-3-8b-instruct/train_wsc_101112_1760351840/checkpoint-252",
+  "epoch": 10.0,
+  "eval_steps": 63,
+  "global_step": 1250,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.04,
+      "grad_norm": 46.0,
+      "learning_rate": 0.00096,
+      "loss": 0.6343,
+      "num_input_tokens_seen": 2176,
+      "step": 5
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 133.0,
+      "learning_rate": 0.0021599999999999996,
+      "loss": 1.7131,
+      "num_input_tokens_seen": 4064,
+      "step": 10
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 60.5,
+      "learning_rate": 0.00336,
+      "loss": 5.78,
+      "num_input_tokens_seen": 5696,
+      "step": 15
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 27.375,
+      "learning_rate": 0.00456,
+      "loss": 1.5326,
+      "num_input_tokens_seen": 7616,
+      "step": 20
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 18.5,
+      "learning_rate": 0.0057599999999999995,
+      "loss": 1.0928,
+      "num_input_tokens_seen": 9728,
+      "step": 25
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 101.5,
+      "learning_rate": 0.00696,
+      "loss": 0.966,
+      "num_input_tokens_seen": 11584,
+      "step": 30
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 5.09375,
+      "learning_rate": 0.00816,
+      "loss": 0.9092,
+      "num_input_tokens_seen": 13696,
+      "step": 35
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 5.25,
+      "learning_rate": 0.00936,
+      "loss": 0.7377,
+      "num_input_tokens_seen": 15712,
+      "step": 40
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 6.0,
+      "learning_rate": 0.010559999999999998,
+      "loss": 0.9322,
+      "num_input_tokens_seen": 17568,
+      "step": 45
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.9375,
+      "learning_rate": 0.01176,
+      "loss": 0.4345,
+      "num_input_tokens_seen": 19648,
+      "step": 50
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 0.6875,
+      "learning_rate": 0.01296,
+      "loss": 1.5337,
+      "num_input_tokens_seen": 21696,
+      "step": 55
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 0.1328125,
+      "learning_rate": 0.014159999999999999,
+      "loss": 0.4762,
+      "num_input_tokens_seen": 23456,
+      "step": 60
+    },
+    {
+      "epoch": 0.504,
+      "eval_loss": 0.40469884872436523,
+      "eval_runtime": 0.9411,
+      "eval_samples_per_second": 59.505,
+      "eval_steps_per_second": 14.876,
+      "num_input_tokens_seen": 24608,
+      "step": 63
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 10.5,
+      "learning_rate": 0.01536,
+      "loss": 0.5655,
+      "num_input_tokens_seen": 25376,
+      "step": 65
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 0.2236328125,
+      "learning_rate": 0.016560000000000002,
+      "loss": 0.4276,
+      "num_input_tokens_seen": 27520,
+      "step": 70
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 0.251953125,
+      "learning_rate": 0.017759999999999998,
+      "loss": 0.3988,
+      "num_input_tokens_seen": 29184,
+      "step": 75
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 0.1162109375,
+      "learning_rate": 0.01896,
+      "loss": 0.3485,
+      "num_input_tokens_seen": 30912,
+      "step": 80
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 0.031982421875,
+      "learning_rate": 0.02016,
+      "loss": 0.6841,
+      "num_input_tokens_seen": 33376,
+      "step": 85
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 0.039794921875,
+      "learning_rate": 0.021359999999999997,
+      "loss": 0.4014,
+      "num_input_tokens_seen": 35520,
+      "step": 90
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 0.07763671875,
+      "learning_rate": 0.02256,
+      "loss": 0.3607,
+      "num_input_tokens_seen": 37760,
+      "step": 95
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.021728515625,
+      "learning_rate": 0.02376,
+      "loss": 0.3482,
+      "num_input_tokens_seen": 39808,
+      "step": 100
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 0.040283203125,
+      "learning_rate": 0.02496,
+      "loss": 0.3514,
+      "num_input_tokens_seen": 41312,
+      "step": 105
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 0.16015625,
+      "learning_rate": 0.02616,
+      "loss": 0.4401,
+      "num_input_tokens_seen": 43296,
+      "step": 110
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 0.0712890625,
+      "learning_rate": 0.02736,
+      "loss": 0.359,
+      "num_input_tokens_seen": 45056,
+      "step": 115
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 0.049072265625,
+      "learning_rate": 0.02856,
+      "loss": 0.3496,
+      "num_input_tokens_seen": 47328,
+      "step": 120
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.029296875,
+      "learning_rate": 0.029759999999999998,
+      "loss": 0.346,
+      "num_input_tokens_seen": 48944,
+      "step": 125
+    },
+    {
+      "epoch": 1.008,
+      "eval_loss": 0.34741348028182983,
+      "eval_runtime": 0.9473,
+      "eval_samples_per_second": 59.114,
+      "eval_steps_per_second": 14.778,
+      "num_input_tokens_seen": 49296,
+      "step": 126
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 0.042236328125,
+      "learning_rate": 0.029999064225016296,
+      "loss": 0.3523,
+      "num_input_tokens_seen": 50672,
+      "step": 130
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 0.031982421875,
+      "learning_rate": 0.029995262839249498,
+      "loss": 0.3733,
+      "num_input_tokens_seen": 52464,
+      "step": 135
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 0.0625,
+      "learning_rate": 0.0299885380972807,
+      "loss": 0.5277,
+      "num_input_tokens_seen": 54288,
+      "step": 140
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 0.049560546875,
+      "learning_rate": 0.02997889131011168,
+      "loss": 0.3683,
+      "num_input_tokens_seen": 56112,
+      "step": 145
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 0.04150390625,
+      "learning_rate": 0.0299663243584027,
+      "loss": 0.3528,
+      "num_input_tokens_seen": 58160,
+      "step": 150
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 0.07861328125,
+      "learning_rate": 0.029950839692105897,
+      "loss": 0.3465,
+      "num_input_tokens_seen": 60368,
+      "step": 155
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 0.0172119140625,
+      "learning_rate": 0.029932440329987653,
+      "loss": 0.2981,
+      "num_input_tokens_seen": 62320,
+      "step": 160
+    },
+    {
+      "epoch": 1.32,
+      "grad_norm": 0.00909423828125,
+      "learning_rate": 0.02991112985904007,
+      "loss": 0.3678,
+      "num_input_tokens_seen": 64368,
+      "step": 165
+    },
+    {
+      "epoch": 1.3599999999999999,
+      "grad_norm": 0.0247802734375,
+      "learning_rate": 0.029886912433781675,
+      "loss": 0.3632,
+      "num_input_tokens_seen": 66544,
+      "step": 170
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 0.052001953125,
+      "learning_rate": 0.02985979277544751,
+      "loss": 0.3643,
+      "num_input_tokens_seen": 68624,
+      "step": 175
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 0.1494140625,
+      "learning_rate": 0.029829776171068707,
+      "loss": 0.3638,
+      "num_input_tokens_seen": 70896,
+      "step": 180
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 0.083984375,
+      "learning_rate": 0.029796868472441763,
+      "loss": 0.5915,
+      "num_input_tokens_seen": 73168,
+      "step": 185
+    },
+    {
+      "epoch": 1.512,
+      "eval_loss": 0.3490610718727112,
+      "eval_runtime": 0.9366,
+      "eval_samples_per_second": 59.791,
+      "eval_steps_per_second": 14.948,
+      "num_input_tokens_seen": 74672,
+      "step": 189
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 0.02685546875,
+      "learning_rate": 0.029761076094987723,
+      "loss": 0.3427,
+      "num_input_tokens_seen": 75024,
+      "step": 190
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 0.035400390625,
+      "learning_rate": 0.02972240601650149,
+      "loss": 0.3229,
+      "num_input_tokens_seen": 76976,
+      "step": 195
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.027587890625,
+      "learning_rate": 0.029680865775791494,
+      "loss": 0.3886,
+      "num_input_tokens_seen": 78800,
+      "step": 200
+    },
+    {
+      "epoch": 1.6400000000000001,
+      "grad_norm": 0.08447265625,
+      "learning_rate": 0.02963646347120996,
+      "loss": 0.3791,
+      "num_input_tokens_seen": 80944,
+      "step": 205
+    },
+    {
+      "epoch": 1.6800000000000002,
+      "grad_norm": 0.043701171875,
+      "learning_rate": 0.029589207759074154,
+      "loss": 0.3534,
+      "num_input_tokens_seen": 82992,
+      "step": 210
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 0.0595703125,
+      "learning_rate": 0.029539107851978778,
+      "loss": 0.3491,
+      "num_input_tokens_seen": 85136,
+      "step": 215
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 0.08349609375,
+      "learning_rate": 0.02948617351699999,
+      "loss": 0.3624,
+      "num_input_tokens_seen": 87280,
+      "step": 220
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 0.078125,
+      "learning_rate": 0.029430415073791287,
+      "loss": 0.3563,
+      "num_input_tokens_seen": 89232,
+      "step": 225
+    },
+    {
+      "epoch": 1.8399999999999999,
+      "grad_norm": 0.10693359375,
+      "learning_rate": 0.029371843392571644,
+      "loss": 0.3746,
+      "num_input_tokens_seen": 90864,
+      "step": 230
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 0.03759765625,
+      "learning_rate": 0.029310469892006367,
+      "loss": 0.368,
+      "num_input_tokens_seen": 92944,
+      "step": 235
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 0.01275634765625,
+      "learning_rate": 0.029246306536981,
+      "loss": 0.3528,
+      "num_input_tokens_seen": 94608,
+      "step": 240
+    },
+    {
+      "epoch": 1.96,
+      "grad_norm": 0.0208740234375,
+      "learning_rate": 0.02917936583626874,
+      "loss": 0.3468,
+      "num_input_tokens_seen": 96336,
+      "step": 245
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.01092529296875,
+      "learning_rate": 0.029109660840091818,
+      "loss": 0.36,
+      "num_input_tokens_seen": 98080,
+      "step": 250
+    },
+    {
+      "epoch": 2.016,
+      "eval_loss": 0.3449070155620575,
+      "eval_runtime": 0.9442,
+      "eval_samples_per_second": 59.307,
+      "eval_steps_per_second": 14.827,
+      "num_input_tokens_seen": 98816,
+      "step": 252
+    },
+    {
+      "epoch": 2.04,
+      "grad_norm": 0.01220703125,
+      "learning_rate": 0.029037205137577363,
+      "loss": 0.358,
+      "num_input_tokens_seen": 99904,
+      "step": 255
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 0.052734375,
+      "learning_rate": 0.02896201285410813,
+      "loss": 0.2806,
+      "num_input_tokens_seen": 101824,
+      "step": 260
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 0.020263671875,
+      "learning_rate": 0.028884098648568782,
+      "loss": 0.4646,
+      "num_input_tokens_seen": 103872,
+      "step": 265
+    },
+    {
+      "epoch": 2.16,
+      "grad_norm": 2.78125,
+      "learning_rate": 0.028803477710488055,
+      "loss": 0.3981,
+      "num_input_tokens_seen": 105856,
+      "step": 270
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 0.01904296875,
+      "learning_rate": 0.028720165757077573,
+      "loss": 0.348,
+      "num_input_tokens_seen": 107776,
+      "step": 275
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 0.04296875,
+      "learning_rate": 0.02863417903016773,
+      "loss": 0.3671,
+      "num_input_tokens_seen": 109760,
+      "step": 280
+    },
+    {
+      "epoch": 2.2800000000000002,
+      "grad_norm": 0.07763671875,
+      "learning_rate": 0.02854553429304131,
+      "loss": 0.3731,
+      "num_input_tokens_seen": 111904,
+      "step": 285
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 0.034423828125,
+      "learning_rate": 0.02845424882716545,
+      "loss": 0.341,
+      "num_input_tokens_seen": 113664,
+      "step": 290
+    },
+    {
+      "epoch": 2.36,
+      "grad_norm": 0.01202392578125,
+      "learning_rate": 0.028360340428822597,
+      "loss": 0.3322,
+      "num_input_tokens_seen": 115712,
+      "step": 295
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 0.06689453125,
+      "learning_rate": 0.028263827405641085,
+      "loss": 0.3308,
+      "num_input_tokens_seen": 117728,
+      "step": 300
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 0.03125,
+      "learning_rate": 0.028164728573026005,
+      "loss": 0.329,
+      "num_input_tokens_seen": 119648,
+      "step": 305
+    },
+    {
+      "epoch": 2.48,
+      "grad_norm": 0.0228271484375,
+      "learning_rate": 0.02806306325049113,
+      "loss": 0.3294,
+      "num_input_tokens_seen": 121920,
+      "step": 310
+    },
+    {
+      "epoch": 2.52,
+      "grad_norm": 18.375,
+      "learning_rate": 0.027958851257892527,
+      "loss": 0.6543,
+      "num_input_tokens_seen": 123680,
+      "step": 315
+    },
+    {
+      "epoch": 2.52,
+      "eval_loss": 0.34932464361190796,
+      "eval_runtime": 0.9487,
+      "eval_samples_per_second": 59.026,
+      "eval_steps_per_second": 14.757,
+      "num_input_tokens_seen": 123680,
+      "step": 315
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 0.049072265625,
+      "learning_rate": 0.02785211291156464,
+      "loss": 0.355,
+      "num_input_tokens_seen": 125504,
+      "step": 320
+    },
+    {
+      "epoch": 2.6,
+      "grad_norm": 0.031005859375,
+      "learning_rate": 0.027742869020359582,
+      "loss": 1.2351,
+      "num_input_tokens_seen": 127264,
+      "step": 325
+    },
+    {
+      "epoch": 2.64,
+      "grad_norm": 0.08056640625,
+      "learning_rate": 0.027631140881590383,
+      "loss": 0.3538,
+      "num_input_tokens_seen": 128960,
+      "step": 330
+    },
+    {
+      "epoch": 2.68,
+      "grad_norm": 0.03369140625,
+      "learning_rate": 0.027516950276879084,
+      "loss": 0.3487,
+      "num_input_tokens_seen": 130752,
+      "step": 335
+    },
+    {
+      "epoch": 2.7199999999999998,
+      "grad_norm": 0.060791015625,
+      "learning_rate": 0.02740031946791033,
+      "loss": 0.4092,
+      "num_input_tokens_seen": 132768,
+      "step": 340
+    },
+    {
+      "epoch": 2.76,
+      "grad_norm": 0.10107421875,
+      "learning_rate": 0.027281271192091415,
+      "loss": 0.432,
+      "num_input_tokens_seen": 134720,
+      "step": 345
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 0.01226806640625,
+      "learning_rate": 0.027159828658119597,
+      "loss": 1.029,
+      "num_input_tokens_seen": 136576,
+      "step": 350
+    },
+    {
+      "epoch": 2.84,
+      "grad_norm": 0.1357421875,
+      "learning_rate": 0.0270360155414575,
+      "loss": 0.3842,
+      "num_input_tokens_seen": 138528,
+      "step": 355
+    },
+    {
+      "epoch": 2.88,
+      "grad_norm": 0.11962890625,
+      "learning_rate": 0.02690985597971753,
+      "loss": 1.6063,
+      "num_input_tokens_seen": 140448,
+      "step": 360
+    },
+    {
+      "epoch": 2.92,
+      "grad_norm": 0.08935546875,
+      "learning_rate": 0.026781374567956224,
+      "loss": 0.3795,
+      "num_input_tokens_seen": 142528,
+      "step": 365
+    },
+    {
+      "epoch": 2.96,
+      "grad_norm": 0.03466796875,
+      "learning_rate": 0.026650596353879386,
+      "loss": 0.3328,
+      "num_input_tokens_seen": 144672,
+      "step": 370
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 0.04736328125,
+      "learning_rate": 0.026517546832958965,
+      "loss": 0.3336,
+      "num_input_tokens_seen": 146624,
+      "step": 375
+    },
+    {
+      "epoch": 3.024,
+      "eval_loss": 0.4224354326725006,
+      "eval_runtime": 0.9449,
+      "eval_samples_per_second": 59.268,
+      "eval_steps_per_second": 14.817,
+      "num_input_tokens_seen": 147776,
+      "step": 378
+    },
+    {
+      "epoch": 3.04,
+      "grad_norm": 0.08154296875,
+      "learning_rate": 0.026382251943462682,
+      "loss": 0.3255,
+      "num_input_tokens_seen": 148576,
+      "step": 380
+    },
+    {
+      "epoch": 3.08,
+      "grad_norm": 0.03857421875,
+      "learning_rate": 0.026244738061397325,
+      "loss": 0.4045,
+      "num_input_tokens_seen": 150496,
+      "step": 385
+    },
+    {
+      "epoch": 3.12,
+      "grad_norm": 0.07373046875,
+      "learning_rate": 0.026105031995366672,
+      "loss": 0.3593,
+      "num_input_tokens_seen": 152448,
+      "step": 390
+    },
+    {
+      "epoch": 3.16,
+      "grad_norm": 0.07568359375,
+      "learning_rate": 0.025963160981345105,
+      "loss": 0.3571,
+      "num_input_tokens_seen": 154368,
+      "step": 395
+    },
+    {
+      "epoch": 3.2,
+      "grad_norm": 0.043701171875,
+      "learning_rate": 0.02581915267736791,
+      "loss": 0.3484,
+      "num_input_tokens_seen": 156480,
+      "step": 400
+    },
+    {
+      "epoch": 3.24,
+      "grad_norm": 0.007568359375,
+      "learning_rate": 0.025673035158139283,
+      "loss": 0.3507,
+      "num_input_tokens_seen": 158336,
+      "step": 405
+    },
+    {
+      "epoch": 3.2800000000000002,
+      "grad_norm": 0.056884765625,
+      "learning_rate": 0.02552483690955911,
+      "loss": 0.3123,
+      "num_input_tokens_seen": 160544,
+      "step": 410
+    },
+    {
+      "epoch": 3.32,
+      "grad_norm": 0.034912109375,
+      "learning_rate": 0.0253745868231696,
+      "loss": 0.3772,
+      "num_input_tokens_seen": 162528,
+      "step": 415
+    },
+    {
+      "epoch": 3.36,
+      "grad_norm": 0.047607421875,
+      "learning_rate": 0.025222314190522798,
+      "loss": 0.4156,
+      "num_input_tokens_seen": 164800,
+      "step": 420
+    },
+    {
+      "epoch": 3.4,
+      "grad_norm": 0.004425048828125,
+      "learning_rate": 0.02506804869747014,
+      "loss": 0.365,
+      "num_input_tokens_seen": 166720,
+      "step": 425
+    },
+    {
+      "epoch": 3.44,
+      "grad_norm": 0.060546875,
+      "learning_rate": 0.024911820418375166,
+      "loss": 0.3529,
+      "num_input_tokens_seen": 169088,
+      "step": 430
+    },
+    {
+      "epoch": 3.48,
+      "grad_norm": 0.020263671875,
+      "learning_rate": 0.02475365981025043,
+      "loss": 0.3489,
+      "num_input_tokens_seen": 170944,
+      "step": 435
+    },
+    {
+      "epoch": 3.52,
+      "grad_norm": 0.01904296875,
+      "learning_rate": 0.02459359770681987,
+      "loss": 0.3359,
+      "num_input_tokens_seen": 173024,
+      "step": 440
+    },
+    {
+      "epoch": 3.528,
+      "eval_loss": 0.36203479766845703,
+      "eval_runtime": 0.9848,
+      "eval_samples_per_second": 56.863,
+      "eval_steps_per_second": 14.216,
+      "num_input_tokens_seen": 173312,
+      "step": 441
+    },
+    {
+      "epoch": 3.56,
+      "grad_norm": 0.0196533203125,
+      "learning_rate": 0.02443166531250769,
+      "loss": 0.3564,
+      "num_input_tokens_seen": 174976,
+      "step": 445
+    },
+    {
+      "epoch": 3.6,
+      "grad_norm": 0.006103515625,
+      "learning_rate": 0.024267894196355017,
+      "loss": 0.355,
+      "num_input_tokens_seen": 177088,
+      "step": 450
+    },
+    {
+      "epoch": 3.64,
+      "grad_norm": 0.0225830078125,
+      "learning_rate": 0.024102316285865434,
+      "loss": 0.3518,
+      "num_input_tokens_seen": 178976,
+      "step": 455
+    },
+    {
+      "epoch": 3.68,
+      "grad_norm": 0.0184326171875,
+      "learning_rate": 0.02393496386078067,
+      "loss": 0.3431,
+      "num_input_tokens_seen": 180608,
+      "step": 460
+    },
+    {
+      "epoch": 3.7199999999999998,
+      "grad_norm": 0.01507568359375,
+      "learning_rate": 0.02376586954678758,
+      "loss": 0.3434,
+      "num_input_tokens_seen": 182496,
+      "step": 465
+    },
+    {
+      "epoch": 3.76,
+      "grad_norm": 0.01519775390625,
+      "learning_rate": 0.02359506630915773,
+      "loss": 0.3854,
+      "num_input_tokens_seen": 184480,
+      "step": 470
+    },
+    {
+      "epoch": 3.8,
+      "grad_norm": 0.01348876953125,
+      "learning_rate": 0.023422587446320715,
+      "loss": 0.3395,
+      "num_input_tokens_seen": 186432,
+      "step": 475
+    },
+    {
+      "epoch": 3.84,
+      "grad_norm": 0.0341796875,
+      "learning_rate": 0.0232484665833726,
+      "loss": 0.3653,
+      "num_input_tokens_seen": 188576,
+      "step": 480
+    },
+    {
+      "epoch": 3.88,
+      "grad_norm": 0.0289306640625,
+      "learning_rate": 0.023072737665520607,
+      "loss": 0.3527,
+      "num_input_tokens_seen": 190816,
+      "step": 485
+    },
+    {
+      "epoch": 3.92,
+      "grad_norm": 0.0216064453125,
+      "learning_rate": 0.022895434951465468,
+      "loss": 0.3521,
+      "num_input_tokens_seen": 192608,
+      "step": 490
+    },
+    {
+      "epoch": 3.96,
+      "grad_norm": 0.033203125,
+      "learning_rate": 0.022716593006722595,
+      "loss": 0.3477,
+      "num_input_tokens_seen": 194624,
+      "step": 495
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 0.0113525390625,
+      "learning_rate": 0.02253624669688347,
+      "loss": 0.386,
+      "num_input_tokens_seen": 196192,
+      "step": 500
+    },
+    {
+      "epoch": 4.032,
+      "eval_loss": 0.3592616617679596,
+      "eval_runtime": 0.9965,
+      "eval_samples_per_second": 56.198,
+      "eval_steps_per_second": 14.05,
+      "num_input_tokens_seen": 197728,
+      "step": 504
+    },
+    {
+      "epoch": 4.04,
+      "grad_norm": 0.0174560546875,
+      "learning_rate": 0.022354431180818528,
+      "loss": 0.3541,
+      "num_input_tokens_seen": 198208,
+      "step": 505
+    },
+    {
+      "epoch": 4.08,
+      "grad_norm": 0.0262451171875,
+      "learning_rate": 0.022171181903822883,
+      "loss": 0.3565,
+      "num_input_tokens_seen": 199776,
+      "step": 510
+    },
+    {
+      "epoch": 4.12,
+      "grad_norm": 0.005157470703125,
+      "learning_rate": 0.021986534590706163,
+      "loss": 0.3515,
+      "num_input_tokens_seen": 201728,
+      "step": 515
+    },
+    {
+      "epoch": 4.16,
+      "grad_norm": 0.004791259765625,
+      "learning_rate": 0.021800525238827927,
+      "loss": 0.3521,
+      "num_input_tokens_seen": 203552,
+      "step": 520
+    },
+    {
+      "epoch": 4.2,
+      "grad_norm": 0.017333984375,
+      "learning_rate": 0.02161319011107988,
+      "loss": 0.3496,
+      "num_input_tokens_seen": 205312,
+      "step": 525
+    },
+    {
+      "epoch": 4.24,
+      "grad_norm": 0.04150390625,
+      "learning_rate": 0.021424565728816354,
+      "loss": 0.3529,
+      "num_input_tokens_seen": 207296,
+      "step": 530
+    },
+    {
+      "epoch": 4.28,
+      "grad_norm": 0.004119873046875,
+      "learning_rate": 0.021234688864734418,
+      "loss": 0.3476,
+      "num_input_tokens_seen": 209344,
+      "step": 535
+    },
+    {
+      "epoch": 4.32,
+      "grad_norm": 0.0194091796875,
+      "learning_rate": 0.02104359653570494,
+      "loss": 0.3574,
+      "num_input_tokens_seen": 211232,
+      "step": 540
+    },
+    {
+      "epoch": 4.36,
+      "grad_norm": 0.01904296875,
+      "learning_rate": 0.020851325995556093,
+      "loss": 0.3341,
+      "num_input_tokens_seen": 213312,
+      "step": 545
+    },
+    {
+      "epoch": 4.4,
+      "grad_norm": 0.0166015625,
+      "learning_rate": 0.020657914727810648,
+      "loss": 0.3558,
+      "num_input_tokens_seen": 215808,
+      "step": 550
+    },
+    {
+      "epoch": 4.44,
+      "grad_norm": 0.015625,
+      "learning_rate": 0.020463400438378472,
+      "loss": 0.3402,
+      "num_input_tokens_seen": 217664,
+      "step": 555
+    },
+    {
+      "epoch": 4.48,
+      "grad_norm": 0.018798828125,
+      "learning_rate": 0.020267821048205698,
+      "loss": 0.3522,
+      "num_input_tokens_seen": 220000,
+      "step": 560
+    },
+    {
+      "epoch": 4.52,
+      "grad_norm": 0.018310546875,
+      "learning_rate": 0.02007121468588196,
+      "loss": 0.3477,
+      "num_input_tokens_seen": 221888,
+      "step": 565
+    },
+    {
+      "epoch": 4.536,
+      "eval_loss": 0.3515055775642395,
+      "eval_runtime": 0.9618,
+      "eval_samples_per_second": 58.222,
+      "eval_steps_per_second": 14.556,
+      "num_input_tokens_seen": 222560,
+      "step": 567
+    },
+    {
+      "epoch": 4.5600000000000005,
+      "grad_norm": 0.027099609375,
+      "learning_rate": 0.019873619680207146,
+      "loss": 0.3613,
+      "num_input_tokens_seen": 224096,
+      "step": 570
+    },
+    {
+      "epoch": 4.6,
+      "grad_norm": 0.020751953125,
+      "learning_rate": 0.019675074552719125,
+      "loss": 0.339,
+      "num_input_tokens_seen": 225952,
+      "step": 575
+    },
+    {
+      "epoch": 4.64,
+      "grad_norm": 0.0201416015625,
+      "learning_rate": 0.019475618010183906,
+      "loss": 0.35,
+      "num_input_tokens_seen": 227680,
+      "step": 580
+    },
+    {
+      "epoch": 4.68,
+      "grad_norm": 0.04736328125,
+      "learning_rate": 0.01927528893704964,
+      "loss": 0.3516,
+      "num_input_tokens_seen": 229696,
+      "step": 585
+    },
+    {
+      "epoch": 4.72,
+      "grad_norm": 0.041259765625,
+      "learning_rate": 0.01907412638786608,
+      "loss": 0.3406,
+      "num_input_tokens_seen": 231360,
+      "step": 590
+    },
+    {
+      "epoch": 4.76,
+      "grad_norm": 0.007293701171875,
+      "learning_rate": 0.018872169579670764,
+      "loss": 0.3363,
+      "num_input_tokens_seen": 233376,
+      "step": 595
+    },
+    {
+      "epoch": 4.8,
+      "grad_norm": 0.01416015625,
+      "learning_rate": 0.01866945788434361,
+      "loss": 0.3393,
+      "num_input_tokens_seen": 235296,
+      "step": 600
+    },
+    {
+      "epoch": 4.84,
+      "grad_norm": 0.038330078125,
+      "learning_rate": 0.018466030820931272,
+      "loss": 0.3559,
+      "num_input_tokens_seen": 237984,
+      "step": 605
+    },
+    {
+      "epoch": 4.88,
+      "grad_norm": 0.009521484375,
+      "learning_rate": 0.01826192804794282,
+      "loss": 0.3434,
+      "num_input_tokens_seen": 239872,
+      "step": 610
+    },
+    {
+      "epoch": 4.92,
+      "grad_norm": 0.00732421875,
+      "learning_rate": 0.018057189355618276,
+      "loss": 0.3767,
+      "num_input_tokens_seen": 241664,
+      "step": 615
+    },
+    {
+      "epoch": 4.96,
+      "grad_norm": 0.019775390625,
+      "learning_rate": 0.01785185465817135,
+      "loss": 0.3542,
+      "num_input_tokens_seen": 243552,
+      "step": 620
+    },
+    {
+      "epoch": 5.0,
+      "grad_norm": 0.04833984375,
+      "learning_rate": 0.017645963986008185,
+      "loss": 0.3574,
+      "num_input_tokens_seen": 245216,
+      "step": 625
+    },
+    {
+      "epoch": 5.04,
+      "grad_norm": 0.019775390625,
+      "learning_rate": 0.017439557477923254,
+      "loss": 0.3466,
+      "num_input_tokens_seen": 246848,
+      "step": 630
+    },
+    {
+      "epoch": 5.04,
+      "eval_loss": 0.3488920032978058,
+      "eval_runtime": 1.0534,
+      "eval_samples_per_second": 53.161,
+      "eval_steps_per_second": 13.29,
+      "num_input_tokens_seen": 246848,
+      "step": 630
+    },
+    {
+      "epoch": 5.08,
+      "grad_norm": 0.004547119140625,
+      "learning_rate": 0.017232675373274282,
+      "loss": 0.3544,
+      "num_input_tokens_seen": 248800,
+      "step": 635
+    },
+    {
+      "epoch": 5.12,
+      "grad_norm": 0.0037078857421875,
+      "learning_rate": 0.017025358004137486,
+      "loss": 0.3479,
+      "num_input_tokens_seen": 250912,
+      "step": 640
+    },
+    {
+      "epoch": 5.16,
+      "grad_norm": 0.018798828125,
+      "learning_rate": 0.016817645787444758,
+      "loss": 0.3484,
+      "num_input_tokens_seen": 252864,
+      "step": 645
+    },
+    {
+      "epoch": 5.2,
+      "grad_norm": 0.00311279296875,
+      "learning_rate": 0.0166095792171043,
+      "loss": 0.3519,
+      "num_input_tokens_seen": 254560,
+      "step": 650
+    },
+    {
+      "epoch": 5.24,
+      "grad_norm": 0.01904296875,
+      "learning_rate": 0.01640119885610626,
+      "loss": 0.3455,
+      "num_input_tokens_seen": 256224,
+      "step": 655
+    },
+    {
+      "epoch": 5.28,
+      "grad_norm": 0.0057373046875,
+      "learning_rate": 0.016192545328614895,
+      "loss": 0.3629,
+      "num_input_tokens_seen": 258400,
+      "step": 660
+    },
+    {
+      "epoch": 5.32,
+      "grad_norm": 0.00506591796875,
+      "learning_rate": 0.015983659312048825,
+      "loss": 0.3434,
+      "num_input_tokens_seen": 260160,
+      "step": 665
+    },
+    {
+      "epoch": 5.36,
+      "grad_norm": 0.042724609375,
+      "learning_rate": 0.015774581529150847,
+      "loss": 0.3526,
+      "num_input_tokens_seen": 261824,
+      "step": 670
+    },
+    {
+      "epoch": 5.4,
+      "grad_norm": 0.0189208984375,
+      "learning_rate": 0.01556535274004902,
+      "loss": 0.3358,
+      "num_input_tokens_seen": 263648,
+      "step": 675
+    },
+    {
+      "epoch": 5.44,
+      "grad_norm": 0.00811767578125,
+      "learning_rate": 0.01535601373431033,
+      "loss": 0.3574,
+      "num_input_tokens_seen": 265824,
+      "step": 680
+    },
+    {
+      "epoch": 5.48,
+      "grad_norm": 0.0152587890625,
+      "learning_rate": 0.015146605322988737,
+      "loss": 0.3481,
+      "num_input_tokens_seen": 267840,
+      "step": 685
+    },
+    {
+      "epoch": 5.52,
+      "grad_norm": 0.0157470703125,
+      "learning_rate": 0.014937168330668944,
+      "loss": 0.3566,
+      "num_input_tokens_seen": 269664,
+      "step": 690
+    },
+    {
+      "epoch": 5.5440000000000005,
+      "eval_loss": 0.3491196036338806,
+      "eval_runtime": 0.956,
+      "eval_samples_per_second": 58.577,
+      "eval_steps_per_second": 14.644,
+      "num_input_tokens_seen": 271008,
+      "step": 693
+    },
+    {
+      "epoch": 5.5600000000000005,
+      "grad_norm": 0.0264892578125,
+      "learning_rate": 0.014727743587507579,
+      "loss": 0.3624,
+      "num_input_tokens_seen": 271648,
+      "step": 695
+    },
+    {
+      "epoch": 5.6,
+      "grad_norm": 0.02587890625,
+      "learning_rate": 0.014518371921273277,
+      "loss": 0.3577,
+      "num_input_tokens_seen": 273504,
+      "step": 700
+    },
+    {
+      "epoch": 5.64,
+      "grad_norm": 0.0233154296875,
+      "learning_rate": 0.014309094149387214,
+      "loss": 0.3441,
+      "num_input_tokens_seen": 275616,
+      "step": 705
+    },
+    {
+      "epoch": 5.68,
+      "grad_norm": 0.0225830078125,
+      "learning_rate": 0.014099951070965693,
+      "loss": 0.3481,
+      "num_input_tokens_seen": 277952,
+      "step": 710
+    },
+    {
+      "epoch": 5.72,
+      "grad_norm": 0.041259765625,
+      "learning_rate": 0.013890983458866225,
+      "loss": 0.348,
+      "num_input_tokens_seen": 279680,
+      "step": 715
+    },
+    {
+      "epoch": 5.76,
+      "grad_norm": 0.0030364990234375,
+      "learning_rate": 0.013682232051738852,
+      "loss": 0.3496,
+      "num_input_tokens_seen": 282432,
+      "step": 720
+    },
+    {
+      "epoch": 5.8,
+      "grad_norm": 0.0037994384765625,
+      "learning_rate": 0.013473737546084006,
+      "loss": 0.3433,
+      "num_input_tokens_seen": 284224,
+      "step": 725
+    },
+    {
+      "epoch": 5.84,
+      "grad_norm": 0.0194091796875,
+      "learning_rate": 0.013265540588318678,
+      "loss": 0.3404,
+      "num_input_tokens_seen": 286368,
+      "step": 730
+    },
+    {
+      "epoch": 5.88,
+      "grad_norm": 0.00506591796875,
+      "learning_rate": 0.013057681766852297,
+      "loss": 0.3484,
+      "num_input_tokens_seen": 288224,
+      "step": 735
+    },
+    {
+      "epoch": 5.92,
+      "grad_norm": 0.0245361328125,
+      "learning_rate": 0.012850201604173958,
+      "loss": 0.3531,
+      "num_input_tokens_seen": 290080,
+      "step": 740
+    },
+    {
+      "epoch": 5.96,
+      "grad_norm": 0.006195068359375,
+      "learning_rate": 0.012643140548952488,
+      "loss": 0.3578,
+      "num_input_tokens_seen": 292160,
+      "step": 745
+    },
+    {
+      "epoch": 6.0,
+      "grad_norm": 0.042236328125,
+      "learning_rate": 0.012436538968150852,
+      "loss": 0.3386,
+      "num_input_tokens_seen": 294128,
+      "step": 750
+    },
+    {
+      "epoch": 6.04,
+      "grad_norm": 0.019287109375,
+      "learning_rate": 0.012230437139156598,
+      "loss": 0.3451,
+      "num_input_tokens_seen": 295664,
+      "step": 755
+    },
+    {
+      "epoch": 6.048,
+      "eval_loss": 0.3467372953891754,
+      "eval_runtime": 0.9579,
+      "eval_samples_per_second": 58.462,
+      "eval_steps_per_second": 14.615,
+      "num_input_tokens_seen": 295984,
+      "step": 756
+    },
+    {
+      "epoch": 6.08,
+      "grad_norm": 0.006866455078125,
+      "learning_rate": 0.012024875241929653,
+      "loss": 0.3461,
+      "num_input_tokens_seen": 297584,
+      "step": 760
+    },
+    {
+      "epoch": 6.12,
+      "grad_norm": 0.006103515625,
+      "learning_rate": 0.011819893351169184,
+      "loss": 0.3396,
+      "num_input_tokens_seen": 299280,
+      "step": 765
+    },
+    {
+      "epoch": 6.16,
+      "grad_norm": 0.007415771484375,
+      "learning_rate": 0.011615531428500938,
+      "loss": 0.3503,
+      "num_input_tokens_seen": 301072,
+      "step": 770
+    },
+    {
+      "epoch": 6.2,
+      "grad_norm": 0.007537841796875,
+      "learning_rate": 0.01141182931468666,
+      "loss": 0.344,
+      "num_input_tokens_seen": 303088,
+      "step": 775
+    },
+    {
+      "epoch": 6.24,
+      "grad_norm": 0.006622314453125,
+      "learning_rate": 0.01120882672185706,
+      "loss": 0.3553,
+      "num_input_tokens_seen": 305008,
+      "step": 780
+    },
+    {
+      "epoch": 6.28,
+      "grad_norm": 0.016845703125,
+      "learning_rate": 0.011006563225769832,
+      "loss": 0.3476,
+      "num_input_tokens_seen": 306960,
+      "step": 785
+    },
+    {
+      "epoch": 6.32,
+      "grad_norm": 0.00604248046875,
+      "learning_rate": 0.010805078258094304,
+      "loss": 0.3521,
+      "num_input_tokens_seen": 308944,
+      "step": 790
+    },
+    {
+      "epoch": 6.36,
+      "grad_norm": 0.00537109375,
+      "learning_rate": 0.01060441109872414,
+      "loss": 0.3503,
+      "num_input_tokens_seen": 310672,
+      "step": 795
+    },
+    {
+      "epoch": 6.4,
+      "grad_norm": 0.025634765625,
+      "learning_rate": 0.01040460086811966,
+      "loss": 0.3531,
+      "num_input_tokens_seen": 312624,
+      "step": 800
+    },
+    {
+      "epoch": 6.44,
+      "grad_norm": 0.0179443359375,
+      "learning_rate": 0.010205686519681232,
+      "loss": 0.3418,
+      "num_input_tokens_seen": 314448,
+      "step": 805
+    },
+    {
+      "epoch": 6.48,
+      "grad_norm": 0.005859375,
+      "learning_rate": 0.0100077068321552,
+      "loss": 0.3507,
+      "num_input_tokens_seen": 316592,
+      "step": 810
+    },
+    {
+      "epoch": 6.52,
+      "grad_norm": 0.02587890625,
+      "learning_rate": 0.009810700402073928,
+      "loss": 0.3555,
+      "num_input_tokens_seen": 318320,
+      "step": 815
+    },
+    {
+      "epoch": 6.552,
+      "eval_loss": 0.3465680480003357,
+      "eval_runtime": 0.9498,
+      "eval_samples_per_second": 58.961,
+      "eval_steps_per_second": 14.74,
+      "num_input_tokens_seen": 320080,
+      "step": 819
+    },
+    {
+      "epoch": 6.5600000000000005,
+      "grad_norm": 0.0250244140625,
+      "learning_rate": 0.009614705636231307,
+      "loss": 0.3599,
+      "num_input_tokens_seen": 320496,
+      "step": 820
+    },
+    {
+      "epoch": 6.6,
+      "grad_norm": 0.04541015625,
+      "learning_rate": 0.009419760744195283,
+      "loss": 0.347,
+      "num_input_tokens_seen": 322224,
+      "step": 825
+    },
+    {
+      "epoch": 6.64,
+      "grad_norm": 0.0034637451171875,
+      "learning_rate": 0.00922590373085881,
+      "loss": 0.345,
+      "num_input_tokens_seen": 324080,
+      "step": 830
+    },
+    {
+      "epoch": 6.68,
+      "grad_norm": 0.041015625,
+      "learning_rate": 0.009033172389030755,
+      "loss": 0.3466,
+      "num_input_tokens_seen": 325872,
+      "step": 835
+    },
+    {
+      "epoch": 6.72,
+      "grad_norm": 0.01904296875,
+      "learning_rate": 0.00884160429206808,
+      "loss": 0.3374,
+      "num_input_tokens_seen": 328336,
+      "step": 840
+    },
+    {
+      "epoch": 6.76,
+      "grad_norm": 0.007080078125,
+      "learning_rate": 0.008651236786550862,
+      "loss": 0.3415,
+      "num_input_tokens_seen": 330800,
+      "step": 845
+    },
+    {
+      "epoch": 6.8,
+      "grad_norm": 0.004180908203125,
+      "learning_rate": 0.00846210698500149,
+      "loss": 0.3531,
+      "num_input_tokens_seen": 332592,
+      "step": 850
+    },
+    {
+      "epoch": 6.84,
+      "grad_norm": 0.006500244140625,
+      "learning_rate": 0.008274251758649518,
+      "loss": 0.3497,
+      "num_input_tokens_seen": 334576,
+      "step": 855
+    },
+    {
+      "epoch": 6.88,
+      "grad_norm": 0.0172119140625,
+      "learning_rate": 0.008087707730243539,
+      "loss": 0.3361,
+      "num_input_tokens_seen": 336816,
+      "step": 860
+    },
+    {
+      "epoch": 6.92,
+      "grad_norm": 0.046875,
+      "learning_rate": 0.007902511266911504,
+      "loss": 0.3705,
+      "num_input_tokens_seen": 338416,
+      "step": 865
+    },
+    {
+      "epoch": 6.96,
+      "grad_norm": 0.007476806640625,
+      "learning_rate": 0.00771869847307089,
+      "loss": 0.3605,
+      "num_input_tokens_seen": 340272,
+      "step": 870
+    },
+    {
+      "epoch": 7.0,
+      "grad_norm": 0.043212890625,
+      "learning_rate": 0.007536305183390062,
+      "loss": 0.3514,
+      "num_input_tokens_seen": 342416,
+      "step": 875
+    },
+    {
+      "epoch": 7.04,
+      "grad_norm": 0.0224609375,
+      "learning_rate": 0.007355366955802234,
+      "loss": 0.3478,
+      "num_input_tokens_seen": 344176,
+      "step": 880
+    },
+    {
+      "epoch": 7.056,
+      "eval_loss": 0.35233280062675476,
+      "eval_runtime": 0.9726,
+      "eval_samples_per_second": 57.576,
+      "eval_steps_per_second": 14.394,
+      "num_input_tokens_seen": 345136,
+      "step": 882
+    },
+    {
+      "epoch": 7.08,
+      "grad_norm": 0.0223388671875,
+      "learning_rate": 0.007175919064573383,
+      "loss": 0.3431,
+      "num_input_tokens_seen": 346224,
+      "step": 885
+    },
+    {
+      "epoch": 7.12,
+      "grad_norm": 0.024169921875,
+      "learning_rate": 0.006997996493425461,
+      "loss": 0.3451,
+      "num_input_tokens_seen": 348272,
+      "step": 890
+    },
+    {
+      "epoch": 7.16,
+      "grad_norm": 0.00482177734375,
+      "learning_rate": 0.0068216339287162486,
+      "loss": 0.3514,
+      "num_input_tokens_seen": 350224,
+      "step": 895
+    },
+    {
+      "epoch": 7.2,
+      "grad_norm": 0.01904296875,
+      "learning_rate": 0.006646865752677185,
+      "loss": 0.3453,
+      "num_input_tokens_seen": 351984,
+      "step": 900
+    },
+    {
+      "epoch": 7.24,
+      "grad_norm": 0.006011962890625,
+      "learning_rate": 0.00647372603671046,
+      "loss": 0.3458,
+      "num_input_tokens_seen": 353808,
+      "step": 905
+    },
+    {
+      "epoch": 7.28,
+      "grad_norm": 0.00634765625,
+      "learning_rate": 0.0063022485347467615,
+      "loss": 0.3393,
+      "num_input_tokens_seen": 356144,
+      "step": 910
+    },
+    {
+      "epoch": 7.32,
+      "grad_norm": 0.0181884765625,
+      "learning_rate": 0.00613246667666487,
+      "loss": 0.3495,
+      "num_input_tokens_seen": 358352,
+      "step": 915
+    },
+    {
+      "epoch": 7.36,
+      "grad_norm": 0.0062255859375,
+      "learning_rate": 0.005964413561774424,
+      "loss": 0.3529,
+      "num_input_tokens_seen": 360272,
+      "step": 920
+    },
+    {
+      "epoch": 7.4,
+      "grad_norm": 0.047607421875,
+      "learning_rate": 0.0057981219523631404,
+      "loss": 0.3594,
+      "num_input_tokens_seen": 362448,
+      "step": 925
+    },
+    {
+      "epoch": 7.44,
+      "grad_norm": 0.045654296875,
+      "learning_rate": 0.005633624267309767,
+      "loss": 0.3634,
+      "num_input_tokens_seen": 364400,
+      "step": 930
+    },
+    {
+      "epoch": 7.48,
+      "grad_norm": 0.0235595703125,
+      "learning_rate": 0.005470952575763933,
+      "loss": 0.3518,
+      "num_input_tokens_seen": 366160,
+      "step": 935
+    },
+    {
+      "epoch": 7.52,
+      "grad_norm": 0.02001953125,
+      "learning_rate": 0.0053101385908942405,
+      "loss": 0.3374,
+      "num_input_tokens_seen": 368400,
+      "step": 940
+    },
+    {
+      "epoch": 7.5600000000000005,
+      "grad_norm": 0.0240478515625,
+      "learning_rate": 0.0051512136637056555,
+      "loss": 0.3544,
+      "num_input_tokens_seen": 370416,
+      "step": 945
+    },
+    {
+      "epoch": 7.5600000000000005,
+      "eval_loss": 0.3476898968219757,
+      "eval_runtime": 0.9345,
+      "eval_samples_per_second": 59.924,
+      "eval_steps_per_second": 14.981,
+      "num_input_tokens_seen": 370416,
+      "step": 945
+    },
+    {
+      "epoch": 7.6,
+      "grad_norm": 0.0211181640625,
+      "learning_rate": 0.004994208776927635,
+      "loss": 0.3511,
+      "num_input_tokens_seen": 372304,
+      "step": 950
+    },
+    {
+      "epoch": 7.64,
+      "grad_norm": 0.0211181640625,
+      "learning_rate": 0.004839154538973943,
+      "loss": 0.3543,
+      "num_input_tokens_seen": 374288,
+      "step": 955
+    },
+    {
+      "epoch": 7.68,
+      "grad_norm": 0.0036468505859375,
+      "learning_rate": 0.00468608117797549,
+      "loss": 0.3496,
+      "num_input_tokens_seen": 376336,
+      "step": 960
+    },
+    {
+      "epoch": 7.72,
+      "grad_norm": 0.0220947265625,
+      "learning_rate": 0.004535018535887305,
+      "loss": 0.3527,
+      "num_input_tokens_seen": 378576,
+      "step": 965
+    },
+    {
+      "epoch": 7.76,
+      "grad_norm": 0.0027008056640625,
+      "learning_rate": 0.004385996062670774,
+      "loss": 0.3419,
+      "num_input_tokens_seen": 380272,
+      "step": 970
+    },
+    {
+      "epoch": 7.8,
+      "grad_norm": 0.020751953125,
+      "learning_rate": 0.0042390428105523225,
+      "loss": 0.3447,
+      "num_input_tokens_seen": 382096,
+      "step": 975
+    },
+    {
+      "epoch": 7.84,
+      "grad_norm": 0.01953125,
+      "learning_rate": 0.004094187428359625,
+      "loss": 0.3451,
+      "num_input_tokens_seen": 383856,
+      "step": 980
+    },
+    {
+      "epoch": 7.88,
+      "grad_norm": 0.0225830078125,
+      "learning_rate": 0.003951458155936452,
+      "loss": 0.356,
+      "num_input_tokens_seen": 385968,
+      "step": 985
+    },
+    {
+      "epoch": 7.92,
+      "grad_norm": 0.0037994384765625,
+      "learning_rate": 0.0038108828186372685,
+      "loss": 0.3484,
+      "num_input_tokens_seen": 387952,
+      "step": 990
+    },
+    {
+      "epoch": 7.96,
+      "grad_norm": 0.005035400390625,
+      "learning_rate": 0.003672488821902614,
+      "loss": 0.3499,
+      "num_input_tokens_seen": 389840,
+      "step": 995
+    },
+    {
+      "epoch": 8.0,
+      "grad_norm": 0.006561279296875,
+      "learning_rate": 0.0035363031459163647,
+      "loss": 0.339,
+      "num_input_tokens_seen": 391552,
+      "step": 1000
+    },
+    {
+      "epoch": 8.04,
+      "grad_norm": 0.0230712890625,
+      "learning_rate": 0.0034023523403458908,
+      "loss": 0.3546,
+      "num_input_tokens_seen": 393536,
+      "step": 1005
+    },
+    {
+      "epoch": 8.064,
+      "eval_loss": 0.34949031472206116,
+      "eval_runtime": 0.9978,
+      "eval_samples_per_second": 56.125,
+      "eval_steps_per_second": 14.031,
+      "num_input_tokens_seen": 394688,
+      "step": 1008
+    },
+    {
+      "epoch": 8.08,
+      "grad_norm": 0.0052490234375,
+      "learning_rate": 0.003270662519166149,
+      "loss": 0.3451,
+      "num_input_tokens_seen": 395456,
+      "step": 1010
+    },
+    {
+      "epoch": 8.12,
+      "grad_norm": 0.0191650390625,
+      "learning_rate": 0.003141259355568705,
+      "loss": 0.3375,
+      "num_input_tokens_seen": 397184,
+      "step": 1015
+    },
+    {
+      "epoch": 8.16,
+      "grad_norm": 0.0255126953125,
+      "learning_rate": 0.003014168076956707,
+      "loss": 0.361,
+      "num_input_tokens_seen": 399456,
+      "step": 1020
+    },
+    {
+      "epoch": 8.2,
+      "grad_norm": 0.020263671875,
+      "learning_rate": 0.002889413460026724,
+      "loss": 0.3451,
+      "num_input_tokens_seen": 401088,
+      "step": 1025
+    },
+    {
+      "epoch": 8.24,
+      "grad_norm": 0.019287109375,
+      "learning_rate": 0.0027670198259385275,
+      "loss": 0.3435,
+      "num_input_tokens_seen": 403008,
+      "step": 1030
+    },
+    {
+      "epoch": 8.28,
+      "grad_norm": 0.024169921875,
+      "learning_rate": 0.0026470110355735882,
+      "loss": 0.3532,
+      "num_input_tokens_seen": 405280,
+      "step": 1035
+    },
+    {
+      "epoch": 8.32,
+      "grad_norm": 0.041015625,
+      "learning_rate": 0.0025294104848833754,
+      "loss": 0.339,
+      "num_input_tokens_seen": 407168,
+      "step": 1040
+    },
+    {
+      "epoch": 8.36,
+      "grad_norm": 0.0235595703125,
+      "learning_rate": 0.002414241100328251,
+      "loss": 0.3594,
+      "num_input_tokens_seen": 408960,
+      "step": 1045
+    },
+    {
+      "epoch": 8.4,
+      "grad_norm": 0.0189208984375,
+      "learning_rate": 0.002301525334407931,
+      "loss": 0.3517,
+      "num_input_tokens_seen": 410880,
+      "step": 1050
+    },
+    {
+      "epoch": 8.44,
+      "grad_norm": 0.0064697265625,
+      "learning_rate": 0.0021912851612843243,
+      "loss": 0.3484,
+      "num_input_tokens_seen": 412800,
+      "step": 1055
+    },
+    {
+      "epoch": 8.48,
+      "grad_norm": 0.0203857421875,
+      "learning_rate": 0.002083542072497606,
+      "loss": 0.3436,
+      "num_input_tokens_seen": 415104,
+      "step": 1060
+    },
+    {
+      "epoch": 8.52,
+      "grad_norm": 0.00653076171875,
+      "learning_rate": 0.001978317072776413,
+      "loss": 0.3451,
+      "num_input_tokens_seen": 416800,
+      "step": 1065
+    },
+    {
+      "epoch": 8.56,
+      "grad_norm": 0.003326416015625,
+      "learning_rate": 0.0018756306759429363,
+      "loss": 0.3436,
+      "num_input_tokens_seen": 418560,
+      "step": 1070
+    },
+    {
+      "epoch": 8.568,
+      "eval_loss": 0.3510778844356537,
+      "eval_runtime": 0.9474,
+      "eval_samples_per_second": 59.112,
+      "eval_steps_per_second": 14.778,
+      "num_input_tokens_seen": 418880,
+      "step": 1071
+    },
+    {
+      "epoch": 8.6,
+      "grad_norm": 0.01953125,
+      "learning_rate": 0.001775502900913697,
+      "loss": 0.3467,
+      "num_input_tokens_seen": 420800,
+      "step": 1075
+    },
+    {
+      "epoch": 8.64,
+      "grad_norm": 0.040771484375,
+      "learning_rate": 0.0016779532677968327,
+      "loss": 0.3392,
+      "num_input_tokens_seen": 422752,
+      "step": 1080
+    },
+    {
+      "epoch": 8.68,
+      "grad_norm": 0.005889892578125,
+      "learning_rate": 0.0015830007940866035,
+      "loss": 0.3499,
+      "num_input_tokens_seen": 424896,
+      "step": 1085
+    },
+    {
+      "epoch": 8.72,
+      "grad_norm": 0.02392578125,
+      "learning_rate": 0.0014906639909558954,
+      "loss": 0.3434,
+      "num_input_tokens_seen": 426880,
+      "step": 1090
+    },
+    {
+      "epoch": 8.76,
+      "grad_norm": 0.02392578125,
+      "learning_rate": 0.0014009608596474348,
+      "loss": 0.3484,
+      "num_input_tokens_seen": 428768,
+      "step": 1095
+    },
+    {
+      "epoch": 8.8,
+      "grad_norm": 0.005462646484375,
+      "learning_rate": 0.001313908887964409,
+      "loss": 0.3499,
+      "num_input_tokens_seen": 430720,
+      "step": 1100
+    },
+    {
+      "epoch": 8.84,
+      "grad_norm": 0.005645751953125,
+      "learning_rate": 0.0012295250468611779,
+      "loss": 0.3484,
+      "num_input_tokens_seen": 432736,
+      "step": 1105
+    },
+    {
+      "epoch": 8.88,
+      "grad_norm": 0.0054931640625,
+      "learning_rate": 0.0011478257871347663,
+      "loss": 0.3436,
+      "num_input_tokens_seen": 434816,
+      "step": 1110
+    },
+    {
+      "epoch": 8.92,
+      "grad_norm": 0.0062255859375,
+      "learning_rate": 0.0010688270362177355,
+      "loss": 0.3451,
+      "num_input_tokens_seen": 437152,
+      "step": 1115
+    },
+    {
+      "epoch": 8.96,
+      "grad_norm": 0.018798828125,
+      "learning_rate": 0.0009925441950730985,
+      "loss": 0.3391,
+      "num_input_tokens_seen": 439232,
+      "step": 1120
+    },
+    {
+      "epoch": 9.0,
+      "grad_norm": 0.00848388671875,
+      "learning_rate": 0.0009189921351918889,
+      "loss": 0.3423,
+      "num_input_tokens_seen": 440848,
+      "step": 1125
+    },
+    {
+      "epoch": 9.04,
+      "grad_norm": 0.01953125,
+      "learning_rate": 0.0008481851956939134,
+      "loss": 0.3359,
+      "num_input_tokens_seen": 442704,
+      "step": 1130
+    },
+    {
+      "epoch": 9.072,
+      "eval_loss": 0.3495887815952301,
+      "eval_runtime": 0.9616,
+      "eval_samples_per_second": 58.234,
+      "eval_steps_per_second": 14.558,
+      "num_input_tokens_seen": 444304,
+      "step": 1134
+    },
+    {
+      "epoch": 9.08,
+      "grad_norm": 0.043701171875,
+      "learning_rate": 0.0007801371805323276,
+      "loss": 0.3499,
+      "num_input_tokens_seen": 444656,
+      "step": 1135
+    },
+    {
+      "epoch": 9.12,
+      "grad_norm": 0.0189208984375,
+      "learning_rate": 0.0007148613558025102,
+      "loss": 0.3422,
+      "num_input_tokens_seen": 447152,
+      "step": 1140
+    },
+    {
+      "epoch": 9.16,
+      "grad_norm": 0.0245361328125,
+      "learning_rate": 0.0006523704471558306,
+      "loss": 0.3498,
+      "num_input_tokens_seen": 448976,
+      "step": 1145
+    },
+    {
+      "epoch": 9.2,
+      "grad_norm": 0.0042724609375,
+      "learning_rate": 0.0005926766373187531,
+      "loss": 0.3437,
+      "num_input_tokens_seen": 450768,
+      "step": 1150
+    },
+    {
+      "epoch": 9.24,
+      "grad_norm": 0.021240234375,
+      "learning_rate": 0.0005357915637177817,
+      "loss": 0.3437,
+      "num_input_tokens_seen": 452688,
+      "step": 1155
+    },
+    {
+      "epoch": 9.28,
+      "grad_norm": 0.0242919921875,
+      "learning_rate": 0.00048172631621072045,
+      "loss": 0.3468,
+      "num_input_tokens_seen": 454448,
+      "step": 1160
+    },
+    {
+      "epoch": 9.32,
+      "grad_norm": 0.004913330078125,
+      "learning_rate": 0.00043049143492470017,
+      "loss": 0.3453,
+      "num_input_tokens_seen": 456304,
+      "step": 1165
+    },
+    {
+      "epoch": 9.36,
+      "grad_norm": 0.004425048828125,
+      "learning_rate": 0.00038209690820134145,
+      "loss": 0.3485,
+      "num_input_tokens_seen": 458128,
+      "step": 1170
+    },
+    {
+      "epoch": 9.4,
+      "grad_norm": 0.004364013671875,
+      "learning_rate": 0.0003365521706495234,
+      "loss": 0.358,
+      "num_input_tokens_seen": 460176,
+      "step": 1175
+    },
+    {
+      "epoch": 9.44,
+      "grad_norm": 0.0034942626953125,
+      "learning_rate": 0.00029386610130606504,
+      "loss": 0.3405,
+      "num_input_tokens_seen": 462288,
+      "step": 1180
+    },
+    {
+      "epoch": 9.48,
+      "grad_norm": 0.0191650390625,
+      "learning_rate": 0.00025404702190476856,
+      "loss": 0.3376,
+      "num_input_tokens_seen": 464048,
+      "step": 1185
+    },
+    {
+      "epoch": 9.52,
+      "grad_norm": 0.005950927734375,
+      "learning_rate": 0.00021710269525405834,
+      "loss": 0.3423,
+      "num_input_tokens_seen": 466128,
+      "step": 1190
+    },
+    {
+      "epoch": 9.56,
+      "grad_norm": 0.0045166015625,
+      "learning_rate": 0.00018304032372361666,
+      "loss": 0.3421,
+      "num_input_tokens_seen": 468336,
+      "step": 1195
+    },
+    {
+      "epoch": 9.576,
+      "eval_loss": 0.3494546711444855,
+      "eval_runtime": 0.9481,
+      "eval_samples_per_second": 59.068,
+      "eval_steps_per_second": 14.767,
+      "num_input_tokens_seen": 469328,
+      "step": 1197
+    },
+    {
+      "epoch": 9.6,
+      "grad_norm": 0.0194091796875,
+      "learning_rate": 0.00015186654784026365,
+      "loss": 0.3514,
+      "num_input_tokens_seen": 470224,
+      "step": 1200
+    },
+    {
+      "epoch": 9.64,
+      "grad_norm": 0.01904296875,
+      "learning_rate": 0.00012358744499337603,
+      "loss": 0.3375,
+      "num_input_tokens_seen": 472208,
+      "step": 1205
+    },
+    {
+      "epoch": 9.68,
+      "grad_norm": 0.01953125,
+      "learning_rate": 9.820852825008664e-05,
+      "loss": 0.3484,
+      "num_input_tokens_seen": 473904,
+      "step": 1210
+    },
+    {
+      "epoch": 9.72,
+      "grad_norm": 0.004150390625,
+      "learning_rate": 7.57347452804974e-05,
+      "loss": 0.3499,
+      "num_input_tokens_seen": 475792,
+      "step": 1215
+    },
+    {
+      "epoch": 9.76,
+      "grad_norm": 0.005340576171875,
+      "learning_rate": 5.6170477393130966e-05,
+      "loss": 0.3485,
+      "num_input_tokens_seen": 477744,
+      "step": 1220
+    },
+    {
+      "epoch": 9.8,
+      "grad_norm": 0.0203857421875,
+      "learning_rate": 3.951953868077229e-05,
+      "loss": 0.3452,
+      "num_input_tokens_seen": 479632,
+      "step": 1225
+    },
+    {
+      "epoch": 9.84,
+      "grad_norm": 0.005126953125,
+      "learning_rate": 2.5785175276920034e-05,
+      "loss": 0.3546,
+      "num_input_tokens_seen": 481392,
+      "step": 1230
+    },
+    {
+      "epoch": 9.88,
+      "grad_norm": 0.00506591796875,
+      "learning_rate": 1.4970064722929499e-05,
+      "loss": 0.3466,
+      "num_input_tokens_seen": 483568,
+      "step": 1235
+    },
+    {
+      "epoch": 9.92,
+      "grad_norm": 0.004730224609375,
+      "learning_rate": 7.076315446033487e-06,
+      "loss": 0.3391,
+      "num_input_tokens_seen": 485296,
+      "step": 1240
+    },
+    {
+      "epoch": 9.96,
+      "grad_norm": 0.006622314453125,
+      "learning_rate": 2.105466348294449e-06,
+      "loss": 0.3424,
+      "num_input_tokens_seen": 486992,
+      "step": 1245
+    },
+    {
+      "epoch": 10.0,
+      "grad_norm": 0.04248046875,
+      "learning_rate": 5.848650659112664e-08,
+      "loss": 0.3453,
+      "num_input_tokens_seen": 488816,
+      "step": 1250
+    },
+    {
+      "epoch": 10.0,
+      "num_input_tokens_seen": 488816,
+      "step": 1250,
+      "total_flos": 2.201116355081011e+16,
+      "train_loss": 0.41895289707183836,
+      "train_runtime": 188.2345,
+      "train_samples_per_second": 26.456,
+      "train_steps_per_second": 6.641
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1250,
+  "num_input_tokens_seen": 488816,
+  "num_train_epochs": 10,
+  "save_steps": 63,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.201116355081011e+16,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}