{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.9936305732484076, "eval_steps": 500, "global_step": 78, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.73125, "completions/max_length": 256.0, "completions/max_terminated_length": 230.2, "completions/mean_length": 241.275, "completions/mean_terminated_length": 145.159130859375, "completions/min_length": 39.4, "completions/min_terminated_length": 39.4, "epoch": 0.06369426751592357, "frac_reward_zero_std": 0.0, "grad_norm": 0.15999643504619598, "kl": 0.0012019407236948608, "learning_rate": 1.0166460952380893e-06, "loss": 0.0348, "num_tokens": 101544.0, "reward": -704.5378051757813, "reward_std": 103.640185546875, "rewards/wrapper/mean": -234.84592828253906, "rewards/wrapper/std": 60.25956506381432, "step": 5 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.7375, "completions/max_length": 256.0, "completions/max_terminated_length": 246.0, "completions/mean_length": 244.825, "completions/mean_terminated_length": 170.27048034667968, "completions/min_length": 55.4, "completions/min_terminated_length": 55.4, "epoch": 0.12738853503184713, "frac_reward_zero_std": 0.0, "grad_norm": 0.17804501950740814, "kl": 0.002049270807765424, "learning_rate": 2.2874537142857005e-06, "loss": 0.0248, "num_tokens": 203454.0, "reward": -721.687255859375, "reward_std": 89.32227020263672, "rewards/wrapper/mean": -240.56242327640453, "rewards/wrapper/std": 57.542199250807364, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.70625, "completions/max_length": 256.0, "completions/max_terminated_length": 230.4, "completions/mean_length": 237.45625, "completions/mean_terminated_length": 129.3524398803711, "completions/min_length": 21.2, "completions/min_terminated_length": 21.2, "epoch": 0.1910828025477707, "frac_reward_zero_std": 0.0, "grad_norm": 0.18392148613929749, "kl": 0.0022296806797385215, "learning_rate": 3.5582613333333122e-06, "loss": 0.0376, "num_tokens": 299538.0, "reward": -717.4067749023437, "reward_std": 99.79640274047851, "rewards/wrapper/mean": -239.13559325635433, "rewards/wrapper/std": 66.15994172419111, "step": 15 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.7375, "completions/max_length": 256.0, "completions/max_terminated_length": 237.8, "completions/mean_length": 240.21875, "completions/mean_terminated_length": 132.5500030517578, "completions/min_length": 18.8, "completions/min_terminated_length": 18.8, "epoch": 0.25477707006369427, "frac_reward_zero_std": 0.0, "grad_norm": 0.1596791297197342, "kl": 0.0022249914007261397, "learning_rate": 4.8290689523809235e-06, "loss": 0.0429, "num_tokens": 400170.0, "reward": -711.0138427734375, "reward_std": 110.16710357666015, "rewards/wrapper/mean": -237.00460715889932, "rewards/wrapper/std": 63.362606817794344, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.75625, "completions/max_length": 256.0, "completions/max_terminated_length": 249.8, "completions/mean_length": 245.984375, "completions/mean_terminated_length": 175.86641540527344, "completions/min_length": 53.0, "completions/min_terminated_length": 53.0, "epoch": 0.3184713375796178, "frac_reward_zero_std": 0.0, "grad_norm": 0.1616552323102951, "kl": 0.002422537445090711, "learning_rate": 6.099876571428536e-06, "loss": 0.0209, "num_tokens": 503751.0, "reward": -714.1826782226562, "reward_std": 79.00167541503906, "rewards/wrapper/mean": -238.06088894257942, "rewards/wrapper/std": 50.63579320944846, "step": 25 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.78125, "completions/max_length": 256.0, "completions/max_terminated_length": 205.8, "completions/mean_length": 243.16875, "completions/mean_terminated_length": 136.35000610351562, "completions/min_length": 74.0, "completions/min_terminated_length": 74.0, "epoch": 0.3821656050955414, "frac_reward_zero_std": 0.0, "grad_norm": 0.16664288938045502, "kl": 0.002880513295531273, "learning_rate": 7.370684190476147e-06, "loss": 0.0305, "num_tokens": 600213.0, "reward": -701.1527465820312, "reward_std": 90.27828216552734, "rewards/wrapper/mean": -233.71757828990619, "rewards/wrapper/std": 58.9955113551269, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.63125, "completions/max_length": 256.0, "completions/max_terminated_length": 244.6, "completions/mean_length": 235.590625, "completions/mean_terminated_length": 144.64424743652344, "completions/min_length": 3.0, "completions/min_terminated_length": 3.0, "epoch": 0.445859872611465, "frac_reward_zero_std": 0.00625, "grad_norm": 0.15922684967517853, "kl": 0.0036258526146411897, "learning_rate": 8.641491809523759e-06, "loss": 0.0406, "num_tokens": 697276.0, "reward": -696.09658203125, "reward_std": 102.45361328125, "rewards/wrapper/mean": -232.03218508164088, "rewards/wrapper/std": 67.28055257995923, "step": 35 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.725, "completions/max_length": 256.0, "completions/max_terminated_length": 228.6, "completions/mean_length": 242.015625, "completions/mean_terminated_length": 150.20833435058594, "completions/min_length": 20.6, "completions/min_terminated_length": 20.6, "epoch": 0.5095541401273885, "frac_reward_zero_std": 0.0, "grad_norm": 0.19204428791999817, "kl": 0.03904692931100726, "learning_rate": 8.87767962349667e-06, "loss": 0.0371, "num_tokens": 798711.0, "reward": -699.6894409179688, "reward_std": 101.79892425537109, "rewards/wrapper/mean": -233.22980764508247, "rewards/wrapper/std": 59.8505485403041, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.73125, "completions/max_length": 256.0, "completions/max_terminated_length": 245.2, "completions/mean_length": 241.159375, "completions/mean_terminated_length": 145.30555725097656, "completions/min_length": 35.0, "completions/min_terminated_length": 35.0, "epoch": 0.5732484076433121, "frac_reward_zero_std": 0.0, "grad_norm": 0.16419248282909393, "kl": 0.0070375889539718624, "learning_rate": 8.804993251128113e-06, "loss": 0.0359, "num_tokens": 899972.0, "reward": -698.934326171875, "reward_std": 109.19157409667969, "rewards/wrapper/mean": -232.97811029901106, "rewards/wrapper/std": 58.439651014966266, "step": 45 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6375, "completions/max_length": 256.0, "completions/max_terminated_length": 244.4, "completions/mean_length": 237.5625, "completions/mean_terminated_length": 155.49524536132813, "completions/min_length": 43.0, "completions/min_terminated_length": 43.0, "epoch": 0.6369426751592356, "frac_reward_zero_std": 0.0, "grad_norm": 0.16871879994869232, "kl": 0.007892952580004931, "learning_rate": 8.67769315322246e-06, "loss": 0.0348, "num_tokens": 1000384.0, "reward": -674.0330444335938, "reward_std": 92.78271789550782, "rewards/wrapper/mean": -224.67768449683984, "rewards/wrapper/std": 62.20398019005855, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.64375, "completions/max_length": 256.0, "completions/max_terminated_length": 245.6, "completions/mean_length": 237.13125, "completions/mean_terminated_length": 152.8044464111328, "completions/min_length": 39.2, "completions/min_terminated_length": 39.2, "epoch": 0.7006369426751592, "frac_reward_zero_std": 0.0, "grad_norm": 0.17889627814292908, "kl": 0.009350586123764515, "learning_rate": 8.497921666572701e-06, "loss": 0.0424, "num_tokens": 1100772.0, "reward": -669.6951293945312, "reward_std": 103.83286743164062, "rewards/wrapper/mean": -223.2317232514421, "rewards/wrapper/std": 60.86215510976811, "step": 55 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.625, "completions/max_length": 256.0, "completions/max_terminated_length": 249.6, "completions/mean_length": 234.55, "completions/mean_terminated_length": 142.2315460205078, "completions/min_length": 2.2, "completions/min_terminated_length": 2.2, "epoch": 0.7643312101910829, "frac_reward_zero_std": 0.0, "grad_norm": 0.15725429356098175, "kl": 0.009933859994634985, "learning_rate": 8.26870417039312e-06, "loss": 0.048, "num_tokens": 1198650.0, "reward": -667.4626831054687, "reward_std": 103.58808135986328, "rewards/wrapper/mean": -222.48756203502415, "rewards/wrapper/std": 64.76753753715505, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.56875, "completions/max_length": 256.0, "completions/max_terminated_length": 245.6, "completions/mean_length": 231.2125, "completions/mean_terminated_length": 140.80873413085936, "completions/min_length": 24.8, "completions/min_terminated_length": 24.8, "epoch": 0.8280254777070064, "frac_reward_zero_std": 0.0, "grad_norm": 0.16986234486103058, "kl": 0.012869389494881034, "learning_rate": 7.993898172130616e-06, "loss": 0.0481, "num_tokens": 1295616.0, "reward": -627.8613159179688, "reward_std": 109.11022491455078, "rewards/wrapper/mean": -209.28710995068153, "rewards/wrapper/std": 66.79779365633925, "step": 65 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.56875, "completions/max_length": 256.0, "completions/max_terminated_length": 236.2, "completions/mean_length": 230.4125, "completions/mean_terminated_length": 137.88794708251953, "completions/min_length": 2.8, "completions/min_terminated_length": 2.8, "epoch": 0.89171974522293, "frac_reward_zero_std": 0.0, "grad_norm": 0.19035795331001282, "kl": 0.021700558252632618, "learning_rate": 7.678128389367485e-06, "loss": 0.0591, "num_tokens": 1391046.0, "reward": -650.4082275390625, "reward_std": 121.22037506103516, "rewards/wrapper/mean": -216.80273101478815, "rewards/wrapper/std": 68.18543163053691, "step": 70 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.525, "completions/max_length": 256.0, "completions/max_terminated_length": 233.6, "completions/mean_length": 225.828125, "completions/mean_terminated_length": 129.7731674194336, "completions/min_length": 25.6, "completions/min_terminated_length": 25.6, "epoch": 0.9554140127388535, "frac_reward_zero_std": 0.0, "grad_norm": 0.18386901915073395, "kl": 0.01846405491232872, "learning_rate": 7.326708920324052e-06, "loss": 0.052, "num_tokens": 1486499.0, "reward": -624.17470703125, "reward_std": 104.77093658447265, "rewards/wrapper/mean": -208.05824314107497, "rewards/wrapper/std": 69.01961156173299, "step": 75 }, { "epoch": 0.9936305732484076, "eval_clip_ratio/high_max": 0.0, "eval_clip_ratio/high_mean": 0.0, "eval_clip_ratio/low_mean": 0.0, "eval_clip_ratio/low_min": 0.0, "eval_clip_ratio/region_mean": 0.0, "eval_completions/clipped_ratio": 0.47, "eval_completions/max_length": 256.0, "eval_completions/max_terminated_length": 168.8, "eval_completions/mean_length": 224.3125, "eval_completions/mean_terminated_length": 129.36600067138673, "eval_completions/min_length": 106.98, "eval_completions/min_terminated_length": 91.62, "eval_frac_reward_zero_std": 0.0, "eval_kl": 0.023264853823930023, "eval_loss": 0.08186180144548416, "eval_num_tokens": 1542474.0, "eval_reward": -608.83853515625, "eval_reward_std": 118.24852516174316, "eval_rewards/wrapper/mean": -202.94617808183034, "eval_rewards/wrapper/std": 66.92328324827055, "eval_runtime": 106.4962, "eval_samples_per_second": 1.878, "eval_steps_per_second": 0.235, "step": 78 } ], "logging_steps": 5, "max_steps": 156, "num_input_tokens_seen": 1542474, "num_train_epochs": 2, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 16, "trial_name": null, "trial_params": null }