Training in progress, step 40000

trainer_log.jsonl

@@ -8157,3 +8157,45 @@
 {"current_steps": 39795, "total_steps": 40000, "loss": 0.4716, "lr": 3.2720109079037443e-09, "epoch": 318.3614457831325, "percentage": 99.49, "elapsed_time": "3:00:10", "remaining_time": "0:00:55", "throughput": 1288.69, "total_tokens": 13931520}
 {"current_steps": 39800, "total_steps": 40000, "loss": 0.6218, "lr": 3.1151063006468193e-09, "epoch": 318.4016064257028, "percentage": 99.5, "elapsed_time": "3:00:11", "remaining_time": "0:00:54", "throughput": 1288.7, "total_tokens": 13933264}
 {"current_steps": 39800, "total_steps": 40000, "eval_loss": 0.6042019724845886, "epoch": 318.4016064257028, "percentage": 99.5, "elapsed_time": "3:00:13", "remaining_time": "0:00:54", "throughput": 1288.54, "total_tokens": 13933264}
+{"current_steps": 39805, "total_steps": 40000, "loss": 0.4262, "lr": 2.962056527169854e-09, "epoch": 318.44176706827307, "percentage": 99.51, "elapsed_time": "3:00:15", "remaining_time": "0:00:52", "throughput": 1288.38, "total_tokens": 13934784}
+{"current_steps": 39810, "total_steps": 40000, "loss": 0.5249, "lr": 2.8128616110761898e-09, "epoch": 318.48192771084337, "percentage": 99.52, "elapsed_time": "3:00:17", "remaining_time": "0:00:51", "throughput": 1288.39, "total_tokens": 13936544}
+{"current_steps": 39815, "total_steps": 40000, "loss": 0.92, "lr": 2.6675215753724223e-09, "epoch": 318.52208835341366, "percentage": 99.54, "elapsed_time": "3:00:18", "remaining_time": "0:00:50", "throughput": 1288.39, "total_tokens": 13938272}
+{"current_steps": 39820, "total_steps": 40000, "loss": 0.5695, "lr": 2.5260364424739557e-09, "epoch": 318.56224899598396, "percentage": 99.55, "elapsed_time": "3:00:19", "remaining_time": "0:00:48", "throughput": 1288.39, "total_tokens": 13939872}
+{"current_steps": 39825, "total_steps": 40000, "loss": 0.5929, "lr": 2.3884062341994475e-09, "epoch": 318.6024096385542, "percentage": 99.56, "elapsed_time": "3:00:20", "remaining_time": "0:00:47", "throughput": 1288.43, "total_tokens": 13941936}
+{"current_steps": 39830, "total_steps": 40000, "loss": 0.4059, "lr": 2.25463097177081e-09, "epoch": 318.6425702811245, "percentage": 99.58, "elapsed_time": "3:00:22", "remaining_time": "0:00:46", "throughput": 1288.43, "total_tokens": 13943648}
+{"current_steps": 39835, "total_steps": 40000, "loss": 0.7287, "lr": 2.1247106758215397e-09, "epoch": 318.6827309236948, "percentage": 99.59, "elapsed_time": "3:00:23", "remaining_time": "0:00:44", "throughput": 1288.43, "total_tokens": 13945344}
+{"current_steps": 39840, "total_steps": 40000, "loss": 0.6015, "lr": 1.998645366382834e-09, "epoch": 318.72289156626505, "percentage": 99.6, "elapsed_time": "3:00:24", "remaining_time": "0:00:43", "throughput": 1288.43, "total_tokens": 13946976}
+{"current_steps": 39845, "total_steps": 40000, "loss": 0.815, "lr": 1.876435062897475e-09, "epoch": 318.76305220883535, "percentage": 99.61, "elapsed_time": "3:00:26", "remaining_time": "0:00:42", "throughput": 1288.43, "total_tokens": 13948592}
+{"current_steps": 39850, "total_steps": 40000, "loss": 0.856, "lr": 1.758079784211497e-09, "epoch": 318.80321285140565, "percentage": 99.62, "elapsed_time": "3:00:27", "remaining_time": "0:00:40", "throughput": 1288.42, "total_tokens": 13950144}
+{"current_steps": 39855, "total_steps": 40000, "loss": 0.4916, "lr": 1.6435795485797434e-09, "epoch": 318.8433734939759, "percentage": 99.64, "elapsed_time": "3:00:28", "remaining_time": "0:00:39", "throughput": 1288.43, "total_tokens": 13951904}
+{"current_steps": 39860, "total_steps": 40000, "loss": 0.8192, "lr": 1.5329343736547596e-09, "epoch": 318.8835341365462, "percentage": 99.65, "elapsed_time": "3:00:29", "remaining_time": "0:00:38", "throughput": 1288.45, "total_tokens": 13953744}
+{"current_steps": 39865, "total_steps": 40000, "loss": 0.7059, "lr": 1.4261442765006739e-09, "epoch": 318.9236947791165, "percentage": 99.66, "elapsed_time": "3:00:31", "remaining_time": "0:00:36", "throughput": 1288.45, "total_tokens": 13955472}
+{"current_steps": 39870, "total_steps": 40000, "loss": 0.8712, "lr": 1.3232092735876445e-09, "epoch": 318.96385542168673, "percentage": 99.67, "elapsed_time": "3:00:32", "remaining_time": "0:00:35", "throughput": 1288.47, "total_tokens": 13957344}
+{"current_steps": 39875, "total_steps": 40000, "loss": 0.7049, "lr": 1.2241293807918607e-09, "epoch": 319.0, "percentage": 99.69, "elapsed_time": "3:00:33", "remaining_time": "0:00:33", "throughput": 1288.46, "total_tokens": 13958896}
+{"current_steps": 39880, "total_steps": 40000, "loss": 0.6555, "lr": 1.128904613387216e-09, "epoch": 319.0401606425703, "percentage": 99.7, "elapsed_time": "3:00:35", "remaining_time": "0:00:32", "throughput": 1288.45, "total_tokens": 13960704}
+{"current_steps": 39885, "total_steps": 40000, "loss": 0.7116, "lr": 1.0375349860591853e-09, "epoch": 319.08032128514054, "percentage": 99.71, "elapsed_time": "3:00:36", "remaining_time": "0:00:31", "throughput": 1288.46, "total_tokens": 13962544}
+{"current_steps": 39890, "total_steps": 40000, "loss": 0.7365, "lr": 9.5002051290205e-10, "epoch": 319.12048192771084, "percentage": 99.72, "elapsed_time": "3:00:37", "remaining_time": "0:00:29", "throughput": 1288.47, "total_tokens": 13964256}
+{"current_steps": 39895, "total_steps": 40000, "loss": 0.572, "lr": 8.663612074077954e-10, "epoch": 319.16064257028114, "percentage": 99.74, "elapsed_time": "3:00:39", "remaining_time": "0:00:28", "throughput": 1288.46, "total_tokens": 13965792}
+{"current_steps": 39900, "total_steps": 40000, "loss": 1.0206, "lr": 7.865570824799884e-10, "epoch": 319.2008032128514, "percentage": 99.75, "elapsed_time": "3:00:40", "remaining_time": "0:00:27", "throughput": 1288.46, "total_tokens": 13967520}
+{"current_steps": 39905, "total_steps": 40000, "loss": 0.6016, "lr": 7.106081504254514e-10, "epoch": 319.2409638554217, "percentage": 99.76, "elapsed_time": "3:00:41", "remaining_time": "0:00:25", "throughput": 1288.5, "total_tokens": 13969536}
+{"current_steps": 39910, "total_steps": 40000, "loss": 0.6749, "lr": 6.385144229570372e-10, "epoch": 319.281124497992, "percentage": 99.78, "elapsed_time": "3:00:43", "remaining_time": "0:00:24", "throughput": 1288.5, "total_tokens": 13971200}
+{"current_steps": 39915, "total_steps": 40000, "loss": 0.8465, "lr": 5.70275911190854e-10, "epoch": 319.3212851405622, "percentage": 99.79, "elapsed_time": "3:00:44", "remaining_time": "0:00:23", "throughput": 1288.5, "total_tokens": 13972960}
+{"current_steps": 39920, "total_steps": 40000, "loss": 0.5, "lr": 5.058926256490403e-10, "epoch": 319.3614457831325, "percentage": 99.8, "elapsed_time": "3:00:45", "remaining_time": "0:00:21", "throughput": 1288.51, "total_tokens": 13974640}
+{"current_steps": 39925, "total_steps": 40000, "loss": 0.7461, "lr": 4.4536457626254134e-10, "epoch": 319.4016064257028, "percentage": 99.81, "elapsed_time": "3:00:46", "remaining_time": "0:00:20", "throughput": 1288.51, "total_tokens": 13976368}
+{"current_steps": 39930, "total_steps": 40000, "loss": 0.6617, "lr": 3.88691772365557e-10, "epoch": 319.44176706827307, "percentage": 99.83, "elapsed_time": "3:00:48", "remaining_time": "0:00:19", "throughput": 1288.51, "total_tokens": 13977984}
+{"current_steps": 39935, "total_steps": 40000, "loss": 0.7625, "lr": 3.358742226955425e-10, "epoch": 319.48192771084337, "percentage": 99.84, "elapsed_time": "3:00:49", "remaining_time": "0:00:17", "throughput": 1288.51, "total_tokens": 13979632}
+{"current_steps": 39940, "total_steps": 40000, "loss": 0.6733, "lr": 2.8691193539875925e-10, "epoch": 319.52208835341366, "percentage": 99.85, "elapsed_time": "3:00:50", "remaining_time": "0:00:16", "throughput": 1288.53, "total_tokens": 13981472}
+{"current_steps": 39945, "total_steps": 40000, "loss": 0.7793, "lr": 2.418049180274995e-10, "epoch": 319.56224899598396, "percentage": 99.86, "elapsed_time": "3:00:52", "remaining_time": "0:00:14", "throughput": 1288.54, "total_tokens": 13983328}
+{"current_steps": 39950, "total_steps": 40000, "loss": 0.6705, "lr": 2.005531775373104e-10, "epoch": 319.6024096385542, "percentage": 99.88, "elapsed_time": "3:00:53", "remaining_time": "0:00:13", "throughput": 1288.55, "total_tokens": 13985040}
+{"current_steps": 39955, "total_steps": 40000, "loss": 0.655, "lr": 1.6315672028699435e-10, "epoch": 319.6425702811245, "percentage": 99.89, "elapsed_time": "3:00:54", "remaining_time": "0:00:12", "throughput": 1288.55, "total_tokens": 13986736}
+{"current_steps": 39960, "total_steps": 40000, "loss": 0.7772, "lr": 1.2961555204693555e-10, "epoch": 319.6827309236948, "percentage": 99.9, "elapsed_time": "3:00:55", "remaining_time": "0:00:10", "throughput": 1288.57, "total_tokens": 13988608}
+{"current_steps": 39965, "total_steps": 40000, "loss": 0.5243, "lr": 9.992967798799768e-11, "epoch": 319.72289156626505, "percentage": 99.91, "elapsed_time": "3:00:57", "remaining_time": "0:00:09", "throughput": 1288.61, "total_tokens": 13990688}
+{"current_steps": 39970, "total_steps": 40000, "loss": 0.549, "lr": 7.409910268707521e-11, "epoch": 319.76305220883535, "percentage": 99.92, "elapsed_time": "3:00:58", "remaining_time": "0:00:08", "throughput": 1288.61, "total_tokens": 13992304}
+{"current_steps": 39975, "total_steps": 40000, "loss": 0.6656, "lr": 5.212383012986877e-11, "epoch": 319.80321285140565, "percentage": 99.94, "elapsed_time": "3:00:59", "remaining_time": "0:00:06", "throughput": 1288.62, "total_tokens": 13994176}
+{"current_steps": 39980, "total_steps": 40000, "loss": 0.5591, "lr": 3.400386370533415e-11, "epoch": 319.8433734939759, "percentage": 99.95, "elapsed_time": "3:01:01", "remaining_time": "0:00:05", "throughput": 1288.62, "total_tokens": 13995824}
+{"current_steps": 39985, "total_steps": 40000, "loss": 0.7281, "lr": 1.9739206205682258e-11, "epoch": 319.8835341365462, "percentage": 99.96, "elapsed_time": "3:01:02", "remaining_time": "0:00:04", "throughput": 1288.63, "total_tokens": 13997632}
+{"current_steps": 39990, "total_steps": 40000, "loss": 0.6478, "lr": 9.329859829154685e-12, "epoch": 319.9236947791165, "percentage": 99.98, "elapsed_time": "3:01:03", "remaining_time": "0:00:02", "throughput": 1288.64, "total_tokens": 13999328}
+{"current_steps": 39995, "total_steps": 40000, "loss": 0.7932, "lr": 2.7758261855748148e-12, "epoch": 319.96385542168673, "percentage": 99.99, "elapsed_time": "3:01:04", "remaining_time": "0:00:01", "throughput": 1288.65, "total_tokens": 14001120}
+{"current_steps": 40000, "total_steps": 40000, "loss": 0.6923, "lr": 7.710628524559838e-14, "epoch": 320.0, "percentage": 100.0, "elapsed_time": "3:01:06", "remaining_time": "0:00:00", "throughput": 1288.64, "total_tokens": 14002704}
+{"current_steps": 40000, "total_steps": 40000, "eval_loss": 0.6042019724845886, "epoch": 320.0, "percentage": 100.0, "elapsed_time": "3:01:07", "remaining_time": "0:00:00", "throughput": 1288.48, "total_tokens": 14002704}
+{"current_steps": 40000, "total_steps": 40000, "epoch": 320.0, "percentage": 100.0, "elapsed_time": "3:01:08", "remaining_time": "0:00:00", "throughput": 1288.35, "total_tokens": 14002704}