End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.15303622418286977,
-    "train_runtime": 105.5929,
+    "train_loss": 88.58008198263124,
+    "train_runtime": 103.6296,
     "train_samples": 28,
-    "train_samples_per_second": 3.031,
-    "train_steps_per_second": 0.189
+    "train_samples_per_second": 3.088,
+    "train_steps_per_second": 0.193
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc1f45e019febf5d9436814623cf7ef5b4a8c38e1d86147c3c3851bf9d8ffade
+oid sha256:2fa6c294c05b730840f4af5f3bcb816b1db7ebda8682a84a81334f9508a50f0b
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.15303622418286977,
-    "train_runtime": 105.5929,
+    "train_loss": 88.58008198263124,
+    "train_runtime": 103.6296,
     "train_samples": 28,
-    "train_samples_per_second": 3.031,
-    "train_steps_per_second": 0.189
+    "train_samples_per_second": 3.088,
+    "train_steps_per_second": 0.193
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 225.09375,
+      "completion_length": 250.375,
       "epoch": 0.5714285714285714,
-      "grad_norm": 20.919921875,
+      "grad_norm": 56.84841537475586,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 3.559715673327446,
-      "reward_std": 1.2320981612429023,
-      "rewards/concensus_correctness_reward_func": 0.8570625055581331,
-      "rewards/consensus_reward_func": 1.0,
+      "reward": 2.6311574075371027,
+      "reward_std": 0.7637105587637052,
+      "rewards/concensus_correctness_reward_func": 0.4665000010281801,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.6169031669851393,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5346261922968552,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.125,
-      "rewards/xmlcount_reward_func": 0.7732500024139881,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.6300312462262809,
       "step": 2
     },
     {
-      "completion_length": 154.16666666666666,
+      "completion_length": 137.625,
       "epoch": 1.0,
-      "grad_norm": 11.730490684509277,
-      "kl": 0.14171995827928185,
+      "grad_norm": 146.0875701904297,
+      "kl": 2.873302392894402,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0001,
-      "reward": 6.33573059240977,
-      "reward_std": 0.5204034863660733,
-      "rewards/concensus_correctness_reward_func": 1.9009999831517537,
-      "rewards/consensus_reward_func": 1.5833333333333333,
+      "loss": 0.0022,
+      "reward": 3.824417700370153,
+      "reward_std": 1.31266384323438,
+      "rewards/concensus_correctness_reward_func": 1.177999993165334,
+      "rewards/consensus_reward_func": 0.8333333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.9295222212870916,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.6210010035041099,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.2135416666666667,
+      "rewards/strict_format_reward_func": 0.22916666666666666,
+      "rewards/xmlcount_reward_func": 0.8795833364129066,
       "step": 4
     },
     {
-      "completion_length": 138.0,
+      "completion_length": 123.78125,
       "epoch": 1.5714285714285714,
-      "grad_norm": 76.44610595703125,
-      "kl": 3.4767233863822185,
+      "grad_norm": 44.83761215209961,
+      "kl": 1.7949985996820033,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0035,
-      "reward": 5.587845887988806,
-      "reward_std": 0.7682387698441744,
-      "rewards/concensus_correctness_reward_func": 1.5034999921917915,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0018,
+      "reward": 4.477854784578085,
+      "reward_std": 0.6243240954063367,
+      "rewards/concensus_correctness_reward_func": 1.3118750005960464,
+      "rewards/consensus_reward_func": 1.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8021896322024986,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6987922478001565,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.1571562513709068,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 0.9828125014901161,
       "step": 6
     },
     {
-      "completion_length": 152.45833333333334,
+      "completion_length": 127.75,
       "epoch": 2.0,
-      "grad_norm": 41634.69140625,
-      "kl": 819.8087679861734,
+      "grad_norm": 76.24980163574219,
+      "kl": 18.988028665383656,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.6149,
-      "reward": 5.618495126565297,
-      "reward_std": 0.9283469424893459,
-      "rewards/concensus_correctness_reward_func": 1.383166675766309,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0142,
+      "reward": 4.308155059814453,
+      "reward_std": 1.7288466555376847,
+      "rewards/concensus_correctness_reward_func": 1.0689166635274887,
+      "rewards/consensus_reward_func": 1.1666666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.8980368028084437,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.7196967353423437,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.2916666666666667,
-      "rewards/xmlcount_reward_func": 1.1289583345254262,
+      "rewards/xmlcount_reward_func": 0.9778749992450079,
       "step": 8
     },
     {
-      "completion_length": 155.71875,
+      "completion_length": 140.3125,
       "epoch": 2.571428571428571,
-      "grad_norm": 5915.94189453125,
-      "kl": 260.16201811283827,
+      "grad_norm": 288.7049865722656,
+      "kl": 17.717969600809738,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.2602,
-      "reward": 5.739752113819122,
-      "reward_std": 0.7196181467734277,
-      "rewards/concensus_correctness_reward_func": 1.599874995648861,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0177,
+      "reward": 3.9995256760157645,
+      "reward_std": 0.8743778481148183,
+      "rewards/concensus_correctness_reward_func": 1.0920624919235706,
+      "rewards/consensus_reward_func": 1.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.8215645651798695,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5601193871116266,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.1308124996721745,
+      "rewards/strict_format_reward_func": 0.234375,
+      "rewards/xmlcount_reward_func": 0.8629687540233135,
       "step": 10
     },
     {
-      "completion_length": 139.91666666666666,
+      "completion_length": 105.25,
       "epoch": 3.0,
-      "grad_norm": 96.66000366210938,
-      "kl": 642.1366088806341,
+      "grad_norm": 80085.2890625,
+      "kl": 4494.306384936829,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.4816,
-      "reward": 5.399486839771271,
-      "reward_std": 0.8307862650641861,
-      "rewards/concensus_correctness_reward_func": 1.2652500073115032,
-      "rewards/consensus_reward_func": 1.5833333333333333,
+      "loss": 3.3707,
+      "reward": 3.7779306173324585,
+      "reward_std": 1.0070178862661123,
+      "rewards/concensus_correctness_reward_func": 0.9075833410024643,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.9183202187220255,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.6610139695306619,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3958333333333333,
-      "rewards/xmlcount_reward_func": 1.1534166634082794,
+      "rewards/strict_format_reward_func": 0.25,
+      "rewards/xmlcount_reward_func": 0.9593333303928375,
       "step": 12
     },
     {
-      "completion_length": 159.21875,
+      "completion_length": 158.71875,
       "epoch": 3.571428571428571,
-      "grad_norm": 55.89032745361328,
-      "kl": 18.68994601815939,
+      "grad_norm": 342.165771484375,
+      "kl": 230.84162104106508,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0187,
-      "reward": 5.988082870841026,
-      "reward_std": 0.451270190969808,
-      "rewards/concensus_correctness_reward_func": 1.598687507212162,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.2308,
+      "reward": 4.614273123443127,
+      "reward_std": 1.2838216330856085,
+      "rewards/concensus_correctness_reward_func": 1.2610000036656857,
+      "rewards/consensus_reward_func": 1.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8969891555607319,
+      "rewards/question_recreation_reward_func": 0.7591793118044734,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.1955312490463257,
+      "rewards/strict_format_reward_func": 0.234375,
+      "rewards/xmlcount_reward_func": 0.9847187455743551,
       "step": 14
     },
     {
-      "completion_length": 145.08333333333334,
+      "completion_length": 124.875,
       "epoch": 4.0,
-      "grad_norm": 194.86911010742188,
-      "kl": 60.706012876083456,
+      "grad_norm": 3599872.25,
+      "kl": 77221.04993572489,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0455,
-      "reward": 5.922488033771515,
-      "reward_std": 0.6264131491382917,
-      "rewards/concensus_correctness_reward_func": 1.705083320538203,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 57.9158,
+      "reward": 4.892165020108223,
+      "reward_std": 0.47424173859568935,
+      "rewards/concensus_correctness_reward_func": 1.3709166596333187,
+      "rewards/consensus_reward_func": 1.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.8970297326644262,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.7545816494772831,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3541666666666667,
-      "rewards/xmlcount_reward_func": 1.1328750004371007,
+      "rewards/strict_format_reward_func": 0.2916666666666667,
+      "rewards/xmlcount_reward_func": 1.0583333323399227,
       "step": 16
     },
     {
-      "completion_length": 129.03125,
+      "completion_length": 130.65625,
       "epoch": 4.571428571428571,
-      "grad_norm": 574.3250732421875,
-      "kl": 61.7873098035343,
+      "grad_norm": 6415.2236328125,
+      "kl": 179.9282298819162,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0618,
-      "reward": 6.002052508294582,
-      "reward_std": 0.5154316779226065,
-      "rewards/concensus_correctness_reward_func": 1.6063125021755695,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.1799,
+      "reward": 4.808562383055687,
+      "reward_std": 0.8287261514924467,
+      "rewards/concensus_correctness_reward_func": 1.2866249941289425,
+      "rewards/consensus_reward_func": 1.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.8903962715994567,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.7774373296415433,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.1459687501192093,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.056999996304512,
       "step": 18
     },
     {
-      "completion_length": 148.5,
+      "completion_length": 174.45833333333334,
       "epoch": 5.0,
-      "grad_norm": 1142.04150390625,
-      "kl": 58.87323649351796,
+      "grad_norm": 41.836551666259766,
+      "kl": 1098756.4305711815,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0442,
-      "reward": 5.781274855136871,
-      "reward_std": 0.7916454064349333,
-      "rewards/concensus_correctness_reward_func": 1.5089999934037526,
-      "rewards/consensus_reward_func": 1.5833333333333333,
+      "loss": 824.0676,
+      "reward": 4.947326357165973,
+      "reward_std": 0.6633909946382724,
+      "rewards/concensus_correctness_reward_func": 1.3980833341677983,
+      "rewards/consensus_reward_func": 1.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8521498764554659,
+      "rewards/question_recreation_reward_func": 0.8026596220831076,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.2117916643619537,
+      "rewards/strict_format_reward_func": 0.25,
+      "rewards/xmlcount_reward_func": 0.9965833326180776,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.15303622418286977,
-      "train_runtime": 105.5929,
-      "train_samples_per_second": 3.031,
-      "train_steps_per_second": 0.189
+      "train_loss": 88.58008198263124,
+      "train_runtime": 103.6296,
+      "train_samples_per_second": 3.088,
+      "train_steps_per_second": 0.193
     }
   ],
   "logging_steps": 2,