End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.829412162303925e-06,
-    "train_runtime": 263.1353,
+    "train_loss": 4.229552814649651e-06,
+    "train_runtime": 224.5386,
     "train_samples": 160,
-    "train_samples_per_second": 1.216,
-    "train_steps_per_second": 0.076
+    "train_samples_per_second": 1.425,
+    "train_steps_per_second": 0.089
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f15d528e2e36f6de62fc6fda08ea4bcac2e8a376fa243f59dc9e7dc03c94aba8
+oid sha256:cd8fbc6de9c364fa5f5a500098b1b31f465b9c6da2caea520c2ac6afd2ea308c
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.829412162303925e-06,
-    "train_runtime": 263.1353,
+    "train_loss": 4.229552814649651e-06,
+    "train_runtime": 224.5386,
     "train_samples": 160,
-    "train_samples_per_second": 1.216,
-    "train_steps_per_second": 0.076
+    "train_samples_per_second": 1.425,
+    "train_steps_per_second": 0.089
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 442.875,
+      "completion_length": 302.125,
       "epoch": 0.1,
-      "grad_norm": 4.908999919891357,
+      "grad_norm": 6.64198112487793,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 2.1628924598917365,
-      "reward_std": 1.6292787790298462,
-      "rewards/concensus_correctness_reward_func": 0.909500002861023,
-      "rewards/consensus_reward_func": 0.25,
+      "loss": -0.0,
+      "reward": 2.1132799829356372,
+      "reward_std": 2.0948233329690993,
+      "rewards/concensus_correctness_reward_func": 0.9079375043511391,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5116424206644297,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.4575299544958398,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.30424999934621155,
+      "rewards/xmlcount_reward_func": 0.3728125037159771,
       "step": 2
     },
     {
-      "completion_length": 410.21875,
+      "completion_length": 257.84375,
       "epoch": 0.2,
-      "grad_norm": 3.8213980197906494,
-      "kl": 0.0011583357190829702,
+      "grad_norm": 5.92756462097168,
+      "kl": 0.0023351105592155363,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.6593156047165394,
-      "reward_std": 0.7778630193788558,
-      "rewards/concensus_correctness_reward_func": 0.21643750183284283,
-      "rewards/consensus_reward_func": 0.25,
+      "reward": 2.712690506130457,
+      "reward_std": 1.4277536073932424,
+      "rewards/concensus_correctness_reward_func": 0.4374999962747097,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.47603437514044344,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.6360654862364754,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5918437484651804,
+      "rewards/xmlcount_reward_func": 0.8266249969601631,
       "step": 4
     },
     {
-      "completion_length": 324.03125,
+      "completion_length": 336.78125,
       "epoch": 0.3,
-      "grad_norm": 4.4785003662109375,
-      "kl": 0.002048558340902673,
+      "grad_norm": 5.250879764556885,
+      "kl": 0.0028426997378119268,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 2.3681117631495,
-      "reward_std": 1.0142140060197562,
-      "rewards/concensus_correctness_reward_func": 0.3446874972432852,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 2.5608479902148247,
+      "reward_std": 1.7966383500024676,
+      "rewards/concensus_correctness_reward_func": 0.4780624993145466,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.46542426804080606,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5151917014736682,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8080000057816505,
+      "rewards/xmlcount_reward_func": 0.7550937533378601,
       "step": 6
     },
     {
-      "completion_length": 454.15625,
+      "completion_length": 281.65625,
       "epoch": 0.4,
-      "grad_norm": 4.800704002380371,
-      "kl": 0.002417401239654282,
+      "grad_norm": 5.771662712097168,
+      "kl": 0.002982961232191883,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.4630326330661774,
-      "reward_std": 1.500903086271137,
-      "rewards/concensus_correctness_reward_func": 0.2278125025331974,
+      "reward": 1.6815071050077677,
+      "reward_std": 1.2123009855858982,
+      "rewards/concensus_correctness_reward_func": 0.20543750189244747,
       "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5519388569518924,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.42456963809672743,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.30828125961124897,
+      "rewards/xmlcount_reward_func": 0.6140000028535724,
       "step": 8
     },
     {
-      "completion_length": 310.40625,
+      "completion_length": 319.875,
       "epoch": 0.5,
-      "grad_norm": 6.513106822967529,
-      "kl": 0.004747492930619046,
+      "grad_norm": 7.018432140350342,
+      "kl": 0.00529094669764163,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 2.8123386055231094,
-      "reward_std": 1.5764636304229498,
-      "rewards/concensus_correctness_reward_func": 0.7538125030696392,
+      "reward": 3.415244299918413,
+      "reward_std": 2.043720346176997,
+      "rewards/concensus_correctness_reward_func": 1.2149999849498272,
       "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5232136324048042,
+      "rewards/question_recreation_reward_func": 0.574900631327182,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7228125035762787,
+      "rewards/xmlcount_reward_func": 0.8128437548875809,
       "step": 10
     },
     {
-      "completion_length": 343.5,
+      "completion_length": 204.90625,
       "epoch": 0.6,
-      "grad_norm": 4.955071449279785,
-      "kl": 0.004099714526091702,
+      "grad_norm": 5.2989678382873535,
+      "kl": 0.00497750002250541,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.8588946387171745,
-      "reward_std": 1.26679428643547,
-      "rewards/concensus_correctness_reward_func": 0.264937499538064,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 1.8684281595051289,
+      "reward_std": 0.6075215226737782,
+      "rewards/concensus_correctness_reward_func": 0.23737500235438347,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5469571151770651,
+      "rewards/question_recreation_reward_func": 0.437771886237897,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7345000132918358,
+      "rewards/xmlcount_reward_func": 0.9432812370359898,
       "step": 12
     },
     {
-      "completion_length": 384.65625,
+      "completion_length": 324.53125,
       "epoch": 0.7,
-      "grad_norm": 3.3973560333251953,
-      "kl": 0.0034129829655284993,
+      "grad_norm": 6.422370910644531,
+      "kl": 0.005500798411958385,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.059538347646594,
-      "reward_std": 1.3035997105762362,
-      "rewards/concensus_correctness_reward_func": 0.36906250193715096,
-      "rewards/consensus_reward_func": 0.375,
+      "reward": 1.6825342141091824,
+      "reward_std": 0.7608490395359695,
+      "rewards/concensus_correctness_reward_func": 0.08293750137090683,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5691633056849241,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.4649092424660921,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6213125018402934,
+      "rewards/xmlcount_reward_func": 0.7596875075250864,
       "step": 14
     },
     {
-      "completion_length": 292.4375,
+      "completion_length": 322.84375,
       "epoch": 0.8,
-      "grad_norm": 4.502716541290283,
-      "kl": 0.007692649000091478,
+      "grad_norm": 4.691440582275391,
+      "kl": 0.007061985146719962,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 3.6721948944032192,
-      "reward_std": 1.5720796391251497,
-      "rewards/concensus_correctness_reward_func": 1.1785624660551548,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 3.5417994149029255,
+      "reward_std": 2.206683919765055,
+      "rewards/concensus_correctness_reward_func": 1.647125005722046,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.6455074540572241,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5895181887317449,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9106250144541264,
+      "rewards/xmlcount_reward_func": 0.6176562458276749,
       "step": 16
     },
     {
-      "completion_length": 332.15625,
+      "completion_length": 319.28125,
       "epoch": 0.9,
-      "grad_norm": 3.9202282428741455,
-      "kl": 0.006438435477321036,
+      "grad_norm": 5.079524517059326,
+      "kl": 0.0047352922993013635,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 2.4832431823015213,
-      "reward_std": 1.7433103364892304,
-      "rewards/concensus_correctness_reward_func": 0.7693750001490116,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 3.7184752356261015,
+      "reward_std": 3.4974633325473405,
+      "rewards/concensus_correctness_reward_func": 1.7295625060796738,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6646182015538216,
+      "rewards/question_recreation_reward_func": 0.6388814090169035,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7992499936372042,
+      "rewards/xmlcount_reward_func": 0.787531275767833,
       "step": 18
     },
     {
-      "completion_length": 398.65625,
+      "completion_length": 441.875,
       "epoch": 1.0,
-      "grad_norm": 5.829668998718262,
-      "kl": 0.00668711714388337,
+      "grad_norm": 5.213651657104492,
+      "kl": 0.0048062925998237915,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 2.2545786015689373,
-      "reward_std": 1.7215171623975039,
-      "rewards/concensus_correctness_reward_func": 0.7259999979287386,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 3.076372131705284,
+      "reward_std": 1.007006654748693,
+      "rewards/concensus_correctness_reward_func": 0.7903749961405993,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5022348794154823,
+      "rewards/question_recreation_reward_func": 0.6119345831684768,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5888437572866678,
+      "rewards/xmlcount_reward_func": 0.611562492325902,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 3.829412162303925e-06,
-      "train_runtime": 263.1353,
-      "train_samples_per_second": 1.216,
-      "train_steps_per_second": 0.076
+      "train_loss": 4.229552814649651e-06,
+      "train_runtime": 224.5386,
+      "train_samples_per_second": 1.425,
+      "train_steps_per_second": 0.089
     }
   ],
   "logging_steps": 2,