End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.0651688575744627e-06,
-    "train_runtime": 6562.8868,
-    "train_samples": 160,
-    "train_samples_per_second": 0.049,
-    "train_steps_per_second": 0.003
+    "train_loss": 2.2225782231544145e-06,
+    "train_runtime": 10464.7973,
+    "train_samples": 110,
+    "train_samples_per_second": 0.031,
+    "train_steps_per_second": 0.002
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f336cb698c058adfd135ed949001fa51779aa3b5305a864ca0db090430fe66c
+oid sha256:f8194f42fa769dc1f6d55f1ca6b88c90d926cc94f6b32377735ceaa0d6f6ba7c
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.0651688575744627e-06,
-    "train_runtime": 6562.8868,
-    "train_samples": 160,
-    "train_samples_per_second": 0.049,
-    "train_steps_per_second": 0.003
+    "train_loss": 2.2225782231544145e-06,
+    "train_runtime": 10464.7973,
+    "train_samples": 110,
+    "train_samples_per_second": 0.031,
+    "train_steps_per_second": 0.002
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 1.4363636363636363,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 382.21875,
-      "epoch": 0.1,
-      "grad_norm": 4.167541980743408,
+      "completion_length": 258.21875,
+      "epoch": 0.14545454545454545,
+      "grad_norm": 41.62550735473633,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
-      "loss": 0.0,
-      "reward": 1.506855795159936,
-      "reward_std": 1.5373446858720854,
-      "rewards/concensus_correctness_reward_func": 0.6624374995008111,
-      "rewards/consensus_reward_func": 0.125,
+      "loss": -0.0,
+      "reward": 1.310115184285678,
+      "reward_std": 1.2714490432990715,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4292933135293424,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.33095891459379345,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2901249974966049,
+      "rewards/xmlcount_reward_func": 0.16665625222958624,
       "step": 2
     },
     {
-      "completion_length": 374.15625,
-      "epoch": 0.2,
-      "grad_norm": 4.083254337310791,
-      "kl": 0.000988674939435441,
+      "completion_length": 275.0625,
+      "epoch": 0.2909090909090909,
+      "grad_norm": 8.184622764587402,
+      "kl": 0.0014130697309155948,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 1.2652242630720139,
-      "reward_std": 0.7753466135982308,
-      "rewards/concensus_correctness_reward_func": 0.023624999448657036,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.44189729867503047,
+      "reward_std": 0.47183452593162656,
+      "rewards/concensus_correctness_reward_func": 0.1146249994635582,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.6518179876729846,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2176785486517474,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.40228125266730785,
+      "rewards/xmlcount_reward_func": 0.10959375335369259,
       "step": 4
     },
     {
-      "completion_length": 384.5,
-      "epoch": 0.3,
-      "grad_norm": 3.960369110107422,
-      "kl": 0.0022889753890922293,
+      "completion_length": 264.15625,
+      "epoch": 0.43636363636363634,
+      "grad_norm": 7.958495140075684,
+      "kl": 0.002850157270586351,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 3.0565542803378776,
-      "reward_std": 1.4951365570741473,
-      "rewards/concensus_correctness_reward_func": 1.9368750005960464,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.6154500190168619,
+      "reward_std": 0.5896279067965224,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.3448979869717732,
+      "rewards/question_recreation_reward_func": 0.32798125920817256,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.46228125039488077,
+      "rewards/xmlcount_reward_func": 0.16246875072829425,
       "step": 6
     },
     {
-      "completion_length": 380.15625,
-      "epoch": 0.4,
-      "grad_norm": 7.687902450561523,
-      "kl": 0.0020300845826568548,
+      "completion_length": 270.34375,
+      "epoch": 0.5818181818181818,
+      "grad_norm": 8.637036323547363,
+      "kl": 0.0017741384599503363,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 1.5852362774312496,
-      "reward_std": 1.9481410621083342,
-      "rewards/concensus_correctness_reward_func": 0.7286875024437904,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.28425077768042684,
+      "reward_std": 0.39603474573232234,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.3305800525704399,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.15184452396351844,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2759687597863376,
+      "rewards/xmlcount_reward_func": 0.0699062505736947,
       "step": 8
     },
     {
-      "completion_length": 348.15625,
-      "epoch": 0.5,
-      "grad_norm": 4.086596965789795,
-      "kl": 0.0032896055272431113,
+      "completion_length": 263.78125,
+      "epoch": 0.7272727272727273,
+      "grad_norm": 44.59328079223633,
+      "kl": 0.0013894537805754226,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 1.0955563625320792,
-      "reward_std": 0.9007547841174528,
-      "rewards/concensus_correctness_reward_func": 0.08056250028312206,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.30447179335169494,
+      "reward_std": 0.32076822535600513,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5132438774453476,
+      "rewards/question_recreation_reward_func": 0.2510342898312956,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3142499979585409,
+      "rewards/xmlcount_reward_func": -0.009062499506399035,
       "step": 10
     },
     {
-      "completion_length": 404.46875,
-      "epoch": 0.6,
-      "grad_norm": 4.184484004974365,
-      "kl": 0.002595337005914189,
+      "completion_length": 302.5625,
+      "epoch": 0.8727272727272727,
+      "grad_norm": 9.689335823059082,
+      "kl": 0.0015908715795376338,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 1.093373519834131,
-      "reward_std": 0.8039959709858522,
-      "rewards/concensus_correctness_reward_func": 0.1400625016540289,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.3704655871260911,
+      "reward_std": 0.20582995464792475,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.44215477036777884,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.22746558138169348,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.323656246997416,
+      "rewards/xmlcount_reward_func": 0.14299999922513962,
       "step": 12
     },
     {
-      "completion_length": 367.59375,
-      "epoch": 0.7,
-      "grad_norm": 5.3371992111206055,
-      "kl": 0.0030056744362809695,
+      "completion_length": 315.57142857142856,
+      "epoch": 1.0,
+      "grad_norm": 4.151434421539307,
+      "kl": 0.001419084849268464,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 2.1255152514204383,
-      "reward_std": 1.6363342870026827,
-      "rewards/concensus_correctness_reward_func": 0.6841875007376075,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 0.3779558068407433,
+      "reward_std": 0.42275290922927006,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.599921464221552,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.21838437845664366,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.3882812522351742,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.15957143077892916,
       "step": 14
     },
     {
-      "completion_length": 341.5,
-      "epoch": 0.8,
-      "grad_norm": 4.51369047164917,
-      "kl": 0.004066580644575879,
+      "completion_length": 314.9375,
+      "epoch": 1.1454545454545455,
+      "grad_norm": 9.743141174316406,
+      "kl": 0.0031447128421859816,
       "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
-      "reward": 1.5783393494784832,
-      "reward_std": 0.8356545143760741,
-      "rewards/concensus_correctness_reward_func": 0.040812499821186066,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.4019067722838372,
+      "reward_std": 0.4524043661658652,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6761830877512693,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.26653177477419376,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5488437403691933,
+      "rewards/xmlcount_reward_func": 0.0728750005364418,
       "step": 16
     },
     {
-      "completion_length": 328.96875,
-      "epoch": 0.9,
-      "grad_norm": 4.285229682922363,
-      "kl": 0.0035460513608995825,
+      "completion_length": 349.625,
+      "epoch": 1.290909090909091,
+      "grad_norm": 6.626418590545654,
+      "kl": 0.004460386695427587,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 1.0606287872651592,
-      "reward_std": 0.5185283102764515,
-      "rewards/concensus_correctness_reward_func": -0.011437499895691872,
+      "reward": 0.3090712802950293,
+      "reward_std": 0.4827816704637371,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.4874725431436673,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2421337771229446,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5220937505364418,
+      "rewards/xmlcount_reward_func": 0.06693749874830246,
       "step": 18
     },
     {
-      "completion_length": 323.25,
-      "epoch": 1.0,
-      "grad_norm": 6.708991527557373,
-      "kl": 0.00451307363982778,
+      "completion_length": 307.84375,
+      "epoch": 1.4363636363636363,
+      "grad_norm": 16.053197860717773,
+      "kl": 0.0044716724842146505,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 2.157086548395455,
-      "reward_std": 1.6307503607531544,
-      "rewards/concensus_correctness_reward_func": 0.7492499984800816,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.49628575635142624,
+      "reward_std": 0.4002031952259131,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.4128365609794855,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.29078574967570603,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.4950000122189522,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.1273749996908009,
       "step": 20
     },
     {
-      "epoch": 1.0,
+      "epoch": 1.4363636363636363,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 3.0651688575744627e-06,
-      "train_runtime": 6562.8868,
-      "train_samples_per_second": 0.049,
-      "train_steps_per_second": 0.003
+      "train_loss": 2.2225782231544145e-06,
+      "train_runtime": 10464.7973,
+      "train_samples_per_second": 0.031,
+      "train_steps_per_second": 0.002
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 2,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e12cc88acda0738eb60ba42148765ab82d9c5cccad4aede636900e88150b15e1
+oid sha256:19c7b2f71fe47069001777accee89e0e0486f2a474360539fa941b7c1af496b4
 size 6008