End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.2225782231544145e-06,
-    "train_runtime": 10464.7973,
-    "train_samples": 110,
-    "train_samples_per_second": 0.031,
-    "train_steps_per_second": 0.002
+    "train_loss": 4.527531564235687e-06,
+    "train_runtime": 2404.5061,
+    "train_samples": 160,
+    "train_samples_per_second": 0.133,
+    "train_steps_per_second": 0.008
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8194f42fa769dc1f6d55f1ca6b88c90d926cc94f6b32377735ceaa0d6f6ba7c
+oid sha256:70243a9a768ca6c72a376d2a9989a37120a13cd018a515f1b9578b56fefd5712
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.2225782231544145e-06,
-    "train_runtime": 10464.7973,
-    "train_samples": 110,
-    "train_samples_per_second": 0.031,
-    "train_steps_per_second": 0.002
+    "train_loss": 4.527531564235687e-06,
+    "train_runtime": 2404.5061,
+    "train_samples": 160,
+    "train_samples_per_second": 0.133,
+    "train_steps_per_second": 0.008
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.4363636363636363,
+  "epoch": 1.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 258.21875,
-      "epoch": 0.14545454545454545,
-      "grad_norm": 41.62550735473633,
+      "completion_length": 316.71875,
+      "epoch": 0.1,
+      "grad_norm": 20.640230178833008,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": -0.0,
-      "reward": 1.310115184285678,
-      "reward_std": 1.2714490432990715,
+      "reward": 1.0204834319010843,
+      "reward_std": 1.2534251582983416,
       "rewards/concensus_correctness_reward_func": 0.625,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.33095891459379345,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.25579588310210966,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16665625222958624,
+      "rewards/xmlcount_reward_func": 0.07718750182539225,
       "step": 2
     },
     {
-      "completion_length": 275.0625,
-      "epoch": 0.2909090909090909,
-      "grad_norm": 8.184622764587402,
-      "kl": 0.0014130697309155948,
+      "completion_length": 333.21875,
+      "epoch": 0.2,
+      "grad_norm": 5.591650009155273,
+      "kl": 0.0014399742321984377,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 0.44189729867503047,
-      "reward_std": 0.47183452593162656,
-      "rewards/concensus_correctness_reward_func": 0.1146249994635582,
+      "reward": 0.6284063304774463,
+      "reward_std": 0.6146276032086462,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2176785486517474,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.294125072658062,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.10959375335369259,
+      "rewards/xmlcount_reward_func": 0.20928124897181988,
       "step": 4
     },
     {
-      "completion_length": 264.15625,
-      "epoch": 0.43636363636363634,
-      "grad_norm": 7.958495140075684,
-      "kl": 0.002850157270586351,
+      "completion_length": 337.15625,
+      "epoch": 0.3,
+      "grad_norm": 8.498673439025879,
+      "kl": 0.002181062922318233,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 0.6154500190168619,
-      "reward_std": 0.5896279067965224,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.8236608348088339,
+      "reward_std": 0.40198390794103034,
+      "rewards/concensus_correctness_reward_func": 0.06043750233948231,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.32798125920817256,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3301920867525041,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16246875072829425,
+      "rewards/xmlcount_reward_func": 0.3705312458187109,
       "step": 6
     },
     {
-      "completion_length": 270.34375,
-      "epoch": 0.5818181818181818,
-      "grad_norm": 8.637036323547363,
-      "kl": 0.0017741384599503363,
+      "completion_length": 315.34375,
+      "epoch": 0.4,
+      "grad_norm": 9.28067398071289,
+      "kl": 0.0012329019318713108,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 0.28425077768042684,
-      "reward_std": 0.39603474573232234,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.9061323929345235,
+      "reward_std": 1.2012234527501278,
+      "rewards/concensus_correctness_reward_func": 0.625,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.15184452396351844,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.14297609927598387,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0699062505736947,
+      "rewards/xmlcount_reward_func": 0.13815624825656414,
       "step": 8
     },
     {
-      "completion_length": 263.78125,
-      "epoch": 0.7272727272727273,
-      "grad_norm": 44.59328079223633,
-      "kl": 0.0013894537805754226,
+      "completion_length": 396.6875,
+      "epoch": 0.5,
+      "grad_norm": 6.3696184158325195,
+      "kl": 0.006439117893023649,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 0.30447179335169494,
-      "reward_std": 0.32076822535600513,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.9482769472524524,
+      "reward_std": 0.6139591162791476,
+      "rewards/concensus_correctness_reward_func": 0.05256250128149986,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.2510342898312956,
+      "rewards/question_recreation_reward_func": 0.3935582006815821,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.009062499506399035,
+      "rewards/xmlcount_reward_func": 0.439656249480322,
       "step": 10
     },
     {
-      "completion_length": 302.5625,
-      "epoch": 0.8727272727272727,
-      "grad_norm": 9.689335823059082,
-      "kl": 0.0015908715795376338,
+      "completion_length": 317.9375,
+      "epoch": 0.6,
+      "grad_norm": 8.388046264648438,
+      "kl": 0.006362274449202232,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 0.3704655871260911,
-      "reward_std": 0.20582995464792475,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.6173857575049624,
+      "reward_std": 0.45685271653928794,
+      "rewards/concensus_correctness_reward_func": 0.018437499180436134,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.22746558138169348,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.30632324831094593,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.14299999922513962,
+      "rewards/xmlcount_reward_func": 0.16762500000186265,
       "step": 12
     },
     {
-      "completion_length": 315.57142857142856,
-      "epoch": 1.0,
-      "grad_norm": 4.151434421539307,
-      "kl": 0.001419084849268464,
+      "completion_length": 328.0625,
+      "epoch": 0.7,
+      "grad_norm": 6.688593864440918,
+      "kl": 0.003958331943067606,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 0.3779558068407433,
-      "reward_std": 0.42275290922927006,
+      "reward": 0.6333054774440825,
+      "reward_std": 0.5454679431859404,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.21838437845664366,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.3044929875759408,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.15957143077892916,
+      "rewards/xmlcount_reward_func": 0.14131249953061342,
       "step": 14
     },
     {
-      "completion_length": 314.9375,
-      "epoch": 1.1454545454545455,
-      "grad_norm": 9.743141174316406,
-      "kl": 0.0031447128421859816,
+      "completion_length": 309.5625,
+      "epoch": 0.8,
+      "grad_norm": 12.809540748596191,
+      "kl": 0.011681057876558043,
       "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
-      "reward": 0.4019067722838372,
-      "reward_std": 0.4524043661658652,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 1.933492950105574,
+      "reward_std": 0.33618732055765577,
+      "rewards/concensus_correctness_reward_func": 1.25,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.26653177477419376,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.32083665631944314,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0728750005364418,
+      "rewards/xmlcount_reward_func": 0.23765625635860488,
       "step": 16
     },
     {
-      "completion_length": 349.625,
-      "epoch": 1.290909090909091,
-      "grad_norm": 6.626418590545654,
-      "kl": 0.004460386695427587,
+      "completion_length": 316.84375,
+      "epoch": 0.9,
+      "grad_norm": 4.517097473144531,
+      "kl": 0.008473690730170347,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 0.3090712802950293,
-      "reward_std": 0.4827816704637371,
+      "reward": 0.4718033701647073,
+      "reward_std": 0.536688668653369,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2421337771229446,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.23252211860381067,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06693749874830246,
+      "rewards/xmlcount_reward_func": 0.17678125109523535,
       "step": 18
     },
     {
-      "completion_length": 307.84375,
-      "epoch": 1.4363636363636363,
-      "grad_norm": 16.053197860717773,
-      "kl": 0.0044716724842146505,
+      "completion_length": 279.1875,
+      "epoch": 1.0,
+      "grad_norm": 61.814781188964844,
+      "kl": 0.003459241639575339,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 0.49628575635142624,
-      "reward_std": 0.4002031952259131,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 1.3219008122105151,
+      "reward_std": 1.3056973450875375,
+      "rewards/concensus_correctness_reward_func": 0.625,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.29078574967570603,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.27558836503885686,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.1273749996908009,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.1713124974630773,
       "step": 20
     },
     {
-      "epoch": 1.4363636363636363,
+      "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.2225782231544145e-06,
-      "train_runtime": 10464.7973,
-      "train_samples_per_second": 0.031,
-      "train_steps_per_second": 0.002
+      "train_loss": 4.527531564235687e-06,
+      "train_runtime": 2404.5061,
+      "train_samples_per_second": 0.133,
+      "train_steps_per_second": 0.008
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 2,
+  "num_train_epochs": 1,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {