Model save

README.md

@@ -1,6 +1,5 @@
 ---
 base_model: JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep
-datasets: trl-lib/ultrafeedback_binarized
 library_name: transformers
 model_name: Qwen_0.5-rDPO_5e-7-3ep_0vpo_const
 tags:
@@ -12,7 +11,7 @@ licence: license
 
 # Model Card for Qwen_0.5-rDPO_5e-7-3ep_0vpo_const
 
-This model is a fine-tuned version of [JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep](https://huggingface.co/JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep) on the [trl-lib/ultrafeedback_binarized](https://huggingface.co/datasets/trl-lib/ultrafeedback_binarized) dataset.
+This model is a fine-tuned version of [JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep](https://huggingface.co/JayHyeon/Qwen2.5-0.5B-SFT-2e-5-2ep).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -28,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/bonin147/huggingface/runs/ezwht1g7)
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/bonin147/huggingface/runs/pmi8lat0)
 
 This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
 

all_results.json

@@ -1,21 +1,21 @@
 {
     "epoch": 2.999098751126561,
-    "eval_log_ratio_diff/mean": 17.375,
-    "eval_logits/chosen": -2.359375,
-    "eval_logits/rejected": -2.4375,
-    "eval_logps/chosen": -346.0,
-    "eval_logps/rejected": -338.0,
-    "eval_loss": 0.2879411578178406,
-    "eval_nll_loss": 1.234375,
+    "eval_log_ratio_diff/mean": 59.0,
+    "eval_logits/chosen": -2.703125,
+    "eval_logits/rejected": -2.8125,
+    "eval_logps/chosen": -444.0,
+    "eval_logps/rejected": -480.0,
+    "eval_loss": -2.3502578735351562,
+    "eval_nll_loss": 1.7734375,
     "eval_ref_probs/chosen": 0.0038271306548267603,
     "eval_ref_probs/rejected": 0.0034569373819977045,
-    "eval_rejected_term/max": -4.116310446988791e-05,
-    "eval_rejected_term/min": -4.116310446988791e-05,
-    "eval_rewards/accuracies": 0.7440000176429749,
-    "eval_rewards/chosen": -2.375,
-    "eval_rewards/margins": 1.7421875,
-    "eval_rewards/rejected": -4.125,
-    "eval_runtime": 25.3378,
-    "eval_samples_per_second": 39.467,
-    "eval_steps_per_second": 9.867
+    "eval_rejected_term/max": -1.9564737385735498e-07,
+    "eval_rejected_term/min": -1.9564737385735498e-07,
+    "eval_rewards/accuracies": 0.6639999747276306,
+    "eval_rewards/chosen": -12.3125,
+    "eval_rewards/margins": 5.90625,
+    "eval_rewards/rejected": -18.25,
+    "eval_runtime": 25.3061,
+    "eval_samples_per_second": 39.516,
+    "eval_steps_per_second": 9.879
 }

eval_results.json

@@ -1,21 +1,21 @@
 {
     "epoch": 2.999098751126561,
-    "eval_log_ratio_diff/mean": 17.375,
-    "eval_logits/chosen": -2.359375,
-    "eval_logits/rejected": -2.4375,
-    "eval_logps/chosen": -346.0,
-    "eval_logps/rejected": -338.0,
-    "eval_loss": 0.2879411578178406,
-    "eval_nll_loss": 1.234375,
+    "eval_log_ratio_diff/mean": 59.0,
+    "eval_logits/chosen": -2.703125,
+    "eval_logits/rejected": -2.8125,
+    "eval_logps/chosen": -444.0,
+    "eval_logps/rejected": -480.0,
+    "eval_loss": -2.3502578735351562,
+    "eval_nll_loss": 1.7734375,
     "eval_ref_probs/chosen": 0.0038271306548267603,
     "eval_ref_probs/rejected": 0.0034569373819977045,
-    "eval_rejected_term/max": -4.116310446988791e-05,
-    "eval_rejected_term/min": -4.116310446988791e-05,
-    "eval_rewards/accuracies": 0.7440000176429749,
-    "eval_rewards/chosen": -2.375,
-    "eval_rewards/margins": 1.7421875,
-    "eval_rewards/rejected": -4.125,
-    "eval_runtime": 25.3378,
-    "eval_samples_per_second": 39.467,
-    "eval_steps_per_second": 9.867
+    "eval_rejected_term/max": -1.9564737385735498e-07,
+    "eval_rejected_term/min": -1.9564737385735498e-07,
+    "eval_rewards/accuracies": 0.6639999747276306,
+    "eval_rewards/chosen": -12.3125,
+    "eval_rewards/margins": 5.90625,
+    "eval_rewards/rejected": -18.25,
+    "eval_runtime": 25.3061,
+    "eval_samples_per_second": 39.516,
+    "eval_steps_per_second": 9.879
 }