Upload folder using huggingface_hub

added_tokens.json

@@ -0,0 +1,13 @@
+{
+  "<|assistant|>": 32001,
+  "<|endoftext|>": 32000,
+  "<|end|>": 32007,
+  "<|placeholder1|>": 32002,
+  "<|placeholder2|>": 32003,
+  "<|placeholder3|>": 32004,
+  "<|placeholder4|>": 32005,
+  "<|placeholder5|>": 32008,
+  "<|placeholder6|>": 32009,
+  "<|system|>": 32006,
+  "<|user|>": 32010
+}

all_results.json

@@ -0,0 +1,25 @@
+{
+    "epoch": 0.9880609304240429,
+    "eval_logits/chosen": 9.246881484985352,
+    "eval_logits/rejected": 10.068053245544434,
+    "eval_logps/chosen": -74.92684936523438,
+    "eval_logps/ref_chosen": -106.16099548339844,
+    "eval_logps/ref_rejected": -119.78473663330078,
+    "eval_logps/rejected": -87.84820556640625,
+    "eval_loss": 0.26139113306999207,
+    "eval_rewards/accuracies": 0.47962963581085205,
+    "eval_rewards/chosen": 0.3123414218425751,
+    "eval_rewards/margins": -0.007023785263299942,
+    "eval_rewards/rejected": 0.3193652033805847,
+    "eval_runtime": 165.9841,
+    "eval_samples": 1080,
+    "eval_samples_per_second": 6.507,
+    "eval_sft_loss": 0.26139113306999207,
+    "eval_steps_per_second": 1.627,
+    "total_flos": 0.0,
+    "train_loss": 0.29937881012757617,
+    "train_runtime": 8120.0447,
+    "train_samples": 9715,
+    "train_samples_per_second": 1.196,
+    "train_steps_per_second": 0.009
+}

checkpoint-12/added_tokens.json

@@ -0,0 +1,13 @@
+{
+  "<|assistant|>": 32001,
+  "<|endoftext|>": 32000,
+  "<|end|>": 32007,
+  "<|placeholder1|>": 32002,
+  "<|placeholder2|>": 32003,
+  "<|placeholder3|>": 32004,
+  "<|placeholder4|>": 32005,
+  "<|placeholder5|>": 32008,
+  "<|placeholder6|>": 32009,
+  "<|system|>": 32006,
+  "<|user|>": 32010
+}

checkpoint-12/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "/home/v-liuzichuan/huggingface/Phi-3-mini-4k-instruct",
+  "architectures": [
+    "Phi3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_phi3.Phi3Config",
+    "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM"
+  },
+  "bos_token_id": 1,
+  "embd_pdrop": 0.0,
+  "eos_token_id": 32000,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 4096,
+  "model_type": "phi3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "original_max_position_embeddings": 4096,
+  "pad_token_id": 32000,
+  "resid_pdrop": 0.0,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "sliding_window": 2047,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.2",
+  "use_cache": false,
+  "vocab_size": 32064
+}

checkpoint-12/generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    32000,
+    32001,
+    32007
+  ],
+  "pad_token_id": 32000,
+  "transformers_version": "4.45.2"
+}

checkpoint-12/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72af21489ab6d8df03873d9da22076fa7c9eca9988eb0383269928eb0aec09a7
+size 4972489328

checkpoint-12/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c4911d56ac9204c2c72ab3199817f27668267737344802b8d5acf99b55f7067
+size 2669692552

checkpoint-12/model.safetensors.index.json

@@ -0,0 +1,202 @@
+{
+  "metadata": {
+    "total_size": 7642159104
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

checkpoint-12/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-12/tokenizer_config.json

@@ -0,0 +1,131 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "32000": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<|placeholder1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<|placeholder2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<|placeholder3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<|placeholder4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "<|placeholder5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "<|placeholder6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "legacy": false,
+  "model_max_length": 4096,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-12/trainer_state.json

@@ -0,0 +1,249 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.15808974886784685,
+  "eval_steps": 500,
+  "global_step": 12,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.013174145738987238,
+      "grad_norm": 1.194186806678772,
+      "learning_rate": 6.25e-08,
+      "logits/chosen": 9.990612030029297,
+      "logits/rejected": 10.698101997375488,
+      "logps/chosen": -102.88545989990234,
+      "logps/ref_chosen": -102.88545989990234,
+      "logps/ref_rejected": -121.84871673583984,
+      "logps/rejected": -121.84871673583984,
+      "loss": 0.3675,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "sft_loss": 0.36753880977630615,
+      "step": 1
+    },
+    {
+      "epoch": 0.026348291477974475,
+      "grad_norm": 0.5353251099586487,
+      "learning_rate": 1.25e-07,
+      "logits/chosen": 10.211905479431152,
+      "logits/rejected": 11.06594467163086,
+      "logps/chosen": -107.70349884033203,
+      "logps/ref_chosen": -107.70349884033203,
+      "logps/ref_rejected": -121.89966583251953,
+      "logps/rejected": -121.89966583251953,
+      "loss": 0.4101,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "sft_loss": 0.41013145446777344,
+      "step": 2
+    },
+    {
+      "epoch": 0.03952243721696171,
+      "grad_norm": 0.7126303315162659,
+      "learning_rate": 1.875e-07,
+      "logits/chosen": 10.032384872436523,
+      "logits/rejected": 11.023520469665527,
+      "logps/chosen": -108.3123779296875,
+      "logps/ref_chosen": -107.98188781738281,
+      "logps/ref_rejected": -124.51527404785156,
+      "logps/rejected": -124.87130737304688,
+      "loss": 0.412,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": -0.003304910147562623,
+      "rewards/margins": 0.0002554532838985324,
+      "rewards/rejected": -0.003560363780707121,
+      "sft_loss": 0.41195932030677795,
+      "step": 3
+    },
+    {
+      "epoch": 0.05269658295594895,
+      "grad_norm": 1.2344533205032349,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": 9.836658477783203,
+      "logits/rejected": 10.855621337890625,
+      "logps/chosen": -109.55919647216797,
+      "logps/ref_chosen": -109.20836639404297,
+      "logps/ref_rejected": -119.23908996582031,
+      "logps/rejected": -119.48279571533203,
+      "loss": 0.4039,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": -0.003508324269205332,
+      "rewards/margins": -0.0010712125804275274,
+      "rewards/rejected": -0.0024371116887778044,
+      "sft_loss": 0.4038863480091095,
+      "step": 4
+    },
+    {
+      "epoch": 0.06587072869493618,
+      "grad_norm": 1.426048994064331,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": 10.212320327758789,
+      "logits/rejected": 10.966379165649414,
+      "logps/chosen": -103.76991271972656,
+      "logps/ref_chosen": -103.87680053710938,
+      "logps/ref_rejected": -118.41618347167969,
+      "logps/rejected": -118.23270416259766,
+      "loss": 0.3697,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": 0.0010687037138268352,
+      "rewards/margins": -0.000766113749705255,
+      "rewards/rejected": 0.0018348174635320902,
+      "sft_loss": 0.3697226345539093,
+      "step": 5
+    },
+    {
+      "epoch": 0.07904487443392343,
+      "grad_norm": 1.413549780845642,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": 10.700042724609375,
+      "logits/rejected": 11.478326797485352,
+      "logps/chosen": -107.56877899169922,
+      "logps/ref_chosen": -107.58968353271484,
+      "logps/ref_rejected": -122.07303619384766,
+      "logps/rejected": -121.85940551757812,
+      "loss": 0.3909,
+      "rewards/accuracies": 0.4609375,
+      "rewards/chosen": 0.0002090137859340757,
+      "rewards/margins": -0.0019273017533123493,
+      "rewards/rejected": 0.00213631521910429,
+      "sft_loss": 0.390906423330307,
+      "step": 6
+    },
+    {
+      "epoch": 0.09221902017291066,
+      "grad_norm": 1.2342580556869507,
+      "learning_rate": 4.375e-07,
+      "logits/chosen": 10.01632308959961,
+      "logits/rejected": 10.7178955078125,
+      "logps/chosen": -107.01339721679688,
+      "logps/ref_chosen": -107.42727661132812,
+      "logps/ref_rejected": -116.87063598632812,
+      "logps/rejected": -116.37357330322266,
+      "loss": 0.3747,
+      "rewards/accuracies": 0.4453125,
+      "rewards/chosen": 0.00413867924362421,
+      "rewards/margins": -0.0008318667532876134,
+      "rewards/rejected": 0.004970546346157789,
+      "sft_loss": 0.3746669888496399,
+      "step": 7
+    },
+    {
+      "epoch": 0.1053931659118979,
+      "grad_norm": 0.6644937992095947,
+      "learning_rate": 5e-07,
+      "logits/chosen": 10.211028099060059,
+      "logits/rejected": 11.11027717590332,
+      "logps/chosen": -104.41184997558594,
+      "logps/ref_chosen": -105.60282135009766,
+      "logps/ref_rejected": -119.53916931152344,
+      "logps/rejected": -118.27430725097656,
+      "loss": 0.3773,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": 0.01190974935889244,
+      "rewards/margins": -0.0007388982339762151,
+      "rewards/rejected": 0.012648648582398891,
+      "sft_loss": 0.37729793787002563,
+      "step": 8
+    },
+    {
+      "epoch": 0.11856731165088513,
+      "grad_norm": 0.9437576532363892,
+      "learning_rate": 4.997252228714278e-07,
+      "logits/chosen": 10.179821014404297,
+      "logits/rejected": 11.147579193115234,
+      "logps/chosen": -104.13174438476562,
+      "logps/ref_chosen": -105.46086120605469,
+      "logps/ref_rejected": -119.00373840332031,
+      "logps/rejected": -117.734130859375,
+      "loss": 0.3807,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": 0.013291322626173496,
+      "rewards/margins": 0.0005952615174464881,
+      "rewards/rejected": 0.012696062214672565,
+      "sft_loss": 0.38070446252822876,
+      "step": 9
+    },
+    {
+      "epoch": 0.13174145738987236,
+      "grad_norm": 0.700039803981781,
+      "learning_rate": 4.989014955054745e-07,
+      "logits/chosen": 10.076737403869629,
+      "logits/rejected": 10.897785186767578,
+      "logps/chosen": -100.81087493896484,
+      "logps/ref_chosen": -104.21009826660156,
+      "logps/ref_rejected": -118.9209213256836,
+      "logps/rejected": -115.75495910644531,
+      "loss": 0.3367,
+      "rewards/accuracies": 0.5546875,
+      "rewards/chosen": 0.033992186188697815,
+      "rewards/margins": 0.0023326175287365913,
+      "rewards/rejected": 0.0316595658659935,
+      "sft_loss": 0.33672136068344116,
+      "step": 10
+    },
+    {
+      "epoch": 0.14491560312885962,
+      "grad_norm": 0.9160856008529663,
+      "learning_rate": 4.975306286336627e-07,
+      "logits/chosen": 9.973880767822266,
+      "logits/rejected": 11.158487319946289,
+      "logps/chosen": -101.3505630493164,
+      "logps/ref_chosen": -105.94319152832031,
+      "logps/ref_rejected": -122.76007843017578,
+      "logps/rejected": -118.6338119506836,
+      "loss": 0.3851,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.045926500111818314,
+      "rewards/margins": 0.004663803614675999,
+      "rewards/rejected": 0.04126270115375519,
+      "sft_loss": 0.3850533962249756,
+      "step": 11
+    },
+    {
+      "epoch": 0.15808974886784685,
+      "grad_norm": 0.9421964883804321,
+      "learning_rate": 4.956156357188939e-07,
+      "logits/chosen": 9.908226013183594,
+      "logits/rejected": 10.598045349121094,
+      "logps/chosen": -103.32762908935547,
+      "logps/ref_chosen": -109.08442687988281,
+      "logps/ref_rejected": -121.41947174072266,
+      "logps/rejected": -115.84996795654297,
+      "loss": 0.3532,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": 0.05756799131631851,
+      "rewards/margins": 0.0018730255542322993,
+      "rewards/rejected": 0.05569496005773544,
+      "sft_loss": 0.3532242476940155,
+      "step": 12
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 75,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 12,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-24/added_tokens.json

@@ -0,0 +1,13 @@
+{
+  "<|assistant|>": 32001,
+  "<|endoftext|>": 32000,
+  "<|end|>": 32007,
+  "<|placeholder1|>": 32002,
+  "<|placeholder2|>": 32003,
+  "<|placeholder3|>": 32004,
+  "<|placeholder4|>": 32005,
+  "<|placeholder5|>": 32008,
+  "<|placeholder6|>": 32009,
+  "<|system|>": 32006,
+  "<|user|>": 32010
+}

checkpoint-24/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "/home/v-liuzichuan/huggingface/Phi-3-mini-4k-instruct",
+  "architectures": [
+    "Phi3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_phi3.Phi3Config",
+    "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM"
+  },
+  "bos_token_id": 1,
+  "embd_pdrop": 0.0,
+  "eos_token_id": 32000,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 4096,
+  "model_type": "phi3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "original_max_position_embeddings": 4096,
+  "pad_token_id": 32000,
+  "resid_pdrop": 0.0,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "sliding_window": 2047,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.2",
+  "use_cache": false,
+  "vocab_size": 32064
+}

checkpoint-24/generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    32000,
+    32001,
+    32007
+  ],
+  "pad_token_id": 32000,
+  "transformers_version": "4.45.2"
+}

checkpoint-24/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47b3c9f88541657956f33299730895624250c6b5b4ecaeeddd0b0d3d43d43080
+size 4972489328

checkpoint-24/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c280fa66f04e7d6c9e782aadb2963493c6952db44e335422768570105eeedd55
+size 2669692552

checkpoint-24/model.safetensors.index.json

@@ -0,0 +1,202 @@
+{
+  "metadata": {
+    "total_size": 7642159104
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

checkpoint-24/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-24/tokenizer_config.json

@@ -0,0 +1,131 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "32000": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<|placeholder1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<|placeholder2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<|placeholder3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<|placeholder4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "<|placeholder5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "<|placeholder6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "legacy": false,
+  "model_max_length": 4096,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-24/trainer_state.json

@@ -0,0 +1,465 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.3161794977356937,
+  "eval_steps": 500,
+  "global_step": 24,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.013174145738987238,
+      "grad_norm": 1.194186806678772,
+      "learning_rate": 6.25e-08,
+      "logits/chosen": 9.990612030029297,
+      "logits/rejected": 10.698101997375488,
+      "logps/chosen": -102.88545989990234,
+      "logps/ref_chosen": -102.88545989990234,
+      "logps/ref_rejected": -121.84871673583984,
+      "logps/rejected": -121.84871673583984,
+      "loss": 0.3675,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "sft_loss": 0.36753880977630615,
+      "step": 1
+    },
+    {
+      "epoch": 0.026348291477974475,
+      "grad_norm": 0.5353251099586487,
+      "learning_rate": 1.25e-07,
+      "logits/chosen": 10.211905479431152,
+      "logits/rejected": 11.06594467163086,
+      "logps/chosen": -107.70349884033203,
+      "logps/ref_chosen": -107.70349884033203,
+      "logps/ref_rejected": -121.89966583251953,
+      "logps/rejected": -121.89966583251953,
+      "loss": 0.4101,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "sft_loss": 0.41013145446777344,
+      "step": 2
+    },
+    {
+      "epoch": 0.03952243721696171,
+      "grad_norm": 0.7126303315162659,
+      "learning_rate": 1.875e-07,
+      "logits/chosen": 10.032384872436523,
+      "logits/rejected": 11.023520469665527,
+      "logps/chosen": -108.3123779296875,
+      "logps/ref_chosen": -107.98188781738281,
+      "logps/ref_rejected": -124.51527404785156,
+      "logps/rejected": -124.87130737304688,
+      "loss": 0.412,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": -0.003304910147562623,
+      "rewards/margins": 0.0002554532838985324,
+      "rewards/rejected": -0.003560363780707121,
+      "sft_loss": 0.41195932030677795,
+      "step": 3
+    },
+    {
+      "epoch": 0.05269658295594895,
+      "grad_norm": 1.2344533205032349,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": 9.836658477783203,
+      "logits/rejected": 10.855621337890625,
+      "logps/chosen": -109.55919647216797,
+      "logps/ref_chosen": -109.20836639404297,
+      "logps/ref_rejected": -119.23908996582031,
+      "logps/rejected": -119.48279571533203,
+      "loss": 0.4039,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": -0.003508324269205332,
+      "rewards/margins": -0.0010712125804275274,
+      "rewards/rejected": -0.0024371116887778044,
+      "sft_loss": 0.4038863480091095,
+      "step": 4
+    },
+    {
+      "epoch": 0.06587072869493618,
+      "grad_norm": 1.426048994064331,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": 10.212320327758789,
+      "logits/rejected": 10.966379165649414,
+      "logps/chosen": -103.76991271972656,
+      "logps/ref_chosen": -103.87680053710938,
+      "logps/ref_rejected": -118.41618347167969,
+      "logps/rejected": -118.23270416259766,
+      "loss": 0.3697,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": 0.0010687037138268352,
+      "rewards/margins": -0.000766113749705255,
+      "rewards/rejected": 0.0018348174635320902,
+      "sft_loss": 0.3697226345539093,
+      "step": 5
+    },
+    {
+      "epoch": 0.07904487443392343,
+      "grad_norm": 1.413549780845642,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": 10.700042724609375,
+      "logits/rejected": 11.478326797485352,
+      "logps/chosen": -107.56877899169922,
+      "logps/ref_chosen": -107.58968353271484,
+      "logps/ref_rejected": -122.07303619384766,
+      "logps/rejected": -121.85940551757812,
+      "loss": 0.3909,
+      "rewards/accuracies": 0.4609375,
+      "rewards/chosen": 0.0002090137859340757,
+      "rewards/margins": -0.0019273017533123493,
+      "rewards/rejected": 0.00213631521910429,
+      "sft_loss": 0.390906423330307,
+      "step": 6
+    },
+    {
+      "epoch": 0.09221902017291066,
+      "grad_norm": 1.2342580556869507,
+      "learning_rate": 4.375e-07,
+      "logits/chosen": 10.01632308959961,
+      "logits/rejected": 10.7178955078125,
+      "logps/chosen": -107.01339721679688,
+      "logps/ref_chosen": -107.42727661132812,
+      "logps/ref_rejected": -116.87063598632812,
+      "logps/rejected": -116.37357330322266,
+      "loss": 0.3747,
+      "rewards/accuracies": 0.4453125,
+      "rewards/chosen": 0.00413867924362421,
+      "rewards/margins": -0.0008318667532876134,
+      "rewards/rejected": 0.004970546346157789,
+      "sft_loss": 0.3746669888496399,
+      "step": 7
+    },
+    {
+      "epoch": 0.1053931659118979,
+      "grad_norm": 0.6644937992095947,
+      "learning_rate": 5e-07,
+      "logits/chosen": 10.211028099060059,
+      "logits/rejected": 11.11027717590332,
+      "logps/chosen": -104.41184997558594,
+      "logps/ref_chosen": -105.60282135009766,
+      "logps/ref_rejected": -119.53916931152344,
+      "logps/rejected": -118.27430725097656,
+      "loss": 0.3773,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": 0.01190974935889244,
+      "rewards/margins": -0.0007388982339762151,
+      "rewards/rejected": 0.012648648582398891,
+      "sft_loss": 0.37729793787002563,
+      "step": 8
+    },
+    {
+      "epoch": 0.11856731165088513,
+      "grad_norm": 0.9437576532363892,
+      "learning_rate": 4.997252228714278e-07,
+      "logits/chosen": 10.179821014404297,
+      "logits/rejected": 11.147579193115234,
+      "logps/chosen": -104.13174438476562,
+      "logps/ref_chosen": -105.46086120605469,
+      "logps/ref_rejected": -119.00373840332031,
+      "logps/rejected": -117.734130859375,
+      "loss": 0.3807,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": 0.013291322626173496,
+      "rewards/margins": 0.0005952615174464881,
+      "rewards/rejected": 0.012696062214672565,
+      "sft_loss": 0.38070446252822876,
+      "step": 9
+    },
+    {
+      "epoch": 0.13174145738987236,
+      "grad_norm": 0.700039803981781,
+      "learning_rate": 4.989014955054745e-07,
+      "logits/chosen": 10.076737403869629,
+      "logits/rejected": 10.897785186767578,
+      "logps/chosen": -100.81087493896484,
+      "logps/ref_chosen": -104.21009826660156,
+      "logps/ref_rejected": -118.9209213256836,
+      "logps/rejected": -115.75495910644531,
+      "loss": 0.3367,
+      "rewards/accuracies": 0.5546875,
+      "rewards/chosen": 0.033992186188697815,
+      "rewards/margins": 0.0023326175287365913,
+      "rewards/rejected": 0.0316595658659935,
+      "sft_loss": 0.33672136068344116,
+      "step": 10
+    },
+    {
+      "epoch": 0.14491560312885962,
+      "grad_norm": 0.9160856008529663,
+      "learning_rate": 4.975306286336627e-07,
+      "logits/chosen": 9.973880767822266,
+      "logits/rejected": 11.158487319946289,
+      "logps/chosen": -101.3505630493164,
+      "logps/ref_chosen": -105.94319152832031,
+      "logps/ref_rejected": -122.76007843017578,
+      "logps/rejected": -118.6338119506836,
+      "loss": 0.3851,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.045926500111818314,
+      "rewards/margins": 0.004663803614675999,
+      "rewards/rejected": 0.04126270115375519,
+      "sft_loss": 0.3850533962249756,
+      "step": 11
+    },
+    {
+      "epoch": 0.15808974886784685,
+      "grad_norm": 0.9421964883804321,
+      "learning_rate": 4.956156357188939e-07,
+      "logits/chosen": 9.908226013183594,
+      "logits/rejected": 10.598045349121094,
+      "logps/chosen": -103.32762908935547,
+      "logps/ref_chosen": -109.08442687988281,
+      "logps/ref_rejected": -121.41947174072266,
+      "logps/rejected": -115.84996795654297,
+      "loss": 0.3532,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": 0.05756799131631851,
+      "rewards/margins": 0.0018730255542322993,
+      "rewards/rejected": 0.05569496005773544,
+      "sft_loss": 0.3532242476940155,
+      "step": 12
+    },
+    {
+      "epoch": 0.17126389460683408,
+      "grad_norm": 0.3328304886817932,
+      "learning_rate": 4.931607263312032e-07,
+      "logits/chosen": 9.964012145996094,
+      "logits/rejected": 11.03992748260498,
+      "logps/chosen": -98.97601318359375,
+      "logps/ref_chosen": -104.62150573730469,
+      "logps/ref_rejected": -119.55384063720703,
+      "logps/rejected": -114.12371826171875,
+      "loss": 0.3686,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": 0.056454867124557495,
+      "rewards/margins": 0.0021536569111049175,
+      "rewards/rejected": 0.05430121719837189,
+      "sft_loss": 0.36859023571014404,
+      "step": 13
+    },
+    {
+      "epoch": 0.1844380403458213,
+      "grad_norm": 0.30642038583755493,
+      "learning_rate": 4.9017129689421e-07,
+      "logits/chosen": 10.519927978515625,
+      "logits/rejected": 11.649580001831055,
+      "logps/chosen": -96.5634765625,
+      "logps/ref_chosen": -106.179443359375,
+      "logps/ref_rejected": -120.73036193847656,
+      "logps/rejected": -110.86133575439453,
+      "loss": 0.3385,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.09615952521562576,
+      "rewards/margins": -0.002530643017962575,
+      "rewards/rejected": 0.09869016706943512,
+      "sft_loss": 0.338548868894577,
+      "step": 14
+    },
+    {
+      "epoch": 0.19761218608480857,
+      "grad_norm": 0.301073282957077,
+      "learning_rate": 4.866539188226085e-07,
+      "logits/chosen": 9.891039848327637,
+      "logits/rejected": 10.824172973632812,
+      "logps/chosen": -95.14861297607422,
+      "logps/ref_chosen": -105.70547485351562,
+      "logps/ref_rejected": -118.89997863769531,
+      "logps/rejected": -108.2326889038086,
+      "loss": 0.3305,
+      "rewards/accuracies": 0.5078125,
+      "rewards/chosen": 0.10556865483522415,
+      "rewards/margins": -0.0011043368140235543,
+      "rewards/rejected": 0.1066729873418808,
+      "sft_loss": 0.3305360674858093,
+      "step": 15
+    },
+    {
+      "epoch": 0.2107863318237958,
+      "grad_norm": 0.36065343022346497,
+      "learning_rate": 4.826163240767716e-07,
+      "logits/chosen": 10.682470321655273,
+      "logits/rejected": 11.299846649169922,
+      "logps/chosen": -96.53520202636719,
+      "logps/ref_chosen": -108.86376953125,
+      "logps/ref_rejected": -122.1635513305664,
+      "logps/rejected": -110.50537872314453,
+      "loss": 0.3484,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.12328556925058365,
+      "rewards/margins": 0.006703883409500122,
+      "rewards/rejected": 0.11658168584108353,
+      "sft_loss": 0.3484281003475189,
+      "step": 16
+    },
+    {
+      "epoch": 0.22396047756278303,
+      "grad_norm": 0.5325565934181213,
+      "learning_rate": 4.780673881662242e-07,
+      "logits/chosen": 10.187503814697266,
+      "logits/rejected": 10.843408584594727,
+      "logps/chosen": -90.1707992553711,
+      "logps/ref_chosen": -102.93986511230469,
+      "logps/ref_rejected": -119.43718719482422,
+      "logps/rejected": -106.76301574707031,
+      "loss": 0.359,
+      "rewards/accuracies": 0.4453125,
+      "rewards/chosen": 0.12769076228141785,
+      "rewards/margins": 0.0009490540251135826,
+      "rewards/rejected": 0.1267417073249817,
+      "sft_loss": 0.3589847683906555,
+      "step": 17
+    },
+    {
+      "epoch": 0.23713462330177026,
+      "grad_norm": 0.4098409116268158,
+      "learning_rate": 4.730171106393466e-07,
+      "logits/chosen": 10.4215669631958,
+      "logits/rejected": 11.216498374938965,
+      "logps/chosen": -90.09894561767578,
+      "logps/ref_chosen": -103.81341552734375,
+      "logps/ref_rejected": -117.45123291015625,
+      "logps/rejected": -104.58552551269531,
+      "loss": 0.3368,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": 0.13714462518692017,
+      "rewards/margins": 0.00848748255521059,
+      "rewards/rejected": 0.12865713238716125,
+      "sft_loss": 0.33678534626960754,
+      "step": 18
+    },
+    {
+      "epoch": 0.2503087690407575,
+      "grad_norm": 0.3302735984325409,
+      "learning_rate": 4.6747659310219757e-07,
+      "logits/chosen": 10.332744598388672,
+      "logits/rejected": 11.005766868591309,
+      "logps/chosen": -94.52428436279297,
+      "logps/ref_chosen": -107.85797119140625,
+      "logps/ref_rejected": -121.88042449951172,
+      "logps/rejected": -108.09265899658203,
+      "loss": 0.3222,
+      "rewards/accuracies": 0.4296875,
+      "rewards/chosen": 0.1333368420600891,
+      "rewards/margins": -0.004540742840617895,
+      "rewards/rejected": 0.13787758350372314,
+      "sft_loss": 0.3221552073955536,
+      "step": 19
+    },
+    {
+      "epoch": 0.2634829147797447,
+      "grad_norm": 0.40531161427497864,
+      "learning_rate": 4.6145801481477433e-07,
+      "logits/chosen": 10.747330665588379,
+      "logits/rejected": 11.561124801635742,
+      "logps/chosen": -89.97228240966797,
+      "logps/ref_chosen": -103.42721557617188,
+      "logps/ref_rejected": -116.7796630859375,
+      "logps/rejected": -103.99850463867188,
+      "loss": 0.3157,
+      "rewards/accuracies": 0.5703125,
+      "rewards/chosen": 0.13454943895339966,
+      "rewards/margins": 0.006737923249602318,
+      "rewards/rejected": 0.1278115212917328,
+      "sft_loss": 0.3156886100769043,
+      "step": 20
+    },
+    {
+      "epoch": 0.276657060518732,
+      "grad_norm": 0.2686282992362976,
+      "learning_rate": 4.549746059183561e-07,
+      "logits/chosen": 9.720458984375,
+      "logits/rejected": 10.846506118774414,
+      "logps/chosen": -92.48249816894531,
+      "logps/ref_chosen": -106.60163879394531,
+      "logps/ref_rejected": -124.56562805175781,
+      "logps/rejected": -109.58876037597656,
+      "loss": 0.3106,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.14119136333465576,
+      "rewards/margins": -0.00857722107321024,
+      "rewards/rejected": 0.14976857602596283,
+      "sft_loss": 0.31064528226852417,
+      "step": 21
+    },
+    {
+      "epoch": 0.28983120625771924,
+      "grad_norm": 0.5989738702774048,
+      "learning_rate": 4.480406183527823e-07,
+      "logits/chosen": 10.225810050964355,
+      "logits/rejected": 11.099544525146484,
+      "logps/chosen": -88.04141998291016,
+      "logps/ref_chosen": -103.77696228027344,
+      "logps/ref_rejected": -118.73616027832031,
+      "logps/rejected": -104.40451049804688,
+      "loss": 0.3321,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.157355397939682,
+      "rewards/margins": 0.014038847759366035,
+      "rewards/rejected": 0.14331655204296112,
+      "sft_loss": 0.3321138620376587,
+      "step": 22
+    },
+    {
+      "epoch": 0.3030053519967065,
+      "grad_norm": 0.1970965415239334,
+      "learning_rate": 4.4067129452759546e-07,
+      "logits/chosen": 10.115339279174805,
+      "logits/rejected": 11.140266418457031,
+      "logps/chosen": -87.26233673095703,
+      "logps/ref_chosen": -104.72956085205078,
+      "logps/ref_rejected": -121.35556030273438,
+      "logps/rejected": -104.43501281738281,
+      "loss": 0.3228,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": 0.17467224597930908,
+      "rewards/margins": 0.005466699134558439,
+      "rewards/rejected": 0.16920553147792816,
+      "sft_loss": 0.3228015899658203,
+      "step": 23
+    },
+    {
+      "epoch": 0.3161794977356937,
+      "grad_norm": 0.2894323468208313,
+      "learning_rate": 4.3288283381591725e-07,
+      "logits/chosen": 10.147160530090332,
+      "logits/rejected": 10.98647689819336,
+      "logps/chosen": -86.99087524414062,
+      "logps/ref_chosen": -105.88758087158203,
+      "logps/ref_rejected": -125.69054412841797,
+      "logps/rejected": -106.15878295898438,
+      "loss": 0.3069,
+      "rewards/accuracies": 0.4765625,
+      "rewards/chosen": 0.1889670193195343,
+      "rewards/margins": -0.0063507393933832645,
+      "rewards/rejected": 0.19531774520874023,
+      "sft_loss": 0.30694928765296936,
+      "step": 24
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 75,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 12,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-36/added_tokens.json

@@ -0,0 +1,13 @@
+{
+  "<|assistant|>": 32001,
+  "<|endoftext|>": 32000,
+  "<|end|>": 32007,
+  "<|placeholder1|>": 32002,
+  "<|placeholder2|>": 32003,
+  "<|placeholder3|>": 32004,
+  "<|placeholder4|>": 32005,
+  "<|placeholder5|>": 32008,
+  "<|placeholder6|>": 32009,
+  "<|system|>": 32006,
+  "<|user|>": 32010
+}

checkpoint-36/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "/home/v-liuzichuan/huggingface/Phi-3-mini-4k-instruct",
+  "architectures": [
+    "Phi3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_phi3.Phi3Config",
+    "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM"
+  },
+  "bos_token_id": 1,
+  "embd_pdrop": 0.0,
+  "eos_token_id": 32000,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 4096,
+  "model_type": "phi3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "original_max_position_embeddings": 4096,
+  "pad_token_id": 32000,
+  "resid_pdrop": 0.0,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "sliding_window": 2047,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.2",
+  "use_cache": false,
+  "vocab_size": 32064
+}

checkpoint-36/generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    32000,
+    32001,
+    32007
+  ],
+  "pad_token_id": 32000,
+  "transformers_version": "4.45.2"
+}

checkpoint-36/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41edd25ec8d485fe7c3430271b04074feddb2d5bf6a0fccfcd3ea1f92b4fe9b9
+size 4972489328

checkpoint-36/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0515a700789f9df0187beee93fb893d40741c8f7880d81a42491983f4fb92646
+size 2669692552

checkpoint-36/model.safetensors.index.json

@@ -0,0 +1,202 @@
+{
+  "metadata": {
+    "total_size": 7642159104
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

checkpoint-36/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-36/tokenizer_config.json

@@ -0,0 +1,131 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "32000": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<|placeholder1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<|placeholder2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<|placeholder3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<|placeholder4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "<|placeholder5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "<|placeholder6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "legacy": false,
+  "model_max_length": 4096,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-36/trainer_state.json

@@ -0,0 +1,681 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.47426924660354053,
+  "eval_steps": 500,
+  "global_step": 36,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.013174145738987238,
+      "grad_norm": 1.194186806678772,
+      "learning_rate": 6.25e-08,
+      "logits/chosen": 9.990612030029297,
+      "logits/rejected": 10.698101997375488,
+      "logps/chosen": -102.88545989990234,
+      "logps/ref_chosen": -102.88545989990234,
+      "logps/ref_rejected": -121.84871673583984,
+      "logps/rejected": -121.84871673583984,
+      "loss": 0.3675,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "sft_loss": 0.36753880977630615,
+      "step": 1
+    },
+    {
+      "epoch": 0.026348291477974475,
+      "grad_norm": 0.5353251099586487,
+      "learning_rate": 1.25e-07,
+      "logits/chosen": 10.211905479431152,
+      "logits/rejected": 11.06594467163086,
+      "logps/chosen": -107.70349884033203,
+      "logps/ref_chosen": -107.70349884033203,
+      "logps/ref_rejected": -121.89966583251953,
+      "logps/rejected": -121.89966583251953,
+      "loss": 0.4101,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "sft_loss": 0.41013145446777344,
+      "step": 2
+    },
+    {
+      "epoch": 0.03952243721696171,
+      "grad_norm": 0.7126303315162659,
+      "learning_rate": 1.875e-07,
+      "logits/chosen": 10.032384872436523,
+      "logits/rejected": 11.023520469665527,
+      "logps/chosen": -108.3123779296875,
+      "logps/ref_chosen": -107.98188781738281,
+      "logps/ref_rejected": -124.51527404785156,
+      "logps/rejected": -124.87130737304688,
+      "loss": 0.412,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": -0.003304910147562623,
+      "rewards/margins": 0.0002554532838985324,
+      "rewards/rejected": -0.003560363780707121,
+      "sft_loss": 0.41195932030677795,
+      "step": 3
+    },
+    {
+      "epoch": 0.05269658295594895,
+      "grad_norm": 1.2344533205032349,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": 9.836658477783203,
+      "logits/rejected": 10.855621337890625,
+      "logps/chosen": -109.55919647216797,
+      "logps/ref_chosen": -109.20836639404297,
+      "logps/ref_rejected": -119.23908996582031,
+      "logps/rejected": -119.48279571533203,
+      "loss": 0.4039,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": -0.003508324269205332,
+      "rewards/margins": -0.0010712125804275274,
+      "rewards/rejected": -0.0024371116887778044,
+      "sft_loss": 0.4038863480091095,
+      "step": 4
+    },
+    {
+      "epoch": 0.06587072869493618,
+      "grad_norm": 1.426048994064331,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": 10.212320327758789,
+      "logits/rejected": 10.966379165649414,
+      "logps/chosen": -103.76991271972656,
+      "logps/ref_chosen": -103.87680053710938,
+      "logps/ref_rejected": -118.41618347167969,
+      "logps/rejected": -118.23270416259766,
+      "loss": 0.3697,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": 0.0010687037138268352,
+      "rewards/margins": -0.000766113749705255,
+      "rewards/rejected": 0.0018348174635320902,
+      "sft_loss": 0.3697226345539093,
+      "step": 5
+    },
+    {
+      "epoch": 0.07904487443392343,
+      "grad_norm": 1.413549780845642,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": 10.700042724609375,
+      "logits/rejected": 11.478326797485352,
+      "logps/chosen": -107.56877899169922,
+      "logps/ref_chosen": -107.58968353271484,
+      "logps/ref_rejected": -122.07303619384766,
+      "logps/rejected": -121.85940551757812,
+      "loss": 0.3909,
+      "rewards/accuracies": 0.4609375,
+      "rewards/chosen": 0.0002090137859340757,
+      "rewards/margins": -0.0019273017533123493,
+      "rewards/rejected": 0.00213631521910429,
+      "sft_loss": 0.390906423330307,
+      "step": 6
+    },
+    {
+      "epoch": 0.09221902017291066,
+      "grad_norm": 1.2342580556869507,
+      "learning_rate": 4.375e-07,
+      "logits/chosen": 10.01632308959961,
+      "logits/rejected": 10.7178955078125,
+      "logps/chosen": -107.01339721679688,
+      "logps/ref_chosen": -107.42727661132812,
+      "logps/ref_rejected": -116.87063598632812,
+      "logps/rejected": -116.37357330322266,
+      "loss": 0.3747,
+      "rewards/accuracies": 0.4453125,
+      "rewards/chosen": 0.00413867924362421,
+      "rewards/margins": -0.0008318667532876134,
+      "rewards/rejected": 0.004970546346157789,
+      "sft_loss": 0.3746669888496399,
+      "step": 7
+    },
+    {
+      "epoch": 0.1053931659118979,
+      "grad_norm": 0.6644937992095947,
+      "learning_rate": 5e-07,
+      "logits/chosen": 10.211028099060059,
+      "logits/rejected": 11.11027717590332,
+      "logps/chosen": -104.41184997558594,
+      "logps/ref_chosen": -105.60282135009766,
+      "logps/ref_rejected": -119.53916931152344,
+      "logps/rejected": -118.27430725097656,
+      "loss": 0.3773,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": 0.01190974935889244,
+      "rewards/margins": -0.0007388982339762151,
+      "rewards/rejected": 0.012648648582398891,
+      "sft_loss": 0.37729793787002563,
+      "step": 8
+    },
+    {
+      "epoch": 0.11856731165088513,
+      "grad_norm": 0.9437576532363892,
+      "learning_rate": 4.997252228714278e-07,
+      "logits/chosen": 10.179821014404297,
+      "logits/rejected": 11.147579193115234,
+      "logps/chosen": -104.13174438476562,
+      "logps/ref_chosen": -105.46086120605469,
+      "logps/ref_rejected": -119.00373840332031,
+      "logps/rejected": -117.734130859375,
+      "loss": 0.3807,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": 0.013291322626173496,
+      "rewards/margins": 0.0005952615174464881,
+      "rewards/rejected": 0.012696062214672565,
+      "sft_loss": 0.38070446252822876,
+      "step": 9
+    },
+    {
+      "epoch": 0.13174145738987236,
+      "grad_norm": 0.700039803981781,
+      "learning_rate": 4.989014955054745e-07,
+      "logits/chosen": 10.076737403869629,
+      "logits/rejected": 10.897785186767578,
+      "logps/chosen": -100.81087493896484,
+      "logps/ref_chosen": -104.21009826660156,
+      "logps/ref_rejected": -118.9209213256836,
+      "logps/rejected": -115.75495910644531,
+      "loss": 0.3367,
+      "rewards/accuracies": 0.5546875,
+      "rewards/chosen": 0.033992186188697815,
+      "rewards/margins": 0.0023326175287365913,
+      "rewards/rejected": 0.0316595658659935,
+      "sft_loss": 0.33672136068344116,
+      "step": 10
+    },
+    {
+      "epoch": 0.14491560312885962,
+      "grad_norm": 0.9160856008529663,
+      "learning_rate": 4.975306286336627e-07,
+      "logits/chosen": 9.973880767822266,
+      "logits/rejected": 11.158487319946289,
+      "logps/chosen": -101.3505630493164,
+      "logps/ref_chosen": -105.94319152832031,
+      "logps/ref_rejected": -122.76007843017578,
+      "logps/rejected": -118.6338119506836,
+      "loss": 0.3851,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.045926500111818314,
+      "rewards/margins": 0.004663803614675999,
+      "rewards/rejected": 0.04126270115375519,
+      "sft_loss": 0.3850533962249756,
+      "step": 11
+    },
+    {
+      "epoch": 0.15808974886784685,
+      "grad_norm": 0.9421964883804321,
+      "learning_rate": 4.956156357188939e-07,
+      "logits/chosen": 9.908226013183594,
+      "logits/rejected": 10.598045349121094,
+      "logps/chosen": -103.32762908935547,
+      "logps/ref_chosen": -109.08442687988281,
+      "logps/ref_rejected": -121.41947174072266,
+      "logps/rejected": -115.84996795654297,
+      "loss": 0.3532,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": 0.05756799131631851,
+      "rewards/margins": 0.0018730255542322993,
+      "rewards/rejected": 0.05569496005773544,
+      "sft_loss": 0.3532242476940155,
+      "step": 12
+    },
+    {
+      "epoch": 0.17126389460683408,
+      "grad_norm": 0.3328304886817932,
+      "learning_rate": 4.931607263312032e-07,
+      "logits/chosen": 9.964012145996094,
+      "logits/rejected": 11.03992748260498,
+      "logps/chosen": -98.97601318359375,
+      "logps/ref_chosen": -104.62150573730469,
+      "logps/ref_rejected": -119.55384063720703,
+      "logps/rejected": -114.12371826171875,
+      "loss": 0.3686,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": 0.056454867124557495,
+      "rewards/margins": 0.0021536569111049175,
+      "rewards/rejected": 0.05430121719837189,
+      "sft_loss": 0.36859023571014404,
+      "step": 13
+    },
+    {
+      "epoch": 0.1844380403458213,
+      "grad_norm": 0.30642038583755493,
+      "learning_rate": 4.9017129689421e-07,
+      "logits/chosen": 10.519927978515625,
+      "logits/rejected": 11.649580001831055,
+      "logps/chosen": -96.5634765625,
+      "logps/ref_chosen": -106.179443359375,
+      "logps/ref_rejected": -120.73036193847656,
+      "logps/rejected": -110.86133575439453,
+      "loss": 0.3385,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.09615952521562576,
+      "rewards/margins": -0.002530643017962575,
+      "rewards/rejected": 0.09869016706943512,
+      "sft_loss": 0.338548868894577,
+      "step": 14
+    },
+    {
+      "epoch": 0.19761218608480857,
+      "grad_norm": 0.301073282957077,
+      "learning_rate": 4.866539188226085e-07,
+      "logits/chosen": 9.891039848327637,
+      "logits/rejected": 10.824172973632812,
+      "logps/chosen": -95.14861297607422,
+      "logps/ref_chosen": -105.70547485351562,
+      "logps/ref_rejected": -118.89997863769531,
+      "logps/rejected": -108.2326889038086,
+      "loss": 0.3305,
+      "rewards/accuracies": 0.5078125,
+      "rewards/chosen": 0.10556865483522415,
+      "rewards/margins": -0.0011043368140235543,
+      "rewards/rejected": 0.1066729873418808,
+      "sft_loss": 0.3305360674858093,
+      "step": 15
+    },
+    {
+      "epoch": 0.2107863318237958,
+      "grad_norm": 0.36065343022346497,
+      "learning_rate": 4.826163240767716e-07,
+      "logits/chosen": 10.682470321655273,
+      "logits/rejected": 11.299846649169922,
+      "logps/chosen": -96.53520202636719,
+      "logps/ref_chosen": -108.86376953125,
+      "logps/ref_rejected": -122.1635513305664,
+      "logps/rejected": -110.50537872314453,
+      "loss": 0.3484,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.12328556925058365,
+      "rewards/margins": 0.006703883409500122,
+      "rewards/rejected": 0.11658168584108353,
+      "sft_loss": 0.3484281003475189,
+      "step": 16
+    },
+    {
+      "epoch": 0.22396047756278303,
+      "grad_norm": 0.5325565934181213,
+      "learning_rate": 4.780673881662242e-07,
+      "logits/chosen": 10.187503814697266,
+      "logits/rejected": 10.843408584594727,
+      "logps/chosen": -90.1707992553711,
+      "logps/ref_chosen": -102.93986511230469,
+      "logps/ref_rejected": -119.43718719482422,
+      "logps/rejected": -106.76301574707031,
+      "loss": 0.359,
+      "rewards/accuracies": 0.4453125,
+      "rewards/chosen": 0.12769076228141785,
+      "rewards/margins": 0.0009490540251135826,
+      "rewards/rejected": 0.1267417073249817,
+      "sft_loss": 0.3589847683906555,
+      "step": 17
+    },
+    {
+      "epoch": 0.23713462330177026,
+      "grad_norm": 0.4098409116268158,
+      "learning_rate": 4.730171106393466e-07,
+      "logits/chosen": 10.4215669631958,
+      "logits/rejected": 11.216498374938965,
+      "logps/chosen": -90.09894561767578,
+      "logps/ref_chosen": -103.81341552734375,
+      "logps/ref_rejected": -117.45123291015625,
+      "logps/rejected": -104.58552551269531,
+      "loss": 0.3368,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": 0.13714462518692017,
+      "rewards/margins": 0.00848748255521059,
+      "rewards/rejected": 0.12865713238716125,
+      "sft_loss": 0.33678534626960754,
+      "step": 18
+    },
+    {
+      "epoch": 0.2503087690407575,
+      "grad_norm": 0.3302735984325409,
+      "learning_rate": 4.6747659310219757e-07,
+      "logits/chosen": 10.332744598388672,
+      "logits/rejected": 11.005766868591309,
+      "logps/chosen": -94.52428436279297,
+      "logps/ref_chosen": -107.85797119140625,
+      "logps/ref_rejected": -121.88042449951172,
+      "logps/rejected": -108.09265899658203,
+      "loss": 0.3222,
+      "rewards/accuracies": 0.4296875,
+      "rewards/chosen": 0.1333368420600891,
+      "rewards/margins": -0.004540742840617895,
+      "rewards/rejected": 0.13787758350372314,
+      "sft_loss": 0.3221552073955536,
+      "step": 19
+    },
+    {
+      "epoch": 0.2634829147797447,
+      "grad_norm": 0.40531161427497864,
+      "learning_rate": 4.6145801481477433e-07,
+      "logits/chosen": 10.747330665588379,
+      "logits/rejected": 11.561124801635742,
+      "logps/chosen": -89.97228240966797,
+      "logps/ref_chosen": -103.42721557617188,
+      "logps/ref_rejected": -116.7796630859375,
+      "logps/rejected": -103.99850463867188,
+      "loss": 0.3157,
+      "rewards/accuracies": 0.5703125,
+      "rewards/chosen": 0.13454943895339966,
+      "rewards/margins": 0.006737923249602318,
+      "rewards/rejected": 0.1278115212917328,
+      "sft_loss": 0.3156886100769043,
+      "step": 20
+    },
+    {
+      "epoch": 0.276657060518732,
+      "grad_norm": 0.2686282992362976,
+      "learning_rate": 4.549746059183561e-07,
+      "logits/chosen": 9.720458984375,
+      "logits/rejected": 10.846506118774414,
+      "logps/chosen": -92.48249816894531,
+      "logps/ref_chosen": -106.60163879394531,
+      "logps/ref_rejected": -124.56562805175781,
+      "logps/rejected": -109.58876037597656,
+      "loss": 0.3106,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.14119136333465576,
+      "rewards/margins": -0.00857722107321024,
+      "rewards/rejected": 0.14976857602596283,
+      "sft_loss": 0.31064528226852417,
+      "step": 21
+    },
+    {
+      "epoch": 0.28983120625771924,
+      "grad_norm": 0.5989738702774048,
+      "learning_rate": 4.480406183527823e-07,
+      "logits/chosen": 10.225810050964355,
+      "logits/rejected": 11.099544525146484,
+      "logps/chosen": -88.04141998291016,
+      "logps/ref_chosen": -103.77696228027344,
+      "logps/ref_rejected": -118.73616027832031,
+      "logps/rejected": -104.40451049804688,
+      "loss": 0.3321,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.157355397939682,
+      "rewards/margins": 0.014038847759366035,
+      "rewards/rejected": 0.14331655204296112,
+      "sft_loss": 0.3321138620376587,
+      "step": 22
+    },
+    {
+      "epoch": 0.3030053519967065,
+      "grad_norm": 0.1970965415239334,
+      "learning_rate": 4.4067129452759546e-07,
+      "logits/chosen": 10.115339279174805,
+      "logits/rejected": 11.140266418457031,
+      "logps/chosen": -87.26233673095703,
+      "logps/ref_chosen": -104.72956085205078,
+      "logps/ref_rejected": -121.35556030273438,
+      "logps/rejected": -104.43501281738281,
+      "loss": 0.3228,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": 0.17467224597930908,
+      "rewards/margins": 0.005466699134558439,
+      "rewards/rejected": 0.16920553147792816,
+      "sft_loss": 0.3228015899658203,
+      "step": 23
+    },
+    {
+      "epoch": 0.3161794977356937,
+      "grad_norm": 0.2894323468208313,
+      "learning_rate": 4.3288283381591725e-07,
+      "logits/chosen": 10.147160530090332,
+      "logits/rejected": 10.98647689819336,
+      "logps/chosen": -86.99087524414062,
+      "logps/ref_chosen": -105.88758087158203,
+      "logps/ref_rejected": -125.69054412841797,
+      "logps/rejected": -106.15878295898438,
+      "loss": 0.3069,
+      "rewards/accuracies": 0.4765625,
+      "rewards/chosen": 0.1889670193195343,
+      "rewards/margins": -0.0063507393933832645,
+      "rewards/rejected": 0.19531774520874023,
+      "sft_loss": 0.30694928765296936,
+      "step": 24
+    },
+    {
+      "epoch": 0.32935364347468093,
+      "grad_norm": 0.31463876366615295,
+      "learning_rate": 4.246923569447104e-07,
+      "logits/chosen": 10.327369689941406,
+      "logits/rejected": 11.063910484313965,
+      "logps/chosen": -87.99880981445312,
+      "logps/ref_chosen": -110.0761489868164,
+      "logps/ref_rejected": -129.10540771484375,
+      "logps/rejected": -107.19017028808594,
+      "loss": 0.2993,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.22077329456806183,
+      "rewards/margins": 0.001621072180569172,
+      "rewards/rejected": 0.21915221214294434,
+      "sft_loss": 0.2992705702781677,
+      "step": 25
+    },
+    {
+      "epoch": 0.34252778921366817,
+      "grad_norm": 0.25855836272239685,
+      "learning_rate": 4.161178683597054e-07,
+      "logits/chosen": 10.388958930969238,
+      "logits/rejected": 11.489179611206055,
+      "logps/chosen": -81.3349609375,
+      "logps/ref_chosen": -103.74571990966797,
+      "logps/ref_rejected": -120.73832702636719,
+      "logps/rejected": -98.57904052734375,
+      "loss": 0.2909,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.22410757839679718,
+      "rewards/margins": 0.0025147469714283943,
+      "rewards/rejected": 0.22159285843372345,
+      "sft_loss": 0.2909452021121979,
+      "step": 26
+    },
+    {
+      "epoch": 0.3557019349526554,
+      "grad_norm": 0.5696946382522583,
+      "learning_rate": 4.0717821664772124e-07,
+      "logits/chosen": 10.086296081542969,
+      "logits/rejected": 11.336379051208496,
+      "logps/chosen": -81.64080810546875,
+      "logps/ref_chosen": -105.47428131103516,
+      "logps/ref_rejected": -120.5193099975586,
+      "logps/rejected": -97.64772033691406,
+      "loss": 0.3163,
+      "rewards/accuracies": 0.5390625,
+      "rewards/chosen": 0.23833464086055756,
+      "rewards/margins": 0.00961877591907978,
+      "rewards/rejected": 0.22871585190296173,
+      "sft_loss": 0.316275417804718,
+      "step": 27
+    },
+    {
+      "epoch": 0.3688760806916426,
+      "grad_norm": 0.14858409762382507,
+      "learning_rate": 3.978930531033806e-07,
+      "logits/chosen": 9.710855484008789,
+      "logits/rejected": 10.872222900390625,
+      "logps/chosen": -80.8717269897461,
+      "logps/ref_chosen": -103.72540283203125,
+      "logps/ref_rejected": -119.79557800292969,
+      "logps/rejected": -96.69274139404297,
+      "loss": 0.2766,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.22853665053844452,
+      "rewards/margins": -0.0024917693808674812,
+      "rewards/rejected": 0.23102842271327972,
+      "sft_loss": 0.2765931785106659,
+      "step": 28
+    },
+    {
+      "epoch": 0.3820502264306299,
+      "grad_norm": 0.2741422951221466,
+      "learning_rate": 3.882827885312998e-07,
+      "logits/chosen": 10.16092586517334,
+      "logits/rejected": 11.23297119140625,
+      "logps/chosen": -85.11812591552734,
+      "logps/ref_chosen": -108.65434265136719,
+      "logps/ref_rejected": -121.46784973144531,
+      "logps/rejected": -98.77241516113281,
+      "loss": 0.2799,
+      "rewards/accuracies": 0.5078125,
+      "rewards/chosen": 0.23536208271980286,
+      "rewards/margins": 0.008407761342823505,
+      "rewards/rejected": 0.22695434093475342,
+      "sft_loss": 0.27991783618927,
+      "step": 29
+    },
+    {
+      "epoch": 0.39522437216961714,
+      "grad_norm": 0.16848187148571014,
+      "learning_rate": 3.7836854837871044e-07,
+      "logits/chosen": 10.2907133102417,
+      "logits/rejected": 11.690597534179688,
+      "logps/chosen": -78.23504638671875,
+      "logps/ref_chosen": -103.62174224853516,
+      "logps/ref_rejected": -126.73807525634766,
+      "logps/rejected": -102.43669128417969,
+      "loss": 0.2962,
+      "rewards/accuracies": 0.546875,
+      "rewards/chosen": 0.2538670301437378,
+      "rewards/margins": 0.010853251442313194,
+      "rewards/rejected": 0.24301378428936005,
+      "sft_loss": 0.2962155342102051,
+      "step": 30
+    },
+    {
+      "epoch": 0.4083985179086044,
+      "grad_norm": 0.1987890601158142,
+      "learning_rate": 3.681721262971413e-07,
+      "logits/chosen": 9.929094314575195,
+      "logits/rejected": 10.946361541748047,
+      "logps/chosen": -80.73751831054688,
+      "logps/ref_chosen": -106.10479736328125,
+      "logps/ref_rejected": -120.6382827758789,
+      "logps/rejected": -96.38467407226562,
+      "loss": 0.2982,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.2536728084087372,
+      "rewards/margins": 0.011136716231703758,
+      "rewards/rejected": 0.24253609776496887,
+      "sft_loss": 0.2981662452220917,
+      "step": 31
+    },
+    {
+      "epoch": 0.4215726636475916,
+      "grad_norm": 0.1829695999622345,
+      "learning_rate": 3.577159362352426e-07,
+      "logits/chosen": 10.097947120666504,
+      "logits/rejected": 11.477932929992676,
+      "logps/chosen": -82.30887603759766,
+      "logps/ref_chosen": -105.99569702148438,
+      "logps/ref_rejected": -128.34303283691406,
+      "logps/rejected": -104.14814758300781,
+      "loss": 0.2848,
+      "rewards/accuracies": 0.5078125,
+      "rewards/chosen": 0.2368682324886322,
+      "rewards/margins": -0.0050805676728487015,
+      "rewards/rejected": 0.24194881319999695,
+      "sft_loss": 0.28480714559555054,
+      "step": 32
+    },
+    {
+      "epoch": 0.43474680938657884,
+      "grad_norm": 0.22964715957641602,
+      "learning_rate": 3.470229631680624e-07,
+      "logits/chosen": 10.105993270874023,
+      "logits/rejected": 10.923880577087402,
+      "logps/chosen": -81.49787902832031,
+      "logps/ref_chosen": -105.72196197509766,
+      "logps/ref_rejected": -121.59507751464844,
+      "logps/rejected": -97.2413101196289,
+      "loss": 0.2686,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.242240771651268,
+      "rewards/margins": -0.0012968010269105434,
+      "rewards/rejected": 0.24353757500648499,
+      "sft_loss": 0.2686034142971039,
+      "step": 33
+    },
+    {
+      "epoch": 0.44792095512556607,
+      "grad_norm": 0.2910502254962921,
+      "learning_rate": 3.361167125710832e-07,
+      "logits/chosen": 10.264101028442383,
+      "logits/rejected": 11.107752799987793,
+      "logps/chosen": -85.68121337890625,
+      "logps/ref_chosen": -111.4834976196289,
+      "logps/ref_rejected": -130.48089599609375,
+      "logps/rejected": -104.08442687988281,
+      "loss": 0.3039,
+      "rewards/accuracies": 0.4609375,
+      "rewards/chosen": 0.2580227851867676,
+      "rewards/margins": -0.005941788665950298,
+      "rewards/rejected": 0.26396459341049194,
+      "sft_loss": 0.3038797080516815,
+      "step": 34
+    },
+    {
+      "epoch": 0.4610951008645533,
+      "grad_norm": 0.2656664550304413,
+      "learning_rate": 3.2502115875008516e-07,
+      "logits/chosen": 10.529006958007812,
+      "logits/rejected": 11.506540298461914,
+      "logps/chosen": -82.62615966796875,
+      "logps/ref_chosen": -108.9183349609375,
+      "logps/ref_rejected": -121.32493591308594,
+      "logps/rejected": -95.86014556884766,
+      "loss": 0.2751,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.26292186975479126,
+      "rewards/margins": 0.008274020627140999,
+      "rewards/rejected": 0.2546478509902954,
+      "sft_loss": 0.2751036286354065,
+      "step": 35
+    },
+    {
+      "epoch": 0.47426924660354053,
+      "grad_norm": 0.4242345690727234,
+      "learning_rate": 3.137606921404191e-07,
+      "logits/chosen": 10.204312324523926,
+      "logits/rejected": 10.856239318847656,
+      "logps/chosen": -81.047607421875,
+      "logps/ref_chosen": -107.1411361694336,
+      "logps/ref_rejected": -118.66165161132812,
+      "logps/rejected": -92.72647094726562,
+      "loss": 0.2871,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.26093533635139465,
+      "rewards/margins": 0.0015834786463528872,
+      "rewards/rejected": 0.25935184955596924,
+      "sft_loss": 0.28712767362594604,
+      "step": 36
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 75,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 12,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-48/added_tokens.json

@@ -0,0 +1,13 @@
+{
+  "<|assistant|>": 32001,
+  "<|endoftext|>": 32000,
+  "<|end|>": 32007,
+  "<|placeholder1|>": 32002,
+  "<|placeholder2|>": 32003,
+  "<|placeholder3|>": 32004,
+  "<|placeholder4|>": 32005,
+  "<|placeholder5|>": 32008,
+  "<|placeholder6|>": 32009,
+  "<|system|>": 32006,
+  "<|user|>": 32010
+}

checkpoint-48/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "/home/v-liuzichuan/huggingface/Phi-3-mini-4k-instruct",
+  "architectures": [
+    "Phi3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_phi3.Phi3Config",
+    "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM"
+  },
+  "bos_token_id": 1,
+  "embd_pdrop": 0.0,
+  "eos_token_id": 32000,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 4096,
+  "model_type": "phi3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "original_max_position_embeddings": 4096,
+  "pad_token_id": 32000,
+  "resid_pdrop": 0.0,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "sliding_window": 2047,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.2",
+  "use_cache": false,
+  "vocab_size": 32064
+}

checkpoint-48/generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    32000,
+    32001,
+    32007
+  ],
+  "pad_token_id": 32000,
+  "transformers_version": "4.45.2"
+}

checkpoint-48/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63b3d3be2e54a68b20d762616db1a086aa2f55ae56806b50ef43501a3face4f8
+size 4972489328

checkpoint-48/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c907407deae185b105fac7c8edb89306e1a7b898aee84dfffaf0e408485b079b
+size 2669692552

checkpoint-48/model.safetensors.index.json

@@ -0,0 +1,202 @@
+{
+  "metadata": {
+    "total_size": 7642159104
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

checkpoint-48/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-48/tokenizer_config.json

@@ -0,0 +1,131 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "32000": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<|placeholder1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<|placeholder2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<|placeholder3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<|placeholder4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "<|placeholder5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "<|placeholder6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "legacy": false,
+  "model_max_length": 4096,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-48/trainer_state.json

@@ -0,0 +1,897 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.6323589954713874,
+  "eval_steps": 500,
+  "global_step": 48,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.013174145738987238,
+      "grad_norm": 1.194186806678772,
+      "learning_rate": 6.25e-08,
+      "logits/chosen": 9.990612030029297,
+      "logits/rejected": 10.698101997375488,
+      "logps/chosen": -102.88545989990234,
+      "logps/ref_chosen": -102.88545989990234,
+      "logps/ref_rejected": -121.84871673583984,
+      "logps/rejected": -121.84871673583984,
+      "loss": 0.3675,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "sft_loss": 0.36753880977630615,
+      "step": 1
+    },
+    {
+      "epoch": 0.026348291477974475,
+      "grad_norm": 0.5353251099586487,
+      "learning_rate": 1.25e-07,
+      "logits/chosen": 10.211905479431152,
+      "logits/rejected": 11.06594467163086,
+      "logps/chosen": -107.70349884033203,
+      "logps/ref_chosen": -107.70349884033203,
+      "logps/ref_rejected": -121.89966583251953,
+      "logps/rejected": -121.89966583251953,
+      "loss": 0.4101,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "sft_loss": 0.41013145446777344,
+      "step": 2
+    },
+    {
+      "epoch": 0.03952243721696171,
+      "grad_norm": 0.7126303315162659,
+      "learning_rate": 1.875e-07,
+      "logits/chosen": 10.032384872436523,
+      "logits/rejected": 11.023520469665527,
+      "logps/chosen": -108.3123779296875,
+      "logps/ref_chosen": -107.98188781738281,
+      "logps/ref_rejected": -124.51527404785156,
+      "logps/rejected": -124.87130737304688,
+      "loss": 0.412,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": -0.003304910147562623,
+      "rewards/margins": 0.0002554532838985324,
+      "rewards/rejected": -0.003560363780707121,
+      "sft_loss": 0.41195932030677795,
+      "step": 3
+    },
+    {
+      "epoch": 0.05269658295594895,
+      "grad_norm": 1.2344533205032349,
+      "learning_rate": 2.5e-07,
+      "logits/chosen": 9.836658477783203,
+      "logits/rejected": 10.855621337890625,
+      "logps/chosen": -109.55919647216797,
+      "logps/ref_chosen": -109.20836639404297,
+      "logps/ref_rejected": -119.23908996582031,
+      "logps/rejected": -119.48279571533203,
+      "loss": 0.4039,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": -0.003508324269205332,
+      "rewards/margins": -0.0010712125804275274,
+      "rewards/rejected": -0.0024371116887778044,
+      "sft_loss": 0.4038863480091095,
+      "step": 4
+    },
+    {
+      "epoch": 0.06587072869493618,
+      "grad_norm": 1.426048994064331,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": 10.212320327758789,
+      "logits/rejected": 10.966379165649414,
+      "logps/chosen": -103.76991271972656,
+      "logps/ref_chosen": -103.87680053710938,
+      "logps/ref_rejected": -118.41618347167969,
+      "logps/rejected": -118.23270416259766,
+      "loss": 0.3697,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": 0.0010687037138268352,
+      "rewards/margins": -0.000766113749705255,
+      "rewards/rejected": 0.0018348174635320902,
+      "sft_loss": 0.3697226345539093,
+      "step": 5
+    },
+    {
+      "epoch": 0.07904487443392343,
+      "grad_norm": 1.413549780845642,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": 10.700042724609375,
+      "logits/rejected": 11.478326797485352,
+      "logps/chosen": -107.56877899169922,
+      "logps/ref_chosen": -107.58968353271484,
+      "logps/ref_rejected": -122.07303619384766,
+      "logps/rejected": -121.85940551757812,
+      "loss": 0.3909,
+      "rewards/accuracies": 0.4609375,
+      "rewards/chosen": 0.0002090137859340757,
+      "rewards/margins": -0.0019273017533123493,
+      "rewards/rejected": 0.00213631521910429,
+      "sft_loss": 0.390906423330307,
+      "step": 6
+    },
+    {
+      "epoch": 0.09221902017291066,
+      "grad_norm": 1.2342580556869507,
+      "learning_rate": 4.375e-07,
+      "logits/chosen": 10.01632308959961,
+      "logits/rejected": 10.7178955078125,
+      "logps/chosen": -107.01339721679688,
+      "logps/ref_chosen": -107.42727661132812,
+      "logps/ref_rejected": -116.87063598632812,
+      "logps/rejected": -116.37357330322266,
+      "loss": 0.3747,
+      "rewards/accuracies": 0.4453125,
+      "rewards/chosen": 0.00413867924362421,
+      "rewards/margins": -0.0008318667532876134,
+      "rewards/rejected": 0.004970546346157789,
+      "sft_loss": 0.3746669888496399,
+      "step": 7
+    },
+    {
+      "epoch": 0.1053931659118979,
+      "grad_norm": 0.6644937992095947,
+      "learning_rate": 5e-07,
+      "logits/chosen": 10.211028099060059,
+      "logits/rejected": 11.11027717590332,
+      "logps/chosen": -104.41184997558594,
+      "logps/ref_chosen": -105.60282135009766,
+      "logps/ref_rejected": -119.53916931152344,
+      "logps/rejected": -118.27430725097656,
+      "loss": 0.3773,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": 0.01190974935889244,
+      "rewards/margins": -0.0007388982339762151,
+      "rewards/rejected": 0.012648648582398891,
+      "sft_loss": 0.37729793787002563,
+      "step": 8
+    },
+    {
+      "epoch": 0.11856731165088513,
+      "grad_norm": 0.9437576532363892,
+      "learning_rate": 4.997252228714278e-07,
+      "logits/chosen": 10.179821014404297,
+      "logits/rejected": 11.147579193115234,
+      "logps/chosen": -104.13174438476562,
+      "logps/ref_chosen": -105.46086120605469,
+      "logps/ref_rejected": -119.00373840332031,
+      "logps/rejected": -117.734130859375,
+      "loss": 0.3807,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": 0.013291322626173496,
+      "rewards/margins": 0.0005952615174464881,
+      "rewards/rejected": 0.012696062214672565,
+      "sft_loss": 0.38070446252822876,
+      "step": 9
+    },
+    {
+      "epoch": 0.13174145738987236,
+      "grad_norm": 0.700039803981781,
+      "learning_rate": 4.989014955054745e-07,
+      "logits/chosen": 10.076737403869629,
+      "logits/rejected": 10.897785186767578,
+      "logps/chosen": -100.81087493896484,
+      "logps/ref_chosen": -104.21009826660156,
+      "logps/ref_rejected": -118.9209213256836,
+      "logps/rejected": -115.75495910644531,
+      "loss": 0.3367,
+      "rewards/accuracies": 0.5546875,
+      "rewards/chosen": 0.033992186188697815,
+      "rewards/margins": 0.0023326175287365913,
+      "rewards/rejected": 0.0316595658659935,
+      "sft_loss": 0.33672136068344116,
+      "step": 10
+    },
+    {
+      "epoch": 0.14491560312885962,
+      "grad_norm": 0.9160856008529663,
+      "learning_rate": 4.975306286336627e-07,
+      "logits/chosen": 9.973880767822266,
+      "logits/rejected": 11.158487319946289,
+      "logps/chosen": -101.3505630493164,
+      "logps/ref_chosen": -105.94319152832031,
+      "logps/ref_rejected": -122.76007843017578,
+      "logps/rejected": -118.6338119506836,
+      "loss": 0.3851,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.045926500111818314,
+      "rewards/margins": 0.004663803614675999,
+      "rewards/rejected": 0.04126270115375519,
+      "sft_loss": 0.3850533962249756,
+      "step": 11
+    },
+    {
+      "epoch": 0.15808974886784685,
+      "grad_norm": 0.9421964883804321,
+      "learning_rate": 4.956156357188939e-07,
+      "logits/chosen": 9.908226013183594,
+      "logits/rejected": 10.598045349121094,
+      "logps/chosen": -103.32762908935547,
+      "logps/ref_chosen": -109.08442687988281,
+      "logps/ref_rejected": -121.41947174072266,
+      "logps/rejected": -115.84996795654297,
+      "loss": 0.3532,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": 0.05756799131631851,
+      "rewards/margins": 0.0018730255542322993,
+      "rewards/rejected": 0.05569496005773544,
+      "sft_loss": 0.3532242476940155,
+      "step": 12
+    },
+    {
+      "epoch": 0.17126389460683408,
+      "grad_norm": 0.3328304886817932,
+      "learning_rate": 4.931607263312032e-07,
+      "logits/chosen": 9.964012145996094,
+      "logits/rejected": 11.03992748260498,
+      "logps/chosen": -98.97601318359375,
+      "logps/ref_chosen": -104.62150573730469,
+      "logps/ref_rejected": -119.55384063720703,
+      "logps/rejected": -114.12371826171875,
+      "loss": 0.3686,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": 0.056454867124557495,
+      "rewards/margins": 0.0021536569111049175,
+      "rewards/rejected": 0.05430121719837189,
+      "sft_loss": 0.36859023571014404,
+      "step": 13
+    },
+    {
+      "epoch": 0.1844380403458213,
+      "grad_norm": 0.30642038583755493,
+      "learning_rate": 4.9017129689421e-07,
+      "logits/chosen": 10.519927978515625,
+      "logits/rejected": 11.649580001831055,
+      "logps/chosen": -96.5634765625,
+      "logps/ref_chosen": -106.179443359375,
+      "logps/ref_rejected": -120.73036193847656,
+      "logps/rejected": -110.86133575439453,
+      "loss": 0.3385,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.09615952521562576,
+      "rewards/margins": -0.002530643017962575,
+      "rewards/rejected": 0.09869016706943512,
+      "sft_loss": 0.338548868894577,
+      "step": 14
+    },
+    {
+      "epoch": 0.19761218608480857,
+      "grad_norm": 0.301073282957077,
+      "learning_rate": 4.866539188226085e-07,
+      "logits/chosen": 9.891039848327637,
+      "logits/rejected": 10.824172973632812,
+      "logps/chosen": -95.14861297607422,
+      "logps/ref_chosen": -105.70547485351562,
+      "logps/ref_rejected": -118.89997863769531,
+      "logps/rejected": -108.2326889038086,
+      "loss": 0.3305,
+      "rewards/accuracies": 0.5078125,
+      "rewards/chosen": 0.10556865483522415,
+      "rewards/margins": -0.0011043368140235543,
+      "rewards/rejected": 0.1066729873418808,
+      "sft_loss": 0.3305360674858093,
+      "step": 15
+    },
+    {
+      "epoch": 0.2107863318237958,
+      "grad_norm": 0.36065343022346497,
+      "learning_rate": 4.826163240767716e-07,
+      "logits/chosen": 10.682470321655273,
+      "logits/rejected": 11.299846649169922,
+      "logps/chosen": -96.53520202636719,
+      "logps/ref_chosen": -108.86376953125,
+      "logps/ref_rejected": -122.1635513305664,
+      "logps/rejected": -110.50537872314453,
+      "loss": 0.3484,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.12328556925058365,
+      "rewards/margins": 0.006703883409500122,
+      "rewards/rejected": 0.11658168584108353,
+      "sft_loss": 0.3484281003475189,
+      "step": 16
+    },
+    {
+      "epoch": 0.22396047756278303,
+      "grad_norm": 0.5325565934181213,
+      "learning_rate": 4.780673881662242e-07,
+      "logits/chosen": 10.187503814697266,
+      "logits/rejected": 10.843408584594727,
+      "logps/chosen": -90.1707992553711,
+      "logps/ref_chosen": -102.93986511230469,
+      "logps/ref_rejected": -119.43718719482422,
+      "logps/rejected": -106.76301574707031,
+      "loss": 0.359,
+      "rewards/accuracies": 0.4453125,
+      "rewards/chosen": 0.12769076228141785,
+      "rewards/margins": 0.0009490540251135826,
+      "rewards/rejected": 0.1267417073249817,
+      "sft_loss": 0.3589847683906555,
+      "step": 17
+    },
+    {
+      "epoch": 0.23713462330177026,
+      "grad_norm": 0.4098409116268158,
+      "learning_rate": 4.730171106393466e-07,
+      "logits/chosen": 10.4215669631958,
+      "logits/rejected": 11.216498374938965,
+      "logps/chosen": -90.09894561767578,
+      "logps/ref_chosen": -103.81341552734375,
+      "logps/ref_rejected": -117.45123291015625,
+      "logps/rejected": -104.58552551269531,
+      "loss": 0.3368,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": 0.13714462518692017,
+      "rewards/margins": 0.00848748255521059,
+      "rewards/rejected": 0.12865713238716125,
+      "sft_loss": 0.33678534626960754,
+      "step": 18
+    },
+    {
+      "epoch": 0.2503087690407575,
+      "grad_norm": 0.3302735984325409,
+      "learning_rate": 4.6747659310219757e-07,
+      "logits/chosen": 10.332744598388672,
+      "logits/rejected": 11.005766868591309,
+      "logps/chosen": -94.52428436279297,
+      "logps/ref_chosen": -107.85797119140625,
+      "logps/ref_rejected": -121.88042449951172,
+      "logps/rejected": -108.09265899658203,
+      "loss": 0.3222,
+      "rewards/accuracies": 0.4296875,
+      "rewards/chosen": 0.1333368420600891,
+      "rewards/margins": -0.004540742840617895,
+      "rewards/rejected": 0.13787758350372314,
+      "sft_loss": 0.3221552073955536,
+      "step": 19
+    },
+    {
+      "epoch": 0.2634829147797447,
+      "grad_norm": 0.40531161427497864,
+      "learning_rate": 4.6145801481477433e-07,
+      "logits/chosen": 10.747330665588379,
+      "logits/rejected": 11.561124801635742,
+      "logps/chosen": -89.97228240966797,
+      "logps/ref_chosen": -103.42721557617188,
+      "logps/ref_rejected": -116.7796630859375,
+      "logps/rejected": -103.99850463867188,
+      "loss": 0.3157,
+      "rewards/accuracies": 0.5703125,
+      "rewards/chosen": 0.13454943895339966,
+      "rewards/margins": 0.006737923249602318,
+      "rewards/rejected": 0.1278115212917328,
+      "sft_loss": 0.3156886100769043,
+      "step": 20
+    },
+    {
+      "epoch": 0.276657060518732,
+      "grad_norm": 0.2686282992362976,
+      "learning_rate": 4.549746059183561e-07,
+      "logits/chosen": 9.720458984375,
+      "logits/rejected": 10.846506118774414,
+      "logps/chosen": -92.48249816894531,
+      "logps/ref_chosen": -106.60163879394531,
+      "logps/ref_rejected": -124.56562805175781,
+      "logps/rejected": -109.58876037597656,
+      "loss": 0.3106,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": 0.14119136333465576,
+      "rewards/margins": -0.00857722107321024,
+      "rewards/rejected": 0.14976857602596283,
+      "sft_loss": 0.31064528226852417,
+      "step": 21
+    },
+    {
+      "epoch": 0.28983120625771924,
+      "grad_norm": 0.5989738702774048,
+      "learning_rate": 4.480406183527823e-07,
+      "logits/chosen": 10.225810050964355,
+      "logits/rejected": 11.099544525146484,
+      "logps/chosen": -88.04141998291016,
+      "logps/ref_chosen": -103.77696228027344,
+      "logps/ref_rejected": -118.73616027832031,
+      "logps/rejected": -104.40451049804688,
+      "loss": 0.3321,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.157355397939682,
+      "rewards/margins": 0.014038847759366035,
+      "rewards/rejected": 0.14331655204296112,
+      "sft_loss": 0.3321138620376587,
+      "step": 22
+    },
+    {
+      "epoch": 0.3030053519967065,
+      "grad_norm": 0.1970965415239334,
+      "learning_rate": 4.4067129452759546e-07,
+      "logits/chosen": 10.115339279174805,
+      "logits/rejected": 11.140266418457031,
+      "logps/chosen": -87.26233673095703,
+      "logps/ref_chosen": -104.72956085205078,
+      "logps/ref_rejected": -121.35556030273438,
+      "logps/rejected": -104.43501281738281,
+      "loss": 0.3228,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": 0.17467224597930908,
+      "rewards/margins": 0.005466699134558439,
+      "rewards/rejected": 0.16920553147792816,
+      "sft_loss": 0.3228015899658203,
+      "step": 23
+    },
+    {
+      "epoch": 0.3161794977356937,
+      "grad_norm": 0.2894323468208313,
+      "learning_rate": 4.3288283381591725e-07,
+      "logits/chosen": 10.147160530090332,
+      "logits/rejected": 10.98647689819336,
+      "logps/chosen": -86.99087524414062,
+      "logps/ref_chosen": -105.88758087158203,
+      "logps/ref_rejected": -125.69054412841797,
+      "logps/rejected": -106.15878295898438,
+      "loss": 0.3069,
+      "rewards/accuracies": 0.4765625,
+      "rewards/chosen": 0.1889670193195343,
+      "rewards/margins": -0.0063507393933832645,
+      "rewards/rejected": 0.19531774520874023,
+      "sft_loss": 0.30694928765296936,
+      "step": 24
+    },
+    {
+      "epoch": 0.32935364347468093,
+      "grad_norm": 0.31463876366615295,
+      "learning_rate": 4.246923569447104e-07,
+      "logits/chosen": 10.327369689941406,
+      "logits/rejected": 11.063910484313965,
+      "logps/chosen": -87.99880981445312,
+      "logps/ref_chosen": -110.0761489868164,
+      "logps/ref_rejected": -129.10540771484375,
+      "logps/rejected": -107.19017028808594,
+      "loss": 0.2993,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.22077329456806183,
+      "rewards/margins": 0.001621072180569172,
+      "rewards/rejected": 0.21915221214294434,
+      "sft_loss": 0.2992705702781677,
+      "step": 25
+    },
+    {
+      "epoch": 0.34252778921366817,
+      "grad_norm": 0.25855836272239685,
+      "learning_rate": 4.161178683597054e-07,
+      "logits/chosen": 10.388958930969238,
+      "logits/rejected": 11.489179611206055,
+      "logps/chosen": -81.3349609375,
+      "logps/ref_chosen": -103.74571990966797,
+      "logps/ref_rejected": -120.73832702636719,
+      "logps/rejected": -98.57904052734375,
+      "loss": 0.2909,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.22410757839679718,
+      "rewards/margins": 0.0025147469714283943,
+      "rewards/rejected": 0.22159285843372345,
+      "sft_loss": 0.2909452021121979,
+      "step": 26
+    },
+    {
+      "epoch": 0.3557019349526554,
+      "grad_norm": 0.5696946382522583,
+      "learning_rate": 4.0717821664772124e-07,
+      "logits/chosen": 10.086296081542969,
+      "logits/rejected": 11.336379051208496,
+      "logps/chosen": -81.64080810546875,
+      "logps/ref_chosen": -105.47428131103516,
+      "logps/ref_rejected": -120.5193099975586,
+      "logps/rejected": -97.64772033691406,
+      "loss": 0.3163,
+      "rewards/accuracies": 0.5390625,
+      "rewards/chosen": 0.23833464086055756,
+      "rewards/margins": 0.00961877591907978,
+      "rewards/rejected": 0.22871585190296173,
+      "sft_loss": 0.316275417804718,
+      "step": 27
+    },
+    {
+      "epoch": 0.3688760806916426,
+      "grad_norm": 0.14858409762382507,
+      "learning_rate": 3.978930531033806e-07,
+      "logits/chosen": 9.710855484008789,
+      "logits/rejected": 10.872222900390625,
+      "logps/chosen": -80.8717269897461,
+      "logps/ref_chosen": -103.72540283203125,
+      "logps/ref_rejected": -119.79557800292969,
+      "logps/rejected": -96.69274139404297,
+      "loss": 0.2766,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.22853665053844452,
+      "rewards/margins": -0.0024917693808674812,
+      "rewards/rejected": 0.23102842271327972,
+      "sft_loss": 0.2765931785106659,
+      "step": 28
+    },
+    {
+      "epoch": 0.3820502264306299,
+      "grad_norm": 0.2741422951221466,
+      "learning_rate": 3.882827885312998e-07,
+      "logits/chosen": 10.16092586517334,
+      "logits/rejected": 11.23297119140625,
+      "logps/chosen": -85.11812591552734,
+      "logps/ref_chosen": -108.65434265136719,
+      "logps/ref_rejected": -121.46784973144531,
+      "logps/rejected": -98.77241516113281,
+      "loss": 0.2799,
+      "rewards/accuracies": 0.5078125,
+      "rewards/chosen": 0.23536208271980286,
+      "rewards/margins": 0.008407761342823505,
+      "rewards/rejected": 0.22695434093475342,
+      "sft_loss": 0.27991783618927,
+      "step": 29
+    },
+    {
+      "epoch": 0.39522437216961714,
+      "grad_norm": 0.16848187148571014,
+      "learning_rate": 3.7836854837871044e-07,
+      "logits/chosen": 10.2907133102417,
+      "logits/rejected": 11.690597534179688,
+      "logps/chosen": -78.23504638671875,
+      "logps/ref_chosen": -103.62174224853516,
+      "logps/ref_rejected": -126.73807525634766,
+      "logps/rejected": -102.43669128417969,
+      "loss": 0.2962,
+      "rewards/accuracies": 0.546875,
+      "rewards/chosen": 0.2538670301437378,
+      "rewards/margins": 0.010853251442313194,
+      "rewards/rejected": 0.24301378428936005,
+      "sft_loss": 0.2962155342102051,
+      "step": 30
+    },
+    {
+      "epoch": 0.4083985179086044,
+      "grad_norm": 0.1987890601158142,
+      "learning_rate": 3.681721262971413e-07,
+      "logits/chosen": 9.929094314575195,
+      "logits/rejected": 10.946361541748047,
+      "logps/chosen": -80.73751831054688,
+      "logps/ref_chosen": -106.10479736328125,
+      "logps/ref_rejected": -120.6382827758789,
+      "logps/rejected": -96.38467407226562,
+      "loss": 0.2982,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.2536728084087372,
+      "rewards/margins": 0.011136716231703758,
+      "rewards/rejected": 0.24253609776496887,
+      "sft_loss": 0.2981662452220917,
+      "step": 31
+    },
+    {
+      "epoch": 0.4215726636475916,
+      "grad_norm": 0.1829695999622345,
+      "learning_rate": 3.577159362352426e-07,
+      "logits/chosen": 10.097947120666504,
+      "logits/rejected": 11.477932929992676,
+      "logps/chosen": -82.30887603759766,
+      "logps/ref_chosen": -105.99569702148438,
+      "logps/ref_rejected": -128.34303283691406,
+      "logps/rejected": -104.14814758300781,
+      "loss": 0.2848,
+      "rewards/accuracies": 0.5078125,
+      "rewards/chosen": 0.2368682324886322,
+      "rewards/margins": -0.0050805676728487015,
+      "rewards/rejected": 0.24194881319999695,
+      "sft_loss": 0.28480714559555054,
+      "step": 32
+    },
+    {
+      "epoch": 0.43474680938657884,
+      "grad_norm": 0.22964715957641602,
+      "learning_rate": 3.470229631680624e-07,
+      "logits/chosen": 10.105993270874023,
+      "logits/rejected": 10.923880577087402,
+      "logps/chosen": -81.49787902832031,
+      "logps/ref_chosen": -105.72196197509766,
+      "logps/ref_rejected": -121.59507751464844,
+      "logps/rejected": -97.2413101196289,
+      "loss": 0.2686,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.242240771651268,
+      "rewards/margins": -0.0012968010269105434,
+      "rewards/rejected": 0.24353757500648499,
+      "sft_loss": 0.2686034142971039,
+      "step": 33
+    },
+    {
+      "epoch": 0.44792095512556607,
+      "grad_norm": 0.2910502254962921,
+      "learning_rate": 3.361167125710832e-07,
+      "logits/chosen": 10.264101028442383,
+      "logits/rejected": 11.107752799987793,
+      "logps/chosen": -85.68121337890625,
+      "logps/ref_chosen": -111.4834976196289,
+      "logps/ref_rejected": -130.48089599609375,
+      "logps/rejected": -104.08442687988281,
+      "loss": 0.3039,
+      "rewards/accuracies": 0.4609375,
+      "rewards/chosen": 0.2580227851867676,
+      "rewards/margins": -0.005941788665950298,
+      "rewards/rejected": 0.26396459341049194,
+      "sft_loss": 0.3038797080516815,
+      "step": 34
+    },
+    {
+      "epoch": 0.4610951008645533,
+      "grad_norm": 0.2656664550304413,
+      "learning_rate": 3.2502115875008516e-07,
+      "logits/chosen": 10.529006958007812,
+      "logits/rejected": 11.506540298461914,
+      "logps/chosen": -82.62615966796875,
+      "logps/ref_chosen": -108.9183349609375,
+      "logps/ref_rejected": -121.32493591308594,
+      "logps/rejected": -95.86014556884766,
+      "loss": 0.2751,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.26292186975479126,
+      "rewards/margins": 0.008274020627140999,
+      "rewards/rejected": 0.2546478509902954,
+      "sft_loss": 0.2751036286354065,
+      "step": 35
+    },
+    {
+      "epoch": 0.47426924660354053,
+      "grad_norm": 0.4242345690727234,
+      "learning_rate": 3.137606921404191e-07,
+      "logits/chosen": 10.204312324523926,
+      "logits/rejected": 10.856239318847656,
+      "logps/chosen": -81.047607421875,
+      "logps/ref_chosen": -107.1411361694336,
+      "logps/ref_rejected": -118.66165161132812,
+      "logps/rejected": -92.72647094726562,
+      "loss": 0.2871,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.26093533635139465,
+      "rewards/margins": 0.0015834786463528872,
+      "rewards/rejected": 0.25935184955596924,
+      "sft_loss": 0.28712767362594604,
+      "step": 36
+    },
+    {
+      "epoch": 0.4874433923425278,
+      "grad_norm": 0.30854037404060364,
+      "learning_rate": 3.0236006569153616e-07,
+      "logits/chosen": 10.416954040527344,
+      "logits/rejected": 11.237515449523926,
+      "logps/chosen": -80.80354309082031,
+      "logps/ref_chosen": -106.6348876953125,
+      "logps/ref_rejected": -121.37834167480469,
+      "logps/rejected": -94.95623779296875,
+      "loss": 0.2868,
+      "rewards/accuracies": 0.5390625,
+      "rewards/chosen": 0.2583135664463043,
+      "rewards/margins": -0.0059075187891721725,
+      "rewards/rejected": 0.26422107219696045,
+      "sft_loss": 0.2867960035800934,
+      "step": 37
+    },
+    {
+      "epoch": 0.500617538081515,
+      "grad_norm": 0.38461658358573914,
+      "learning_rate": 2.9084434045463254e-07,
+      "logits/chosen": 9.907350540161133,
+      "logits/rejected": 10.992050170898438,
+      "logps/chosen": -76.08082580566406,
+      "logps/ref_chosen": -104.01033782958984,
+      "logps/ref_rejected": -119.02666473388672,
+      "logps/rejected": -90.15015411376953,
+      "loss": 0.2865,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.2792952358722687,
+      "rewards/margins": -0.009469768032431602,
+      "rewards/rejected": 0.28876498341560364,
+      "sft_loss": 0.28654617071151733,
+      "step": 38
+    },
+    {
+      "epoch": 0.5137916838205022,
+      "grad_norm": 0.32149407267570496,
+      "learning_rate": 2.7923883049302066e-07,
+      "logits/chosen": 10.409524917602539,
+      "logits/rejected": 11.193167686462402,
+      "logps/chosen": -82.37915802001953,
+      "logps/ref_chosen": -109.76485443115234,
+      "logps/ref_rejected": -122.25163269042969,
+      "logps/rejected": -96.5480728149414,
+      "loss": 0.2919,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.2738569676876068,
+      "rewards/margins": 0.016821369528770447,
+      "rewards/rejected": 0.25703561305999756,
+      "sft_loss": 0.2919383943080902,
+      "step": 39
+    },
+    {
+      "epoch": 0.5269658295594895,
+      "grad_norm": 0.48350030183792114,
+      "learning_rate": 2.6756904723632324e-07,
+      "logits/chosen": 10.301675796508789,
+      "logits/rejected": 11.467926025390625,
+      "logps/chosen": -79.19075012207031,
+      "logps/ref_chosen": -107.18782806396484,
+      "logps/ref_rejected": -124.24542236328125,
+      "logps/rejected": -96.9850082397461,
+      "loss": 0.2805,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.2799707055091858,
+      "rewards/margins": 0.007366571109741926,
+      "rewards/rejected": 0.27260416746139526,
+      "sft_loss": 0.2805452048778534,
+      "step": 40
+    },
+    {
+      "epoch": 0.5401399752984768,
+      "grad_norm": 0.27216726541519165,
+      "learning_rate": 2.5586064340081516e-07,
+      "logits/chosen": 10.603267669677734,
+      "logits/rejected": 11.226158142089844,
+      "logps/chosen": -78.87698364257812,
+      "logps/ref_chosen": -106.42051696777344,
+      "logps/ref_rejected": -122.25247192382812,
+      "logps/rejected": -94.32707977294922,
+      "loss": 0.2862,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": 0.275435209274292,
+      "rewards/margins": -0.003818750847131014,
+      "rewards/rejected": 0.2792539596557617,
+      "sft_loss": 0.28622862696647644,
+      "step": 41
+    },
+    {
+      "epoch": 0.553314121037464,
+      "grad_norm": 0.33358198404312134,
+      "learning_rate": 2.4413935659918487e-07,
+      "logits/chosen": 9.588984489440918,
+      "logits/rejected": 10.6452054977417,
+      "logps/chosen": -74.2744369506836,
+      "logps/ref_chosen": -103.1148452758789,
+      "logps/ref_rejected": -116.55464935302734,
+      "logps/rejected": -88.39811706542969,
+      "loss": 0.2641,
+      "rewards/accuracies": 0.546875,
+      "rewards/chosen": 0.28840407729148865,
+      "rewards/margins": 0.0068388087674975395,
+      "rewards/rejected": 0.28156527876853943,
+      "sft_loss": 0.2640990614891052,
+      "step": 42
+    },
+    {
+      "epoch": 0.5664882667764513,
+      "grad_norm": 0.47038576006889343,
+      "learning_rate": 2.3243095276367684e-07,
+      "logits/chosen": 9.663254737854004,
+      "logits/rejected": 10.8170747756958,
+      "logps/chosen": -76.7999267578125,
+      "logps/ref_chosen": -104.21064758300781,
+      "logps/ref_rejected": -118.7614974975586,
+      "logps/rejected": -91.26052856445312,
+      "loss": 0.2744,
+      "rewards/accuracies": 0.4609375,
+      "rewards/chosen": 0.2741071879863739,
+      "rewards/margins": -0.0009024296887218952,
+      "rewards/rejected": 0.2750096321105957,
+      "sft_loss": 0.2744351625442505,
+      "step": 43
+    },
+    {
+      "epoch": 0.5796624125154385,
+      "grad_norm": 0.30214762687683105,
+      "learning_rate": 2.2076116950697937e-07,
+      "logits/chosen": 9.781536102294922,
+      "logits/rejected": 10.607444763183594,
+      "logps/chosen": -72.30926513671875,
+      "logps/ref_chosen": -100.59449005126953,
+      "logps/ref_rejected": -115.95166778564453,
+      "logps/rejected": -87.6107177734375,
+      "loss": 0.2461,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": 0.28285229206085205,
+      "rewards/margins": -0.0005572582595050335,
+      "rewards/rejected": 0.2834095358848572,
+      "sft_loss": 0.24606820940971375,
+      "step": 44
+    },
+    {
+      "epoch": 0.5928365582544257,
+      "grad_norm": 0.26689764857292175,
+      "learning_rate": 2.091556595453674e-07,
+      "logits/chosen": 9.98751449584961,
+      "logits/rejected": 10.849931716918945,
+      "logps/chosen": -78.55535888671875,
+      "logps/ref_chosen": -106.96060943603516,
+      "logps/ref_rejected": -125.49449157714844,
+      "logps/rejected": -98.11154174804688,
+      "loss": 0.2706,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.2840524911880493,
+      "rewards/margins": 0.010223127901554108,
+      "rewards/rejected": 0.2738293409347534,
+      "sft_loss": 0.2706489562988281,
+      "step": 45
+    },
+    {
+      "epoch": 0.606010703993413,
+      "grad_norm": 0.24426043033599854,
+      "learning_rate": 1.9763993430846392e-07,
+      "logits/chosen": 10.033075332641602,
+      "logits/rejected": 10.728992462158203,
+      "logps/chosen": -76.60491180419922,
+      "logps/ref_chosen": -107.08544158935547,
+      "logps/ref_rejected": -120.38542175292969,
+      "logps/rejected": -89.13396453857422,
+      "loss": 0.2738,
+      "rewards/accuracies": 0.4609375,
+      "rewards/chosen": 0.30480533838272095,
+      "rewards/margins": -0.00770913390442729,
+      "rewards/rejected": 0.31251445412635803,
+      "sft_loss": 0.2737652063369751,
+      "step": 46
+    },
+    {
+      "epoch": 0.6191848497324002,
+      "grad_norm": 0.22300882637500763,
+      "learning_rate": 1.862393078595809e-07,
+      "logits/chosen": 9.950118064880371,
+      "logits/rejected": 11.169652938842773,
+      "logps/chosen": -77.8197021484375,
+      "logps/ref_chosen": -105.74787902832031,
+      "logps/ref_rejected": -122.93606567382812,
+      "logps/rejected": -96.1205062866211,
+      "loss": 0.2652,
+      "rewards/accuracies": 0.5078125,
+      "rewards/chosen": 0.2792818248271942,
+      "rewards/margins": 0.011126276105642319,
+      "rewards/rejected": 0.2681555151939392,
+      "sft_loss": 0.26522505283355713,
+      "step": 47
+    },
+    {
+      "epoch": 0.6323589954713874,
+      "grad_norm": 0.2568327784538269,
+      "learning_rate": 1.7497884124991485e-07,
+      "logits/chosen": 10.451184272766113,
+      "logits/rejected": 11.392836570739746,
+      "logps/chosen": -76.05364990234375,
+      "logps/ref_chosen": -105.3005599975586,
+      "logps/ref_rejected": -123.93569946289062,
+      "logps/rejected": -93.72474670410156,
+      "loss": 0.2697,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.29246923327445984,
+      "rewards/margins": -0.009640296921133995,
+      "rewards/rejected": 0.3021095395088196,
+      "sft_loss": 0.26966598629951477,
+      "step": 48
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 75,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 12,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-60/added_tokens.json

@@ -0,0 +1,13 @@
+{
+  "<|assistant|>": 32001,
+  "<|endoftext|>": 32000,
+  "<|end|>": 32007,
+  "<|placeholder1|>": 32002,
+  "<|placeholder2|>": 32003,
+  "<|placeholder3|>": 32004,
+  "<|placeholder4|>": 32005,
+  "<|placeholder5|>": 32008,
+  "<|placeholder6|>": 32009,
+  "<|system|>": 32006,
+  "<|user|>": 32010
+}

checkpoint-60/config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "/home/v-liuzichuan/huggingface/Phi-3-mini-4k-instruct",
+  "architectures": [
+    "Phi3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_phi3.Phi3Config",
+    "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM"
+  },
+  "bos_token_id": 1,
+  "embd_pdrop": 0.0,
+  "eos_token_id": 32000,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 4096,
+  "model_type": "phi3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "original_max_position_embeddings": 4096,
+  "pad_token_id": 32000,
+  "resid_pdrop": 0.0,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "sliding_window": 2047,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.2",
+  "use_cache": false,
+  "vocab_size": 32064
+}

checkpoint-60/generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    32000,
+    32001,
+    32007
+  ],
+  "pad_token_id": 32000,
+  "transformers_version": "4.45.2"
+}

checkpoint-60/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb3f8d348a1a76ac48fbb63c3eae324259735c08f1cd1575e1cbcd9123810c5f
+size 4972489328

checkpoint-60/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:226d9c00608040eaf7bd64c4ea443a3200fb5e396ab5bc09cf37f7a5ef3a3629
+size 2669692552

checkpoint-60/model.safetensors.index.json

@@ -0,0 +1,202 @@
+{
+  "metadata": {
+    "total_size": 7642159104
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

checkpoint-60/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}