Upload 20 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,60 @@
----
-license: mit
----
+---
+library_name: peft
+license: other
+base_model: google/gemma-3-12b-it
+tags:
+- llama-factory
+- lora
+- generated_from_trainer
+model-index:
+- name: train_2025-05-14-10-29-44
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# train_2025-05-14-10-29-44
+
+This model is a fine-tuned version of [google/gemma-3-12b-it](https://huggingface.co/google/gemma-3-12b-it) on the top_5_training_dataset dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 5
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 5
+- total_train_batch_size: 25
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine_with_restarts
+- lr_scheduler_warmup_steps: 150
+- num_epochs: 4.0
+- label_smoothing_factor: 0.05
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.15.1
+- Transformers 4.51.3
+- Pytorch 2.6.0+cu124
+- Datasets 3.5.0
+- Tokenizers 0.21.1

adapter_config.json

@@ -0,0 +1,180 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "google/gemma-3-12b-it",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 64,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "44.self_attn.q_proj",
+    "40.self_attn.q_proj",
+    "27.self_attn.k_proj",
+    "31.self_attn.q_proj",
+    "39.self_attn.k_proj",
+    "model.layers.18.self_attn.k_proj",
+    "41.self_attn.k_proj",
+    "31.self_attn.v_proj",
+    "model.layers.23.self_attn.q_proj",
+    "34.self_attn.q_proj",
+    "model.layers.15.self_attn.q_proj",
+    "model.layers.23.self_attn.v_proj",
+    "27.self_attn.v_proj",
+    "model.layers.21.self_attn.v_proj",
+    "30.self_attn.q_proj",
+    "model.layers.11.self_attn.k_proj",
+    "model.layers.22.self_attn.q_proj",
+    "model.layers.10.self_attn.k_proj",
+    "model.layers.2.self_attn.k_proj",
+    "37.self_attn.q_proj",
+    "27.self_attn.q_proj",
+    "37.self_attn.k_proj",
+    "model.layers.25.self_attn.v_proj",
+    "model.layers.19.self_attn.q_proj",
+    "model.layers.25.self_attn.k_proj",
+    "34.self_attn.v_proj",
+    "42.self_attn.k_proj",
+    "model.layers.21.self_attn.q_proj",
+    "45.self_attn.k_proj",
+    "model.layers.14.self_attn.v_proj",
+    "35.self_attn.q_proj",
+    "model.layers.6.self_attn.v_proj",
+    "model.layers.15.self_attn.k_proj",
+    "model.layers.25.self_attn.q_proj",
+    "35.self_attn.k_proj",
+    "model.layers.20.self_attn.k_proj",
+    "model.layers.13.self_attn.k_proj",
+    "model.layers.17.self_attn.k_proj",
+    "32.self_attn.v_proj",
+    "model.layers.9.self_attn.q_proj",
+    "o_proj",
+    "35.self_attn.v_proj",
+    "model.layers.26.self_attn.q_proj",
+    "28.self_attn.v_proj",
+    "model.layers.7.self_attn.q_proj",
+    "37.self_attn.v_proj",
+    "40.self_attn.v_proj",
+    "model.layers.13.self_attn.q_proj",
+    "model.layers.1.self_attn.q_proj",
+    "33.self_attn.v_proj",
+    "33.self_attn.q_proj",
+    "42.self_attn.v_proj",
+    "model.layers.10.self_attn.q_proj",
+    "45.self_attn.v_proj",
+    "44.self_attn.k_proj",
+    "model.layers.14.self_attn.k_proj",
+    "model.layers.8.self_attn.k_proj",
+    "model.layers.12.self_attn.k_proj",
+    "model.layers.0.self_attn.v_proj",
+    "model.layers.20.self_attn.v_proj",
+    "model.layers.26.self_attn.v_proj",
+    "33.self_attn.k_proj",
+    "model.layers.12.self_attn.v_proj",
+    "36.self_attn.v_proj",
+    "model.layers.24.self_attn.v_proj",
+    "model.layers.4.self_attn.v_proj",
+    "46.self_attn.k_proj",
+    "46.self_attn.q_proj",
+    "29.self_attn.v_proj",
+    "32.self_attn.q_proj",
+    "model.layers.7.self_attn.k_proj",
+    "model.layers.16.self_attn.k_proj",
+    "model.layers.22.self_attn.k_proj",
+    "model.layers.26.self_attn.k_proj",
+    "41.self_attn.v_proj",
+    "model.layers.24.self_attn.k_proj",
+    "29.self_attn.q_proj",
+    "39.self_attn.q_proj",
+    "39.self_attn.v_proj",
+    "model.layers.12.self_attn.q_proj",
+    "47.self_attn.v_proj",
+    "model.layers.1.self_attn.k_proj",
+    "model.layers.11.self_attn.q_proj",
+    "model.layers.13.self_attn.v_proj",
+    "38.self_attn.k_proj",
+    "model.layers.4.self_attn.q_proj",
+    "model.layers.9.self_attn.k_proj",
+    "43.self_attn.q_proj",
+    "model.layers.17.self_attn.v_proj",
+    "38.self_attn.v_proj",
+    "model.layers.0.self_attn.k_proj",
+    "47.self_attn.k_proj",
+    "model.layers.6.self_attn.k_proj",
+    "model.layers.5.self_attn.q_proj",
+    "model.layers.24.self_attn.q_proj",
+    "model.layers.18.self_attn.v_proj",
+    "30.self_attn.k_proj",
+    "40.self_attn.k_proj",
+    "model.layers.14.self_attn.q_proj",
+    "model.layers.3.self_attn.q_proj",
+    "model.layers.21.self_attn.k_proj",
+    "36.self_attn.k_proj",
+    "34.self_attn.k_proj",
+    "model.layers.1.self_attn.v_proj",
+    "up_proj",
+    "model.layers.20.self_attn.q_proj",
+    "43.self_attn.k_proj",
+    "model.layers.3.self_attn.v_proj",
+    "model.layers.11.self_attn.v_proj",
+    "model.layers.23.self_attn.k_proj",
+    "31.self_attn.k_proj",
+    "model.layers.19.self_attn.k_proj",
+    "47.self_attn.q_proj",
+    "model.layers.22.self_attn.v_proj",
+    "32.self_attn.k_proj",
+    "model.layers.15.self_attn.v_proj",
+    "42.self_attn.q_proj",
+    "model.layers.8.self_attn.v_proj",
+    "model.layers.2.self_attn.v_proj",
+    "46.self_attn.v_proj",
+    "model.layers.4.self_attn.k_proj",
+    "model.layers.2.self_attn.q_proj",
+    "28.self_attn.q_proj",
+    "30.self_attn.v_proj",
+    "model.layers.18.self_attn.q_proj",
+    "model.layers.17.self_attn.q_proj",
+    "model.layers.10.self_attn.v_proj",
+    "44.self_attn.v_proj",
+    "down_proj",
+    "model.layers.5.self_attn.v_proj",
+    "model.layers.16.self_attn.q_proj",
+    "model.layers.19.self_attn.v_proj",
+    "gate_proj",
+    "model.layers.16.self_attn.v_proj",
+    "29.self_attn.k_proj",
+    "36.self_attn.q_proj",
+    "model.layers.8.self_attn.q_proj",
+    "model.layers.7.self_attn.v_proj",
+    "28.self_attn.k_proj",
+    "model.layers.6.self_attn.q_proj",
+    "model.layers.0.self_attn.q_proj",
+    "38.self_attn.q_proj",
+    "45.self_attn.q_proj",
+    "model.layers.3.self_attn.k_proj",
+    "43.self_attn.v_proj",
+    "model.layers.9.self_attn.v_proj",
+    "41.self_attn.q_proj",
+    "model.layers.5.self_attn.k_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76ddf7912ece96c7aecac3d8beac981c665f5bfff70274b114b37081f2b61e50
+size 523864568

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image_soft_token>": 262144
+}

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.9809417040358746,
+    "num_input_tokens_seen": 9067184,
+    "total_flos": 6.151178707859082e+17,
+    "train_loss": 1.3522842801019046,
+    "train_runtime": 6892.301,
+    "train_samples_per_second": 2.588,
+    "train_steps_per_second": 0.103
+}

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "{{ bos_token }}\n{%- if messages[0]['role'] == 'system' -%}\n    {%- if messages[0]['content'] is string -%}\n        {%- set first_user_prefix = messages[0]['content'] + '\n\n' -%}\n    {%- else -%}\n        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' -%}\n    {%- endif -%}\n    {%- set loop_messages = messages[1:] -%}\n{%- else -%}\n    {%- set first_user_prefix = \"\" -%}\n    {%- set loop_messages = messages -%}\n{%- endif -%}\n{%- for message in loop_messages -%}\n    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}\n        {{ raise_exception(\"Conversation roles must alternate user/assistant/user/assistant/...\") }}\n    {%- endif -%}\n    {%- if (message['role'] == 'assistant') -%}\n        {%- set role = \"model\" -%}\n    {%- else -%}\n        {%- set role = message['role'] -%}\n    {%- endif -%}\n    {{ '<start_of_turn>' + role + '\n' + (first_user_prefix if loop.first else \"\") }}\n    {%- if message['content'] is string -%}\n        {{ message['content'] | trim }}\n    {%- elif message['content'] is iterable -%}\n        {%- for item in message['content'] -%}\n            {%- if item['type'] == 'image' -%}\n                {{ '<start_of_image>' }}\n            {%- elif item['type'] == 'text' -%}\n                {{ item['text'] | trim }}\n            {%- endif -%}\n        {%- endfor -%}\n    {%- else -%}\n        {{ raise_exception(\"Invalid content type\") }}\n    {%- endif -%}\n    {{ '<end_of_turn>\n' }}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n    {{'<start_of_turn>model\n'}}\n{%- endif -%}\n"
+}

llamaboard_config.yaml

@@ -0,0 +1,78 @@
+top.booster: auto
+top.checkpoint_path: []
+top.finetuning_type: lora
+top.model_name: Gemma-3-12B-Instruct
+top.quantization_bit: none
+top.quantization_method: bnb
+top.rope_scaling: none
+top.template: alpaca
+train.additional_target: ''
+train.apollo_rank: 16
+train.apollo_scale: 32
+train.apollo_target: all
+train.apollo_update_interval: 200
+train.badam_mode: layer
+train.badam_switch_interval: 50
+train.badam_switch_mode: ascending
+train.badam_update_ratio: 0.05
+train.batch_size: 5
+train.compute_type: bf16
+train.create_new_adapter: true
+train.cutoff_len: 512
+train.dataset:
+- top_5_training_dataset
+train.dataset_dir: data
+train.ds_offload: false
+train.ds_stage: none
+train.extra_args: '{"optim":"adamw_torch","weight_decay":0.01,"label_smoothing_factor":0.05}'
+train.freeze_extra_modules: ''
+train.freeze_trainable_layers: 4
+train.freeze_trainable_modules: all
+train.galore_rank: 16
+train.galore_scale: 2
+train.galore_target: all
+train.galore_update_interval: 200
+train.gradient_accumulation_steps: 5
+train.learning_rate: 2e-5
+train.logging_steps: 25
+train.lora_alpha: 64
+train.lora_dropout: 0.05
+train.lora_rank: 32
+train.lora_target: q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj
+train.loraplus_lr_ratio: 8
+train.lr_scheduler_type: cosine_with_restarts
+train.mask_history: false
+train.max_grad_norm: '0.3'
+train.max_samples: '100000'
+train.neat_packing: false
+train.neftune_alpha: 0
+train.num_train_epochs: '4'
+train.packing: false
+train.ppo_score_norm: false
+train.ppo_whiten_rewards: false
+train.pref_beta: 0.1
+train.pref_ftx: 0
+train.pref_loss: sigmoid
+train.report_to:
+- none
+train.resize_vocab: false
+train.reward_model: []
+train.save_steps: 250
+train.swanlab_api_key: ''
+train.swanlab_link: ''
+train.swanlab_mode: cloud
+train.swanlab_project: llamafactory
+train.swanlab_run_name: ''
+train.swanlab_workspace: ''
+train.train_on_prompt: false
+train.training_stage: Supervised Fine-Tuning
+train.use_apollo: false
+train.use_badam: false
+train.use_dora: false
+train.use_galore: false
+train.use_llama_pro: false
+train.use_pissa: false
+train.use_rslora: false
+train.use_swanlab: false
+train.val_size: 0
+train.warmup_steps: 150

preprocessor_config.json

@@ -0,0 +1,29 @@
+{
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_pan_and_scan": null,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Gemma3ImageProcessor",
+  "image_seq_length": 256,
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "pan_and_scan_max_num_crops": null,
+  "pan_and_scan_min_crop_size": null,
+  "pan_and_scan_min_ratio_to_activate": null,
+  "processor_class": "Gemma3Processor",
+  "resample": 2,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 896,
+    "width": 896
+  }
+}

processor_config.json

@@ -0,0 +1,4 @@
+{
+  "image_seq_length": 256,
+  "processor_class": "Gemma3Processor"
+}

special_tokens_map.json

@@ -0,0 +1,33 @@
+{
+  "boi_token": "<start_of_image>",
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eoi_token": "<end_of_image>",
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "image_token": "<image_soft_token>",
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
+size 33384568

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1299c11d7cf632ef3b4e11937501358ada021bbdf7c47638d13c0ee982f2e79c
+size 4689074

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.9809417040358746,
+    "num_input_tokens_seen": 9067184,
+    "total_flos": 6.151178707859082e+17,
+    "train_loss": 1.3522842801019046,
+    "train_runtime": 6892.301,
+    "train_samples_per_second": 2.588,
+    "train_steps_per_second": 0.103
+}

trainer_log.jsonl

@@ -0,0 +1,29 @@
+{"current_steps": 25, "total_steps": 712, "loss": 7.1534, "lr": 3.2000000000000003e-06, "epoch": 0.14013452914798205, "percentage": 3.51, "elapsed_time": "0:04:01", "remaining_time": "1:50:44", "throughput": 1316.08, "total_tokens": 318240}
+{"current_steps": 50, "total_steps": 712, "loss": 1.9472, "lr": 6.533333333333334e-06, "epoch": 0.2802690582959641, "percentage": 7.02, "elapsed_time": "0:08:02", "remaining_time": "1:46:30", "throughput": 1319.26, "total_tokens": 636760}
+{"current_steps": 75, "total_steps": 712, "loss": 1.5046, "lr": 9.866666666666668e-06, "epoch": 0.4204035874439462, "percentage": 10.53, "elapsed_time": "0:12:04", "remaining_time": "1:42:30", "throughput": 1320.9, "total_tokens": 956520}
+{"current_steps": 100, "total_steps": 712, "loss": 1.3971, "lr": 1.3200000000000002e-05, "epoch": 0.5605381165919282, "percentage": 14.04, "elapsed_time": "0:16:05", "remaining_time": "1:38:30", "throughput": 1321.21, "total_tokens": 1275920}
+{"current_steps": 125, "total_steps": 712, "loss": 1.3768, "lr": 1.6533333333333333e-05, "epoch": 0.7006726457399103, "percentage": 17.56, "elapsed_time": "0:20:07", "remaining_time": "1:34:28", "throughput": 1321.53, "total_tokens": 1595280}
+{"current_steps": 150, "total_steps": 712, "loss": 1.3278, "lr": 1.9866666666666667e-05, "epoch": 0.8408071748878924, "percentage": 21.07, "elapsed_time": "0:24:08", "remaining_time": "1:30:27", "throughput": 1321.82, "total_tokens": 1914720}
+{"current_steps": 175, "total_steps": 712, "loss": 1.2927, "lr": 1.9910139651840497e-05, "epoch": 0.9809417040358744, "percentage": 24.58, "elapsed_time": "0:28:09", "remaining_time": "1:26:24", "throughput": 1322.18, "total_tokens": 2233680}
+{"current_steps": 200, "total_steps": 712, "loss": 1.2029, "lr": 1.962720313575358e-05, "epoch": 1.1177130044843049, "percentage": 28.09, "elapsed_time": "0:32:04", "remaining_time": "1:22:07", "throughput": 1322.22, "total_tokens": 2544928}
+{"current_steps": 225, "total_steps": 712, "loss": 1.2199, "lr": 1.915655103523529e-05, "epoch": 1.257847533632287, "percentage": 31.6, "elapsed_time": "0:36:06", "remaining_time": "1:18:08", "throughput": 1322.25, "total_tokens": 2864008}
+{"current_steps": 250, "total_steps": 712, "loss": 1.2112, "lr": 1.8507360338956896e-05, "epoch": 1.397982062780269, "percentage": 35.11, "elapsed_time": "0:40:07", "remaining_time": "1:14:08", "throughput": 1322.45, "total_tokens": 3183248}
+{"current_steps": 275, "total_steps": 712, "loss": 1.221, "lr": 1.7692289262315e-05, "epoch": 1.5381165919282511, "percentage": 38.62, "elapsed_time": "0:44:20", "remaining_time": "1:10:27", "throughput": 1316.5, "total_tokens": 3502528}
+{"current_steps": 300, "total_steps": 712, "loss": 1.1758, "lr": 1.6727230431791816e-05, "epoch": 1.6782511210762332, "percentage": 42.13, "elapsed_time": "0:48:21", "remaining_time": "1:06:25", "throughput": 1316.99, "total_tokens": 3821648}
+{"current_steps": 325, "total_steps": 712, "loss": 1.1828, "lr": 1.563100100329731e-05, "epoch": 1.8183856502242153, "percentage": 45.65, "elapsed_time": "0:52:22", "remaining_time": "1:02:21", "throughput": 1317.42, "total_tokens": 4140008}
+{"current_steps": 350, "total_steps": 712, "loss": 1.1948, "lr": 1.442497575670668e-05, "epoch": 1.9585201793721974, "percentage": 49.16, "elapsed_time": "0:56:23", "remaining_time": "0:58:19", "throughput": 1317.74, "total_tokens": 4458448}
+{"current_steps": 375, "total_steps": 712, "loss": 1.07, "lr": 1.313267032068285e-05, "epoch": 2.0952914798206277, "percentage": 52.67, "elapsed_time": "1:00:18", "remaining_time": "0:54:12", "throughput": 1318.08, "total_tokens": 4770136}
+{"current_steps": 400, "total_steps": 712, "loss": 1.0581, "lr": 1.1779282654255685e-05, "epoch": 2.2354260089686098, "percentage": 56.18, "elapsed_time": "1:04:19", "remaining_time": "0:50:10", "throughput": 1318.28, "total_tokens": 5087896}
+{"current_steps": 425, "total_steps": 712, "loss": 1.048, "lr": 1.0391201725558842e-05, "epoch": 2.375560538116592, "percentage": 59.69, "elapsed_time": "1:08:21", "remaining_time": "0:46:09", "throughput": 1318.52, "total_tokens": 5407296}
+{"current_steps": 450, "total_steps": 712, "loss": 1.0218, "lr": 8.99549296772945e-06, "epoch": 2.515695067264574, "percentage": 63.2, "elapsed_time": "1:12:22", "remaining_time": "0:42:08", "throughput": 1318.81, "total_tokens": 5726896}
+{"current_steps": 475, "total_steps": 712, "loss": 1.0206, "lr": 7.619370544785608e-06, "epoch": 2.655829596412556, "percentage": 66.71, "elapsed_time": "1:16:23", "remaining_time": "0:38:07", "throughput": 1319.13, "total_tokens": 6046856}
+{"current_steps": 500, "total_steps": 712, "loss": 1.0277, "lr": 6.289666717481497e-06, "epoch": 2.795964125560538, "percentage": 70.22, "elapsed_time": "1:20:25", "remaining_time": "0:34:05", "throughput": 1319.31, "total_tokens": 6366216}
+{"current_steps": 525, "total_steps": 712, "loss": 1.0494, "lr": 5.032308655686011e-06, "epoch": 2.93609865470852, "percentage": 73.74, "elapsed_time": "1:24:39", "remaining_time": "0:30:09", "throughput": 1316.22, "total_tokens": 6685296}
+{"current_steps": 550, "total_steps": 712, "loss": 0.9487, "lr": 3.8718128986350154e-06, "epoch": 3.0728699551569507, "percentage": 77.25, "elapsed_time": "1:28:34", "remaining_time": "0:26:05", "throughput": 1316.47, "total_tokens": 6996704}
+{"current_steps": 575, "total_steps": 712, "loss": 0.9103, "lr": 2.8308073203011667e-06, "epoch": 3.213004484304933, "percentage": 80.76, "elapsed_time": "1:32:36", "remaining_time": "0:22:03", "throughput": 1316.77, "total_tokens": 7316144}
+{"current_steps": 600, "total_steps": 712, "loss": 0.9102, "lr": 1.929589920817806e-06, "epoch": 3.353139013452915, "percentage": 84.27, "elapsed_time": "1:36:37", "remaining_time": "0:18:02", "throughput": 1317.02, "total_tokens": 7635704}
+{"current_steps": 625, "total_steps": 712, "loss": 0.9034, "lr": 1.1857330468424466e-06, "epoch": 3.493273542600897, "percentage": 87.78, "elapsed_time": "1:40:39", "remaining_time": "0:14:00", "throughput": 1317.22, "total_tokens": 7955304}
+{"current_steps": 650, "total_steps": 712, "loss": 0.8949, "lr": 6.137407579511212e-07, "epoch": 3.633408071748879, "percentage": 91.29, "elapsed_time": "1:44:40", "remaining_time": "0:09:59", "throughput": 1317.47, "total_tokens": 8274744}
+{"current_steps": 675, "total_steps": 712, "loss": 0.9073, "lr": 2.2476601988947965e-07, "epoch": 3.773542600896861, "percentage": 94.8, "elapsed_time": "1:48:42", "remaining_time": "0:05:57", "throughput": 1317.67, "total_tokens": 8594304}
+{"current_steps": 700, "total_steps": 712, "loss": 0.8967, "lr": 2.639323897518975e-08, "epoch": 3.913677130044843, "percentage": 98.31, "elapsed_time": "1:52:43", "remaining_time": "0:01:55", "throughput": 1317.87, "total_tokens": 8913944}
+{"current_steps": 712, "total_steps": 712, "epoch": 3.9809417040358746, "percentage": 100.0, "elapsed_time": "1:54:52", "remaining_time": "0:00:00", "throughput": 1315.55, "total_tokens": 9067184}

trainer_state.json

@@ -0,0 +1,268 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.9809417040358746,
+  "eval_steps": 500,
+  "global_step": 712,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.14013452914798205,
+      "grad_norm": 14.509355545043945,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 7.1534,
+      "num_input_tokens_seen": 318240,
+      "step": 25
+    },
+    {
+      "epoch": 0.2802690582959641,
+      "grad_norm": 6.642310619354248,
+      "learning_rate": 6.533333333333334e-06,
+      "loss": 1.9472,
+      "num_input_tokens_seen": 636760,
+      "step": 50
+    },
+    {
+      "epoch": 0.4204035874439462,
+      "grad_norm": 3.3247640132904053,
+      "learning_rate": 9.866666666666668e-06,
+      "loss": 1.5046,
+      "num_input_tokens_seen": 956520,
+      "step": 75
+    },
+    {
+      "epoch": 0.5605381165919282,
+      "grad_norm": 2.3405237197875977,
+      "learning_rate": 1.3200000000000002e-05,
+      "loss": 1.3971,
+      "num_input_tokens_seen": 1275920,
+      "step": 100
+    },
+    {
+      "epoch": 0.7006726457399103,
+      "grad_norm": 3.028830051422119,
+      "learning_rate": 1.6533333333333333e-05,
+      "loss": 1.3768,
+      "num_input_tokens_seen": 1595280,
+      "step": 125
+    },
+    {
+      "epoch": 0.8408071748878924,
+      "grad_norm": 1.428653597831726,
+      "learning_rate": 1.9866666666666667e-05,
+      "loss": 1.3278,
+      "num_input_tokens_seen": 1914720,
+      "step": 150
+    },
+    {
+      "epoch": 0.9809417040358744,
+      "grad_norm": 1.9624038934707642,
+      "learning_rate": 1.9910139651840497e-05,
+      "loss": 1.2927,
+      "num_input_tokens_seen": 2233680,
+      "step": 175
+    },
+    {
+      "epoch": 1.1177130044843049,
+      "grad_norm": 1.6478056907653809,
+      "learning_rate": 1.962720313575358e-05,
+      "loss": 1.2029,
+      "num_input_tokens_seen": 2544928,
+      "step": 200
+    },
+    {
+      "epoch": 1.257847533632287,
+      "grad_norm": 2.1640427112579346,
+      "learning_rate": 1.915655103523529e-05,
+      "loss": 1.2199,
+      "num_input_tokens_seen": 2864008,
+      "step": 225
+    },
+    {
+      "epoch": 1.397982062780269,
+      "grad_norm": 2.563964605331421,
+      "learning_rate": 1.8507360338956896e-05,
+      "loss": 1.2112,
+      "num_input_tokens_seen": 3183248,
+      "step": 250
+    },
+    {
+      "epoch": 1.5381165919282511,
+      "grad_norm": 1.9103755950927734,
+      "learning_rate": 1.7692289262315e-05,
+      "loss": 1.221,
+      "num_input_tokens_seen": 3502528,
+      "step": 275
+    },
+    {
+      "epoch": 1.6782511210762332,
+      "grad_norm": 2.564152240753174,
+      "learning_rate": 1.6727230431791816e-05,
+      "loss": 1.1758,
+      "num_input_tokens_seen": 3821648,
+      "step": 300
+    },
+    {
+      "epoch": 1.8183856502242153,
+      "grad_norm": 1.57261061668396,
+      "learning_rate": 1.563100100329731e-05,
+      "loss": 1.1828,
+      "num_input_tokens_seen": 4140008,
+      "step": 325
+    },
+    {
+      "epoch": 1.9585201793721974,
+      "grad_norm": 1.8220309019088745,
+      "learning_rate": 1.442497575670668e-05,
+      "loss": 1.1948,
+      "num_input_tokens_seen": 4458448,
+      "step": 350
+    },
+    {
+      "epoch": 2.0952914798206277,
+      "grad_norm": 2.028656005859375,
+      "learning_rate": 1.313267032068285e-05,
+      "loss": 1.07,
+      "num_input_tokens_seen": 4770136,
+      "step": 375
+    },
+    {
+      "epoch": 2.2354260089686098,
+      "grad_norm": 3.713899850845337,
+      "learning_rate": 1.1779282654255685e-05,
+      "loss": 1.0581,
+      "num_input_tokens_seen": 5087896,
+      "step": 400
+    },
+    {
+      "epoch": 2.375560538116592,
+      "grad_norm": 2.000035285949707,
+      "learning_rate": 1.0391201725558842e-05,
+      "loss": 1.048,
+      "num_input_tokens_seen": 5407296,
+      "step": 425
+    },
+    {
+      "epoch": 2.515695067264574,
+      "grad_norm": 1.5742937326431274,
+      "learning_rate": 8.99549296772945e-06,
+      "loss": 1.0218,
+      "num_input_tokens_seen": 5726896,
+      "step": 450
+    },
+    {
+      "epoch": 2.655829596412556,
+      "grad_norm": 1.7964842319488525,
+      "learning_rate": 7.619370544785608e-06,
+      "loss": 1.0206,
+      "num_input_tokens_seen": 6046856,
+      "step": 475
+    },
+    {
+      "epoch": 2.795964125560538,
+      "grad_norm": 1.5667575597763062,
+      "learning_rate": 6.289666717481497e-06,
+      "loss": 1.0277,
+      "num_input_tokens_seen": 6366216,
+      "step": 500
+    },
+    {
+      "epoch": 2.93609865470852,
+      "grad_norm": 1.8910105228424072,
+      "learning_rate": 5.032308655686011e-06,
+      "loss": 1.0494,
+      "num_input_tokens_seen": 6685296,
+      "step": 525
+    },
+    {
+      "epoch": 3.0728699551569507,
+      "grad_norm": 1.1843669414520264,
+      "learning_rate": 3.8718128986350154e-06,
+      "loss": 0.9487,
+      "num_input_tokens_seen": 6996704,
+      "step": 550
+    },
+    {
+      "epoch": 3.213004484304933,
+      "grad_norm": 1.5657708644866943,
+      "learning_rate": 2.8308073203011667e-06,
+      "loss": 0.9103,
+      "num_input_tokens_seen": 7316144,
+      "step": 575
+    },
+    {
+      "epoch": 3.353139013452915,
+      "grad_norm": 1.8331918716430664,
+      "learning_rate": 1.929589920817806e-06,
+      "loss": 0.9102,
+      "num_input_tokens_seen": 7635704,
+      "step": 600
+    },
+    {
+      "epoch": 3.493273542600897,
+      "grad_norm": 1.651666283607483,
+      "learning_rate": 1.1857330468424466e-06,
+      "loss": 0.9034,
+      "num_input_tokens_seen": 7955304,
+      "step": 625
+    },
+    {
+      "epoch": 3.633408071748879,
+      "grad_norm": 1.1031330823898315,
+      "learning_rate": 6.137407579511212e-07,
+      "loss": 0.8949,
+      "num_input_tokens_seen": 8274744,
+      "step": 650
+    },
+    {
+      "epoch": 3.773542600896861,
+      "grad_norm": 3.233630895614624,
+      "learning_rate": 2.2476601988947965e-07,
+      "loss": 0.9073,
+      "num_input_tokens_seen": 8594304,
+      "step": 675
+    },
+    {
+      "epoch": 3.913677130044843,
+      "grad_norm": 1.2947144508361816,
+      "learning_rate": 2.639323897518975e-08,
+      "loss": 0.8967,
+      "num_input_tokens_seen": 8913944,
+      "step": 700
+    },
+    {
+      "epoch": 3.9809417040358746,
+      "num_input_tokens_seen": 9067184,
+      "step": 712,
+      "total_flos": 6.151178707859082e+17,
+      "train_loss": 1.3522842801019046,
+      "train_runtime": 6892.301,
+      "train_samples_per_second": 2.588,
+      "train_steps_per_second": 0.103
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 712,
+  "num_input_tokens_seen": 9067184,
+  "num_train_epochs": 4,
+  "save_steps": 250,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.151178707859082e+17,
+  "train_batch_size": 5,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:178998ea29350ea4266d5c48226840b2656edd32cdff377cd45b2dc2b85dace9
+size 5752

training_args.yaml

@@ -0,0 +1,37 @@
+bf16: true
+create_new_adapter: true
+cutoff_len: 512
+dataset: top_5_training_dataset
+dataset_dir: data
+ddp_timeout: 180000000
+do_train: true
+finetuning_type: lora
+flash_attn: auto
+gradient_accumulation_steps: 5
+include_num_input_tokens_seen: true
+label_smoothing_factor: 0.05
+learning_rate: 2.0e-05
+logging_steps: 25
+lora_alpha: 64
+lora_dropout: 0.05
+lora_rank: 32
+lora_target: q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj
+loraplus_lr_ratio: 8
+lr_scheduler_type: cosine_with_restarts
+max_grad_norm: 0.3
+max_samples: 100000
+model_name_or_path: google/gemma-3-12b-it
+num_train_epochs: 4.0
+optim: adamw_torch
+output_dir: saves/Gemma-3-12B-Instruct/lora/train_2025-05-14-10-29-44
+packing: false
+per_device_train_batch_size: 5
+plot_loss: true
+preprocessing_num_workers: 16
+report_to: none
+save_steps: 250
+stage: sft
+template: alpaca
+trust_remote_code: true
+warmup_steps: 150
+weight_decay: 0.01