Upload 11 files

adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen2.5-0.5B-Instruct",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "k_proj",
+    "down_proj",
+    "up_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

all_results.json

@@ -0,0 +1,13 @@
+{
+    "epoch": 0.9998876025626616,
+    "eval_accuracy": 0.9947849346546027,
+    "eval_loss": 0.012421553954482079,
+    "eval_runtime": 845.4917,
+    "eval_samples_per_second": 175.686,
+    "eval_steps_per_second": 2.745,
+    "total_flos": 2.593644880773251e+18,
+    "train_loss": 0.03529202291290823,
+    "train_runtime": 23401.2469,
+    "train_samples_per_second": 72.997,
+    "train_steps_per_second": 0.143
+}

eval_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.9998876025626616,
+    "eval_accuracy": 0.9947849346546027,
+    "eval_loss": 0.012421553954482079,
+    "eval_runtime": 845.4917,
+    "eval_samples_per_second": 175.686,
+    "eval_steps_per_second": 2.745
+}

llamaboard_config.yaml

@@ -0,0 +1,67 @@
+top.booster: auto
+top.checkpoint_path: []
+top.finetuning_type: lora
+top.model_name: Qwen2.5-0.5B-Instruct
+top.quantization_bit: none
+top.quantization_method: bitsandbytes
+top.rope_scaling: none
+top.template: qwen
+train.additional_target: ''
+train.badam_mode: layer
+train.badam_switch_interval: 50
+train.badam_switch_mode: ascending
+train.badam_update_ratio: 0.05
+train.batch_size: 8
+train.compute_type: fp16
+train.create_new_adapter: false
+train.cutoff_len: 1024
+train.dataset:
+- JB_Detect_v2
+train.dataset_dir: data
+train.ds_offload: false
+train.ds_stage: none
+train.extra_args: "{\"optim\": \"adamw_torch\", \"do_sample\": \"false\",  \"max_new_tokens\"\
+  :\n  2,  \"compute_accuracy\": \"true\"}"
+train.freeze_extra_modules: ''
+train.freeze_trainable_layers: 2
+train.freeze_trainable_modules: all
+train.galore_rank: 16
+train.galore_scale: 0.25
+train.galore_target: all
+train.galore_update_interval: 200
+train.gradient_accumulation_steps: 8
+train.learning_rate: 5e-5
+train.logging_steps: 100
+train.lora_alpha: 128
+train.lora_dropout: 0.05
+train.lora_rank: 64
+train.lora_target: ''
+train.loraplus_lr_ratio: 0
+train.lr_scheduler_type: cosine
+train.mask_history: false
+train.max_grad_norm: '1.0'
+train.max_samples: '10000000000'
+train.neat_packing: false
+train.neftune_alpha: 0
+train.num_train_epochs: '1.0'
+train.packing: false
+train.ppo_score_norm: false
+train.ppo_whiten_rewards: false
+train.pref_beta: 0.1
+train.pref_ftx: 0
+train.pref_loss: sigmoid
+train.report_to: false
+train.resize_vocab: false
+train.reward_model: null
+train.save_steps: 5000
+train.shift_attn: false
+train.train_on_prompt: false
+train.training_stage: Supervised Fine-Tuning
+train.use_badam: false
+train.use_dora: false
+train.use_galore: false
+train.use_llama_pro: false
+train.use_pissa: false
+train.use_rslora: false
+train.val_size: 0.08
+train.warmup_steps: 1239

running_log.txt

@@ -0,0 +1,374 @@
+[WARNING|2025-05-29 20:33:25] logging.py:162 >> `ddp_find_unused_parameters` needs to be set as False for LoRA in DDP training.
+
+[INFO|2025-05-29 20:33:25] parser.py:355 >> Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: torch.float16
+
+[INFO|2025-05-29 20:33:25] configuration_utils.py:679 >> loading configuration file config.json from cache at /home/aiscuser/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/config.json
+
+[INFO|2025-05-29 20:33:25] configuration_utils.py:746 >> Model config Qwen2Config {
+  "_name_or_path": "Qwen/Qwen2.5-0.5B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}
+
+
+[INFO|2025-05-29 20:33:26] parser.py:355 >> Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: torch.float16
+
+[INFO|2025-05-29 20:33:26] parser.py:355 >> Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: torch.float16
+
+[INFO|2025-05-29 20:33:26] parser.py:355 >> Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: torch.float16
+
+[INFO|2025-05-29 20:33:26] parser.py:355 >> Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: torch.float16
+
+[INFO|2025-05-29 20:33:26] parser.py:355 >> Process rank: 7, device: cuda:7, n_gpu: 1, distributed training: True, compute dtype: torch.float16
+
+[INFO|2025-05-29 20:33:26] parser.py:355 >> Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: torch.float16
+
+[INFO|2025-05-29 20:33:26] parser.py:355 >> Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: torch.float16
+
+[INFO|2025-05-29 20:33:25] tokenization_utils_base.py:2211 >> loading file vocab.json from cache at /home/aiscuser/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/vocab.json
+
+[INFO|2025-05-29 20:33:25] tokenization_utils_base.py:2211 >> loading file merges.txt from cache at /home/aiscuser/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/merges.txt
+
+[INFO|2025-05-29 20:33:25] tokenization_utils_base.py:2211 >> loading file tokenizer.json from cache at /home/aiscuser/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/tokenizer.json
+
+[INFO|2025-05-29 20:33:25] tokenization_utils_base.py:2211 >> loading file added_tokens.json from cache at None
+
+[INFO|2025-05-29 20:33:25] tokenization_utils_base.py:2211 >> loading file special_tokens_map.json from cache at None
+
+[INFO|2025-05-29 20:33:25] tokenization_utils_base.py:2211 >> loading file tokenizer_config.json from cache at /home/aiscuser/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/tokenizer_config.json
+
+[INFO|2025-05-29 20:33:26] tokenization_utils_base.py:2475 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|2025-05-29 20:33:26] configuration_utils.py:679 >> loading configuration file config.json from cache at /home/aiscuser/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/config.json
+
+[INFO|2025-05-29 20:33:26] configuration_utils.py:746 >> Model config Qwen2Config {
+  "_name_or_path": "Qwen/Qwen2.5-0.5B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}
+
+
+[INFO|2025-05-29 20:33:26] tokenization_utils_base.py:2211 >> loading file vocab.json from cache at /home/aiscuser/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/vocab.json
+
+[INFO|2025-05-29 20:33:26] tokenization_utils_base.py:2211 >> loading file merges.txt from cache at /home/aiscuser/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/merges.txt
+
+[INFO|2025-05-29 20:33:26] tokenization_utils_base.py:2211 >> loading file tokenizer.json from cache at /home/aiscuser/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/tokenizer.json
+
+[INFO|2025-05-29 20:33:26] tokenization_utils_base.py:2211 >> loading file added_tokens.json from cache at None
+
+[INFO|2025-05-29 20:33:26] tokenization_utils_base.py:2211 >> loading file special_tokens_map.json from cache at None
+
+[INFO|2025-05-29 20:33:26] tokenization_utils_base.py:2211 >> loading file tokenizer_config.json from cache at /home/aiscuser/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/tokenizer_config.json
+
+[INFO|2025-05-29 20:33:26] tokenization_utils_base.py:2475 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|2025-05-29 20:33:26] logging.py:157 >> Replace eos token: <|im_end|>
+
+[INFO|2025-05-29 20:33:26] logging.py:157 >> Loading dataset JB_Detect_v2.json...
+
+[INFO|2025-05-29 20:35:38] configuration_utils.py:679 >> loading configuration file config.json from cache at /home/aiscuser/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/config.json
+
+[INFO|2025-05-29 20:35:38] configuration_utils.py:746 >> Model config Qwen2Config {
+  "_name_or_path": "Qwen/Qwen2.5-0.5B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}
+
+
+[INFO|2025-05-29 20:35:39] modeling_utils.py:3937 >> loading weights file model.safetensors from cache at /home/aiscuser/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/model.safetensors
+
+[INFO|2025-05-29 20:35:39] modeling_utils.py:1670 >> Instantiating Qwen2ForCausalLM model under default dtype torch.float16.
+
+[INFO|2025-05-29 20:35:39] configuration_utils.py:1096 >> Generate config GenerationConfig {
+  "bos_token_id": 151643,
+  "eos_token_id": 151645
+}
+
+
+[INFO|2025-05-29 20:35:41] modeling_utils.py:4800 >> All model checkpoint weights were used when initializing Qwen2ForCausalLM.
+
+
+[INFO|2025-05-29 20:35:41] modeling_utils.py:4808 >> All the weights of Qwen2ForCausalLM were initialized from the model checkpoint at Qwen/Qwen2.5-0.5B-Instruct.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use Qwen2ForCausalLM for predictions without further training.
+
+[INFO|2025-05-29 20:35:41] configuration_utils.py:1051 >> loading configuration file generation_config.json from cache at /home/aiscuser/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/generation_config.json
+
+[INFO|2025-05-29 20:35:41] configuration_utils.py:1096 >> Generate config GenerationConfig {
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8
+}
+
+
+[INFO|2025-05-29 20:35:41] logging.py:157 >> Gradient checkpointing enabled.
+
+[INFO|2025-05-29 20:35:41] logging.py:157 >> Using torch SDPA for faster training and inference.
+
+[INFO|2025-05-29 20:35:41] logging.py:157 >> Upcasting trainable params to float32.
+
+[INFO|2025-05-29 20:35:41] logging.py:157 >> Fine-tuning method: LoRA
+
+[INFO|2025-05-29 20:35:41] logging.py:157 >> Found linear modules: gate_proj,v_proj,o_proj,k_proj,down_proj,up_proj,q_proj
+
+[INFO|2025-05-29 20:35:41] logging.py:157 >> trainable params: 35,192,832 || all params: 529,225,600 || trainable%: 6.6499
+
+[INFO|2025-05-29 20:35:41] trainer.py:698 >> Using auto half precision backend
+
+[INFO|2025-05-29 20:35:43] trainer.py:2313 >> ***** Running training *****
+
+[INFO|2025-05-29 20:35:43] trainer.py:2314 >>   Num examples = 1,708,215
+
+[INFO|2025-05-29 20:35:43] trainer.py:2315 >>   Num Epochs = 1
+
+[INFO|2025-05-29 20:35:43] trainer.py:2316 >>   Instantaneous batch size per device = 8
+
+[INFO|2025-05-29 20:35:43] trainer.py:2319 >>   Total train batch size (w. parallel, distributed & accumulation) = 512
+
+[INFO|2025-05-29 20:35:43] trainer.py:2320 >>   Gradient Accumulation steps = 8
+
+[INFO|2025-05-29 20:35:43] trainer.py:2321 >>   Total optimization steps = 3,336
+
+[INFO|2025-05-29 20:35:43] trainer.py:2322 >>   Number of trainable parameters = 35,192,832
+
+[INFO|2025-05-29 20:47:26] logging.py:157 >> {'loss': 0.5614, 'learning_rate': 3.8337e-06, 'epoch': 0.03}
+
+[INFO|2025-05-29 20:59:03] logging.py:157 >> {'loss': 0.0905, 'learning_rate': 7.8692e-06, 'epoch': 0.06}
+
+[INFO|2025-05-29 21:10:50] logging.py:157 >> {'loss': 0.0370, 'learning_rate': 1.1905e-05, 'epoch': 0.09}
+
+[INFO|2025-05-29 21:22:41] logging.py:157 >> {'loss': 0.0295, 'learning_rate': 1.5940e-05, 'epoch': 0.12}
+
+[INFO|2025-05-29 21:34:27] logging.py:157 >> {'loss': 0.0266, 'learning_rate': 1.9976e-05, 'epoch': 0.15}
+
+[INFO|2025-05-29 21:46:06] logging.py:157 >> {'loss': 0.0224, 'learning_rate': 2.4011e-05, 'epoch': 0.18}
+
+[INFO|2025-05-29 21:57:49] logging.py:157 >> {'loss': 0.0203, 'learning_rate': 2.8047e-05, 'epoch': 0.21}
+
+[INFO|2025-05-29 22:09:37] logging.py:157 >> {'loss': 0.0187, 'learning_rate': 3.2082e-05, 'epoch': 0.24}
+
+[INFO|2025-05-29 22:21:20] logging.py:157 >> {'loss': 0.0186, 'learning_rate': 3.6118e-05, 'epoch': 0.27}
+
+[INFO|2025-05-29 22:32:56] logging.py:157 >> {'loss': 0.0180, 'learning_rate': 4.0153e-05, 'epoch': 0.30}
+
+[INFO|2025-05-29 22:44:39] logging.py:157 >> {'loss': 0.0168, 'learning_rate': 4.4189e-05, 'epoch': 0.33}
+
+[INFO|2025-05-29 22:56:14] logging.py:157 >> {'loss': 0.0173, 'learning_rate': 4.8224e-05, 'epoch': 0.36}
+
+[INFO|2025-05-29 23:07:51] logging.py:157 >> {'loss': 0.0167, 'learning_rate': 4.9912e-05, 'epoch': 0.39}
+
+[INFO|2025-05-29 23:19:38] logging.py:157 >> {'loss': 0.0168, 'learning_rate': 4.9320e-05, 'epoch': 0.42}
+
+[INFO|2025-05-29 23:31:13] logging.py:157 >> {'loss': 0.0163, 'learning_rate': 4.8184e-05, 'epoch': 0.45}
+
+[INFO|2025-05-29 23:42:47] logging.py:157 >> {'loss': 0.0152, 'learning_rate': 4.6528e-05, 'epoch': 0.48}
+
+[INFO|2025-05-29 23:54:24] logging.py:157 >> {'loss': 0.0149, 'learning_rate': 4.4390e-05, 'epoch': 0.51}
+
+[INFO|2025-05-30 00:06:14] logging.py:157 >> {'loss': 0.0147, 'learning_rate': 4.1817e-05, 'epoch': 0.54}
+
+[INFO|2025-05-30 00:17:56] logging.py:157 >> {'loss': 0.0150, 'learning_rate': 3.8868e-05, 'epoch': 0.57}
+
+[INFO|2025-05-30 00:29:40] logging.py:157 >> {'loss': 0.0148, 'learning_rate': 3.5608e-05, 'epoch': 0.60}
+
+[INFO|2025-05-30 00:41:19] logging.py:157 >> {'loss': 0.0135, 'learning_rate': 3.2110e-05, 'epoch': 0.63}
+
+[INFO|2025-05-30 00:52:53] logging.py:157 >> {'loss': 0.0140, 'learning_rate': 2.8453e-05, 'epoch': 0.66}
+
+[INFO|2025-05-30 01:04:33] logging.py:157 >> {'loss': 0.0145, 'learning_rate': 2.4719e-05, 'epoch': 0.69}
+
+[INFO|2025-05-30 01:16:20] logging.py:157 >> {'loss': 0.0139, 'learning_rate': 2.0991e-05, 'epoch': 0.72}
+
+[INFO|2025-05-30 01:27:56] logging.py:157 >> {'loss': 0.0135, 'learning_rate': 1.7353e-05, 'epoch': 0.75}
+
+[INFO|2025-05-30 01:39:51] logging.py:157 >> {'loss': 0.0132, 'learning_rate': 1.3886e-05, 'epoch': 0.78}
+
+[INFO|2025-05-30 01:51:27] logging.py:157 >> {'loss': 0.0133, 'learning_rate': 1.0668e-05, 'epoch': 0.81}
+
+[INFO|2025-05-30 02:03:01] logging.py:157 >> {'loss': 0.0127, 'learning_rate': 7.7714e-06, 'epoch': 0.84}
+
+[INFO|2025-05-30 02:14:38] logging.py:157 >> {'loss': 0.0126, 'learning_rate': 5.2606e-06, 'epoch': 0.87}
+
+[INFO|2025-05-30 02:26:21] logging.py:157 >> {'loss': 0.0123, 'learning_rate': 3.1919e-06, 'epoch': 0.90}
+
+[INFO|2025-05-30 02:38:08] logging.py:157 >> {'loss': 0.0129, 'learning_rate': 1.6118e-06, 'epoch': 0.93}
+
+[INFO|2025-05-30 02:49:49] logging.py:157 >> {'loss': 0.0126, 'learning_rate': 5.5569e-07, 'epoch': 0.96}
+
+[INFO|2025-05-30 03:01:32] logging.py:157 >> {'loss': 0.0129, 'learning_rate': 4.7146e-08, 'epoch': 0.99}
+
+[INFO|2025-05-30 03:05:43] trainer.py:3801 >> Saving model checkpoint to saves/Qwen2.5-0.5B-Instruct/lora/train_2025-05-29-20-20-04_2/checkpoint-3336
+
+[INFO|2025-05-30 03:05:43] configuration_utils.py:679 >> loading configuration file config.json from cache at /home/aiscuser/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/config.json
+
+[INFO|2025-05-30 03:05:43] configuration_utils.py:746 >> Model config Qwen2Config {
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}
+
+
+[INFO|2025-05-30 03:05:43] tokenization_utils_base.py:2646 >> tokenizer config file saved in saves/Qwen2.5-0.5B-Instruct/lora/train_2025-05-29-20-20-04_2/checkpoint-3336/tokenizer_config.json
+
+[INFO|2025-05-30 03:05:43] tokenization_utils_base.py:2655 >> Special tokens file saved in saves/Qwen2.5-0.5B-Instruct/lora/train_2025-05-29-20-20-04_2/checkpoint-3336/special_tokens_map.json
+
+[INFO|2025-05-30 03:05:44] trainer.py:2584 >> 
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
+[INFO|2025-05-30 03:05:44] trainer.py:3801 >> Saving model checkpoint to saves/Qwen2.5-0.5B-Instruct/lora/train_2025-05-29-20-20-04_2
+
+[INFO|2025-05-30 03:05:44] configuration_utils.py:679 >> loading configuration file config.json from cache at /home/aiscuser/.cache/huggingface/hub/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775/config.json
+
+[INFO|2025-05-30 03:05:44] configuration_utils.py:746 >> Model config Qwen2Config {
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}
+
+
+[INFO|2025-05-30 03:05:44] tokenization_utils_base.py:2646 >> tokenizer config file saved in saves/Qwen2.5-0.5B-Instruct/lora/train_2025-05-29-20-20-04_2/tokenizer_config.json
+
+[INFO|2025-05-30 03:05:44] tokenization_utils_base.py:2655 >> Special tokens file saved in saves/Qwen2.5-0.5B-Instruct/lora/train_2025-05-29-20-20-04_2/special_tokens_map.json
+
+[WARNING|2025-05-30 03:05:45] logging.py:162 >> No metric eval_loss to plot.
+
+[WARNING|2025-05-30 03:05:45] logging.py:162 >> No metric eval_accuracy to plot.
+
+[INFO|2025-05-30 03:05:45] trainer.py:4117 >> 
+***** Running Evaluation *****
+
+[INFO|2025-05-30 03:05:45] trainer.py:4119 >>   Num examples = 148541
+
+[INFO|2025-05-30 03:05:45] trainer.py:4122 >>   Batch size = 8
+
+[INFO|2025-05-30 03:19:50] modelcard.py:449 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}, 'metrics': [{'name': 'Accuracy', 'type': 'accuracy', 'value': 0.9947849346546027}]}
+

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.9998876025626616,
+    "total_flos": 2.593644880773251e+18,
+    "train_loss": 0.03529202291290823,
+    "train_runtime": 23401.2469,
+    "train_samples_per_second": 72.997,
+    "train_steps_per_second": 0.143
+}

trainer_log.jsonl

@@ -0,0 +1,34 @@
+{"current_steps": 100, "total_steps": 3336, "loss": 0.5614, "lr": 3.833736884584342e-06, "epoch": 0.029972649956914316, "percentage": 3.0, "elapsed_time": "0:11:43", "remaining_time": "6:19:30"}
+{"current_steps": 200, "total_steps": 3336, "loss": 0.0905, "lr": 7.869249394673125e-06, "epoch": 0.05994529991382863, "percentage": 6.0, "elapsed_time": "0:23:20", "remaining_time": "6:05:55"}
+{"current_steps": 300, "total_steps": 3336, "loss": 0.037, "lr": 1.1904761904761905e-05, "epoch": 0.08991794987074295, "percentage": 8.99, "elapsed_time": "0:35:07", "remaining_time": "5:55:31"}
+{"current_steps": 400, "total_steps": 3336, "loss": 0.0295, "lr": 1.5940274414850686e-05, "epoch": 0.11989059982765726, "percentage": 11.99, "elapsed_time": "0:46:58", "remaining_time": "5:44:47"}
+{"current_steps": 500, "total_steps": 3336, "loss": 0.0266, "lr": 1.9975786924939468e-05, "epoch": 0.14986324978457158, "percentage": 14.99, "elapsed_time": "0:58:44", "remaining_time": "5:33:11"}
+{"current_steps": 600, "total_steps": 3336, "loss": 0.0224, "lr": 2.401129943502825e-05, "epoch": 0.1798358997414859, "percentage": 17.99, "elapsed_time": "1:10:23", "remaining_time": "5:20:59"}
+{"current_steps": 700, "total_steps": 3336, "loss": 0.0203, "lr": 2.804681194511703e-05, "epoch": 0.2098085496984002, "percentage": 20.98, "elapsed_time": "1:22:06", "remaining_time": "5:09:10"}
+{"current_steps": 800, "total_steps": 3336, "loss": 0.0187, "lr": 3.2082324455205814e-05, "epoch": 0.23978119965531453, "percentage": 23.98, "elapsed_time": "1:33:53", "remaining_time": "4:57:39"}
+{"current_steps": 900, "total_steps": 3336, "loss": 0.0186, "lr": 3.6117836965294597e-05, "epoch": 0.26975384961222887, "percentage": 26.98, "elapsed_time": "1:45:37", "remaining_time": "4:45:52"}
+{"current_steps": 1000, "total_steps": 3336, "loss": 0.018, "lr": 4.015334947538338e-05, "epoch": 0.29972649956914316, "percentage": 29.98, "elapsed_time": "1:57:13", "remaining_time": "4:33:51"}
+{"current_steps": 1100, "total_steps": 3336, "loss": 0.0168, "lr": 4.4188861985472154e-05, "epoch": 0.3296991495260575, "percentage": 32.97, "elapsed_time": "2:08:56", "remaining_time": "4:22:06"}
+{"current_steps": 1200, "total_steps": 3336, "loss": 0.0173, "lr": 4.8224374495560936e-05, "epoch": 0.3596717994829718, "percentage": 35.97, "elapsed_time": "2:20:31", "remaining_time": "4:10:07"}
+{"current_steps": 1300, "total_steps": 3336, "loss": 0.0167, "lr": 4.99120705806192e-05, "epoch": 0.38964444943988613, "percentage": 38.97, "elapsed_time": "2:32:08", "remaining_time": "3:58:17"}
+{"current_steps": 1400, "total_steps": 3336, "loss": 0.0168, "lr": 4.932035139761111e-05, "epoch": 0.4196170993968004, "percentage": 41.97, "elapsed_time": "2:43:55", "remaining_time": "3:46:41"}
+{"current_steps": 1500, "total_steps": 3336, "loss": 0.0163, "lr": 4.8183803112054984e-05, "epoch": 0.44958974935371476, "percentage": 44.96, "elapsed_time": "2:55:30", "remaining_time": "3:34:49"}
+{"current_steps": 1600, "total_steps": 3336, "loss": 0.0152, "lr": 4.652788689317074e-05, "epoch": 0.47956239931062905, "percentage": 47.96, "elapsed_time": "3:07:04", "remaining_time": "3:22:58"}
+{"current_steps": 1700, "total_steps": 3336, "loss": 0.0149, "lr": 4.438969888865293e-05, "epoch": 0.5095350492675433, "percentage": 50.96, "elapsed_time": "3:18:41", "remaining_time": "3:11:12"}
+{"current_steps": 1800, "total_steps": 3336, "loss": 0.0147, "lr": 4.1817139189768106e-05, "epoch": 0.5395076992244577, "percentage": 53.96, "elapsed_time": "3:30:31", "remaining_time": "2:59:38"}
+{"current_steps": 1900, "total_steps": 3336, "loss": 0.015, "lr": 3.886783876446901e-05, "epoch": 0.569480349181372, "percentage": 56.95, "elapsed_time": "3:42:13", "remaining_time": "2:47:57"}
+{"current_steps": 2000, "total_steps": 3336, "loss": 0.0148, "lr": 3.560786839757242e-05, "epoch": 0.5994529991382863, "percentage": 59.95, "elapsed_time": "3:53:56", "remaining_time": "2:36:16"}
+{"current_steps": 2100, "total_steps": 3336, "loss": 0.0135, "lr": 3.211025856056666e-05, "epoch": 0.6294256490952006, "percentage": 62.95, "elapsed_time": "4:05:36", "remaining_time": "2:24:33"}
+{"current_steps": 2200, "total_steps": 3336, "loss": 0.014, "lr": 2.8453363369204606e-05, "epoch": 0.659398299052115, "percentage": 65.95, "elapsed_time": "4:17:10", "remaining_time": "2:12:47"}
+{"current_steps": 2300, "total_steps": 3336, "loss": 0.0145, "lr": 2.4719105279813023e-05, "epoch": 0.6893709490090293, "percentage": 68.94, "elapsed_time": "4:28:50", "remaining_time": "2:01:05"}
+{"current_steps": 2400, "total_steps": 3336, "loss": 0.0139, "lr": 2.0991139846962775e-05, "epoch": 0.7193435989659436, "percentage": 71.94, "elapsed_time": "4:40:37", "remaining_time": "1:49:26"}
+{"current_steps": 2500, "total_steps": 3336, "loss": 0.0135, "lr": 1.7352981655945167e-05, "epoch": 0.7493162489228579, "percentage": 74.94, "elapsed_time": "4:52:13", "remaining_time": "1:37:43"}
+{"current_steps": 2600, "total_steps": 3336, "loss": 0.0132, "lr": 1.3886133413268552e-05, "epoch": 0.7792888988797723, "percentage": 77.94, "elapsed_time": "5:04:08", "remaining_time": "1:26:05"}
+{"current_steps": 2700, "total_steps": 3336, "loss": 0.0133, "lr": 1.0668260107643305e-05, "epoch": 0.8092615488366866, "percentage": 80.94, "elapsed_time": "5:15:44", "remaining_time": "1:14:22"}
+{"current_steps": 2800, "total_steps": 3336, "loss": 0.0127, "lr": 7.771449144245214e-06, "epoch": 0.8392341987936008, "percentage": 83.93, "elapsed_time": "5:27:17", "remaining_time": "1:02:39"}
+{"current_steps": 2900, "total_steps": 3336, "loss": 0.0126, "lr": 5.260595429058021e-06, "epoch": 0.8692068487505151, "percentage": 86.93, "elapsed_time": "5:38:55", "remaining_time": "0:50:57"}
+{"current_steps": 3000, "total_steps": 3336, "loss": 0.0123, "lr": 3.191947580940455e-06, "epoch": 0.8991794987074295, "percentage": 89.93, "elapsed_time": "5:50:38", "remaining_time": "0:39:16"}
+{"current_steps": 3100, "total_steps": 3336, "loss": 0.0129, "lr": 1.61184783944949e-06, "epoch": 0.9291521486643438, "percentage": 92.93, "elapsed_time": "6:02:25", "remaining_time": "0:27:35"}
+{"current_steps": 3200, "total_steps": 3336, "loss": 0.0126, "lr": 5.556938972425402e-07, "epoch": 0.9591247986212581, "percentage": 95.92, "elapsed_time": "6:14:06", "remaining_time": "0:15:53"}
+{"current_steps": 3300, "total_steps": 3336, "loss": 0.0129, "lr": 4.714591428512938e-08, "epoch": 0.9890974485781724, "percentage": 98.92, "elapsed_time": "6:25:49", "remaining_time": "0:04:12"}
+{"current_steps": 3336, "total_steps": 3336, "epoch": 0.9998876025626616, "percentage": 100.0, "elapsed_time": "6:30:01", "remaining_time": "0:00:00"}

trainer_state.json

@@ -0,0 +1,273 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9998876025626616,
+  "eval_steps": 5000,
+  "global_step": 3336,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.029972649956914316,
+      "grad_norm": 4.414547443389893,
+      "learning_rate": 3.833736884584342e-06,
+      "loss": 0.5614,
+      "step": 100
+    },
+    {
+      "epoch": 0.05994529991382863,
+      "grad_norm": 7.150412559509277,
+      "learning_rate": 7.869249394673125e-06,
+      "loss": 0.0905,
+      "step": 200
+    },
+    {
+      "epoch": 0.08991794987074295,
+      "grad_norm": 7.323289394378662,
+      "learning_rate": 1.1904761904761905e-05,
+      "loss": 0.037,
+      "step": 300
+    },
+    {
+      "epoch": 0.11989059982765726,
+      "grad_norm": 2.585188627243042,
+      "learning_rate": 1.5940274414850686e-05,
+      "loss": 0.0295,
+      "step": 400
+    },
+    {
+      "epoch": 0.14986324978457158,
+      "grad_norm": 2.407606840133667,
+      "learning_rate": 1.9975786924939468e-05,
+      "loss": 0.0266,
+      "step": 500
+    },
+    {
+      "epoch": 0.1798358997414859,
+      "grad_norm": 2.0257647037506104,
+      "learning_rate": 2.401129943502825e-05,
+      "loss": 0.0224,
+      "step": 600
+    },
+    {
+      "epoch": 0.2098085496984002,
+      "grad_norm": 1.106999397277832,
+      "learning_rate": 2.804681194511703e-05,
+      "loss": 0.0203,
+      "step": 700
+    },
+    {
+      "epoch": 0.23978119965531453,
+      "grad_norm": 0.4099288582801819,
+      "learning_rate": 3.2082324455205814e-05,
+      "loss": 0.0187,
+      "step": 800
+    },
+    {
+      "epoch": 0.26975384961222887,
+      "grad_norm": 0.54881751537323,
+      "learning_rate": 3.6117836965294597e-05,
+      "loss": 0.0186,
+      "step": 900
+    },
+    {
+      "epoch": 0.29972649956914316,
+      "grad_norm": 0.1522761881351471,
+      "learning_rate": 4.015334947538338e-05,
+      "loss": 0.018,
+      "step": 1000
+    },
+    {
+      "epoch": 0.3296991495260575,
+      "grad_norm": 0.1916244626045227,
+      "learning_rate": 4.4188861985472154e-05,
+      "loss": 0.0168,
+      "step": 1100
+    },
+    {
+      "epoch": 0.3596717994829718,
+      "grad_norm": 0.36830204725265503,
+      "learning_rate": 4.8224374495560936e-05,
+      "loss": 0.0173,
+      "step": 1200
+    },
+    {
+      "epoch": 0.38964444943988613,
+      "grad_norm": 0.3123892843723297,
+      "learning_rate": 4.99120705806192e-05,
+      "loss": 0.0167,
+      "step": 1300
+    },
+    {
+      "epoch": 0.4196170993968004,
+      "grad_norm": 0.16122741997241974,
+      "learning_rate": 4.932035139761111e-05,
+      "loss": 0.0168,
+      "step": 1400
+    },
+    {
+      "epoch": 0.44958974935371476,
+      "grad_norm": 0.28243348002433777,
+      "learning_rate": 4.8183803112054984e-05,
+      "loss": 0.0163,
+      "step": 1500
+    },
+    {
+      "epoch": 0.47956239931062905,
+      "grad_norm": 0.16183069348335266,
+      "learning_rate": 4.652788689317074e-05,
+      "loss": 0.0152,
+      "step": 1600
+    },
+    {
+      "epoch": 0.5095350492675433,
+      "grad_norm": 0.1692025065422058,
+      "learning_rate": 4.438969888865293e-05,
+      "loss": 0.0149,
+      "step": 1700
+    },
+    {
+      "epoch": 0.5395076992244577,
+      "grad_norm": 0.14949053525924683,
+      "learning_rate": 4.1817139189768106e-05,
+      "loss": 0.0147,
+      "step": 1800
+    },
+    {
+      "epoch": 0.569480349181372,
+      "grad_norm": 0.39853450655937195,
+      "learning_rate": 3.886783876446901e-05,
+      "loss": 0.015,
+      "step": 1900
+    },
+    {
+      "epoch": 0.5994529991382863,
+      "grad_norm": 0.17293961346149445,
+      "learning_rate": 3.560786839757242e-05,
+      "loss": 0.0148,
+      "step": 2000
+    },
+    {
+      "epoch": 0.6294256490952006,
+      "grad_norm": 0.14277803897857666,
+      "learning_rate": 3.211025856056666e-05,
+      "loss": 0.0135,
+      "step": 2100
+    },
+    {
+      "epoch": 0.659398299052115,
+      "grad_norm": 0.2294313907623291,
+      "learning_rate": 2.8453363369204606e-05,
+      "loss": 0.014,
+      "step": 2200
+    },
+    {
+      "epoch": 0.6893709490090293,
+      "grad_norm": 0.12665383517742157,
+      "learning_rate": 2.4719105279813023e-05,
+      "loss": 0.0145,
+      "step": 2300
+    },
+    {
+      "epoch": 0.7193435989659436,
+      "grad_norm": 0.17277060449123383,
+      "learning_rate": 2.0991139846962775e-05,
+      "loss": 0.0139,
+      "step": 2400
+    },
+    {
+      "epoch": 0.7493162489228579,
+      "grad_norm": 0.1914021372795105,
+      "learning_rate": 1.7352981655945167e-05,
+      "loss": 0.0135,
+      "step": 2500
+    },
+    {
+      "epoch": 0.7792888988797723,
+      "grad_norm": 0.18266354501247406,
+      "learning_rate": 1.3886133413268552e-05,
+      "loss": 0.0132,
+      "step": 2600
+    },
+    {
+      "epoch": 0.8092615488366866,
+      "grad_norm": 0.1917112022638321,
+      "learning_rate": 1.0668260107643305e-05,
+      "loss": 0.0133,
+      "step": 2700
+    },
+    {
+      "epoch": 0.8392341987936008,
+      "grad_norm": 0.13778215646743774,
+      "learning_rate": 7.771449144245214e-06,
+      "loss": 0.0127,
+      "step": 2800
+    },
+    {
+      "epoch": 0.8692068487505151,
+      "grad_norm": 0.2762675881385803,
+      "learning_rate": 5.260595429058021e-06,
+      "loss": 0.0126,
+      "step": 2900
+    },
+    {
+      "epoch": 0.8991794987074295,
+      "grad_norm": 0.1501600295305252,
+      "learning_rate": 3.191947580940455e-06,
+      "loss": 0.0123,
+      "step": 3000
+    },
+    {
+      "epoch": 0.9291521486643438,
+      "grad_norm": 0.21935300529003143,
+      "learning_rate": 1.61184783944949e-06,
+      "loss": 0.0129,
+      "step": 3100
+    },
+    {
+      "epoch": 0.9591247986212581,
+      "grad_norm": 0.18737538158893585,
+      "learning_rate": 5.556938972425402e-07,
+      "loss": 0.0126,
+      "step": 3200
+    },
+    {
+      "epoch": 0.9890974485781724,
+      "grad_norm": 0.17027494311332703,
+      "learning_rate": 4.714591428512938e-08,
+      "loss": 0.0129,
+      "step": 3300
+    },
+    {
+      "epoch": 0.9998876025626616,
+      "step": 3336,
+      "total_flos": 2.593644880773251e+18,
+      "train_loss": 0.03529202291290823,
+      "train_runtime": 23401.2469,
+      "train_samples_per_second": 72.997,
+      "train_steps_per_second": 0.143
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 3336,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 5000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.593644880773251e+18,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d148aa73f06a1bf63006c2a81d020364084537c8071629cfa138c444c978304f
+size 5432

training_args.yaml

@@ -0,0 +1,38 @@
+compute_accuracy: 'true'
+cutoff_len: 1024
+dataset: JB_Detect_v2
+dataset_dir: data
+ddp_timeout: 180000000
+do_sample: 'false'
+do_train: true
+eval_steps: 5000
+eval_strategy: steps
+finetuning_type: lora
+flash_attn: auto
+fp16: true
+gradient_accumulation_steps: 8
+learning_rate: 5.0e-05
+logging_steps: 100
+lora_alpha: 128
+lora_dropout: 0.05
+lora_rank: 64
+lora_target: all
+lr_scheduler_type: cosine
+max_grad_norm: 1.0
+max_new_tokens: 2
+max_samples: 10000000000
+model_name_or_path: Qwen/Qwen2.5-0.5B-Instruct
+num_train_epochs: 1.0
+optim: adamw_torch
+output_dir: saves/Qwen2.5-0.5B-Instruct/lora/train_2025-05-29-20-20-04_2
+packing: false
+per_device_eval_batch_size: 8
+per_device_train_batch_size: 8
+plot_loss: true
+preprocessing_num_workers: 16
+report_to: none
+save_steps: 5000
+stage: sft
+template: qwen
+val_size: 0.08
+warmup_steps: 1239