End of training

README.md

@@ -1,12 +1,12 @@
 ---
 library_name: peft
 license: apache-2.0
-base_model: unsloth/SmolLM2-360M-Instruct
+base_model: NousResearch/Yarn-Solar-10b-32k
 tags:
 - axolotl
 - generated_from_trainer
 model-index:
-- name: bece4e2a-1acf-4d5f-9255-9dc0500ce188
+- name: bb4b141f-2c27-4c46-88a8-b7997d6eb489
   results: []
 ---
 
@@ -19,20 +19,19 @@ should probably proofread and complete it, then remove this comment. -->
 axolotl version: `0.4.1`
 ```yaml
 adapter: lora
-base_model: unsloth/SmolLM2-360M-Instruct
+base_model: NousResearch/Yarn-Solar-10b-32k
 bf16: auto
 dataset_prepared_path: null
 datasets:
 - data_files:
-  - 65b7b1b934182bde_train_data.json
+  - 2791f4c87cd58c7c_train_data.json
   ds_type: json
   format: custom
-  path: /root/G.O.D-test/core/data/65b7b1b934182bde_train_data.json
+  path: /root/G.O.D-test/core/data/2791f4c87cd58c7c_train_data.json
   type:
-    field_input: input
     field_instruction: instruction
     field_output: output
-    format: '{instruction} {input}'
+    format: '{instruction}'
     no_input_format: '{instruction}'
     system_format: '{system}'
     system_prompt: ''
@@ -42,13 +41,13 @@ eval_max_new_tokens: 128
 eval_steps: 0
 evals_per_epoch: null
 flash_attention: true
-fp16: false
+fp16: null
 fsdp: null
 fsdp_config: null
-gradient_accumulation_steps: 4
+gradient_accumulation_steps: 6
 gradient_checkpointing: false
 group_by_length: false
-hub_model_id: souging/bece4e2a-1acf-4d5f-9255-9dc0500ce188
+hub_model_id: souging/bb4b141f-2c27-4c46-88a8-b7997d6eb489
 hub_repo: null
 hub_strategy: checkpoint
 hub_token: null
@@ -64,9 +63,9 @@ lora_model_dir: null
 lora_r: 32
 lora_target_linear: true
 lr_scheduler: cosine
-max_steps: 500
-micro_batch_size: 10
-mlflow_experiment_name: /tmp/65b7b1b934182bde_train_data.json
+max_steps: 700
+micro_batch_size: 1
+mlflow_experiment_name: /tmp/2791f4c87cd58c7c_train_data.json
 model_type: AutoModelForCausalLM
 num_epochs: 10
 optimizer: adamw_bnb_8bit
@@ -77,8 +76,9 @@ s2_attention: null
 sample_packing: false
 save_steps: 0
 saves_per_epoch: null
-seed: 20
-sequence_len: 2048
+sequence_len: 1920
+special_tokens:
+  pad_token: </s>
 strict: false
 tf32: false
 tokenizer_type: AutoTokenizer
@@ -86,21 +86,21 @@ train_on_inputs: false
 trust_remote_code: true
 wandb_entity: null
 wandb_mode: online
-wandb_name: 42d66002-df23-4eff-b50f-ada526fd7375
+wandb_name: 0d6c6775-16be-47ce-928c-5e74f826d12a
 wandb_project: Gradients-On-Demand
 wandb_run: your_name
-wandb_runid: 42d66002-df23-4eff-b50f-ada526fd7375
+wandb_runid: 0d6c6775-16be-47ce-928c-5e74f826d12a
 warmup_steps: 100
-weight_decay: 0.0
-xformers_attention: false
+weight_decay: 0.01
+xformers_attention: null
 
 ```
 
 </details><br>
 
-# bece4e2a-1acf-4d5f-9255-9dc0500ce188
+# bb4b141f-2c27-4c46-88a8-b7997d6eb489
 
-This model is a fine-tuned version of [unsloth/SmolLM2-360M-Instruct](https://huggingface.co/unsloth/SmolLM2-360M-Instruct) on the None dataset.
+This model is a fine-tuned version of [NousResearch/Yarn-Solar-10b-32k](https://huggingface.co/NousResearch/Yarn-Solar-10b-32k) on the None dataset.
 
 ## Model description
 
@@ -120,18 +120,18 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.000202
-- train_batch_size: 10
-- eval_batch_size: 10
-- seed: 20
+- train_batch_size: 1
+- eval_batch_size: 1
+- seed: 42
 - distributed_type: multi-GPU
 - num_devices: 8
-- gradient_accumulation_steps: 4
-- total_train_batch_size: 320
-- total_eval_batch_size: 80
+- gradient_accumulation_steps: 6
+- total_train_batch_size: 48
+- total_eval_batch_size: 8
 - optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_steps: 100
-- training_steps: 500
+- training_steps: 700
 
 ### Training results
 

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:abd2868dbb0e5c7bf55bea07e85af59be50add159ee543d2e717a25d30ce4a96
-size 69629450
+oid sha256:f3994fdea994ac2ad5f46d32b8e6e5211aff3e00b064b989500712c0b21ef5ea
+size 503559370