Update README.md

README.md

@@ -190,6 +190,8 @@ details summary:hover {
 - Amount of Tokens: 500 Million
 </div>
 
+
+
 </div>
 
 <div style="border: 2px solid #6e00ff; border-radius: 10px; padding: 20px; margin: 20px 0; box-shadow: 0 0 15px rgba(110, 0, 255, 0.5);">
@@ -197,30 +199,45 @@ details summary:hover {
 ## Axolotl Config  ꒰(˶• ᴗ •˶)꒱
 
 <details>
-```
-base_model: NewEden_Phi4-PT-merged
+  
+```yaml
+base_model: NewEden_Phi-PT-merged-LIT
 model_type: AutoModelForCausalLM
 tokenizer_type: AutoTokenizer
+
 plugins:
   - axolotl.integrations.liger.LigerPlugin
 liger_rope: true
 liger_rms_norm: true
 liger_swiglu: true
 liger_fused_linear_cross_entropy: true
+
+
 load_in_8bit: false
 load_in_4bit: false
 strict: false
+
 datasets:
-  - path: NewEden/Orion-LIT
-    type: completion
-    field: text
+  - path: PocketDoc/Dans-MemoryCore-CoreCurriculum-Small
+    type: dan-chat-advanced
+  - path: Nitral-AI/ARES-ShareGPT
+    type: dan-chat-advanced
+  - path: Gryphe/Sonnet3.5-SlimOrcaDedupCleaned-20k
+    type: dan-chat-advanced
+  - path: NewEden/Claude-Instruct-2.7K
+    type: dan-chat-advanced
+  - path: NewEden/Claude-Instruct-5K
+    type: dan-chat-advanced
+
 shuffle_merged_datasets: true
 dataset_prepared_path: prepared_data
 val_set_size: 0.0
-output_dir: ./phi4-ptv2-out-r1
+output_dir: ./phi4-inst-out-r2
+
 sequence_len: 16384
 sample_packing: true
 pad_to_sequence_len: true
+
 adapter: lora
 lora_model_dir:
 lora_r: 128
@@ -234,25 +251,31 @@ lora_target_modules:
   - v_proj
   - k_proj
   - o_proj
+
 lora_modules_to_save:
  - embed_tokens
  - lm_head
+
+
 wandb_project: mag-phi
 wandb_entity:
 wandb_watch:
-wandb_name: comp-v2-attempt-01
+wandb_name: inst-attempt-02
 wandb_log_model:
+
 gradient_accumulation_steps: 4
 micro_batch_size: 2
-num_epochs: 1
+num_epochs: 4
 optimizer: paged_ademamix_8bit
 lr_scheduler: cosine
-learning_rate: 0.00002
+learning_rate: 0.000025
+
 train_on_inputs: false
 group_by_length: false
 bf16: auto
 fp16:
 tf32: false
+
 gradient_checkpointing: unsloth
 early_stopping_patience:
 resume_from_checkpoint:
@@ -260,17 +283,19 @@ local_rank:
 logging_steps: 1
 xformers_attention:
 flash_attention: true
+
 warmup_steps: 15
 evals_per_epoch: 4
 eval_table_size:
 eval_max_new_tokens: 128
-saves_per_epoch: 4
+saves_per_epoch: 2
 debug:
 deepspeed: /workspace/axolotl/deepspeed_configs/zero3_bf16_cpuoffload_params.json
 weight_decay: 0.01
 fsdp:
 fsdp_config:
 ```
+
 </details>
 </div>