{"mode": "finetune", "wandb": "True", "batch_size": "12", "context_length": "2048", "flash_atten": "True", "tokenizer": "TIGER-Lab/MAmmoTH-13B", "model": "TIGER-Lab/MAmmoTH-13B", "dataset": "approach0/retrieval-augment-finetune", "dataset_shuffle": "True", "dataset_map_fn": "datamap_double_train_for_query_and_answer", "collate_fn": "collate_final_dataset_for_generalist", "collate_add_eos": "True", "collate_debug": "True", "eval_during_train": "no", "peft": "{\n\"peft_attach_new\": true,\n\"peft_lora_rank\": 16,\n\"peft_lora_alpha\": 16,\n\"peft_lora_targets\": [\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\", \"gate_proj\", \"down_proj\", \"up_proj\"]\n}", "trainer": "[\n\"--output_dir\", \"<exp_outdir>\",\n\"--save_strategy\", \"steps\",\n\"--save_steps\", 100,\n\"--save_total_limit\", 2,\n\"--evaluation_strategy\", \"no\",\n\"--eval_steps\", 10,\n\"--report_to\", \"wandb\",\n\"--logging_steps\", 1,\n\"--per_device_train_batch_size\", 1,\n\"--remove_unused_columns\", false,\n\"--gradient_accumulation_steps\", 12,\n\"--learning_rate\", 2e-5,\n\"--warmup_steps\", 10,\n\"--optim\", \"adamw_torch\",\n\"--num_train_epochs\", 2,\n\"--fp16\", false,\n\"--bf16\", true,\n\"--seed\", 70\n]", "load_in_8bit": "True", "device_map": null, "deepspeed": "{\n\"en_param_offload\": true\n}", "local_rank": "0", "run": "GCR-13B-mammoth-generalist", "7b_vicuna_v1_5": "lmsys/vicuna-7b-v1.5", "7b_vicuna_v1_5_32k": "lmsys/longchat-7b-v1.5-32k", "test_lora_repo": "Lajonbot/vicuna-7b-v1.5-PL-lora_adapter_model", "13b_mathy_fft": "approach0/mathy-vicuna-13B-FFT", "7b_wizardmath": "WizardLM/WizardMath-7B-V1.0", "13b_wizardmath": "WizardLM/WizardMath-13B-V1.0", "70b_wizardmath": "WizardLM/WizardMath-70B-V1.0", "7b_mammoth": "TIGER-Lab/MAmmoTH-7B", "13b_mammoth": "TIGER-Lab/MAmmoTH-13B", "34b_mammoth_code": "TIGER-Lab/MAmmoTH-Coder-13B", "70b_mammoth": "TIGER-Lab/MAmmoTH-70B", "7b_tora": "llm-agents/tora-7b-v1.0", "13b_tora": "llm-agents/tora-13b-v1.0", "seed": "70", "output_dir": "./output", "add_sys_paths": "[\"../Progressive-Hint\", \"../math/modeling\"]"}