{"model_cache_dir": null, "dataset_cache_dir": null, "data_root": "/data/long-llm", "train_data": ["/share/qhj/rags/data/long_context/train/0830.all.30k.jsonl"], "eval_data": null, "model_name_or_path": "/share/qhj/Activation-Beacon/outputs/beacon-qwen2-full_coverage-interleave-2048_2048-2,4,8,16,32-step_random-qkv-redpajama/checkpoint-11436", "padding_side": "left", "no_use_fast": false, "access_token": null, "attn_impl": "flash_attention_2", "max_length": 40000, "chat_template": "qwen", "max_position_embeddings": null, "mistral_sliding_window": null, "rope_theta": null, "rope_method": null, "rope_factor": 1.0, "lora": null, "lora_unload": true, "load_in_4_bit": false, "dtype": "bf16", "device_map": null, "batch_size": 1, "cpu": false, "enable_tp": false, "enable_vllm": false, "vllm_mem": 0.9, "vllm_tp": 1, "vllm_len": null, "vllm_disable_ar": false, "enable_beacon": true, "beacon_window": 2048, "beacon_stride": 2048, "beacon_attn": "full-coverage", "beacon_ratio": [2, 4, 8], "beacon_ratio_mix": "step-random", "beacon_param": ["q", "k", "v"], "beacon_embed_init": "eos", "beacon_sink_size": 0, "beacon_attend_prev": true, "beacon_pos": "interleave", "beacon_parallel_window": null, "retrieval_method": null, "retrieval_topk": null, "retrieval_key_length": null, "max_new_tokens": null, "do_sample": null, "temperature": null, "top_p": null} |