Model save

README.md

@@ -0,0 +1,56 @@
+---
+tags:
+- generated_from_trainer
+model-index:
+- name: ft-moe-llava-qwen1.5-1.8b-vista-1ep
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# ft-moe-llava-qwen1.5-1.8b-vista-1ep
+
+This model was trained from scratch on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 128
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.03
+- num_epochs: 1.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.37.0
+- Pytorch 2.0.1+cu117
+- Datasets 2.20.0
+- Tokenizers 0.15.1

config.json

@@ -1,7 +1,7 @@
 {
   "_name_or_path": "/home/users/astar/ares/stunvat/scratch/checkpoints/moe-llava-qwen1.5-1.8b-ft-mergedlora",
   "architectures": [
-    "MoELLaVAQwen1_5ForCausalLM"
+    "LlavaQwen1_5ForCausalLM"
   ],
   "attention_dropout": 0.0,
   "bos_token_id": 151643,
@@ -79,10 +79,10 @@
   "sliding_window": 32768,
   "tie_word_embeddings": false,
   "tokenizer_padding_side": "right",
-  "torch_dtype": "bfloat16",
+  "torch_dtype": "float16",
   "transformers_version": "4.37.0",
   "tune_mm_mlp_adapter": false,
-  "use_cache": false,
+  "use_cache": true,
   "use_mm_proj": true,
   "use_sliding_window": false,
   "video_global_proj": false,

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.37.0"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d8b64014a55c0ee57c309887a809ba63a7c3eb44a8505474220f87dd50dff0e
+size 4995418608

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0e34ee4f386d58d941ca6e846a1a5313c44938c46677ed5c097281bc0471975
+size 1308341240

model.safetensors.index.json

@@ -0,0 +1,630 @@
+{
+  "metadata": {
+    "total_size": 6303674880
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.embeddings.patch_embedding.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.embeddings.patch_embedding.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.embeddings.position_embedding.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.head.attention.in_proj_bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.head.attention.in_proj_weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.head.attention.out_proj.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.head.attention.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.head.layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.head.layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.head.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.head.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.head.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.head.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.head.probe": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.post_layernorm.bias": "model-00002-of-00002.safetensors",
+    "model.image_tower.image_tower.vision_model.post_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.deepspeed_moe.experts.deepspeed_experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.deepspeed_moe.experts.deepspeed_experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.deepspeed_moe.experts.deepspeed_experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.deepspeed_moe.experts.deepspeed_experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.deepspeed_moe.experts.deepspeed_experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.deepspeed_moe.experts.deepspeed_experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.deepspeed_moe.experts.deepspeed_experts.2.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.deepspeed_moe.experts.deepspeed_experts.2.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.deepspeed_moe.experts.deepspeed_experts.2.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.deepspeed_moe.experts.deepspeed_experts.3.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.deepspeed_moe.experts.deepspeed_experts.3.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.deepspeed_moe.experts.deepspeed_experts.3.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.deepspeed_moe.gate.wg.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.deepspeed_moe.experts.deepspeed_experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.deepspeed_moe.experts.deepspeed_experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.deepspeed_moe.experts.deepspeed_experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.deepspeed_moe.experts.deepspeed_experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.deepspeed_moe.experts.deepspeed_experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.deepspeed_moe.experts.deepspeed_experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.deepspeed_moe.experts.deepspeed_experts.2.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.deepspeed_moe.experts.deepspeed_experts.2.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.deepspeed_moe.experts.deepspeed_experts.2.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.deepspeed_moe.experts.deepspeed_experts.3.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.deepspeed_moe.experts.deepspeed_experts.3.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.deepspeed_moe.experts.deepspeed_experts.3.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.deepspeed_moe.gate.wg.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.deepspeed_moe.experts.deepspeed_experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.deepspeed_moe.experts.deepspeed_experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.deepspeed_moe.experts.deepspeed_experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.deepspeed_moe.experts.deepspeed_experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.deepspeed_moe.experts.deepspeed_experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.deepspeed_moe.experts.deepspeed_experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.deepspeed_moe.experts.deepspeed_experts.2.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.deepspeed_moe.experts.deepspeed_experts.2.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.deepspeed_moe.experts.deepspeed_experts.2.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.deepspeed_moe.experts.deepspeed_experts.3.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.deepspeed_moe.experts.deepspeed_experts.3.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.deepspeed_moe.experts.deepspeed_experts.3.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.deepspeed_moe.gate.wg.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.deepspeed_moe.experts.deepspeed_experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.deepspeed_moe.experts.deepspeed_experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.deepspeed_moe.experts.deepspeed_experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.deepspeed_moe.experts.deepspeed_experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.deepspeed_moe.experts.deepspeed_experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.deepspeed_moe.experts.deepspeed_experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.deepspeed_moe.experts.deepspeed_experts.2.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.deepspeed_moe.experts.deepspeed_experts.2.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.deepspeed_moe.experts.deepspeed_experts.2.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.deepspeed_moe.experts.deepspeed_experts.3.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.deepspeed_moe.experts.deepspeed_experts.3.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.deepspeed_moe.experts.deepspeed_experts.3.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.deepspeed_moe.gate.wg.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.deepspeed_moe.experts.deepspeed_experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.deepspeed_moe.experts.deepspeed_experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.deepspeed_moe.experts.deepspeed_experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.deepspeed_moe.experts.deepspeed_experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.deepspeed_moe.experts.deepspeed_experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.deepspeed_moe.experts.deepspeed_experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.deepspeed_moe.experts.deepspeed_experts.2.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.deepspeed_moe.experts.deepspeed_experts.2.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.deepspeed_moe.experts.deepspeed_experts.2.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.deepspeed_moe.experts.deepspeed_experts.3.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.deepspeed_moe.experts.deepspeed_experts.3.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.deepspeed_moe.experts.deepspeed_experts.3.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.deepspeed_moe.gate.wg.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.deepspeed_moe.experts.deepspeed_experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.deepspeed_moe.experts.deepspeed_experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.deepspeed_moe.experts.deepspeed_experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.deepspeed_moe.experts.deepspeed_experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.deepspeed_moe.experts.deepspeed_experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.deepspeed_moe.experts.deepspeed_experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.deepspeed_moe.experts.deepspeed_experts.2.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.deepspeed_moe.experts.deepspeed_experts.2.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.deepspeed_moe.experts.deepspeed_experts.2.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.deepspeed_moe.experts.deepspeed_experts.3.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.deepspeed_moe.experts.deepspeed_experts.3.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.deepspeed_moe.experts.deepspeed_experts.3.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.deepspeed_moe.gate.wg.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.deepspeed_moe.experts.deepspeed_experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.deepspeed_moe.experts.deepspeed_experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.deepspeed_moe.experts.deepspeed_experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.deepspeed_moe.experts.deepspeed_experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.deepspeed_moe.experts.deepspeed_experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.deepspeed_moe.experts.deepspeed_experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.deepspeed_moe.experts.deepspeed_experts.2.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.deepspeed_moe.experts.deepspeed_experts.2.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.deepspeed_moe.experts.deepspeed_experts.2.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.deepspeed_moe.experts.deepspeed_experts.3.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.deepspeed_moe.experts.deepspeed_experts.3.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.deepspeed_moe.experts.deepspeed_experts.3.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.deepspeed_moe.gate.wg.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.deepspeed_moe.experts.deepspeed_experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.deepspeed_moe.experts.deepspeed_experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.deepspeed_moe.experts.deepspeed_experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.deepspeed_moe.experts.deepspeed_experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.deepspeed_moe.experts.deepspeed_experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.deepspeed_moe.experts.deepspeed_experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.deepspeed_moe.experts.deepspeed_experts.2.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.deepspeed_moe.experts.deepspeed_experts.2.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.deepspeed_moe.experts.deepspeed_experts.2.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.deepspeed_moe.experts.deepspeed_experts.3.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.deepspeed_moe.experts.deepspeed_experts.3.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.deepspeed_moe.experts.deepspeed_experts.3.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.deepspeed_moe.gate.wg.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.deepspeed_moe.experts.deepspeed_experts.0.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.deepspeed_moe.experts.deepspeed_experts.0.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.deepspeed_moe.experts.deepspeed_experts.0.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.deepspeed_moe.experts.deepspeed_experts.1.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.deepspeed_moe.experts.deepspeed_experts.1.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.deepspeed_moe.experts.deepspeed_experts.1.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.deepspeed_moe.experts.deepspeed_experts.2.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.deepspeed_moe.experts.deepspeed_experts.2.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.deepspeed_moe.experts.deepspeed_experts.2.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.deepspeed_moe.experts.deepspeed_experts.3.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.deepspeed_moe.experts.deepspeed_experts.3.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.deepspeed_moe.experts.deepspeed_experts.3.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.deepspeed_moe.gate.wg.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.deepspeed_moe.experts.deepspeed_experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.deepspeed_moe.experts.deepspeed_experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.deepspeed_moe.experts.deepspeed_experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.deepspeed_moe.experts.deepspeed_experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.deepspeed_moe.experts.deepspeed_experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.deepspeed_moe.experts.deepspeed_experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.deepspeed_moe.experts.deepspeed_experts.2.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.deepspeed_moe.experts.deepspeed_experts.2.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.deepspeed_moe.experts.deepspeed_experts.2.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.deepspeed_moe.experts.deepspeed_experts.3.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.deepspeed_moe.experts.deepspeed_experts.3.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.deepspeed_moe.experts.deepspeed_experts.3.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.deepspeed_moe.gate.wg.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.deepspeed_moe.experts.deepspeed_experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.deepspeed_moe.experts.deepspeed_experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.deepspeed_moe.experts.deepspeed_experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.deepspeed_moe.experts.deepspeed_experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.deepspeed_moe.experts.deepspeed_experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.deepspeed_moe.experts.deepspeed_experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.deepspeed_moe.experts.deepspeed_experts.2.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.deepspeed_moe.experts.deepspeed_experts.2.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.deepspeed_moe.experts.deepspeed_experts.2.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.deepspeed_moe.experts.deepspeed_experts.3.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.deepspeed_moe.experts.deepspeed_experts.3.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.deepspeed_moe.experts.deepspeed_experts.3.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.deepspeed_moe.gate.wg.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.deepspeed_moe.experts.deepspeed_experts.0.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.deepspeed_moe.experts.deepspeed_experts.0.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.deepspeed_moe.experts.deepspeed_experts.0.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.deepspeed_moe.experts.deepspeed_experts.1.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.deepspeed_moe.experts.deepspeed_experts.1.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.deepspeed_moe.experts.deepspeed_experts.1.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.deepspeed_moe.experts.deepspeed_experts.2.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.deepspeed_moe.experts.deepspeed_experts.2.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.deepspeed_moe.experts.deepspeed_experts.2.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.deepspeed_moe.experts.deepspeed_experts.3.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.deepspeed_moe.experts.deepspeed_experts.3.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.deepspeed_moe.experts.deepspeed_experts.3.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.deepspeed_moe.gate.wg.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "model.mm_projector.image_spatial_proj.0.bias": "model-00002-of-00002.safetensors",
+    "model.mm_projector.image_spatial_proj.0.weight": "model-00002-of-00002.safetensors",
+    "model.mm_projector.image_spatial_proj.2.bias": "model-00002-of-00002.safetensors",
+    "model.mm_projector.image_spatial_proj.2.weight": "model-00002-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06a690a4cc93095148d000c2a476170ba0c799fc09cb0e5d36fc3ba6155ffc04
+size 6303861801

trainer_state.json

@@ -0,0 +1,1650 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9999075529259499,
+  "eval_steps": 500,
+  "global_step": 5408,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.45398773006135e-06,
+      "loss": 1.4655,
+      "step": 20
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 4.9079754601227e-06,
+      "loss": 1.4595,
+      "step": 40
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.36196319018405e-06,
+      "loss": 1.4546,
+      "step": 60
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 9.8159509202454e-06,
+      "loss": 1.4591,
+      "step": 80
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.226993865030675e-05,
+      "loss": 1.4607,
+      "step": 100
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.47239263803681e-05,
+      "loss": 1.4737,
+      "step": 120
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.717791411042945e-05,
+      "loss": 1.4726,
+      "step": 140
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.96319018404908e-05,
+      "loss": 1.4924,
+      "step": 160
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.999948159061518e-05,
+      "loss": 1.4928,
+      "step": 180
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.9997544361478652e-05,
+      "loss": 1.4881,
+      "step": 200
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.999417244621822e-05,
+      "loss": 1.4819,
+      "step": 220
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.9989366328716704e-05,
+      "loss": 1.4705,
+      "step": 240
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 1.998312669867048e-05,
+      "loss": 1.4718,
+      "step": 260
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 1.99754544514905e-05,
+      "loss": 1.4461,
+      "step": 280
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.9966350688173777e-05,
+      "loss": 1.448,
+      "step": 300
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.9955816715145415e-05,
+      "loss": 1.4414,
+      "step": 320
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 1.994385404407112e-05,
+      "loss": 1.4374,
+      "step": 340
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.993046439164027e-05,
+      "loss": 1.4346,
+      "step": 360
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.9915649679319553e-05,
+      "loss": 1.4331,
+      "step": 380
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 1.9899412033077257e-05,
+      "loss": 1.4279,
+      "step": 400
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.9881753783078163e-05,
+      "loss": 1.4332,
+      "step": 420
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 1.986267746334916e-05,
+      "loss": 1.44,
+      "step": 440
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.98421858114156e-05,
+      "loss": 1.4149,
+      "step": 460
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.982028176790847e-05,
+      "loss": 1.4163,
+      "step": 480
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 1.9796968476142383e-05,
+      "loss": 1.4,
+      "step": 500
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.97722492816645e-05,
+      "loss": 1.4111,
+      "step": 520
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.9746127731774444e-05,
+      "loss": 1.3904,
+      "step": 540
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 1.971860757501524e-05,
+      "loss": 1.3973,
+      "step": 560
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.9689692760635374e-05,
+      "loss": 1.3839,
+      "step": 580
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.9659387438022094e-05,
+      "loss": 1.3851,
+      "step": 600
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 1.9627695956105913e-05,
+      "loss": 1.3816,
+      "step": 620
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.9594622862736563e-05,
+      "loss": 1.3843,
+      "step": 640
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 1.9560172904030326e-05,
+      "loss": 1.3828,
+      "step": 660
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.952435102368897e-05,
+      "loss": 1.3724,
+      "step": 680
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.94871623622903e-05,
+      "loss": 1.3615,
+      "step": 700
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 1.9448612256550472e-05,
+      "loss": 1.3732,
+      "step": 720
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.940870623855814e-05,
+      "loss": 1.3633,
+      "step": 740
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.93674500349806e-05,
+      "loss": 1.3749,
+      "step": 760
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 1.932484956624197e-05,
+      "loss": 1.3663,
+      "step": 780
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.9280910945673598e-05,
+      "loss": 1.3612,
+      "step": 800
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 1.923564047863678e-05,
+      "loss": 1.3705,
+      "step": 820
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.9189044661617905e-05,
+      "loss": 1.3677,
+      "step": 840
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.9141130181296192e-05,
+      "loss": 1.3624,
+      "step": 860
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 1.9091903913584117e-05,
+      "loss": 1.3527,
+      "step": 880
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.9041372922640708e-05,
+      "loss": 1.3579,
+      "step": 900
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.8989544459857798e-05,
+      "loss": 1.3553,
+      "step": 920
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 1.893642596281943e-05,
+      "loss": 1.3467,
+      "step": 940
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.8882025054234518e-05,
+      "loss": 1.365,
+      "step": 960
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.8826349540842992e-05,
+      "loss": 1.3452,
+      "step": 980
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 1.8769407412295465e-05,
+      "loss": 1.3342,
+      "step": 1000
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.871120684000672e-05,
+      "loss": 1.3463,
+      "step": 1020
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 1.8651756175983044e-05,
+      "loss": 1.3358,
+      "step": 1040
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.859106395162372e-05,
+      "loss": 1.331,
+      "step": 1060
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.852913887649671e-05,
+      "loss": 1.3252,
+      "step": 1080
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 1.8465989837088817e-05,
+      "loss": 1.3223,
+      "step": 1100
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.8401625895530434e-05,
+      "loss": 1.3139,
+      "step": 1120
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.8336056288295085e-05,
+      "loss": 1.3319,
+      "step": 1140
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 1.8269290424873963e-05,
+      "loss": 1.3289,
+      "step": 1160
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.820133788642564e-05,
+      "loss": 1.3295,
+      "step": 1180
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 1.8132208424401116e-05,
+      "loss": 1.3288,
+      "step": 1200
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.8061911959144457e-05,
+      "loss": 1.3214,
+      "step": 1220
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.79904585784692e-05,
+      "loss": 1.317,
+      "step": 1240
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 1.791785853621069e-05,
+      "loss": 1.3261,
+      "step": 1260
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 1.7844122250754625e-05,
+      "loss": 1.3129,
+      "step": 1280
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 1.7769260303541985e-05,
+      "loss": 1.2986,
+      "step": 1300
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 1.769328343755054e-05,
+      "loss": 1.3116,
+      "step": 1320
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.7616202555753212e-05,
+      "loss": 1.3113,
+      "step": 1340
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 1.7538028719553425e-05,
+      "loss": 1.3028,
+      "step": 1360
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.7458773147197788e-05,
+      "loss": 1.3102,
+      "step": 1380
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.73784472121662e-05,
+      "loss": 1.3053,
+      "step": 1400
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 1.729706244153974e-05,
+      "loss": 1.3086,
+      "step": 1420
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.721463051434646e-05,
+      "loss": 1.2989,
+      "step": 1440
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.7131163259885424e-05,
+      "loss": 1.2857,
+      "step": 1460
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 1.7046672656029143e-05,
+      "loss": 1.2999,
+      "step": 1480
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.6961170827504698e-05,
+      "loss": 1.2916,
+      "step": 1500
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.6874670044153826e-05,
+      "loss": 1.2824,
+      "step": 1520
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 1.678718271917211e-05,
+      "loss": 1.3138,
+      "step": 1540
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 1.6698721407327666e-05,
+      "loss": 1.2841,
+      "step": 1560
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 1.6609298803159478e-05,
+      "loss": 1.2949,
+      "step": 1580
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.651892773915567e-05,
+      "loss": 1.2945,
+      "step": 1600
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.6427621183912018e-05,
+      "loss": 1.3093,
+      "step": 1620
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 1.6335392240270888e-05,
+      "loss": 1.284,
+      "step": 1640
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.6242254143440936e-05,
+      "loss": 1.288,
+      "step": 1660
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.6148220259097804e-05,
+      "loss": 1.2787,
+      "step": 1680
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 1.6053304081466086e-05,
+      "loss": 1.2926,
+      "step": 1700
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.5957519231382868e-05,
+      "loss": 1.2766,
+      "step": 1720
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 1.5860879454343074e-05,
+      "loss": 1.281,
+      "step": 1740
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.576339861852694e-05,
+      "loss": 1.2734,
+      "step": 1760
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.5665090712809887e-05,
+      "loss": 1.2747,
+      "step": 1780
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 1.5565969844755033e-05,
+      "loss": 1.2716,
+      "step": 1800
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.5466050238588738e-05,
+      "loss": 1.2757,
+      "step": 1820
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.5365346233159343e-05,
+      "loss": 1.273,
+      "step": 1840
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 1.5263872279879528e-05,
+      "loss": 1.2667,
+      "step": 1860
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.5161642940652428e-05,
+      "loss": 1.2663,
+      "step": 1880
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.5058672885781993e-05,
+      "loss": 1.2681,
+      "step": 1900
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 1.4954976891867705e-05,
+      "loss": 1.2667,
+      "step": 1920
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.4850569839684108e-05,
+      "loss": 1.2885,
+      "step": 1940
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 1.4745466712045335e-05,
+      "loss": 1.2759,
+      "step": 1960
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.4639682591655035e-05,
+      "loss": 1.2712,
+      "step": 1980
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.4533232658941938e-05,
+      "loss": 1.2746,
+      "step": 2000
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 1.4426132189881409e-05,
+      "loss": 1.2712,
+      "step": 2020
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.4318396553803279e-05,
+      "loss": 1.2749,
+      "step": 2040
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.4210041211186295e-05,
+      "loss": 1.2711,
+      "step": 2060
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 1.4101081711439484e-05,
+      "loss": 1.2544,
+      "step": 2080
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.399153369067074e-05,
+      "loss": 1.2696,
+      "step": 2100
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 1.3881412869443001e-05,
+      "loss": 1.2654,
+      "step": 2120
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.3770735050518262e-05,
+      "loss": 1.2495,
+      "step": 2140
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.3659516116589852e-05,
+      "loss": 1.2573,
+      "step": 2160
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 1.354777202800317e-05,
+      "loss": 1.2604,
+      "step": 2180
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.3435518820465343e-05,
+      "loss": 1.2489,
+      "step": 2200
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.3322772602744018e-05,
+      "loss": 1.2566,
+      "step": 2220
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 1.3209549554355717e-05,
+      "loss": 1.2648,
+      "step": 2240
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.309586592324398e-05,
+      "loss": 1.2533,
+      "step": 2260
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 1.2981738023447761e-05,
+      "loss": 1.2448,
+      "step": 2280
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.2867182232760266e-05,
+      "loss": 1.2558,
+      "step": 2300
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.275221499037871e-05,
+      "loss": 1.2529,
+      "step": 2320
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 1.2636852794545185e-05,
+      "loss": 1.2503,
+      "step": 2340
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.2521112200179148e-05,
+      "loss": 1.2617,
+      "step": 2360
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.2405009816501694e-05,
+      "loss": 1.239,
+      "step": 2380
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 1.2288562304652078e-05,
+      "loss": 1.2561,
+      "step": 2400
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.2171786375296776e-05,
+      "loss": 1.2695,
+      "step": 2420
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.205469878623144e-05,
+      "loss": 1.2479,
+      "step": 2440
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 1.1937316339976098e-05,
+      "loss": 1.2493,
+      "step": 2460
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 1.1819655881363907e-05,
+      "loss": 1.2428,
+      "step": 2480
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 1.1701734295123888e-05,
+      "loss": 1.2476,
+      "step": 2500
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.158356850345786e-05,
+      "loss": 1.2372,
+      "step": 2520
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.1465175463612079e-05,
+      "loss": 1.2382,
+      "step": 2540
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 1.134657216544378e-05,
+      "loss": 1.2491,
+      "step": 2560
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.1227775628983085e-05,
+      "loss": 1.2229,
+      "step": 2580
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.110880290199054e-05,
+      "loss": 1.218,
+      "step": 2600
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 1.0989671057510732e-05,
+      "loss": 1.2533,
+      "step": 2620
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.0870397191422208e-05,
+      "loss": 1.2424,
+      "step": 2640
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 1.075099841998416e-05,
+      "loss": 1.2491,
+      "step": 2660
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 1.063149187738019e-05,
+      "loss": 1.2375,
+      "step": 2680
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 1.0511894713259462e-05,
+      "loss": 1.2402,
+      "step": 2700
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 1.0392224090275698e-05,
+      "loss": 1.2331,
+      "step": 2720
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 1.0272497181624225e-05,
+      "loss": 1.233,
+      "step": 2740
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 1.0152731168577603e-05,
+      "loss": 1.2545,
+      "step": 2760
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 1.0032943238020003e-05,
+      "loss": 1.2387,
+      "step": 2780
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 9.91315057998086e-06,
+      "loss": 1.233,
+      "step": 2800
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 9.793370385168012e-06,
+      "loss": 1.2289,
+      "step": 2820
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 9.673619842500781e-06,
+      "loss": 1.2291,
+      "step": 2840
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 9.553916136643283e-06,
+      "loss": 1.2206,
+      "step": 2860
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 9.434276445538378e-06,
+      "loss": 1.23,
+      "step": 2880
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 9.314717937942565e-06,
+      "loss": 1.2371,
+      "step": 2900
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 9.195257770962194e-06,
+      "loss": 1.2334,
+      "step": 2920
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 9.075913087591355e-06,
+      "loss": 1.2284,
+      "step": 2940
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 8.956701014251793e-06,
+      "loss": 1.2271,
+      "step": 2960
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 8.837638658335217e-06,
+      "loss": 1.2242,
+      "step": 2980
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 8.718743105748294e-06,
+      "loss": 1.2268,
+      "step": 3000
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 8.600031418460777e-06,
+      "loss": 1.2266,
+      "step": 3020
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 8.481520632057034e-06,
+      "loss": 1.2224,
+      "step": 3040
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 8.363227753291398e-06,
+      "loss": 1.2249,
+      "step": 3060
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 8.2451697576476e-06,
+      "loss": 1.2086,
+      "step": 3080
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 8.127363586902748e-06,
+      "loss": 1.2295,
+      "step": 3100
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 8.0098261466961e-06,
+      "loss": 1.2267,
+      "step": 3120
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 7.89257430410305e-06,
+      "loss": 1.2231,
+      "step": 3140
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 7.775624885214637e-06,
+      "loss": 1.2123,
+      "step": 3160
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 7.658994672722934e-06,
+      "loss": 1.2216,
+      "step": 3180
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 7.5427004035126604e-06,
+      "loss": 1.2156,
+      "step": 3200
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 7.426758766259381e-06,
+      "loss": 1.2215,
+      "step": 3220
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 7.3111863990346274e-06,
+      "loss": 1.2196,
+      "step": 3240
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 7.19599988691825e-06,
+      "loss": 1.2167,
+      "step": 3260
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 7.081215759618404e-06,
+      "loss": 1.2195,
+      "step": 3280
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 6.966850489099475e-06,
+      "loss": 1.2062,
+      "step": 3300
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 6.852920487218295e-06,
+      "loss": 1.2151,
+      "step": 3320
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 6.739442103368962e-06,
+      "loss": 1.2096,
+      "step": 3340
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 6.626431622136646e-06,
+      "loss": 1.2101,
+      "step": 3360
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 6.513905260960691e-06,
+      "loss": 1.217,
+      "step": 3380
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 6.4018791678073545e-06,
+      "loss": 1.2227,
+      "step": 3400
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 6.290369418852498e-06,
+      "loss": 1.2091,
+      "step": 3420
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 6.179392016174603e-06,
+      "loss": 1.2129,
+      "step": 3440
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 6.0689628854584115e-06,
+      "loss": 1.2206,
+      "step": 3460
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 5.959097873709522e-06,
+      "loss": 1.2119,
+      "step": 3480
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 5.8498127469803015e-06,
+      "loss": 1.214,
+      "step": 3500
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 5.741123188107371e-06,
+      "loss": 1.2106,
+      "step": 3520
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 5.633044794461071e-06,
+      "loss": 1.2186,
+      "step": 3540
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 5.525593075707184e-06,
+      "loss": 1.2018,
+      "step": 3560
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 5.418783451581226e-06,
+      "loss": 1.2093,
+      "step": 3580
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 5.312631249675668e-06,
+      "loss": 1.201,
+      "step": 3600
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 5.207151703240349e-06,
+      "loss": 1.1973,
+      "step": 3620
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 5.102359948996464e-06,
+      "loss": 1.1953,
+      "step": 3640
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 4.998271024964381e-06,
+      "loss": 1.1997,
+      "step": 3660
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 4.894899868305625e-06,
+      "loss": 1.2124,
+      "step": 3680
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 4.792261313179331e-06,
+      "loss": 1.2027,
+      "step": 3700
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.690370088613498e-06,
+      "loss": 1.2081,
+      "step": 3720
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.589240816391314e-06,
+      "loss": 1.2003,
+      "step": 3740
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.488888008952871e-06,
+      "loss": 1.1984,
+      "step": 3760
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.3893260673125605e-06,
+      "loss": 1.2067,
+      "step": 3780
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 4.290569278992505e-06,
+      "loss": 1.2035,
+      "step": 3800
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 4.192631815972201e-06,
+      "loss": 1.1964,
+      "step": 3820
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 4.095527732654841e-06,
+      "loss": 1.2172,
+      "step": 3840
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 3.999270963850389e-06,
+      "loss": 1.2078,
+      "step": 3860
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 3.903875322775932e-06,
+      "loss": 1.2017,
+      "step": 3880
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 3.8093544990733943e-06,
+      "loss": 1.2046,
+      "step": 3900
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 3.7157220568450613e-06,
+      "loss": 1.21,
+      "step": 3920
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 3.6229914327070317e-06,
+      "loss": 1.1955,
+      "step": 3940
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 3.531175933861051e-06,
+      "loss": 1.1841,
+      "step": 3960
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 3.4402887361848435e-06,
+      "loss": 1.1971,
+      "step": 3980
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 3.3503428823413643e-06,
+      "loss": 1.191,
+      "step": 4000
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 3.261351279907078e-06,
+      "loss": 1.2104,
+      "step": 4020
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 3.173326699519714e-06,
+      "loss": 1.1835,
+      "step": 4040
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 3.0862817730455986e-06,
+      "loss": 1.1964,
+      "step": 4060
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 3.000228991766967e-06,
+      "loss": 1.2008,
+      "step": 4080
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.9151807045893765e-06,
+      "loss": 1.2,
+      "step": 4100
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 2.8311491162696302e-06,
+      "loss": 1.1789,
+      "step": 4120
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.7481462856643128e-06,
+      "loss": 1.1906,
+      "step": 4140
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.6661841239993245e-06,
+      "loss": 1.1959,
+      "step": 4160
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 2.585274393160565e-06,
+      "loss": 1.1973,
+      "step": 4180
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.5054287040060543e-06,
+      "loss": 1.1906,
+      "step": 4200
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.4266585146997255e-06,
+      "loss": 1.1884,
+      "step": 4220
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 2.3489751290671513e-06,
+      "loss": 1.1874,
+      "step": 4240
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.272389694973386e-06,
+      "loss": 1.1904,
+      "step": 4260
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 2.196913202723211e-06,
+      "loss": 1.1858,
+      "step": 4280
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.122556483483982e-06,
+      "loss": 1.2015,
+      "step": 4300
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 2.049330207731309e-06,
+      "loss": 1.2048,
+      "step": 4320
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.977244883717815e-06,
+      "loss": 1.2035,
+      "step": 4340
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.9063108559651478e-06,
+      "loss": 1.2008,
+      "step": 4360
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.836538303779516e-06,
+      "loss": 1.1919,
+      "step": 4380
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.7679372397908966e-06,
+      "loss": 1.1841,
+      "step": 4400
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.7005175085162107e-06,
+      "loss": 1.1967,
+      "step": 4420
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.6342887849465794e-06,
+      "loss": 1.197,
+      "step": 4440
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.5692605731589338e-06,
+      "loss": 1.1981,
+      "step": 4460
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.5054422049521332e-06,
+      "loss": 1.2042,
+      "step": 4480
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 1.4428428385078296e-06,
+      "loss": 1.1903,
+      "step": 4500
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.3814714570762244e-06,
+      "loss": 1.1929,
+      "step": 4520
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.3213368676869443e-06,
+      "loss": 1.1981,
+      "step": 4540
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 1.2624476998851832e-06,
+      "loss": 1.1972,
+      "step": 4560
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.2048124044933518e-06,
+      "loss": 1.1924,
+      "step": 4580
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.1484392523983322e-06,
+      "loss": 1.1877,
+      "step": 4600
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 1.0933363333645907e-06,
+      "loss": 1.2004,
+      "step": 4620
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 1.0395115548732447e-06,
+      "loss": 1.1868,
+      "step": 4640
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 9.869726409873316e-07,
+      "loss": 1.194,
+      "step": 4660
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 9.35727131243358e-07,
+      "loss": 1.189,
+      "step": 4680
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 8.857823795693621e-07,
+      "loss": 1.1884,
+      "step": 4700
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 8.371455532295769e-07,
+      "loss": 1.187,
+      "step": 4720
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 7.898236317959229e-07,
+      "loss": 1.1926,
+      "step": 4740
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 7.438234061463967e-07,
+      "loss": 1.1892,
+      "step": 4760
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.991514774905639e-07,
+      "loss": 1.1846,
+      "step": 4780
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 6.55814256422247e-07,
+      "loss": 1.1816,
+      "step": 4800
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 6.138179619995965e-07,
+      "loss": 1.201,
+      "step": 4820
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.731686208526165e-07,
+      "loss": 1.187,
+      "step": 4840
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5.338720663183394e-07,
+      "loss": 1.1868,
+      "step": 4860
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 4.959339376036976e-07,
+      "loss": 1.1943,
+      "step": 4880
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.593596789762911e-07,
+      "loss": 1.1988,
+      "step": 4900
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 4.2415453898310365e-07,
+      "loss": 1.1907,
+      "step": 4920
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 3.903235696973329e-07,
+      "loss": 1.1926,
+      "step": 4940
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.5787162599337343e-07,
+      "loss": 1.1902,
+      "step": 4960
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 3.26803364850149e-07,
+      "loss": 1.1949,
+      "step": 4980
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 2.971232446827943e-07,
+      "loss": 1.2,
+      "step": 5000
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 2.6883552470287957e-07,
+      "loss": 1.202,
+      "step": 5020
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 2.4194426430717567e-07,
+      "loss": 1.1932,
+      "step": 5040
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 2.164533224951304e-07,
+      "loss": 1.1928,
+      "step": 5060
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 1.923663573150747e-07,
+      "loss": 1.1856,
+      "step": 5080
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 1.6968682533929358e-07,
+      "loss": 1.1855,
+      "step": 5100
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 1.4841798116797758e-07,
+      "loss": 1.1991,
+      "step": 5120
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 1.285628769621905e-07,
+      "loss": 1.1817,
+      "step": 5140
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 1.1012436200586007e-07,
+      "loss": 1.1917,
+      "step": 5160
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 9.310508229690374e-08,
+      "loss": 1.185,
+      "step": 5180
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 7.750748016751575e-08,
+      "loss": 1.1819,
+      "step": 5200
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 6.333379393367756e-08,
+      "loss": 1.1931,
+      "step": 5220
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 5.058605757395807e-08,
+      "loss": 1.1848,
+      "step": 5240
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 3.926610043762602e-08,
+      "loss": 1.189,
+      "step": 5260
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.9375546982131075e-08,
+      "loss": 1.1945,
+      "step": 5280
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 2.091581653999142e-08,
+      "loss": 1.1959,
+      "step": 5300
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.3888123115110009e-08,
+      "loss": 1.1986,
+      "step": 5320
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 8.29347520856172e-09,
+      "loss": 1.1877,
+      "step": 5340
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.132675673870212e-09,
+      "loss": 1.1981,
+      "step": 5360
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.4063216017956572e-09,
+      "loss": 1.1844,
+      "step": 5380
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.1480423464882429e-10,
+      "loss": 1.194,
+      "step": 5400
+    },
+    {
+      "epoch": 1.0,
+      "step": 5408,
+      "total_flos": 4.334944925142483e+18,
+      "train_loss": 1.2657453445993232,
+      "train_runtime": 25911.696,
+      "train_samples_per_second": 26.717,
+      "train_steps_per_second": 0.209
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 5408,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 5000,
+  "total_flos": 4.334944925142483e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}