Upload folder using huggingface_hub

config.json

@@ -0,0 +1,175 @@
+{
+  "architectures": [
+    "QQMMForCausalLM"
+  ],
+  "auto_map": {
+      "AutoConfig": "configuration_qqmm.QQMMConfig",
+      "AutoModel": "modeling_qqmm.QQMMForCausalLM"
+  },
+  "image_token_id": 151646,
+  "language_model_config": {
+    "_name_or_path": "qqmm/models/Qwen2_5-7B-Instruct",
+    "add_cross_attention": false,
+    "architectures": [
+      "Qwen2ForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 151643,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 151645,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "hidden_size": 3584,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 18944,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 32768,
+    "max_window_layers": 28,
+    "min_length": 0,
+    "model_type": "qwen2",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 28,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 4,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 1000000.0,
+    "sep_token_id": null,
+    "sliding_window": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "use_sliding_window": false,
+    "vocab_size": 152064
+  },
+  "model_type": "qqmm",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.2",
+  "vision_abstractor_config": {
+    "exist_cls_embedding": false,
+    "projection_type": "mlp2x_gelu"
+  },
+  "vision_model_config": {
+    "_name_or_path": "qqmm/models/Qwen2-VL-7B-Instruct-NaViT",
+    "add_cross_attention": false,
+    "architectures": [
+      "Qwen2VLNaViT"
+    ],
+    "auto_map": {
+      "AutoConfig": "configuration_qwen2vl_navit.Qwen2VLNaViTConfig",
+      "AutoModel": "modeling_qwen2vl_navit.Qwen2VLNaViT"
+    },
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "depth": 32,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "quick_gelu",
+    "hidden_size": 1280,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "in_channels": 3,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "mlp_ratio": 4,
+    "model_type": "qwen2vl_navit",
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_heads": 16,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 14,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false
+  },
+  "vision_output_key": null
+}

configuration_qqmm.py

@@ -0,0 +1,57 @@
+from transformers import PretrainedConfig, AutoConfig, CONFIG_MAPPING
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+
+class QQMMConfig(PretrainedConfig):
+    model_type = "qqmm"
+    is_composition = True
+
+    def __init__(self,
+                 language_model_config=None,
+                 vision_model_config=None,
+                 vision_abstractor_config=None,
+                 vision_output_key='last_hidden_state',
+                 image_token_id=None,
+                 **kwargs):
+        super().__init__(**kwargs)
+
+        if isinstance(language_model_config, dict):
+            if '_name_or_path' not in language_model_config:
+                language_model_config['_name_or_path'] = self._name_or_path
+            language_model_type = language_model_config.get('model_type', '')
+            is_remote_code = '.' in language_model_config.get('auto_map', {}).get('AutoConfig', '')
+            if language_model_type in CONFIG_MAPPING and not is_remote_code:
+                language_model_config = AutoConfig.for_model(**language_model_config)
+            elif language_model_type:
+                Config = get_class_from_dynamic_module(language_model_config["auto_map"]["AutoConfig"],
+                                                       language_model_config['_name_or_path'])
+                language_model_config = Config(**language_model_config)
+        self.language_model_config = language_model_config
+
+        if isinstance(vision_model_config, dict):
+            if '_name_or_path' not in vision_model_config:
+                vision_model_config['_name_or_path'] = self._name_or_path
+            vision_model_type = vision_model_config.get('model_type', '')
+            is_remote_code = '.' in vision_model_config.get('auto_map', {}).get('AutoConfig', '')
+            if vision_model_type in CONFIG_MAPPING and not is_remote_code:
+                vision_model_config = AutoConfig.for_model(**vision_model_config)
+            elif vision_model_type:
+                Config = get_class_from_dynamic_module(vision_model_config["auto_map"]["AutoConfig"],
+                                                       vision_model_config['_name_or_path'])
+                vision_model_config = Config(**vision_model_config)
+        self.vision_model_config = vision_model_config
+
+        self.vision_abstractor_config = vision_abstractor_config
+
+        self.vision_output_key = vision_output_key
+        self.image_token_id = image_token_id
+
+    @property
+    def hidden_size(self):
+        return self.language_model_config.hidden_size
+
+    @classmethod
+    def from_dict(cls, config_dict, **kwargs):
+        if 'name_or_path' in kwargs:
+            config_dict['_name_or_path'] = kwargs.pop('name_or_path')
+        return super().from_dict(config_dict, **kwargs)

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.44.2"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c4d36d613d2313f24a29c6b6c6d1543c9e31666da8e591ca1df91bcdb15f25b
+size 4938421320

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50d3b7e5903c3d38458f37656231cc86eb3ba6cb60be5e833a9ef3575e7f381f
+size 4991497784

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37de034fdcf198a3772eea43fe251fe0de3a97d3733eb8f9fd806dcea45f65ce
+size 4932752872

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19bb2facb0ac48a68a80235aa9254769d83163f31a7fb01af20c5d8470208c5e
+size 1691924640

model.safetensors.index.json

@@ -0,0 +1,736 @@
+{
+  "metadata": {
+    "total_size": 16554509312
+  },
+  "weight_map": {
+    "language_model.lm_head.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.16.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.16.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.16.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.26.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.27.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.27.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.27.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.27.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.5.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.5.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.norm.weight": "model-00004-of-00004.safetensors",
+    "vision_abstractor.image_embedding": "model-00001-of-00004.safetensors",
+    "vision_abstractor.projection.0.bias": "model-00001-of-00004.safetensors",
+    "vision_abstractor.projection.0.weight": "model-00001-of-00004.safetensors",
+    "vision_abstractor.projection.2.bias": "model-00001-of-00004.safetensors",
+    "vision_abstractor.projection.2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.0.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.0.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.0.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.0.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.0.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.0.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.0.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.0.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.0.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.0.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.0.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.0.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.1.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.1.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.1.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.1.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.1.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.1.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.1.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.1.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.1.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.1.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.1.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.1.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.10.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.10.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.10.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.10.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.10.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.10.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.10.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.10.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.10.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.10.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.10.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.10.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.11.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.11.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.11.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.11.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.11.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.11.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.11.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.11.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.11.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.11.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.11.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.11.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.12.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.12.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.12.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.12.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.12.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.12.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.12.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.12.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.12.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.12.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.12.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.12.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.13.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.13.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.13.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.13.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.13.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.13.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.13.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.13.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.13.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.13.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.13.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.13.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.14.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.14.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.14.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.14.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.14.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.14.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.14.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.14.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.14.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.14.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.14.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.14.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.15.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.15.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.15.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.15.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.15.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.15.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.15.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.15.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.15.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.15.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.15.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.15.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.16.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.16.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.16.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.16.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.16.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.16.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.16.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.16.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.16.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.16.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.16.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.16.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.17.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.17.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.17.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.17.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.17.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.17.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.17.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.17.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.17.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.17.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.17.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.17.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.18.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.18.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.18.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.18.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.18.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.18.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.18.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.18.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.18.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.18.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.18.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.18.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.19.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.19.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.19.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.19.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.19.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.19.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.19.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.19.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.19.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.19.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.19.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.19.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.2.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.2.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.2.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.2.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.2.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.2.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.2.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.2.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.2.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.2.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.2.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.2.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.20.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.20.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.20.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.20.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.20.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.20.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.20.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.20.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.20.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.20.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.20.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.20.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.21.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.21.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.21.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.21.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.21.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.21.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.21.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.21.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.21.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.21.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.21.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.21.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.22.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.22.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.22.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.22.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.22.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.22.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.22.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.22.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.22.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.22.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.22.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.22.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.23.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.23.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.23.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.23.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.23.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.23.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.23.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.23.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.23.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.23.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.23.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.23.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.24.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.24.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.24.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.24.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.24.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.24.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.24.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.24.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.24.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.24.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.24.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.24.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.25.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.25.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.25.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.25.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.25.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.25.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.25.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.25.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.25.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.25.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.25.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.25.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.26.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.26.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.26.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.26.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.26.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.26.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.26.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.26.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.26.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.26.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.26.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.26.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.27.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.27.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.27.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.27.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.27.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.27.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.27.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.27.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.27.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.27.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.27.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.27.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.28.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.28.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.28.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.28.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.28.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.28.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.28.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.28.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.28.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.28.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.28.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.28.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.29.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.29.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.29.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.29.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.29.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.29.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.29.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.29.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.29.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.29.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.29.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.29.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.3.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.3.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.3.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.3.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.3.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.3.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.3.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.3.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.3.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.3.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.3.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.3.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.30.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.30.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.30.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.30.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.30.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.30.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.30.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.30.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.30.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.30.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.30.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.30.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.31.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.31.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.31.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.31.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.31.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.31.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.31.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.31.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.31.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.31.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.31.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.31.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.4.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.4.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.4.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.4.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.4.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.4.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.4.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.4.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.4.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.4.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.4.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.4.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.5.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.5.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.5.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.5.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.5.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.5.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.5.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.5.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.5.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.5.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.5.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.5.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.6.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.6.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.6.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.6.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.6.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.6.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.6.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.6.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.6.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.6.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.6.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.6.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.7.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.7.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.7.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.7.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.7.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.7.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.7.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.7.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.7.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.7.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.7.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.7.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.8.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.8.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.8.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.8.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.8.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.8.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.8.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.8.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.8.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.8.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.8.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.8.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.9.attn.proj.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.9.attn.proj.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.9.attn.qkv.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.9.attn.qkv.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.9.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.9.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.9.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.9.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.9.norm1.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.9.norm1.weight": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.9.norm2.bias": "model-00001-of-00004.safetensors",
+    "vision_model.blocks.9.norm2.weight": "model-00001-of-00004.safetensors",
+    "vision_model.patch_embed.weight": "model-00001-of-00004.safetensors"
+  }
+}

modeling_qqmm.py

@@ -0,0 +1,231 @@
+from typing import Optional, List
+import torch
+from torch.nn import functional as F
+from transformers import PreTrainedModel, AutoModel, AutoModelForCausalLM
+
+from .configuration_qqmm import QQMMConfig
+from .modeling_abstractor import PerceiverProjection
+
+
+def build_vision_model(config, model=None):
+    if model is None:
+        model = AutoModel.from_config(config, trust_remote_code=True)
+
+    model_type = model.config.model_type
+    assert 'navit' in model_type, "Only support navit vision models."
+
+    return model
+
+
+class QQMMPreTrainedModel(PreTrainedModel):
+    config_class = QQMMConfig
+    supports_gradient_checkpointing = True
+    _skip_keys_device_placement = "past_key_values"
+    _supports_cache_class = True
+    _supports_flash_attn_2 = True
+    _supports_sdpa = True
+
+
+class QQMMForCausalLM(QQMMPreTrainedModel):
+    def __init__(self,
+                 config: QQMMConfig,
+                 language_model=None,
+                 vision_model=None):
+        super().__init__(config)
+
+        vision_model = build_vision_model(config.vision_model_config, vision_model)
+
+        vision_abstractor = PerceiverProjection(**config.vision_abstractor_config,
+                                                in_dim=self.config.vision_model_config.hidden_size,
+                                                out_dim=self.config.language_model_config.hidden_size)
+
+        if language_model is None:
+            kwargs_ = {}
+            if config._attn_implementation_internal is not None:
+                kwargs_['attn_implementation'] = config._attn_implementation_internal
+            language_model = AutoModelForCausalLM.from_config(config.language_model_config, trust_remote_code=True,
+                                                              **kwargs_)
+
+        self.vision_model = vision_model
+        self.vision_abstractor = vision_abstractor
+        self.language_model = language_model
+
+        self.vision_output_key = parse_output_key(self.config.vision_output_key)
+
+    def forward_vision(self, pixel_values, grid_sizes):
+        is_dummy_input = pixel_values.size(0) == 0
+        if is_dummy_input:
+            pixel_values = torch.zeros((4,) + pixel_values.shape[1:],
+                                       dtype=pixel_values.dtype, device=pixel_values.device)
+            grid_sizes = torch.full((1,) + grid_sizes.shape[1:], fill_value=2,
+                                    dtype=grid_sizes.dtype, device=grid_sizes.device)
+
+        outputs = self.vision_model(pixel_values, grid_sizes)
+
+        for k in self.vision_output_key:
+            outputs = outputs[k]
+        vision_embeds = outputs
+
+        if is_dummy_input:
+            vision_embeds = vision_embeds[:0]
+            grid_sizes = grid_sizes[:0]
+
+        vision_embeds = self.vision_abstractor(vision_embeds, grid_sizes)
+
+        return vision_embeds
+
+    def prepare_for_lm(self, input_ids, vision_embeds):
+        inputs_embeds = self.get_input_embeddings()(input_ids)
+
+        if vision_embeds is not None:
+            vision_mask = input_ids == self.config.image_token_id
+            # assert torch.count_nonzero(vision_mask).item() == vision_embeds.shape[:-1].numel(), \
+            #     "vision embeddings mismatch input embeddings: " \
+            #     f"vision_mask shape={vision_mask.shape}; " \
+            #     f"vision_mask count={torch.count_nonzero(vision_mask)}; " \
+            #     f"vision_embeds shape={vision_embeds.shape}"
+            inputs_embeds = torch.masked_scatter(inputs_embeds, vision_mask.unsqueeze(-1),
+                                                 vision_embeds.to(inputs_embeds.dtype).view(-1, vision_embeds.size(-1)))
+
+        return {'inputs_embeds': inputs_embeds}
+
+    def forward(self,
+                input_ids: torch.LongTensor = None,
+                attention_mask: Optional[torch.BoolTensor] = None,
+                position_ids: Optional[torch.LongTensor] = None,
+                *,
+                pixel_values: Optional[torch.Tensor] = None,
+                grid_sizes: Optional[torch.Tensor] = None,
+                vision_embeds: Optional[torch.FloatTensor] = None,
+                past_key_values: Optional[List[torch.FloatTensor]] = None,
+                labels: Optional[torch.LongTensor] = None,
+                return_dict: bool = True,
+                **kwargs):
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # ========Get visual embedding========
+        if pixel_values is not None and vision_embeds is None:
+            vision_embeds = self.forward_vision(pixel_values, grid_sizes)
+        # if self._gradient_checkpointing and vision_embeds is not None and not vision_embeds.requires_grad:
+        #     vision_embeds.requires_grad_(True)
+
+        # ========Prepare inputs for LM========
+        kwargs_ = self.prepare_for_lm(input_ids, vision_embeds)
+        kwargs.update(kwargs_)
+        inputs_embeds = kwargs.pop('inputs_embeds')
+
+        if self.is_gradient_checkpointing and torch.is_grad_enabled():
+            inputs_embeds.requires_grad_(True)
+
+        # ========Forward into LM========
+        outputs = self.language_model(
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            return_dict=return_dict,
+            **kwargs)
+
+        # ========Compute Loss========
+        if labels is not None:
+            logits = outputs['logits'] if return_dict else outputs[0]
+            loss = self.loss_function(logits=logits, labels=labels)
+            if return_dict:
+                outputs['loss'] = loss
+            else:
+                outputs = (loss,) + outputs
+
+        return outputs
+
+    def prepare_inputs_for_generation(self,
+                                      input_ids,
+                                      past_key_values=None,
+                                      attention_mask=None,
+                                      inputs_embeds=None,
+                                      cache_position=None,
+                                      position_ids=None,
+                                      use_cache=True,
+                                      *,
+                                      pixel_values: Optional[torch.Tensor] = None,
+                                      grid_sizes: Optional[torch.Tensor] = None,
+                                      vision_embeds: Optional[torch.FloatTensor] = None,
+                                      **kwargs):
+        cur_position = cache_position[0].item()
+        if attention_mask is not None and position_ids is None:
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+        if past_key_values is not None:
+            position_ids = (position_ids[:, -(attention_mask.size(1) - cur_position):]
+                            + (attention_mask.size(1) - position_ids.size(1)))
+
+        model_inputs = self.language_model.prepare_inputs_for_generation(
+            input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            inputs_embeds=inputs_embeds,
+            cache_position=cache_position,
+            position_ids=position_ids,
+            use_cache=use_cache,
+            **kwargs,
+        )
+
+        if cur_position == 0:
+            model_inputs['pixel_values'] = pixel_values
+            model_inputs['grid_sizes'] = grid_sizes
+            model_inputs['vision_embeds'] = vision_embeds
+
+        return model_inputs
+
+    def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None):
+        super().gradient_checkpointing_enable(gradient_checkpointing_kwargs)
+        self.language_model.enable_input_require_grads()
+
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.language_model.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.language_model.get_output_embeddings()
+
+    def set_output_embeddings(self, new_embeddings):
+        self.language_model.set_output_embeddings(new_embeddings)
+
+    def set_decoder(self, decoder):
+        self.language_model.set_decoder(decoder)
+
+    def get_decoder(self):
+        return self.language_model.get_decoder()
+
+    def tie_weights(self):
+        return self.language_model.tie_weights()
+
+
+def parse_output_key(output_key):
+    output_key_ = []
+    if not output_key:
+        return output_key_
+    ks = [k.strip() for k in output_key.split(',')]
+    i = 0
+    while i < len(ks):
+        k = ks[i]
+        if k.startswith('['):
+            for j in range(len(ks)-1, i, -1):
+                if ks[j].endswith(']'):
+                    break
+            else:
+                raise ValueError(output_key)
+            ns = tuple(parse_output_key(','.join([k.lstrip('[')] + ks[i+1:j] + [ks[j].rstrip(']')])))
+            output_key_.append(ns)
+            i = j
+        elif k.lstrip('-').isdigit():
+            output_key_.append(int(k))
+        elif ':' in k:
+            ns = [int(n) if n != '' else None for n in k.split(':')]
+            output_key_.append(slice(*ns))
+        else:
+            output_key_.append(k)
+        i += 1
+
+    return output_key_