Upload folder using huggingface_hub

config.json

@@ -0,0 +1,134 @@
+{
+  "_name_or_path": "mesolitica/Malaysian-Audio-Qwen2.5-7B-Instruct",
+  "architectures": [
+    "Model"
+  ],
+  "attention_dropout": 0.0,
+  "audio_encoder_config": {
+    "_attn_implementation_autoset": true,
+    "_name_or_path": "huseinzol05/whisper-large-v3-encoder",
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "add_cross_attention": false,
+    "apply_spec_augment": false,
+    "architectures": [
+      "WhisperEncoder"
+    ],
+    "attention_dropout": 0.0,
+    "attn_implementation": null,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": [
+      220,
+      50257
+    ],
+    "bos_token_id": 50257,
+    "chunk_size_feed_forward": 0,
+    "classifier_proj_size": 256,
+    "cross_attention_hidden_size": null,
+    "d_model": 1280,
+    "decoder_attention_heads": 20,
+    "decoder_ffn_dim": 5120,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 32,
+    "decoder_start_token_id": 50258,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "dropout": 0.0,
+    "early_stopping": false,
+    "encoder_attention_heads": 20,
+    "encoder_ffn_dim": 5120,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 32,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 50257,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "init_std": 0.02,
+    "is_decoder": false,
+    "is_encoder_decoder": true,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_prob": 0.05,
+    "max_length": 448,
+    "max_source_positions": 1500,
+    "max_target_positions": 448,
+    "median_filter_width": 7,
+    "min_length": 0,
+    "model_type": "whisper",
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 32,
+    "num_mel_bins": 128,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 50256,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "scale_embedding": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "float32",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 51866
+  },
+  "audio_token_index": 151665,
+  "auto_map": {
+    "AutoModelForCausalLM": "qwen_model.Model"
+  },
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151668
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.46.0"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac3f77da00e91ba3a0d63b559ec70ea6d2eb106b5824da64e54df5b8b29074bb
+size 4874822248

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b31140ad3e3f73bd56c03e20f6f0d2e658febdcb2c23713894666bd45286f716
+size 4932751008

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63444d92c88ef4a5eecbe18a12c10b4d40c82d94c7007caa7ef00c4b17951cd2
+size 4330865200

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:440d8b140ed2fa404b5985f34945ab5da2f29f5ac310b4ce83692ffd75193f02
+size 2370335624

model.safetensors.index.json

@@ -0,0 +1,835 @@
+{
+  "metadata": {
+    "total_size": 16508680928
+  },
+  "weight_map": {
+    "encoder.conv1.bias": "model-00004-of-00004.safetensors",
+    "encoder.conv1.weight": "model-00004-of-00004.safetensors",
+    "encoder.conv2.bias": "model-00004-of-00004.safetensors",
+    "encoder.conv2.weight": "model-00004-of-00004.safetensors",
+    "encoder.embed_positions.weight": "model-00004-of-00004.safetensors",
+    "encoder.layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.0.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.0.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.0.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.0.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.0.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.0.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.0.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.0.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.0.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.0.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.0.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.0.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.1.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.1.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.1.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.1.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.1.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.1.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.1.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.1.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.1.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.1.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.1.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.1.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.10.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.10.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.10.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.10.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.10.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.10.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.10.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.10.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.10.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.10.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.10.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.10.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.10.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.10.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.10.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.11.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.11.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.11.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.11.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.11.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.11.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.11.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.11.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.11.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.11.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.11.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.11.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.11.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.11.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.11.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.12.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.12.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.12.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.12.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.12.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.12.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.12.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.12.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.12.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.12.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.12.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.12.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.12.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.12.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.12.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.13.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.13.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.13.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.13.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.13.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.13.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.13.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.13.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.13.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.13.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.13.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.13.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.13.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.13.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.13.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.14.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.14.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.14.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.14.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.14.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.14.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.14.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.14.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.14.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.14.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.14.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.14.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.14.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.14.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.14.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.15.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.15.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.15.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.15.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.15.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.15.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.15.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.15.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.15.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.15.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.15.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.15.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.15.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.15.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.15.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.16.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.16.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.16.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.16.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.16.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.16.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.16.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.16.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.16.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.16.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.16.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.16.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.16.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.16.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.16.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.17.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.17.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.17.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.17.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.17.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.17.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.17.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.17.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.17.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.17.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.17.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.17.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.17.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.17.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.17.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.18.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.18.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.18.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.18.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.18.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.18.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.18.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.18.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.18.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.18.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.18.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.18.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.18.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.18.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.18.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.19.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.19.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.19.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.19.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.19.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.19.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.19.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.19.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.19.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.19.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.19.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.19.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.19.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.19.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.19.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.2.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.2.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.2.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.2.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.2.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.2.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.2.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.2.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.2.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.2.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.2.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.2.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.20.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.20.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.20.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.20.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.20.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.20.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.20.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.20.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.20.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.20.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.20.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.20.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.20.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.20.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.20.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.21.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.21.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.21.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.21.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.21.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.21.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.21.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.21.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.21.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.21.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.21.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.21.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.21.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.21.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.21.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.22.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.22.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.22.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.22.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.22.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.22.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.22.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.22.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.22.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.22.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.22.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.22.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.22.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.22.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.22.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.23.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.23.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.23.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.23.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.23.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.23.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.23.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.23.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.23.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.23.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.23.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.23.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.23.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.23.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.23.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.24.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.24.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.24.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.24.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.24.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.24.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.24.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.24.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.24.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.24.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.24.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.24.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.24.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.24.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.24.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.25.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.25.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.25.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.25.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.25.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.25.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.25.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.25.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.25.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.25.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.25.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.25.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.25.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.25.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.25.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.26.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.26.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.26.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.26.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.26.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.26.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.26.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.26.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.26.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.26.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.26.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.26.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.26.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.26.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.27.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.27.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.27.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.27.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.27.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.27.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.27.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.27.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.27.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.27.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.27.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.27.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.27.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.27.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.27.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.28.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.28.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.28.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.28.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.28.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.28.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.28.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.28.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.28.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.28.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.28.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.28.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.28.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.28.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.28.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.29.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.29.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.29.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.29.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.29.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.29.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.29.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.29.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.29.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.29.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.29.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.29.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.29.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.29.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.29.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.3.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.3.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.3.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.3.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.3.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.3.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.3.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.3.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.3.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.3.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.3.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.3.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.30.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.30.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.30.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.30.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.30.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.30.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.30.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.30.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.30.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.30.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.30.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.30.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.30.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.30.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.30.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.31.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.31.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.31.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.31.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.31.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.31.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.31.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.31.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.31.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.31.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.31.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.31.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.31.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.31.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.31.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.4.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.4.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.4.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.4.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.4.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.4.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.4.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.4.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.4.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.4.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.4.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.4.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.4.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.4.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.4.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.5.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.5.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.5.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.5.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.5.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.5.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.5.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.5.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.5.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.5.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.5.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.5.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.5.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.5.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.5.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.6.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.6.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.6.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.6.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.6.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.6.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.6.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.6.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.6.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.6.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.6.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.6.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.6.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.6.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.6.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.7.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.7.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.7.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.7.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.7.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.7.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.7.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.7.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.7.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.7.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.7.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.7.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.7.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.7.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.7.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.8.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.8.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.8.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.8.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.8.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.8.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.8.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.8.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.8.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.8.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.8.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.8.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.8.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.8.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.8.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.9.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.9.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.9.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.9.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.9.final_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.9.final_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.9.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.9.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.9.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.9.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.9.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.9.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.9.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder.layers.9.self_attn_layer_norm.bias": "model-00004-of-00004.safetensors",
+    "encoder.layers.9.self_attn_layer_norm.weight": "model-00004-of-00004.safetensors",
+    "encoder.range_max_source_positions": "model-00004-of-00004.safetensors",
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors",
+    "projection.weight": "model-00004-of-00004.safetensors"
+  }
+}

qwen_model.py

@@ -0,0 +1,209 @@
+from transformers import Qwen2ForCausalLM, AutoModel
+from transformers.models.whisper.modeling_whisper import WhisperEncoderLayer
+from transformers import WhisperPreTrainedModel, WhisperConfig
+from transformers.modeling_outputs import BaseModelOutput, CausalLMOutputWithPast
+from torch import nn
+import torch
+import math
+import logging
+
+class WhisperEncoder(WhisperPreTrainedModel):
+    
+    def __init__(self, config: WhisperConfig):
+        super().__init__(config)
+        self.dropout = config.dropout
+        self.layerdrop = config.encoder_layerdrop
+
+        embed_dim = config.d_model
+        self.num_mel_bins = config.num_mel_bins
+        self.padding_idx = config.pad_token_id
+        self.max_source_positions = config.max_source_positions
+        self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0
+
+        self.conv1 = nn.Conv1d(self.num_mel_bins, embed_dim, kernel_size=3, padding=1)
+        self.conv2 = nn.Conv1d(embed_dim, embed_dim, kernel_size=3, stride=2, padding=1)
+        
+        self.register_buffer('range_max_source_positions', torch.arange(self.max_source_positions))
+
+        self.embed_positions = nn.Embedding(self.max_source_positions, embed_dim)
+        self.embed_positions.requires_grad_(False)
+
+        self.layers = nn.ModuleList([WhisperEncoderLayer(config) for _ in range(config.encoder_layers)])
+        self.layer_norm = nn.LayerNorm(config.d_model)
+
+        self.gradient_checkpointing = False
+        self.post_init()
+
+    def _freeze_parameters(self):
+        for param in self.parameters():
+            param.requires_grad = False
+        self._requires_grad = False
+
+    def get_input_embeddings(self) -> nn.Module:
+        return self.conv1
+
+    def set_input_embeddings(self, value: nn.Module):
+        self.conv1 = value
+
+    def forward(
+        self,
+        input_features,
+        attention_mask=None,
+        head_mask=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+    ):
+
+        expected_seq_length = self.config.max_source_positions * self.conv1.stride[0] * self.conv2.stride[0]
+        if input_features.shape[-1] != expected_seq_length:
+            raise ValueError(
+                f"Whisper expects the mel input features to be of length {expected_seq_length}, but found {input_features.shape[-1]}. Make sure to pad the input mel features to {expected_seq_length}."
+            )
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        inputs_embeds = nn.functional.gelu(self.conv1(input_features))
+        inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds))
+
+        inputs_embeds = inputs_embeds.permute(0, 2, 1)
+        embed_pos = self.embed_positions(self.range_max_source_positions)
+
+        hidden_states = inputs_embeds + embed_pos
+        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
+
+        encoder_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+
+        # check if head_mask has a correct number of layers specified if desired
+        if head_mask is not None:
+            assert head_mask.size()[0] == (len(self.layers)), (
+                f"The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.size()[0]}."
+            )
+
+        for idx, encoder_layer in enumerate(self.layers):
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
+            to_drop = False
+            if self.training:
+                dropout_probability = torch.rand([])
+                if dropout_probability < self.layerdrop:  # skip the layer
+                    to_drop = True
+
+            if to_drop:
+                layer_outputs = (None, None)
+            else:
+                if self.gradient_checkpointing and self.training:
+                    layer_outputs = self._gradient_checkpointing_func(
+                        encoder_layer.__call__,
+                        hidden_states,
+                        None,
+                        (head_mask[idx] if head_mask is not None else None),
+                        output_attentions,
+                    )
+                else:
+                    layer_outputs = encoder_layer(
+                        hidden_states,
+                        None,
+                        layer_head_mask=(head_mask[idx] if head_mask is not None else None),
+                        output_attentions=output_attentions,
+                    )
+
+                hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+        hidden_states = self.layer_norm(hidden_states)
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
+        return BaseModelOutput(
+            last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
+        )
+
+class Model(Qwen2ForCausalLM):
+    def __init__(self, config):
+        super().__init__(config)
+        try:
+            audio_config = WhisperConfig.from_dict(config.audio_encoder_config)
+            self.encoder = WhisperEncoder(audio_config)
+            self.projection = nn.Linear(self.encoder.config.d_model, self.config.hidden_size, bias=False)
+        except Exception as e:
+            logging.warning(f'{e}, cannot load encoder.')
+            self.encoder = None
+            self.projection = None
+    
+    def forward(
+        self, 
+        input_ids, 
+        attention_mask = None,
+        position_ids = None,
+        input_features = None, 
+        feature_attention_mask = None, 
+        past_key_values = None,
+        print_input_features_shape = False,
+        **kwargs,
+    ):
+        inputs_embeds = self.get_input_embeddings()(input_ids)
+        if input_features is not None and input_ids.shape[1] != 1:
+            
+            if self.encoder is None:
+                raise Exception('encoder is None.')
+            
+            if feature_attention_mask is None:
+                raise Exception('feature_attention_mask should not empty')
+                
+            batch_size, _, max_mel_seq_len = input_features.shape
+            max_seq_len = (max_mel_seq_len - 2) // 2 + 1
+            audio_feat_lengths = self.encoder._get_feat_extract_output_lengths(feature_attention_mask.sum(-1))
+            seq_range = (
+                torch.arange(0, max_seq_len, dtype=audio_feat_lengths.dtype, device=audio_feat_lengths.device)
+                .unsqueeze(0)
+                .expand(batch_size, max_seq_len)
+            )
+            lengths_expand = audio_feat_lengths.unsqueeze(1).expand(batch_size, max_seq_len)
+            padding_mask = seq_range >= lengths_expand
+
+            audio_attention_mask_ = padding_mask.view(batch_size, 1, 1, max_seq_len).expand(
+                batch_size, 1, max_seq_len, max_seq_len
+            )
+            audio_attention_mask = audio_attention_mask_.to(
+                dtype=self.encoder.conv1.weight.dtype, device=self.encoder.conv1.weight.device
+            )
+            audio_attention_mask[audio_attention_mask_] = float("-inf")
+            audio_outputs = self.encoder(input_features, attention_mask=audio_attention_mask)
+            selected_audio_feature = audio_outputs.last_hidden_state
+            audio_features = self.projection(selected_audio_feature)
+            num_audio_tokens = audio_feat_lengths
+            num_audios, max_audio_tokens, embed_dim = audio_features.shape
+            audio_features_mask = torch.arange(max_audio_tokens).expand(num_audios, max_audio_tokens).to(
+                num_audio_tokens.device
+            ) < num_audio_tokens.unsqueeze(1)
+            masked_audio_features = audio_features[audio_features_mask].view(-1, embed_dim)
+            if print_input_features_shape:
+                print(masked_audio_features.shape, masked_audio_features.contiguous())
+            inputs_embeds[input_ids == self.config.audio_token_index] = masked_audio_features.contiguous()
+        
+        outputs = self.model.forward(
+            inputs_embeds = inputs_embeds, 
+            attention_mask = attention_mask,
+            position_ids = position_ids,
+            past_key_values=past_key_values,
+            output_hidden_states = True,
+        )
+        hidden_states = outputs.last_hidden_state
+        logits = self.lm_head(hidden_states)
+        return CausalLMOutputWithPast(
+            loss=None,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )