Upload folder using huggingface_hub

config.json

@@ -1,7 +1,7 @@
 {
-  "_name_or_path": "./7b-speech-instruct",
+  "_name_or_path": "./7b-speech-instruct-v1",
   "architectures": [
-    "Model"
+    "LLMAudioForConditionalGeneration"
   ],
   "attention_dropout": 0.0,
   "audio_encoder_config": {
@@ -107,6 +107,7 @@
   },
   "audio_token_index": 151665,
   "auto_map": {
+    "AutoModel": "qwen_model_v2.LLMAudioForConditionalGeneration",
     "AutoModelForCausalLM": "qwen_model.Model"
   },
   "bos_token_id": 151643,

generation_config.json

@@ -1,14 +1,6 @@
 {
+  "_from_model_config": true,
   "bos_token_id": 151643,
-  "do_sample": true,
-  "eos_token_id": [
-    151645,
-    151643
-  ],
-  "pad_token_id": 151643,
-  "repetition_penalty": 1.05,
-  "temperature": 0.7,
-  "top_k": 20,
-  "top_p": 0.8,
+  "eos_token_id": 151645,
   "transformers_version": "4.46.0"
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8564759f422ecee4fb6c94f0d428eb320d6bd2f3469c783885a083361c3988f8
-size 4874822248
+oid sha256:73f9484636ebafcc25b696b1f7511adf253dd14ca2153652de2e9953823cf98a
+size 4895442400

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:08a0ab69d649c1293d7f1c49749feb59260a2c8e28e1ec9a4b4b18e5cb385b0e
-size 4932751008
+oid sha256:109e1723323dcf5ea303e87b4450181c2b4df5446085ff49f06de7b3cdee3ad7
+size 4991497784

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e470ca361e99f6a85af127ecb4f80391ed078d7aed6fc2554a649f400f2afe6
-size 4330865200
+oid sha256:5f11714fcd99ef9f2aecb013958e70165b6dff5803801ddd6761042f69a2f4e0
+size 4932752872

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3eb3387772c3fbe8e9aa349ed4cbafdd46b118d3418f93e06a8e2d785d145351
-size 2370335624
+oid sha256:394652a278904bd992eddaec542ca2540aa5ae4ebce8c580c2615b03bd9abb00
+size 1689086112

qwen_model_v2.py

@@ -0,0 +1,253 @@
+from transformers import Qwen2ForCausalLM, AutoModel, Qwen2Config
+from transformers.models.whisper.modeling_whisper import WhisperEncoderLayer
+from transformers import WhisperPreTrainedModel, WhisperConfig
+from transformers.modeling_outputs import BaseModelOutput, CausalLMOutputWithPast
+from transformers.modeling_utils import PreTrainedModel
+from transformers.generation import GenerationMixin
+from transformers.models.auto import AutoModel, AutoModelForCausalLM
+from torch import nn
+import torch
+import math
+import logging
+
+class WhisperEncoder(WhisperPreTrainedModel):
+    
+    def __init__(self, config: WhisperConfig):
+        super().__init__(config)
+        self.dropout = config.dropout
+        self.layerdrop = config.encoder_layerdrop
+
+        embed_dim = config.d_model
+        self.num_mel_bins = config.num_mel_bins
+        self.padding_idx = config.pad_token_id
+        self.max_source_positions = config.max_source_positions
+        self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0
+
+        self.conv1 = nn.Conv1d(self.num_mel_bins, embed_dim, kernel_size=3, padding=1)
+        self.conv2 = nn.Conv1d(embed_dim, embed_dim, kernel_size=3, stride=2, padding=1)
+        
+        self.register_buffer('range_max_source_positions', torch.arange(self.max_source_positions))
+
+        self.embed_positions = nn.Embedding(self.max_source_positions, embed_dim)
+        self.embed_positions.requires_grad_(False)
+
+        self.layers = nn.ModuleList([WhisperEncoderLayer(config) for _ in range(config.encoder_layers)])
+        self.layer_norm = nn.LayerNorm(config.d_model)
+
+        self.gradient_checkpointing = False
+        self.post_init()
+
+    def _freeze_parameters(self):
+        for param in self.parameters():
+            param.requires_grad = False
+        self._requires_grad = False
+
+    def get_input_embeddings(self) -> nn.Module:
+        return self.conv1
+
+    def set_input_embeddings(self, value: nn.Module):
+        self.conv1 = value
+
+    def forward(
+        self,
+        input_features,
+        attention_mask=None,
+        head_mask=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+    ):
+
+        expected_seq_length = self.config.max_source_positions * self.conv1.stride[0] * self.conv2.stride[0]
+        if input_features.shape[-1] != expected_seq_length:
+            raise ValueError(
+                f"Whisper expects the mel input features to be of length {expected_seq_length}, but found {input_features.shape[-1]}. Make sure to pad the input mel features to {expected_seq_length}."
+            )
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        inputs_embeds = nn.functional.gelu(self.conv1(input_features))
+        inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds))
+
+        inputs_embeds = inputs_embeds.permute(0, 2, 1)
+        embed_pos = self.embed_positions(self.range_max_source_positions)
+
+        hidden_states = inputs_embeds + embed_pos
+        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
+
+        encoder_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+
+        # check if head_mask has a correct number of layers specified if desired
+        if head_mask is not None:
+            assert head_mask.size()[0] == (len(self.layers)), (
+                f"The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.size()[0]}."
+            )
+
+        for idx, encoder_layer in enumerate(self.layers):
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description)
+            to_drop = False
+            if self.training:
+                dropout_probability = torch.rand([])
+                if dropout_probability < self.layerdrop:  # skip the layer
+                    to_drop = True
+
+            if to_drop:
+                layer_outputs = (None, None)
+            else:
+                if self.gradient_checkpointing and self.training:
+                    layer_outputs = self._gradient_checkpointing_func(
+                        encoder_layer.__call__,
+                        hidden_states,
+                        None,
+                        (head_mask[idx] if head_mask is not None else None),
+                        output_attentions,
+                    )
+                else:
+                    layer_outputs = encoder_layer(
+                        hidden_states,
+                        None,
+                        layer_head_mask=(head_mask[idx] if head_mask is not None else None),
+                        output_attentions=output_attentions,
+                    )
+
+                hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+        hidden_states = self.layer_norm(hidden_states)
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
+        return BaseModelOutput(
+            last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
+        )
+
+class LLMAudioPreTrainedModel(PreTrainedModel):
+    config_class = Qwen2Config
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _skip_keys_device_placement = "past_key_values"
+    _supports_flash_attn_2 = True
+    _supports_sdpa = True
+
+    def _init_weights(self, module):
+        std = (
+            self.config.initializer_range
+            if hasattr(self.config, "initializer_range")
+            else self.config.audio_config.initializer_range
+        )
+
+        if isinstance(module, (nn.Linear, nn.Conv1d)):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+                
+class LLMAudioForConditionalGeneration(LLMAudioPreTrainedModel, GenerationMixin):
+    def __init__(self, config):
+        super().__init__(config)
+        
+        audio_config = WhisperConfig.from_dict(config.audio_encoder_config)
+        self.encoder = WhisperEncoder(audio_config)
+        self.projection = nn.Linear(self.encoder.config.d_model, self.config.hidden_size, bias=False)
+        self.language_model = AutoModelForCausalLM.from_config(config)
+        if self.language_model._tied_weights_keys is not None:
+            self._tied_weights_keys = [f"language_model.{k}" for k in self.language_model._tied_weights_keys]
+    
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.language_model.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.language_model.get_output_embeddings()
+
+    def set_output_embeddings(self, new_embeddings):
+        self.language_model.set_output_embeddings(new_embeddings)
+
+    def set_decoder(self, decoder):
+        self.language_model.set_decoder(decoder)
+
+    def get_decoder(self):
+        return self.language_model.get_decoder()
+    
+    def forward(
+        self, 
+        input_ids, 
+        attention_mask = None,
+        position_ids = None,
+        input_features = None, 
+        feature_attention_mask = None, 
+        past_key_values = None,
+        print_input_features_shape = False,
+        inputs_embeds = None,
+        **kwargs,
+    ):
+        target_device = self.encoder.device
+
+        if input_features is not None:
+            input_features = input_features.to(target_device)
+            feature_attention_mask = feature_attention_mask.to(target_device)
+           
+        if inputs_embeds is None:
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+            if input_features is not None and input_ids.shape[1] != 1:
+                batch_size, _, max_mel_seq_len = input_features.shape
+                max_seq_len = (max_mel_seq_len - 2) // 2 + 1
+                audio_feat_lengths = self.encoder._get_feat_extract_output_lengths(feature_attention_mask.sum(-1))
+                seq_range = (
+                    torch.arange(0, max_seq_len, dtype=audio_feat_lengths.dtype, device=audio_feat_lengths.device)
+                    .unsqueeze(0)
+                    .expand(batch_size, max_seq_len)
+                )
+                lengths_expand = audio_feat_lengths.unsqueeze(1).expand(batch_size, max_seq_len)
+                padding_mask = seq_range >= lengths_expand
+
+                audio_attention_mask_ = padding_mask.view(batch_size, 1, 1, max_seq_len).expand(
+                    batch_size, 1, max_seq_len, max_seq_len
+                )
+                audio_attention_mask = audio_attention_mask_.to(
+                    dtype=self.encoder.conv1.weight.dtype, device=self.encoder.conv1.weight.device
+                )
+                audio_attention_mask[audio_attention_mask_] = float("-inf")
+                audio_outputs = self.encoder(input_features, attention_mask=audio_attention_mask)
+                selected_audio_feature = audio_outputs.last_hidden_state
+                audio_features = self.projection(selected_audio_feature)
+                num_audio_tokens = audio_feat_lengths
+                num_audios, max_audio_tokens, embed_dim = audio_features.shape
+                audio_features_mask = torch.arange(max_audio_tokens).expand(num_audios, max_audio_tokens).to(
+                    num_audio_tokens.device
+                ) < num_audio_tokens.unsqueeze(1)
+                masked_audio_features = audio_features[audio_features_mask].view(-1, embed_dim)
+                if print_input_features_shape:
+                    print(masked_audio_features.shape, masked_audio_features.contiguous())
+                inputs_embeds[input_ids == self.config.audio_token_index] = masked_audio_features.contiguous()
+        
+        outputs = self.language_model(
+            inputs_embeds=inputs_embeds, 
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+        )
+
+        logits = outputs[0]
+        return CausalLMOutputWithPast(
+            loss=None,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )