Upload DeepSeekV2Lite DenseMixer model

DeepSeekV2Lite model with DenseMixer architecture

modeling_deepseek.py

@@ -666,7 +666,7 @@ class DeepseekV2MoE(nn.Module):
                 matches = (topk_idx == expert_idx)
                 if matches.any():
                     token_indices, k_indices = torch.where(matches)
-                    weights_topk = topk_weight[token_indices, k_indices].unsqueeze(-1)  # (num_matches, 1)
+                    weights_topk = topk_weight[token_indices, k_indices].unsqueeze(-1).to(sparse_outputs.dtype)  # (num_matches, 1)
                     sparse_outputs[token_indices] = sparse_outputs[token_indices] + expert_output[token_indices] * weights_topk
         else:
             # 推理模式：使用原始的稀疏计算逻辑