Upload folder using huggingface_hub

README.md

@@ -0,0 +1,130 @@
+# **Scaling Reasoning without Attention** 
+
+
+[![ArXiv](https://img.shields.io/badge/arXiv-2505.22425-red)](http://arxiv.org/abs/2505.22425)
+[![GitHub](https://img.shields.io/badge/GitHub-PromptCoT-blue)](https://github.com/inclusionAI/PromptCoT)
+
+---
+
+## 🚀 Overview
+
+**PromptCoT-Mamba** establishes the first **attention-free foundation model** capable of surpassing strong Transformer baselines across a broad suite of competition-level math and code reasoning tasks. Built on the **Mamba-2** architecture and trained through a structured, two-stage curriculum using the [**PromptCoT**](http://arxiv.org/abs/2503.02324) pipeline, it delivers **high accuracy with constant-memory inference**, eliminating the need for KV caching.
+
+
+---
+
+## 📈 Key Results
+
+### 🔹 General Performance
+
+| Model                  | MATH-500 | AIME 24  | AIME 25  | OlympiadBench | HumanEval | HumanEval+ | Livecodebench |
+| ---------------------- | -------- | -------- | -------- | ------------- | --------- | ---------- | ------------- |
+| **PromptCoT-Mamba-7B** | 84.6     | **35.2** | **24.6** | 50.7          | 81.7      | 75.0       | **29.9**      |
+| Gemma3-27B             | **89.0** | 32.6     | 24.0     | **54.2**      | **86.0**  | **78.0**   | 26.9          |
+| Gemma3-12B             | 83.8     | 22.9     | 19.2     | 49.9          | 81.1      | 73.2       | 22.2          |
+| Sky-T1-7B              | 85.0     | 19.2     | 19.2     | 49.2          | 41.5      | 37.2       | 18.3          |
+| S1.1-7B                | 82.0     | 19.2     | 17.5     | 43.1          | 64.0      | 56.7       | 13.3          |
+| Bespoke-Stratos-7B     | 81.2     | 18.3     | 16.3     | 45.0          | 73.2      | 68.3       | 8.6           |
+| Nemotron-H-8B          | 77.6     | --       | --       | --            | 79.3      | 74.4       | --            |
+| M1-3B                  | 81.7     | 23.0     | 22.0     | 43.6          | --        | --         | --            |
+
+> 🔍 **PromptCoT-Mamba-7B** consistently outperforms all 7B-scale Transformer and hybrid Mamba-Transformer baselines across all tasks.
+
+---
+
+### 🔹 Math Specialization vs. Generalist
+
+| Model                       | MATH-500 | AIME 24  | AIME 25  | OlympiadBench | HumanEval | HumanEval+ | Livecodebench |
+| --------------------------- | -------- | -------- | -------- | ------------- | --------- | ---------- | ------------- |
+| **PromptCoT-Mamba-Math-7B** | **88.0** | **42.9** | **30.8** | **52.1**      | 71.3      | 66.5       | 20.3          |
+| PromptCoT-Mamba-7B          | 84.6     | 35.2     | 24.6     | 50.7          | **81.7**  | **75.0**   | **29.9**      |
+
+> 🎯 The math-specialized variant improves AIME 24 by **+7.7%** and AIME 25 by **+6.2%**, with a slight trade-off in code-related performance.
+
+---
+
+### ⚡ Inference Efficiency
+
+Using `vLLM` under constrained memory, PromptCoT-Mamba-7B demonstrates substantial speedups over the S1.1-7B Transformer baseline:
+
+* 💡 **3.66× faster** at long-sequence generation on **24GB GPU**
+* 💡 **1.69× faster** under **72GB memory**
+
+> ⚙️ Practical for cost-sensitive or long-context inference workloads at scale.
+
+---
+
+## 🧪 Quick Start
+
+### 🔧 Install Requirements
+
+```bash
+pip install transformers vllm torch accelerate
+```
+
+### 🧠 Load and Run the Model
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model_name = "xl-zhao/PromptCoT-Mamba-7B"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name).to("cuda")
+
+problem_statement = (
+    "A robe takes 2 bolts of blue fiber and half that much white fiber.  How many bolts in total does it take?"
+)
+
+prompt = (
+    f"<|im_start|>user\n{problem_statement}\nPlease reason step by step, and put your final answer within \\boxed{{}}.<|im_end|>\n"
+    "<|im_start|>assistant\n"
+)
+
+inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
+
+with torch.no_grad():
+    output = model.generate(**inputs, max_length=65536, temperature=0.8)
+
+generated_solution = tokenizer.decode(output[0], skip_special_tokens=True)
+print(generated_solution)
+```
+
+---
+
+## ⚡ Fast Inference with vLLM
+
+```python
+from vllm import LLM, SamplingParams
+
+model_name = "xl-zhao/PromptCoT-Mamba-7B"
+llm = LLM(model=model_name, tensor_parallel_size=1)
+
+problem_statement = (
+    "A robe takes 2 bolts of blue fiber and half that much white fiber.  How many bolts in total does it take?"
+)
+
+prompt = (
+    f"<|im_start|>user\n{problem_statement}\nPlease reason step by step, and put your final answer within \\boxed{{}}.<|im_end|>\n"
+    "<|im_start|>assistant\n"
+)
+
+sampling_params = SamplingParams(temperature=0.8, max_tokens=65536)
+outputs = llm.generate([prompt], sampling_params)
+
+print(outputs[0].outputs[0].text)
+```
+
+---
+
+## 📜 Citation
+
+```bibtex
+@article{zhao2025scaling,
+  author    = {Xueliang Zhao and Wei Wu and Lingpeng Kong},
+  title     = {Scaling Reasoning without Attention},
+  journal   = {arXiv preprint arXiv:2505.22425},
+  year      = {2025},
+  url       = {https://arxiv.org/abs/2505.22425}
+}
+```
+

config.json

@@ -0,0 +1,43 @@
+{
+  "architectures": [
+    "Mamba2ForCausalLM"
+  ],
+  "bos_token_id": 0,
+  "chunk_size": 256,
+  "conv_kernel": 4,
+  "eos_token_id": 0,
+  "expand": 2,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.1,
+  "intermediate_size": 8192,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "mamba2",
+  "n_groups": 8,
+  "norm_before_gate": true,
+  "num_heads": 128,
+  "num_hidden_layers": 64,
+  "pad_token_id": 0,
+  "rescale_prenorm_residual": false,
+  "residual_in_fp32": true,
+  "rms_norm": true,
+  "state_size": 128,
+  "tie_word_embeddings": false,
+  "time_step_floor": 0.0001,
+  "time_step_init_scheme": "random",
+  "time_step_limit": [
+    0.0,
+    Infinity
+  ],
+  "time_step_max": 0.1,
+  "time_step_min": 0.001,
+  "time_step_rank": 256,
+  "time_step_scale": 1.0,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.49.0",
+  "use_bias": false,
+  "use_cache": false,
+  "use_conv_bias": true,
+  "vocab_size": 32768
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "pad_token_id": 1,
+  "transformers_version": "4.49.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:114bed2638f47387c22f9f74eebed655c969e260282512829c4b0ecf15ccbd8d
+size 4873450856

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e6f13bbdd9691e0a78bcf9d02fd42b9506b077db0c80e63822d3f8b3a32d0c8
+size 4976244624

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edb717707a82f8e42e96aeab68e9cd095e1f34688c85187526cf9049278fada8
+size 4721175056

model.safetensors.index.json

@@ -0,0 +1,586 @@
+{
+  "metadata": {
+    "total_size": 14570807296
+  },
+  "weight_map": {
+    "backbone.embeddings.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.0.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.0.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.0.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.0.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.0.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.0.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.0.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.0.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.0.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.1.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.1.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.1.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.1.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.1.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.1.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.1.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.1.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.1.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.10.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.10.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.10.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.10.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.10.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.10.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.10.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.10.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.10.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.11.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.11.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.11.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.11.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.11.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.11.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.11.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.11.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.11.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.12.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.12.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.12.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.12.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.12.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.12.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.12.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.12.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.12.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.13.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.13.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.13.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.13.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.13.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.13.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.13.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.13.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.13.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.14.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.14.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.14.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.14.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.14.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.14.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.14.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.14.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.14.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.15.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.15.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.15.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.15.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.15.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.15.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.15.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.15.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.15.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.16.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.16.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.16.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.16.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.16.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.16.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.16.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.16.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.16.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.17.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.17.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.17.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.17.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.17.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.17.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.17.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.17.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.17.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.18.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.18.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.18.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.18.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.18.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.18.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.18.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.18.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.18.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.19.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.19.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.19.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.19.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.19.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.19.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.19.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.19.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.19.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.2.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.2.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.2.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.2.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.2.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.2.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.2.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.2.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.2.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.20.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.20.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.20.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.20.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.20.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.20.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.20.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.20.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.20.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.21.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.21.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.21.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.21.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.21.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.21.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.21.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.21.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.21.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.22.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.22.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.22.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.22.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.22.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.22.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.22.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.22.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.22.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.23.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.23.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.23.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.23.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.23.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.23.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.23.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.23.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.23.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.24.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.24.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.24.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.24.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.24.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.24.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.24.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.24.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.24.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.25.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.25.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.25.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.25.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.25.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.25.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.25.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.25.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.25.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.26.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.26.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.26.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.26.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.26.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.26.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.26.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.26.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.26.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.27.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.27.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.27.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.27.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.27.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.27.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.27.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.27.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.27.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.28.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.28.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.28.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.28.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.28.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.28.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.28.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.28.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.28.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.29.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.29.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.29.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.29.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.29.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.29.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.29.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.29.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.29.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.3.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.3.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.3.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.3.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.3.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.3.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.3.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.3.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.3.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.30.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.30.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.30.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.30.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.30.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.30.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.30.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.30.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.30.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.31.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.31.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.31.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.31.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.31.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.31.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.31.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.31.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.31.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.32.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.32.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.32.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.32.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.32.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.32.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.32.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.32.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.32.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.33.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.33.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.33.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.33.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.33.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.33.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.33.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.33.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.33.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.34.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.34.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.34.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.34.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.34.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.34.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.34.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.34.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.34.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.35.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.35.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.35.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.35.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.35.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.35.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.35.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.35.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.35.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.36.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.36.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.36.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.36.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.36.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.36.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.36.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.36.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.36.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.37.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.37.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.37.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.37.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.37.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.37.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.37.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.37.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.37.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.38.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.38.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.38.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.38.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.38.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.38.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.38.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.38.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.38.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.39.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.39.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.39.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.39.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.39.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.39.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.39.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.39.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.39.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.4.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.4.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.4.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.4.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.4.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.4.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.4.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.4.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.4.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.40.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.40.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.40.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.40.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.40.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.40.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.40.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.40.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.40.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.41.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.41.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.41.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.41.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.41.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.41.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.41.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.41.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.41.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.42.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.42.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.42.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.42.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.42.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.42.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.42.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.42.mixer.out_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.42.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.43.mixer.A_log": "model-00002-of-00003.safetensors",
+    "backbone.layers.43.mixer.D": "model-00002-of-00003.safetensors",
+    "backbone.layers.43.mixer.conv1d.bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.43.mixer.conv1d.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.43.mixer.dt_bias": "model-00002-of-00003.safetensors",
+    "backbone.layers.43.mixer.in_proj.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.43.mixer.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.43.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.43.norm.weight": "model-00002-of-00003.safetensors",
+    "backbone.layers.44.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.44.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.44.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.44.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.44.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.44.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.44.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.44.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.44.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.45.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.45.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.45.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.45.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.45.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.45.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.45.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.45.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.45.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.46.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.46.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.46.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.46.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.46.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.46.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.46.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.46.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.46.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.47.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.47.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.47.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.47.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.47.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.47.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.47.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.47.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.47.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.48.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.48.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.48.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.48.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.48.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.48.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.48.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.48.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.48.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.49.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.49.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.49.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.49.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.49.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.49.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.49.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.49.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.49.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.5.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.5.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.5.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.5.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.5.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.5.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.5.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.5.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.5.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.50.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.50.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.50.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.50.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.50.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.50.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.50.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.50.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.50.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.51.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.51.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.51.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.51.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.51.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.51.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.51.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.51.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.51.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.52.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.52.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.52.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.52.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.52.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.52.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.52.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.52.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.52.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.53.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.53.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.53.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.53.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.53.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.53.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.53.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.53.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.53.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.54.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.54.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.54.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.54.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.54.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.54.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.54.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.54.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.54.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.55.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.55.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.55.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.55.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.55.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.55.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.55.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.55.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.55.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.56.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.56.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.56.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.56.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.56.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.56.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.56.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.56.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.56.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.57.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.57.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.57.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.57.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.57.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.57.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.57.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.57.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.57.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.58.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.58.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.58.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.58.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.58.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.58.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.58.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.58.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.58.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.59.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.59.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.59.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.59.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.59.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.59.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.59.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.59.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.59.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.6.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.6.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.6.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.6.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.6.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.6.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.6.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.6.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.6.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.60.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.60.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.60.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.60.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.60.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.60.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.60.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.60.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.60.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.61.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.61.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.61.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.61.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.61.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.61.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.61.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.61.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.61.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.62.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.62.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.62.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.62.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.62.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.62.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.62.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.62.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.62.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.63.mixer.A_log": "model-00003-of-00003.safetensors",
+    "backbone.layers.63.mixer.D": "model-00003-of-00003.safetensors",
+    "backbone.layers.63.mixer.conv1d.bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.63.mixer.conv1d.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.63.mixer.dt_bias": "model-00003-of-00003.safetensors",
+    "backbone.layers.63.mixer.in_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.63.mixer.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.63.mixer.out_proj.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.63.norm.weight": "model-00003-of-00003.safetensors",
+    "backbone.layers.7.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.7.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.7.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.7.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.7.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.7.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.7.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.7.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.7.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.8.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.8.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.8.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.8.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.8.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.8.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.8.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.8.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.8.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.9.mixer.A_log": "model-00001-of-00003.safetensors",
+    "backbone.layers.9.mixer.D": "model-00001-of-00003.safetensors",
+    "backbone.layers.9.mixer.conv1d.bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.9.mixer.conv1d.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.9.mixer.dt_bias": "model-00001-of-00003.safetensors",
+    "backbone.layers.9.mixer.in_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.9.mixer.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.9.mixer.out_proj.weight": "model-00001-of-00003.safetensors",
+    "backbone.layers.9.norm.weight": "model-00001-of-00003.safetensors",
+    "backbone.norm_f.weight": "model-00003-of-00003.safetensors",
+    "lm_head.weight": "model-00003-of-00003.safetensors"
+  }
+}

requirements.txt

@@ -0,0 +1,186 @@
+accelerate==1.6.0
+addict==2.4.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.11.14
+aiohttp-cors==0.8.0
+aiosignal==1.3.2
+airportsdata==20250224
+annotated-types==0.7.0
+anyio==4.9.0
+astor==0.8.1
+attrs==25.3.0
+blake3==1.0.4
+cachetools==5.5.2
+causal-conv1d==1.5.0
+certifi==2025.1.31
+charset-normalizer==3.4.1
+click==8.1.8
+cloudpickle==3.1.1
+cmake==3.31.6
+colorful==0.5.6
+compressed-tensors==0.9.1
+cupy-cuda12x==13.4.1
+datasets==3.2.0
+deepspeed==0.16.7
+depyf==0.18.0
+dill==0.3.8
+diskcache==5.6.3
+distlib==0.3.9
+distro==1.9.0
+dnspython==2.7.0
+einops==0.8.1
+email_validator==2.2.0
+fastapi==0.115.11
+fastapi-cli==0.0.7
+fastrlock==0.8.3
+filelock==3.18.0
+fire==0.7.0
+flash_attn==2.7.4.post1
+frozenlist==1.5.0
+fsspec==2024.9.0
+gguf==0.10.0
+google-api-core==2.24.2
+google-auth==2.38.0
+googleapis-common-protos==1.69.2
+grpcio==1.71.0
+h11==0.14.0
+hjson==3.1.0
+httpcore==1.0.7
+httptools==0.6.4
+httpx==0.28.1
+huggingface-hub==0.29.3
+idna==3.10
+immutables==0.20
+importlib_metadata==8.6.1
+iniconfig==2.1.0
+intel-cmplr-lib-ur==2025.0.5
+intel-openmp==2025.0.5
+interegular==0.3.3
+Jinja2==3.1.6
+jiter==0.9.0
+joblib==1.4.2
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+lark==1.2.2
+llvmlite==0.43.0
+lm-format-enforcer==0.10.11
+mamba-ssm==2.2.3
+mambapy==1.2.0
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+mdurl==0.1.2
+mistral_common==1.5.4
+mkl==2025.0.1
+mkl-include==2025.0.1
+modelscope==1.24.0
+mpmath==1.3.0
+msgpack==1.1.0
+msgspec==0.19.0
+multidict==6.2.0
+multiprocess==0.70.16
+nest-asyncio==1.6.0
+networkx==3.4.2
+ninja==1.11.1.3
+numba==0.60.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-ml-py==12.570.86
+nvidia-nccl-cu12==2.21.5
+nvidia-nvjitlink-cu12==12.8.93
+nvidia-nvtx-cu12==12.1.105
+openai==1.66.3
+opencensus==0.11.4
+opencensus-context==0.1.3
+opencv-python-headless==4.11.0.86
+outlines==0.1.11
+outlines_core==0.1.26
+packaging==24.2
+pandas==2.2.3
+partial-json-parser==0.2.1.1.post5
+peft==0.15.2
+pillow==11.1.0
+platformdirs==4.3.6
+pluggy==1.5.0
+prometheus-fastapi-instrumentator==7.0.2
+prometheus_client==0.21.1
+propcache==0.3.0
+proto-plus==1.26.1
+protobuf==6.30.1
+psutil==7.0.0
+py-cpuinfo==9.0.0
+py-spy==0.4.0
+pyarrow==19.0.1
+pyasn1==0.6.1
+pyasn1_modules==0.4.1
+pybind11==2.13.6
+pycountry==24.6.1
+pydantic==2.10.6
+pydantic_core==2.27.2
+Pygments==2.19.1
+pytest==8.3.5
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-multipart==0.0.20
+pytz==2025.1
+PyYAML==6.0.2
+pyzmq==26.3.0
+ray==2.40.0
+referencing==0.36.2
+regex==2024.11.6
+requests==2.32.3
+rich==14.0.0
+rich-toolkit==0.14.1
+rpds-py==0.23.1
+rsa==4.9
+safetensors==0.5.3
+scikit-learn==1.6.1
+scipy==1.15.2
+sentencepiece==0.2.0
+shellingham==1.5.4
+simplejson==3.20.1
+six==1.17.0
+smart-open==7.1.0
+sniffio==1.3.1
+sortedcontainers==2.4.0
+starlette==0.46.1
+str2bool==1.1
+sympy==1.13.1
+tbb==2022.0.0
+tcmlib==1.2.0
+termcolor==2.5.0
+threadpoolctl==3.6.0
+tiktoken==0.9.0
+timm==1.0.15
+tokenizers==0.21.1
+torch==2.5.1
+torchaudio==2.5.1
+torchvision==0.20.1
+tqdm==4.66.5
+transformers==4.49.0
+triton==3.1.0
+trl==0.16.1
+typer==0.15.2
+typing_extensions==4.12.2
+tzdata==2025.1
+umf==0.9.1
+urllib3==2.3.0
+uvicorn==0.34.0
+uvloop==0.21.0
+virtualenv==20.29.3
+vllm==0.7.3
+watchfiles==1.0.4
+websockets==15.0.1
+wrapt==1.17.2
+xformers==0.0.28.post3
+xgrammar==0.1.11
+xxhash==3.5.0
+yarl==1.18.3
+zipp==3.21.0

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59f95e28944c062244741268596badc900df86c7f5ded05088d2da22a7379e06
+size 587583