Duplicate from TRI-ML/DCLM-1B

Co-authored-by: Achal Dave <[email protected]>

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,181 @@
+---
+license: apache-2.0
+---
+
+
+
+<img src="https://cdn-uploads.huggingface.co/production/uploads/63118add64939fabc0108b28/BB42g4V8HTxb5dR4tcy8A.png" alt="DCLM Logo" width="300" style="margin-left:'auto' margin-right:'auto' display:'block'"/>
+
+
+# Model Card for DCLM-1B
+
+DCLM-1B is a 1.4 billion parameter language model trained on the DCLM-Baseline dataset, which was curated as part of the DataComp for Language Models (DCLM) benchmark. This model is designed to showcase the effectiveness of systematic data curation techniques for improving language model performance.
+
+The instruction tuned version of this model is available here: https://huggingface.co/TRI-ML/DCLM-1B-IT
+
+## Quickstart
+First install open_lm
+```
+pip install git+https://github.com/mlfoundations/open_lm.git
+```
+
+Then you can load the model using HF's Auto classes as follows:
+```python
+from open_lm.hf import *
+from transformers import AutoTokenizer, AutoModelForCausalLM
+tokenizer = AutoTokenizer.from_pretrained("TRI-ML/DCLM-1B")
+model = AutoModelForCausalLM.from_pretrained("TRI-ML/DCLM-1B")
+
+inputs = tokenizer(["Machine learning is"], return_tensors="pt")
+gen_kwargs = {"max_new_tokens": 50, "top_p": 0.8, "temperature": 0.8, "do_sample": True, "repetition_penalty": 1.1}
+output = model.generate(inputs['input_ids'], **gen_kwargs)
+output = tokenizer.decode(output[0].tolist(), skip_special_tokens=True)
+print(output)
+```
+
+
+## Evaluation
+
+We evaluate DCLM-1B using the [llm-foundry](https://github.com/mosaicml/llm-foundry) eval suite, and compare to recently released small models on key benchmarks.
+As described in the paper, Core accuracy is the average of centered accuracy on
+22 tasks (including HellaSwag and ARC-E), Extended is centered accuracy averaged
+over 53 tasks.
+
+
+| Model                             | Params | Tokens | Open dataset? | Core  | MMLU 5-shot    | Extended |
+|-----------------------------------|:--------:|:--------:|:---------------:|:----------:|:----------:|:-----------:|
+| **Open weights, closed datasets** |        |        |               |          |          |           |
+| Qwen2-1.5B                        | 1.5B   | 7T      | ❌             | 42.1     | **56.4** | **32.4**  |
+| Gemma-2B                          | 2.5B   | 3T     | ❌             | **43.3** | 40.8     | 26.6      |
+| **Open weights, open datasets**   |        |        |               |          |          |           |
+| OLMo-1B                           | 1.2B   | 3T     | ✅             | 29.7     | 26.0     | 16.1      |
+| SmolLM                            | 1.7B   | 1T     | ✅             | 36.3     | 30.0     | 21.2      |
+| DCLM-1B                           | 1.4B   | 4.3T   | ✅             | **45.2** | **47.5** | **28.1**  |
+
+## Model Details
+
+| Size | Training Tokens | Layers | Hidden Size | Attention Heads | Context Length |
+|:------:|:-----------------:|:--------:|:-------------:|:-----------------:|:----------------:|
+| 1.4B   | 4.3T            | 24     | 2048        | 16              | 2048           |
+
+
+### Model Description
+
+- **Developed by:** DataComp for Language Models (DCLM) Team
+- **Model type:** Decoder-only Transformer language model
+- **Language(s):** English (primarily)
+- **License:** Apache 2.0
+- **Contact:** [email protected]
+- **Date:** July 2024
+
+### Model Sources
+
+- **Repository:** https://github.com/mlfoundations/dclm
+- **Dataset:** https://huggingface.co/datasets/mlfoundations/dclm-baseline-1.0
+- **Paper:** [DataComp-LM: In search of the next generation of training sets for language models](https://arxiv.org/abs/2406.11794)
+
+
+### Training Details
+
+The model was trained using the following setup:
+
+- **Architecture:** Decoder-only Transformer 
+- **Framework:** PyTorch with OpenLM
+- **Optimizer:** AdamW
+- **Learning Rate:** 1e-2 (peak)
+- **Weight Decay:** 1e-2
+- **Batch Size:** 2048 sequences
+- **Sequence Length:** 2048 tokens
+- **Total Training Tokens:** 4.3T
+- **Hardware:** Trained on H100 GPUs
+
+
+We train our 1.4B model for 4.3T tokens on DCLM-Baseline, combined with the
+StarCoder and ProofPile2 datasets.
+We will update our paper soon with more training details.
+
+
+### Detailed evaluation
+
+| Task                                     | Score   |
+|------------------------------------------|---------|
+| AGI Eval LSAT AR                         |  0.2652 |
+| AGI Eval LSAT LR                         |  0.3314 |
+| AGI Eval LSAT RC                         |  0.4179 |
+| AGI Eval SAT English                     |  0.4709 |
+| AGI Eval SAT Math (CoT)                  |  0.0318 |
+| AQuA (CoT)                               |  0.0245 |
+| ARC (challenge)                          |  0.4744 |
+| ARC (easy)                               |  0.7462 |
+| BBQ                                      |  0.5151 |
+| BigBench Conceptual Combinations         |  0.5437 |
+| BigBench Conlang Translation             |  0.0793 |
+| BigBench CS Algorithms                   |  0.4720 |
+| BigBench Dyck Languages                  |  0.2210 |
+| BigBench Elementary Math QA              |  0.2598 |
+| BigBench Language Identification         |  0.3284 |
+| BigBench Logical Deduction               |  0.2473 |
+| BigBench Misconceptions                  |  0.5662 |
+| BigBench Novel Concepts                  |  0.5000 |
+| BigBench Operators                       |  0.3476 |
+| BigBench QA Wikidata                     |  0.6852 |
+| BigBench Repeat Copy Logic               |  0.1250 |
+| BigBench Strange Stories                 |  0.6724 |
+| BigBench Strategy QA                     |  0.5671 |
+| BigBench Understanding Fables            |  0.4603 |
+| BoolQ                                    |  0.7382 |
+| CommonSenseQA                            |  0.6708 |
+| COPA                                     |  0.8200 |
+| CoQA                                     |  0.4314 |
+| Enterprise PII Classification            |  0.5246 |
+| GPQA Diamond                             |  0.2424 |
+| GPQA                                     |  0.2500 |
+| GSM8K (CoT)                              |  0.0629 |
+| HellaSwag                                |  0.7285 |
+| HellaSwag (zero-shot)                    |  0.7162 |
+| Jeopardy                                 |  0.4514 |
+| LAMBADA (OpenAI)                         |  0.6992 |
+| LogiQA                                   |  0.3103 |
+| MathQA                                   |  0.2682 |
+| MMLU (few-shot)                          |  0.4752 |
+| MMLU (zero-shot)                         |  0.4175 |
+| OpenBookQA                               |  0.4280 |
+| PIQA                                     |  0.7829 |
+| PubMedQA (labeled)                       |  0.3790 |
+| Simple Arithmetic (no spaces)            |  0.0650 |
+| Simple Arithmetic (with spaces)          |  0.0700 |
+| SIQA                                     |  0.6868 |
+| SQuAD                                    |  0.5494 |
+| SVAMP (CoT)                              |  0.2733 |
+| TriviaQA (small subset)                  |  0.4133 |
+| Winogender (MC female)                   |  0.4667 |
+| Winogender (MC male)                     |  0.4000 |
+| Winograd                                 |  0.8608 |
+| Winogrande                               |  0.6630 |
+
+
+## Limitations and Biases
+
+While DCLM-1B demonstrates strong performance across a range of tasks, it's important to note:
+
+1. The model may exhibit biases present in its training data, which is derived from web crawl data.
+2. It has not undergone specific alignment or safety fine-tuning, so outputs should be used with caution.
+3. Performance on tasks not included in the evaluation suite may vary.
+4. The model's knowledge is limited to its training data cutoff date.
+
+## Ethical Considerations
+
+Users should be aware that this model, like all large language models, can potentially generate harmful or biased content. It should not be used for making decisions about individuals or in sensitive applications without appropriate safeguards and human oversight.
+
+## Citation
+
+If you use this model in your research, please cite:
+
+```
+@article{Li2024DataCompLM,
+  title={DataComp-LM: In search of the next generation of training sets for language models},
+  author={Jeffrey Li and Alex Fang and Georgios Smyrnis and Maor Ivgi and Matt Jordan and Samir Gadre and Hritik Bansal and Etash Guha and Sedrick Keh and Kushal Arora and [... full author list]},
+  journal={arXiv preprint arXiv:2406.11794},
+  year={2024}
+}
+```

config.json

@@ -0,0 +1,34 @@
+{
+  "architectures": [
+    "OpenLMModel"
+  ],
+  "model_type": "openlm",
+  "params": null,
+    "apply_qk_norm": true,
+    "attn_activation": null,
+    "attn_name": "torch_attn",
+    "attn_seq_scalar": null,
+    "attn_seq_scalar_alpha": null,
+    "dim": 2048,
+    "ffn_type": "swiglu_torch",
+    "model": "open_lm_1b_swiglutorch",
+    "norm_type": "gain_only_lp_layer_norm",
+    "moe_capacity_factor": 1.25,
+    "moe_expert_model_parallelism": false,
+    "moe_freq": 0,
+    "moe_loss_weight": 0.1,
+    "moe_num_experts": null,
+    "moe_top_k": 2,
+    "moe_weight_parallelism": false,
+    "n_heads": 16,
+    "n_layers": 24,
+    "norm_eps": 1e-05,
+    "positional_embedding_type": "rotary",
+    "post_embed_norm": false,
+    "qk_norm": true,
+    "seq_len": 2048,
+    "vocab_size": 50432,
+    "weight_tying": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.40.2"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97eadb24e4c75851e56e30b34f6b8fc6b2eaf32e69d01c83f189c6625569c4aa
+size 4985679432

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29ca61873460f22081cc0f71c2a97a12bc9b314a943d528d6f6ac3184a185865
+size 773925424

model.safetensors.index.json

@@ -0,0 +1,226 @@
+{
+  "metadata": {
+    "total_size": 5759580160
+  },
+  "weight_map": {
+    "model.layers.0.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.0.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.1.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.10.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.11.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.12.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.13.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.14.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.15.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.16.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.17.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.18.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.19.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.2.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.20.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.21.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.22.attention.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.attention.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.attention.pos_embed.inv_freq": "model-00002-of-00002.safetensors",
+    "model.layers.22.attention.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.feed_forward.w12.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.attention.in_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.attention.k_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.attention.out_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.attention.pos_embed.inv_freq": "model-00002-of-00002.safetensors",
+    "model.layers.23.attention.q_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.attention_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.feed_forward.w12.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.feed_forward.w3.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.ffn_norm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.3.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.4.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.5.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.6.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.7.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.8.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.attention.in_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.attention.k_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.attention.out_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.attention.pos_embed.inv_freq": "model-00001-of-00002.safetensors",
+    "model.layers.9.attention.q_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.attention_norm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.feed_forward.w12.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.feed_forward.w3.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.ffn_norm.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors",
+    "model.output.weight": "model-00002-of-00002.safetensors",
+    "model.tok_embeddings.weight": "model-00001-of-00002.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "tokenizer_class": "GPTNeoXTokenizer"}