Upload folder using huggingface_hub

config.json

@@ -0,0 +1,34 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoTokenizer": [
+      "gptx_tokenizer.SPTokenizer",
+      null
+    ]
+  },
+  "bos_token_id": 1,
+  "eos_token_id": 4,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.0158,
+  "intermediate_size": 13440,
+  "max_position_embeddings": 4096,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 2,
+  "pad_token_id": 3,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": true,
+  "tokenizer_class": "SPTokenizer",
+  "transformers_version": "4.38.1",
+  "use_cache": true,
+  "vocab_size": 250880
+}

gptx_tokenizer.py

@@ -0,0 +1,34 @@
+"""
+This module supplies `transformers`-compatible wrappers for
+`GPTXTokenizer`s.
+
+The tokenizers in this do not conform to the `PreTrainedTokenizer` API,
+but allow for better practical usage.
+"""
+
+from typing import List
+
+from gptx_tokenizer.hf_wrappers import (
+    HFTokenizer as _HFTokenizer,
+    SPTokenizer as _SPTokenizer,
+)
+
+class HFTokenizer(_HFTokenizer):
+    # The tokenizer is ridiculously slow without this; however, this
+    # doesn't implement all APIs of `PreTrainedTokenizer`.
+    def encode(self, text: str, **kwargs) -> List[int]:
+        return_tokens = kwargs.pop('return_tokens', False)
+        return self._tok.encode(text, return_tokens=return_tokens)
+
+
+class SPTokenizer(_SPTokenizer):
+    # `is_continuation` does not work without this, but it doesn't
+    # implement all APIs of `PreTrainedTokenizer`.
+    def encode(self, text: str, **kwargs) -> List[int]:
+        return_tokens = kwargs.pop('return_tokens', False)
+        is_continuation = kwargs.pop('is_continuation', False)
+        return self._tok.encode(
+            text,
+            return_tokens=return_tokens,
+            is_continuation=is_continuation,
+        )

pytorch_model-00001-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05319fd8f2ca0b076cd179e250c73f7cfd7ac4a1c57525db057bc94690ae58b3
+size 3953224892

pytorch_model-00002-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9aff5a1f04d4e861ea1b08a7e1e61afe6cdbf5e9ba7b1ddeac9eaba74134c31d
+size 4016233683

pytorch_model-00003-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9db1331d9645b8fa62a353c6b6a8620fee3daa6474f297d730dabb9e9bbcee3d
+size 4016233683

pytorch_model-00004-of-00004.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6595d0f85fcc78f53ea2ff3084801d72e9c29003a627f5f03dc1bee2a9e4a31c
+size 2921472588

pytorch_model.bin.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14772871168
+  },
+  "weight_map": {
+    "lm_head.weight": "pytorch_model-00001-of-00004.bin",
+    "model.embed_tokens.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.10.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.15.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.15.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.15.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.16.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.16.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.16.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.17.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.17.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.17.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.18.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.18.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.18.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.18.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.18.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.18.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.20.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.mlp.down_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.input_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.24.mlp.gate_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.mlp.up_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.post_attention_layernorm.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.self_attn.k_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00003-of-00004.bin",
+    "model.layers.25.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.25.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.25.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.25.mlp.up_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.25.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.25.self_attn.k_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.26.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.26.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.26.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.26.mlp.up_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.26.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.26.self_attn.k_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.26.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.26.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.27.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.27.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.27.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.27.mlp.up_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.27.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.27.self_attn.k_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.27.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.27.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.28.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.28.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.28.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.28.mlp.up_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.28.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.28.self_attn.k_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.28.self_attn.o_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.28.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.28.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.29.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.29.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.29.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.29.mlp.up_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.29.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.29.self_attn.k_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.29.self_attn.o_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.29.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.29.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.30.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.30.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.30.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.30.mlp.up_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.30.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.30.self_attn.k_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.30.self_attn.o_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.30.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.30.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.31.input_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.31.mlp.down_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.31.mlp.gate_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.31.mlp.up_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.31.post_attention_layernorm.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.31.self_attn.k_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.31.self_attn.o_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.31.self_attn.q_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.31.self_attn.v_proj.weight": "pytorch_model-00004-of-00004.bin",
+    "model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00004.bin",
+    "model.layers.5.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.5.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.5.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.6.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.6.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.6.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.7.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.7.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.7.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.8.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.8.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.8.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.input_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.mlp.down_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.mlp.up_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00002-of-00004.bin",
+    "model.norm.weight": "pytorch_model-00004-of-00004.bin"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,264 @@
+{
+  "additional_special_tokens": [
+    "</s>",
+    "<placeholder_tok_0>",
+    "<placeholder_tok_1>",
+    "<placeholder_tok_2>",
+    "<placeholder_tok_3>",
+    "<placeholder_tok_4>",
+    "<placeholder_tok_5>",
+    "<placeholder_tok_6>",
+    "<placeholder_tok_7>",
+    "<placeholder_tok_8>",
+    "<placeholder_tok_9>",
+    "<placeholder_tok_10>",
+    "<placeholder_tok_11>",
+    "<placeholder_tok_12>",
+    "<placeholder_tok_13>",
+    "<placeholder_tok_14>",
+    "<placeholder_tok_15>",
+    "<placeholder_tok_16>",
+    "<placeholder_tok_17>",
+    "<placeholder_tok_18>",
+    "<placeholder_tok_19>",
+    "<placeholder_tok_20>",
+    "<placeholder_tok_21>",
+    "<placeholder_tok_22>",
+    "<placeholder_tok_23>",
+    "<placeholder_tok_24>",
+    "<placeholder_tok_25>",
+    "<placeholder_tok_26>",
+    "<placeholder_tok_27>",
+    "<placeholder_tok_28>",
+    "<placeholder_tok_29>",
+    "<placeholder_tok_30>",
+    "<placeholder_tok_31>",
+    "<placeholder_tok_32>",
+    "<placeholder_tok_33>",
+    "<placeholder_tok_34>",
+    "<placeholder_tok_35>",
+    "<placeholder_tok_36>",
+    "<placeholder_tok_37>",
+    "<placeholder_tok_38>",
+    "<placeholder_tok_39>",
+    "<placeholder_tok_40>",
+    "<placeholder_tok_41>",
+    "<placeholder_tok_42>",
+    "<placeholder_tok_43>",
+    "<placeholder_tok_44>",
+    "<placeholder_tok_45>",
+    "<placeholder_tok_46>",
+    "<placeholder_tok_47>",
+    "<placeholder_tok_48>",
+    "<placeholder_tok_49>",
+    "<placeholder_tok_50>",
+    "<placeholder_tok_51>",
+    "<placeholder_tok_52>",
+    "<placeholder_tok_53>",
+    "<placeholder_tok_54>",
+    "<placeholder_tok_55>",
+    "<placeholder_tok_56>",
+    "<placeholder_tok_57>",
+    "<placeholder_tok_58>",
+    "<placeholder_tok_59>",
+    "<placeholder_tok_60>",
+    "<placeholder_tok_61>",
+    "<placeholder_tok_62>",
+    "<placeholder_tok_63>",
+    "<placeholder_tok_64>",
+    "<placeholder_tok_65>",
+    "<placeholder_tok_66>",
+    "<placeholder_tok_67>",
+    "<placeholder_tok_68>",
+    "<placeholder_tok_69>",
+    "<placeholder_tok_70>",
+    "<placeholder_tok_71>",
+    "<placeholder_tok_72>",
+    "<placeholder_tok_73>",
+    "<placeholder_tok_74>",
+    "<placeholder_tok_75>",
+    "<placeholder_tok_76>",
+    "<placeholder_tok_77>",
+    "<placeholder_tok_78>",
+    "<placeholder_tok_79>",
+    "<placeholder_tok_80>",
+    "<placeholder_tok_81>",
+    "<placeholder_tok_82>",
+    "<placeholder_tok_83>",
+    "<placeholder_tok_84>",
+    "<placeholder_tok_85>",
+    "<placeholder_tok_86>",
+    "<placeholder_tok_87>",
+    "<placeholder_tok_88>",
+    "<placeholder_tok_89>",
+    "<placeholder_tok_90>",
+    "<placeholder_tok_91>",
+    "<placeholder_tok_92>",
+    "<placeholder_tok_93>",
+    "<placeholder_tok_94>",
+    "<placeholder_tok_95>",
+    "<placeholder_tok_96>",
+    "<placeholder_tok_97>",
+    "<placeholder_tok_98>",
+    "<placeholder_tok_99>",
+    "<placeholder_tok_100>",
+    "<placeholder_tok_101>",
+    "<placeholder_tok_102>",
+    "<placeholder_tok_103>",
+    "<placeholder_tok_104>",
+    "<placeholder_tok_105>",
+    "<placeholder_tok_106>",
+    "<placeholder_tok_107>",
+    "<placeholder_tok_108>",
+    "<placeholder_tok_109>",
+    "<placeholder_tok_110>",
+    "<placeholder_tok_111>",
+    "<placeholder_tok_112>",
+    "<placeholder_tok_113>",
+    "<placeholder_tok_114>",
+    "<placeholder_tok_115>",
+    "<placeholder_tok_116>",
+    "<placeholder_tok_117>",
+    "<placeholder_tok_118>",
+    "<placeholder_tok_119>",
+    "<placeholder_tok_120>",
+    "<placeholder_tok_121>",
+    "<placeholder_tok_122>",
+    "<placeholder_tok_123>",
+    "<placeholder_tok_124>",
+    "<placeholder_tok_125>",
+    "<placeholder_tok_126>",
+    "<placeholder_tok_127>",
+    "<placeholder_tok_128>",
+    "<placeholder_tok_129>",
+    "<placeholder_tok_130>",
+    "<placeholder_tok_131>",
+    "<placeholder_tok_132>",
+    "<placeholder_tok_133>",
+    "<placeholder_tok_134>",
+    "<placeholder_tok_135>",
+    "<placeholder_tok_136>",
+    "<placeholder_tok_137>",
+    "<placeholder_tok_138>",
+    "<placeholder_tok_139>",
+    "<placeholder_tok_140>",
+    "<placeholder_tok_141>",
+    "<placeholder_tok_142>",
+    "<placeholder_tok_143>",
+    "<placeholder_tok_144>",
+    "<placeholder_tok_145>",
+    "<placeholder_tok_146>",
+    "<placeholder_tok_147>",
+    "<placeholder_tok_148>",
+    "<placeholder_tok_149>",
+    "<placeholder_tok_150>",
+    "<placeholder_tok_151>",
+    "<placeholder_tok_152>",
+    "<placeholder_tok_153>",
+    "<placeholder_tok_154>",
+    "<placeholder_tok_155>",
+    "<placeholder_tok_156>",
+    "<placeholder_tok_157>",
+    "<placeholder_tok_158>",
+    "<placeholder_tok_159>",
+    "<placeholder_tok_160>",
+    "<placeholder_tok_161>",
+    "<placeholder_tok_162>",
+    "<placeholder_tok_163>",
+    "<placeholder_tok_164>",
+    "<placeholder_tok_165>",
+    "<placeholder_tok_166>",
+    "<placeholder_tok_167>",
+    "<placeholder_tok_168>",
+    "<placeholder_tok_169>",
+    "<placeholder_tok_170>",
+    "<placeholder_tok_171>",
+    "<placeholder_tok_172>",
+    "<placeholder_tok_173>",
+    "<placeholder_tok_174>",
+    "<placeholder_tok_175>",
+    "<placeholder_tok_176>",
+    "<placeholder_tok_177>",
+    "<placeholder_tok_178>",
+    "<placeholder_tok_179>",
+    "<placeholder_tok_180>",
+    "<placeholder_tok_181>",
+    "<placeholder_tok_182>",
+    "<placeholder_tok_183>",
+    "<placeholder_tok_184>",
+    "<placeholder_tok_185>",
+    "<placeholder_tok_186>",
+    "<placeholder_tok_187>",
+    "<placeholder_tok_188>",
+    "<placeholder_tok_189>",
+    "<placeholder_tok_190>",
+    "<placeholder_tok_191>",
+    "<placeholder_tok_192>",
+    "<placeholder_tok_193>",
+    "<placeholder_tok_194>",
+    "<placeholder_tok_195>",
+    "<placeholder_tok_196>",
+    "<placeholder_tok_197>",
+    "<placeholder_tok_198>",
+    "<placeholder_tok_199>",
+    "<placeholder_tok_200>",
+    "<placeholder_tok_201>",
+    "<placeholder_tok_202>",
+    "<placeholder_tok_203>",
+    "<placeholder_tok_204>",
+    "<placeholder_tok_205>",
+    "<placeholder_tok_206>",
+    "<placeholder_tok_207>",
+    "<placeholder_tok_208>",
+    "<placeholder_tok_209>",
+    "<placeholder_tok_210>",
+    "<placeholder_tok_211>",
+    "<placeholder_tok_212>",
+    "<placeholder_tok_213>",
+    "<placeholder_tok_214>",
+    "<placeholder_tok_215>",
+    "<placeholder_tok_216>",
+    "<placeholder_tok_217>",
+    "<placeholder_tok_218>",
+    "<placeholder_tok_219>",
+    "<placeholder_tok_220>",
+    "<placeholder_tok_221>",
+    "<placeholder_tok_222>",
+    "<placeholder_tok_223>",
+    "<placeholder_tok_224>",
+    "<placeholder_tok_225>",
+    "<placeholder_tok_226>",
+    "<placeholder_tok_227>",
+    "<placeholder_tok_228>",
+    "<placeholder_tok_229>",
+    "<placeholder_tok_230>",
+    "<placeholder_tok_231>",
+    "<placeholder_tok_232>",
+    "<placeholder_tok_233>",
+    "<placeholder_tok_234>",
+    "<placeholder_tok_235>",
+    "<placeholder_tok_236>",
+    "<placeholder_tok_237>",
+    "<placeholder_tok_238>",
+    "<placeholder_tok_239>",
+    "<placeholder_tok_240>",
+    "<placeholder_tok_241>",
+    "<placeholder_tok_242>",
+    "<placeholder_tok_243>",
+    "<placeholder_tok_244>",
+    "<placeholder_tok_245>",
+    "<placeholder_tok_246>",
+    "<placeholder_tok_247>",
+    "<placeholder_tok_248>",
+    "<placeholder_tok_249>",
+    "<placeholder_tok_250>",
+    "<placeholder_tok_251>",
+    "<placeholder_tok_252>",
+    "<placeholder_tok_253>",
+    "<placeholder_tok_254>",
+    "<placeholder_tok_255>"
+  ],
+  "bos_token": "<s>",
+  "eos_token": "<eod>",
+  "pad_token": "<pad>"
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08d0c8316539a853f2fe6e14f51f0df583011dfb078fa08c8b6dc5c15a19a7e6
+size 4719922

tokenizer_config.json

@@ -0,0 +1,292 @@
+{
+    "num_threads": 224,
+    "split_by_whitespace": true,
+    "model_type": "unigram",
+    "vocab_size": 250680,
+    "character_coverage": 0.9999,
+    "byte_fallback": true,
+    "split_by_number": true,
+    "split_digits": true,
+    "normalization_rule_name": "nfkc",
+    "max_sentence_length": 4096,
+    "shuffle_input_sentence": true,
+    "input_sentence_size": 0,
+    "train_extremely_large_corpus": true,
+    "allow_whitespace_only_pieces": true,
+    "required_chars": "",
+    "remove_extra_whitespaces": false,
+    "user_defined_symbols": [
+        "<s>",
+        "</s>",
+        "<pad>",
+        "<eod>",
+        "<placeholder_tok_0>",
+        "<placeholder_tok_1>",
+        "<placeholder_tok_2>",
+        "<placeholder_tok_3>",
+        "<placeholder_tok_4>",
+        "<placeholder_tok_5>",
+        "<placeholder_tok_6>",
+        "<placeholder_tok_7>",
+        "<placeholder_tok_8>",
+        "<placeholder_tok_9>",
+        "<placeholder_tok_10>",
+        "<placeholder_tok_11>",
+        "<placeholder_tok_12>",
+        "<placeholder_tok_13>",
+        "<placeholder_tok_14>",
+        "<placeholder_tok_15>",
+        "<placeholder_tok_16>",
+        "<placeholder_tok_17>",
+        "<placeholder_tok_18>",
+        "<placeholder_tok_19>",
+        "<placeholder_tok_20>",
+        "<placeholder_tok_21>",
+        "<placeholder_tok_22>",
+        "<placeholder_tok_23>",
+        "<placeholder_tok_24>",
+        "<placeholder_tok_25>",
+        "<placeholder_tok_26>",
+        "<placeholder_tok_27>",
+        "<placeholder_tok_28>",
+        "<placeholder_tok_29>",
+        "<placeholder_tok_30>",
+        "<placeholder_tok_31>",
+        "<placeholder_tok_32>",
+        "<placeholder_tok_33>",
+        "<placeholder_tok_34>",
+        "<placeholder_tok_35>",
+        "<placeholder_tok_36>",
+        "<placeholder_tok_37>",
+        "<placeholder_tok_38>",
+        "<placeholder_tok_39>",
+        "<placeholder_tok_40>",
+        "<placeholder_tok_41>",
+        "<placeholder_tok_42>",
+        "<placeholder_tok_43>",
+        "<placeholder_tok_44>",
+        "<placeholder_tok_45>",
+        "<placeholder_tok_46>",
+        "<placeholder_tok_47>",
+        "<placeholder_tok_48>",
+        "<placeholder_tok_49>",
+        "<placeholder_tok_50>",
+        "<placeholder_tok_51>",
+        "<placeholder_tok_52>",
+        "<placeholder_tok_53>",
+        "<placeholder_tok_54>",
+        "<placeholder_tok_55>",
+        "<placeholder_tok_56>",
+        "<placeholder_tok_57>",
+        "<placeholder_tok_58>",
+        "<placeholder_tok_59>",
+        "<placeholder_tok_60>",
+        "<placeholder_tok_61>",
+        "<placeholder_tok_62>",
+        "<placeholder_tok_63>",
+        "<placeholder_tok_64>",
+        "<placeholder_tok_65>",
+        "<placeholder_tok_66>",
+        "<placeholder_tok_67>",
+        "<placeholder_tok_68>",
+        "<placeholder_tok_69>",
+        "<placeholder_tok_70>",
+        "<placeholder_tok_71>",
+        "<placeholder_tok_72>",
+        "<placeholder_tok_73>",
+        "<placeholder_tok_74>",
+        "<placeholder_tok_75>",
+        "<placeholder_tok_76>",
+        "<placeholder_tok_77>",
+        "<placeholder_tok_78>",
+        "<placeholder_tok_79>",
+        "<placeholder_tok_80>",
+        "<placeholder_tok_81>",
+        "<placeholder_tok_82>",
+        "<placeholder_tok_83>",
+        "<placeholder_tok_84>",
+        "<placeholder_tok_85>",
+        "<placeholder_tok_86>",
+        "<placeholder_tok_87>",
+        "<placeholder_tok_88>",
+        "<placeholder_tok_89>",
+        "<placeholder_tok_90>",
+        "<placeholder_tok_91>",
+        "<placeholder_tok_92>",
+        "<placeholder_tok_93>",
+        "<placeholder_tok_94>",
+        "<placeholder_tok_95>",
+        "<placeholder_tok_96>",
+        "<placeholder_tok_97>",
+        "<placeholder_tok_98>",
+        "<placeholder_tok_99>",
+        "<placeholder_tok_100>",
+        "<placeholder_tok_101>",
+        "<placeholder_tok_102>",
+        "<placeholder_tok_103>",
+        "<placeholder_tok_104>",
+        "<placeholder_tok_105>",
+        "<placeholder_tok_106>",
+        "<placeholder_tok_107>",
+        "<placeholder_tok_108>",
+        "<placeholder_tok_109>",
+        "<placeholder_tok_110>",
+        "<placeholder_tok_111>",
+        "<placeholder_tok_112>",
+        "<placeholder_tok_113>",
+        "<placeholder_tok_114>",
+        "<placeholder_tok_115>",
+        "<placeholder_tok_116>",
+        "<placeholder_tok_117>",
+        "<placeholder_tok_118>",
+        "<placeholder_tok_119>",
+        "<placeholder_tok_120>",
+        "<placeholder_tok_121>",
+        "<placeholder_tok_122>",
+        "<placeholder_tok_123>",
+        "<placeholder_tok_124>",
+        "<placeholder_tok_125>",
+        "<placeholder_tok_126>",
+        "<placeholder_tok_127>",
+        "<placeholder_tok_128>",
+        "<placeholder_tok_129>",
+        "<placeholder_tok_130>",
+        "<placeholder_tok_131>",
+        "<placeholder_tok_132>",
+        "<placeholder_tok_133>",
+        "<placeholder_tok_134>",
+        "<placeholder_tok_135>",
+        "<placeholder_tok_136>",
+        "<placeholder_tok_137>",
+        "<placeholder_tok_138>",
+        "<placeholder_tok_139>",
+        "<placeholder_tok_140>",
+        "<placeholder_tok_141>",
+        "<placeholder_tok_142>",
+        "<placeholder_tok_143>",
+        "<placeholder_tok_144>",
+        "<placeholder_tok_145>",
+        "<placeholder_tok_146>",
+        "<placeholder_tok_147>",
+        "<placeholder_tok_148>",
+        "<placeholder_tok_149>",
+        "<placeholder_tok_150>",
+        "<placeholder_tok_151>",
+        "<placeholder_tok_152>",
+        "<placeholder_tok_153>",
+        "<placeholder_tok_154>",
+        "<placeholder_tok_155>",
+        "<placeholder_tok_156>",
+        "<placeholder_tok_157>",
+        "<placeholder_tok_158>",
+        "<placeholder_tok_159>",
+        "<placeholder_tok_160>",
+        "<placeholder_tok_161>",
+        "<placeholder_tok_162>",
+        "<placeholder_tok_163>",
+        "<placeholder_tok_164>",
+        "<placeholder_tok_165>",
+        "<placeholder_tok_166>",
+        "<placeholder_tok_167>",
+        "<placeholder_tok_168>",
+        "<placeholder_tok_169>",
+        "<placeholder_tok_170>",
+        "<placeholder_tok_171>",
+        "<placeholder_tok_172>",
+        "<placeholder_tok_173>",
+        "<placeholder_tok_174>",
+        "<placeholder_tok_175>",
+        "<placeholder_tok_176>",
+        "<placeholder_tok_177>",
+        "<placeholder_tok_178>",
+        "<placeholder_tok_179>",
+        "<placeholder_tok_180>",
+        "<placeholder_tok_181>",
+        "<placeholder_tok_182>",
+        "<placeholder_tok_183>",
+        "<placeholder_tok_184>",
+        "<placeholder_tok_185>",
+        "<placeholder_tok_186>",
+        "<placeholder_tok_187>",
+        "<placeholder_tok_188>",
+        "<placeholder_tok_189>",
+        "<placeholder_tok_190>",
+        "<placeholder_tok_191>",
+        "<placeholder_tok_192>",
+        "<placeholder_tok_193>",
+        "<placeholder_tok_194>",
+        "<placeholder_tok_195>",
+        "<placeholder_tok_196>",
+        "<placeholder_tok_197>",
+        "<placeholder_tok_198>",
+        "<placeholder_tok_199>",
+        "<placeholder_tok_200>",
+        "<placeholder_tok_201>",
+        "<placeholder_tok_202>",
+        "<placeholder_tok_203>",
+        "<placeholder_tok_204>",
+        "<placeholder_tok_205>",
+        "<placeholder_tok_206>",
+        "<placeholder_tok_207>",
+        "<placeholder_tok_208>",
+        "<placeholder_tok_209>",
+        "<placeholder_tok_210>",
+        "<placeholder_tok_211>",
+        "<placeholder_tok_212>",
+        "<placeholder_tok_213>",
+        "<placeholder_tok_214>",
+        "<placeholder_tok_215>",
+        "<placeholder_tok_216>",
+        "<placeholder_tok_217>",
+        "<placeholder_tok_218>",
+        "<placeholder_tok_219>",
+        "<placeholder_tok_220>",
+        "<placeholder_tok_221>",
+        "<placeholder_tok_222>",
+        "<placeholder_tok_223>",
+        "<placeholder_tok_224>",
+        "<placeholder_tok_225>",
+        "<placeholder_tok_226>",
+        "<placeholder_tok_227>",
+        "<placeholder_tok_228>",
+        "<placeholder_tok_229>",
+        "<placeholder_tok_230>",
+        "<placeholder_tok_231>",
+        "<placeholder_tok_232>",
+        "<placeholder_tok_233>",
+        "<placeholder_tok_234>",
+        "<placeholder_tok_235>",
+        "<placeholder_tok_236>",
+        "<placeholder_tok_237>",
+        "<placeholder_tok_238>",
+        "<placeholder_tok_239>",
+        "<placeholder_tok_240>",
+        "<placeholder_tok_241>",
+        "<placeholder_tok_242>",
+        "<placeholder_tok_243>",
+        "<placeholder_tok_244>",
+        "<placeholder_tok_245>",
+        "<placeholder_tok_246>",
+        "<placeholder_tok_247>",
+        "<placeholder_tok_248>",
+        "<placeholder_tok_249>",
+        "<placeholder_tok_250>",
+        "<placeholder_tok_251>",
+        "<placeholder_tok_252>",
+        "<placeholder_tok_253>",
+        "<placeholder_tok_254>",
+        "<placeholder_tok_255>"
+    ],
+    "datasets_dir": "/home/fhgiais/gptx_ablations/bias_analysis/data/tokenizer/temp/",
+    "save_dir": "/home/fhgiais/gptx_ablations/bias_analysis/tokenizer/24",
+    "text_key": "text",
+    "cache_dir": "/home/fhgiais/gptx_ablations/bias_analysis/tokenizer/24/cache",
+    "library": "sentencepiece",
+    "auto_map": {
+        "AutoTokenizer": [
+            "gptx_tokenizer.SPTokenizer",
+            null
+        ]
+    },
+    "tokenizer_class": "SPTokenizer"
+}