Upload folder using huggingface_hub

config.json

@@ -4,9 +4,17 @@
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
-  "bos_token_id": 100257,
+  "bos_token_id": [
+    100257,
+    100272
+  ],
   "end_token_id": 100257,
-  "eos_token_id": 100257,
+  "eos_token_id": [
+    100257,
+    100273,
+    100274,
+    100275
+  ],
   "head_dim": 128,
   "hidden_act": "silu",
   "hidden_size": 3072,
@@ -30,4 +38,4 @@
   "transformers_version": "4.51.3",
   "use_cache": true,
   "vocab_size": 110592
-}
+}

special_tokens_map.json

@@ -56,13 +56,19 @@
     "<PASSWORD>"
   ],
   "bos_token": {
-    "content": "<|endoftext|>",
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
-  "eos_token": "<|endofturn|>",
   "pad_token": {
     "content": "<|endoftext|>",
     "lstrip": false,
@@ -77,4 +83,4 @@
     "rstrip": false,
     "single_word": false
   }
-}
+}

tokenizer_config.json

@@ -490,13 +490,13 @@
     "<KEY>",
     "<PASSWORD>"
   ],
-  "bos_token": "<|endoftext|>",
+  "bos_token": "<|im_start|>",
   "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": true,
-  "eos_token": "<|endofturn|>",
+  "eos_token": "<|im_end|>",
   "extra_special_tokens": {},
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<|endoftext|>",
   "tokenizer_class": "GPT2Tokenizer",
   "unk_token": "<|endoftext|>"
-}
+}