axolotl-ai-co
/

rrt-smollm2-1_7b-checkpoint-4B-tokens

winglian commited on 8 days ago

Commit

bb2e3a6

verified ·

1 Parent(s): 29f5dc1

Upload folder using huggingface_hub

Files changed (1) hide show

tokenizer_config.json CHANGED Viewed

@@ -162,7 +162,7 @@
   "eos_token": "<|endoftext|>",
   "extra_special_tokens": {},
   "model_max_length": 8192,
-  "pad_token": "<|endoftext|>",
   "tokenizer_class": "GPT2Tokenizer",
   "unk_token": "<|endoftext|>",
   "vocab_size": 49152

   "eos_token": "<|endoftext|>",
   "extra_special_tokens": {},
   "model_max_length": 8192,
+  "pad_token": "<|empty_output|>",
   "tokenizer_class": "GPT2Tokenizer",
   "unk_token": "<|endoftext|>",
   "vocab_size": 49152