Huseyin commited on
Commit
e17cc9b
·
verified ·
1 Parent(s): 54c88d7

Upload tokenizer_config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +15 -0
tokenizer_config.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "PreTrainedTokenizerFast",
3
+ "model_type": "qwen3",
4
+ "vocab_size": 151936,
5
+ "model_max_length": 8192,
6
+ "padding_side": "left",
7
+ "truncation_side": "right",
8
+ "chat_template": "{% for message in messages %}{{ message.content }}{% endfor %}",
9
+ "clean_up_tokenization_spaces": false,
10
+ "split_special_tokens": false,
11
+ "bos_token": "<|im_start|>",
12
+ "eos_token": "<|im_end|>",
13
+ "unk_token": "<UNK>",
14
+ "pad_token": "<PAD>"
15
+ }