Goekdeniz-Guelmez commited on
Commit
541ef2d
·
verified ·
1 Parent(s): 18cd860

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -7
tokenizer_config.json CHANGED
@@ -199,14 +199,9 @@
199
  "clean_up_tokenization_spaces": false,
200
  "eos_token": "<|im_end|>",
201
  "errors": "replace",
202
- "extra_special_tokens": {},
203
- "max_length": null,
204
  "model_max_length": 32768,
205
- "pad_to_multiple_of": null,
206
- "pad_token": "<|vision_pad|>",
207
- "pad_token_type_id": 0,
208
- "padding_side": "left",
209
  "split_special_tokens": false,
210
  "tokenizer_class": "Qwen2Tokenizer",
211
  "unk_token": null
212
- }
 
199
  "clean_up_tokenization_spaces": false,
200
  "eos_token": "<|im_end|>",
201
  "errors": "replace",
 
 
202
  "model_max_length": 32768,
203
+ "pad_token": "<|endoftext|>",
 
 
 
204
  "split_special_tokens": false,
205
  "tokenizer_class": "Qwen2Tokenizer",
206
  "unk_token": null
207
+ }