{ "add_bos_token": true, "add_eos_token": false, "add_prefix_space": true, "added_tokens_decoder": { "0": { "content": "<unk>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "1": { "content": "<s>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "</s>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128111": { "content": "<restate>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128112": { "content": "</restate>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128113": { "content": "<planning>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128114": { "content": "</planning>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128115": { "content": "<recollect>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128116": { "content": "</recollect>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128117": { "content": "<execution>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128118": { "content": "</execution>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128119": { "content": "<review>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128120": { "content": "</review>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128121": { "content": "<summarize>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128122": { "content": "</summarize>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128123": { "content": "<retry>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128124": { "content": "</retry>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128125": { "content": "<conclude>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128126": { "content": "</conclude>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128127": { "content": "<|plugin|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128128": { "content": "<|interpreter|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128129": { "content": "<|action_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128130": { "content": "<|action_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128131": { "content": "<|im_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "128132": { "content": "<|im_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "additional_special_tokens": [ "<|im_start|>", "<|im_end|>", "<|action_start|>", "<|action_end|>", "<|interpreter|>", "<|plugin|>", "<restate>", "</restate>", "<planning>", "</planning>", "<recollect>", "</recollect>", "<execution>", "</execution>", "<review>", "</review>", "<summarize>", "</summarize>", "<retry>", "</retry>", "<conclude>", "</conclude>" ], "auto_map": { "AutoTokenizer": [ "internlm/internlm3-8b-instruct--tokenization_internlm3.InternLM3Tokenizer", null ] }, "bos_token": "<s>", "chat_template": "{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", "clean_up_tokenization_spaces": false, "eos_token": "</s>", "extra_special_tokens": {}, "model_max_length": 1000000000000000019884624838656, "pad_token": "</s>", "sp_model_kwargs": {}, "spaces_between_special_tokens": false, "tokenizer_class": "InternLM3Tokenizer", "unk_token": "<unk>", "use_default_system_prompt": false }