tokenizer_config.json

{
    "add_bos_token": true,
    "add_eos_token": false,
    "bos_token": {
      "__type": "AddedToken",
      "content": "<｜begin▁of▁sentence｜>",
      "lstrip": false,
      "normalized": true,
      "rstrip": false,
      "single_word": false
    },
    "clean_up_tokenization_spaces": false,
    "eos_token": {
      "__type": "AddedToken",
      "content": "<｜end▁of▁sentence｜>",
      "lstrip": false,
      "normalized": true,
      "rstrip": false,
      "single_word": false
    },
    "legacy": true,
    "model_max_length": 16384,
    "sp_model_kwargs": {},
    "unk_token": null,
    "tokenizer_class": "LlamaTokenizerFast",
    "chat_template": "{%- set found_item = false -%}\n{%- for message in messages -%}\n    {%- if message['role'] == 'system' -%}\n        {%- set found_item = true -%}\n    {%- endif -%}\n{%- endfor -%}\n{%- if not found_item -%}\n{{''}}\n{%- endif %}\n{%- for message in messages %}\n    {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n    {%- else %}\n        {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n        {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{{'### Response:\\n'}}\n"
  }