special_tokens_map.json

{
  "additional_special_tokens": [
    {
      "content": "<|ip|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|endoftext|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|startoftranscript|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|en|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|zh|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|de|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|es|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ru|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ko|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|fr|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ja|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|pt|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|tr|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|pl|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ca|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|nl|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ar|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|sv|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|it|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|id|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|hi|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|fi|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|vi|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|he|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|uk|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|el|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ms|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|cs|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ro|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|da|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|hu|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ta|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|no|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|th|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ur|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|hr|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|bg|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|lt|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|la|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|mi|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ml|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|cy|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|sk|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|te|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|fa|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|lv|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|bn|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|sr|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|az|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|sl|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|kn|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|et|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|mk|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|br|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|eu|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|is|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|hy|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ne|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|mn|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|bs|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|kk|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|sq|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|sw|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|gl|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|mr|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|pa|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|si|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|km|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|sn|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|yo|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|so|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|af|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|oc|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ka|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|be|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|tg|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|sd|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|gu|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|am|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|yi|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|lo|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|uz|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|fo|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ht|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ps|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|tk|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|nn|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|mt|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|sa|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|lb|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|my|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|bo|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|tl|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|mg|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|as|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|tt|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|haw|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ln|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ha|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|ba|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|jw|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|su|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|translate|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|transcribe|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|startoflm|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|startofprev|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|nocaptions|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    },
    {
      "content": "<|notimestamps|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": false
    }
  ],
  "bos_token": {
    "content": "<|endoftext|>",
    "lstrip": false,
    "normalized": false,
    "rstrip": false,
    "single_word": false
  },
  "eos_token": {
    "content": "<|endoftext|>",
    "lstrip": false,
    "normalized": false,
    "rstrip": false,
    "single_word": false
  },
  "pad_token": {
    "content": "<|endoftext|>",
    "lstrip": false,
    "normalized": false,
    "rstrip": false,
    "single_word": false
  },
  "unk_token": {
    "content": "<|endoftext|>",
    "lstrip": false,
    "normalized": false,
    "rstrip": false,
    "single_word": false
  }
}