{ "version": "1.0", "truncation": null, "padding": null, "added_tokens": [ { "id": 15, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true } ], "normalizer": null, "pre_tokenizer": { "type": "Whitespace" }, "post_processor": null, "decoder": null, "model": { "type": "WordLevel", "vocab": { "r": 0, "u": 1, "f": 2, "l": 3, "d": 4, "b": 5, "R": 6, "RR": 7, "RRR": 8, "U": 9, "UU": 10, "UUU": 11, "F": 12, "FF": 13, "FFF": 14, "<|endoftext|>": 15 }, "unk_token": "<|endoftext|>" } }