Safetensors
codebyzeb's picture
Upload folder using huggingface_hub
9e9685b verified
raw
history blame contribute delete
2.62 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "UNK",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 1,
"content": "PAD",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 2,
"content": "WORD_BOUNDARY",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
},
{
"id": 3,
"content": "UTT_BOUNDARY",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "Sequence",
"normalizers": [
{
"type": "Strip",
"strip_left": true,
"strip_right": true
}
]
},
"pre_tokenizer": {
"type": "Whitespace"
},
"post_processor": {
"type": "TemplateProcessing",
"single": [
{
"SpecialToken": {
"id": "UTT_BOUNDARY",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
}
],
"pair": [
{
"SpecialToken": {
"id": "UTT_BOUNDARY",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "UTT_BOUNDARY",
"type_id": 0
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
}
],
"special_tokens": {
"UTT_BOUNDARY": {
"id": "UTT_BOUNDARY",
"ids": [
3
],
"tokens": [
"UTT_BOUNDARY"
]
}
}
},
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {
"UNK": 0,
"PAD": 1,
"WORD_BOUNDARY": 2,
"UTT_BOUNDARY": 3,
"蓻": 4,
"l": 5,
"尾": 6,
"瑟": 7,
"n": 8,
"a": 9,
"s": 10,
"t": 11,
"r": 12,
"d": 13,
"a藧": 14,
"t虪蕛": 15,
"m": 16,
"蓴": 17,
"h": 18,
"p": 19,
"蕣": 20,
"伞": 21,
"k": 22,
"q": 23,
"f": 24,
"j": 25,
"w": 26,
"蕩": 27,
"p始": 28,
"蕯": 29,
"t始": 30,
"t虪蕛始": 31,
"k始": 32,
"瑟藧": 33,
"q始": 34,
"蓻藧": 35
},
"unk_token": "UNK"
}
}