Commit
·
e0543ab
1
Parent(s):
02d3e22
Remove unnused token
Browse files- special_tokens_map.json +0 -1
- tokenizer.json +5 -12
special_tokens_map.json
CHANGED
@@ -14,7 +14,6 @@
|
|
14 |
"<start_system>",
|
15 |
"<start_user>",
|
16 |
"<start_assistant>",
|
17 |
-
"<start_example>",
|
18 |
"<end_message>"
|
19 |
],
|
20 |
"eos_token": "<|endoftext|>"
|
|
|
14 |
"<start_system>",
|
15 |
"<start_user>",
|
16 |
"<start_assistant>",
|
|
|
17 |
"<end_message>"
|
18 |
],
|
19 |
"eos_token": "<|endoftext|>"
|
tokenizer.json
CHANGED
@@ -141,15 +141,6 @@
|
|
141 |
{
|
142 |
"id": 65020,
|
143 |
"special": true,
|
144 |
-
"content": "<start_example>",
|
145 |
-
"single_word": false,
|
146 |
-
"lstrip": false,
|
147 |
-
"rstrip": false,
|
148 |
-
"normalized": false
|
149 |
-
},
|
150 |
-
{
|
151 |
-
"id": 65019,
|
152 |
-
"special": true,
|
153 |
"content": "<end_message>",
|
154 |
"single_word": false,
|
155 |
"lstrip": false,
|
@@ -65217,8 +65208,8 @@
|
|
65217 |
"vee": 65016,
|
65218 |
"ĠSEN": 65017,
|
65219 |
"adero": 65018,
|
65220 |
-
"
|
65221 |
-
"<
|
65222 |
"<start_assistant>": 65021,
|
65223 |
"<start_user>": 65022,
|
65224 |
"<start_system>": 65023
|
@@ -130001,7 +129992,9 @@
|
|
130001 |
"s now",
|
130002 |
"Ġd end",
|
130003 |
"ve e",
|
130004 |
-
"ĠS EN"
|
|
|
|
|
130005 |
]
|
130006 |
}
|
130007 |
}
|
|
|
141 |
{
|
142 |
"id": 65020,
|
143 |
"special": true,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
144 |
"content": "<end_message>",
|
145 |
"single_word": false,
|
146 |
"lstrip": false,
|
|
|
65208 |
"vee": 65016,
|
65209 |
"ĠSEN": 65017,
|
65210 |
"adero": 65018,
|
65211 |
+
"ĠAFF": 65019,
|
65212 |
+
"<end_message>": 65020,
|
65213 |
"<start_assistant>": 65021,
|
65214 |
"<start_user>": 65022,
|
65215 |
"<start_system>": 65023
|
|
|
129992 |
"s now",
|
129993 |
"Ġd end",
|
129994 |
"ve e",
|
129995 |
+
"ĠS EN",
|
129996 |
+
"ad ero",
|
129997 |
+
"ĠA FF"
|
129998 |
]
|
129999 |
}
|
130000 |
}
|