ohallstrom commited on
Commit
e0543ab
·
1 Parent(s): 02d3e22

Remove unnused token

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +0 -1
  2. tokenizer.json +5 -12
special_tokens_map.json CHANGED
@@ -14,7 +14,6 @@
14
  "<start_system>",
15
  "<start_user>",
16
  "<start_assistant>",
17
- "<start_example>",
18
  "<end_message>"
19
  ],
20
  "eos_token": "<|endoftext|>"
 
14
  "<start_system>",
15
  "<start_user>",
16
  "<start_assistant>",
 
17
  "<end_message>"
18
  ],
19
  "eos_token": "<|endoftext|>"
tokenizer.json CHANGED
@@ -141,15 +141,6 @@
141
  {
142
  "id": 65020,
143
  "special": true,
144
- "content": "<start_example>",
145
- "single_word": false,
146
- "lstrip": false,
147
- "rstrip": false,
148
- "normalized": false
149
- },
150
- {
151
- "id": 65019,
152
- "special": true,
153
  "content": "<end_message>",
154
  "single_word": false,
155
  "lstrip": false,
@@ -65217,8 +65208,8 @@
65217
  "vee": 65016,
65218
  "ĠSEN": 65017,
65219
  "adero": 65018,
65220
- "<end_message>": 65019,
65221
- "<start_example>": 65020,
65222
  "<start_assistant>": 65021,
65223
  "<start_user>": 65022,
65224
  "<start_system>": 65023
@@ -130001,7 +129992,9 @@
130001
  "s now",
130002
  "Ġd end",
130003
  "ve e",
130004
- "ĠS EN"
 
 
130005
  ]
130006
  }
130007
  }
 
141
  {
142
  "id": 65020,
143
  "special": true,
 
 
 
 
 
 
 
 
 
144
  "content": "<end_message>",
145
  "single_word": false,
146
  "lstrip": false,
 
65208
  "vee": 65016,
65209
  "ĠSEN": 65017,
65210
  "adero": 65018,
65211
+ "ĠAFF": 65019,
65212
+ "<end_message>": 65020,
65213
  "<start_assistant>": 65021,
65214
  "<start_user>": 65022,
65215
  "<start_system>": 65023
 
129992
  "s now",
129993
  "Ġd end",
129994
  "ve e",
129995
+ "ĠS EN",
129996
+ "ad ero",
129997
+ "ĠA FF"
129998
  ]
129999
  }
130000
  }