Hugging Face's logo

minpeter
/
Llama-3.2-1B-AlternateTokenizer-tool-chatml

Text Generation
Transformers
Safetensors
PyTorch
llama
facebook
meta
llama-3
text-generation-inference
Inference Endpoints
Model card Files Community
minpeter commited on
Commit
10f4b4d
·
verified ·
1 Parent(s): 6d37dae

Upload folder using huggingface_hub

Browse files
Files changed (2) hide show
  1. tokenizer.json +6 -6
  2. tokenizer_config.json +6 -6
tokenizer.json CHANGED
@@ -109,7 +109,7 @@
109
  "lstrip": false,
110
  "rstrip": false,
111
  "normalized": false,
112
- "special": true
113
  },
114
  {
115
  "id": 128012,
@@ -118,7 +118,7 @@
118
  "lstrip": false,
119
  "rstrip": false,
120
  "normalized": false,
121
- "special": true
122
  },
123
  {
124
  "id": 128013,
@@ -127,7 +127,7 @@
127
  "lstrip": false,
128
  "rstrip": false,
129
  "normalized": false,
130
- "special": true
131
  },
132
  {
133
  "id": 128014,
@@ -136,7 +136,7 @@
136
  "lstrip": false,
137
  "rstrip": false,
138
  "normalized": false,
139
- "special": true
140
  },
141
  {
142
  "id": 128015,
@@ -145,7 +145,7 @@
145
  "lstrip": false,
146
  "rstrip": false,
147
  "normalized": false,
148
- "special": true
149
  },
150
  {
151
  "id": 128016,
@@ -154,7 +154,7 @@
154
  "lstrip": false,
155
  "rstrip": false,
156
  "normalized": false,
157
- "special": true
158
  },
159
  {
160
  "id": 128017,
 
109
  "lstrip": false,
110
  "rstrip": false,
111
  "normalized": false,
112
+ "special": false
113
  },
114
  {
115
  "id": 128012,
 
118
  "lstrip": false,
119
  "rstrip": false,
120
  "normalized": false,
121
+ "special": false
122
  },
123
  {
124
  "id": 128013,
 
127
  "lstrip": false,
128
  "rstrip": false,
129
  "normalized": false,
130
+ "special": false
131
  },
132
  {
133
  "id": 128014,
 
136
  "lstrip": false,
137
  "rstrip": false,
138
  "normalized": false,
139
+ "special": false
140
  },
141
  {
142
  "id": 128015,
 
145
  "lstrip": false,
146
  "rstrip": false,
147
  "normalized": false,
148
+ "special": false
149
  },
150
  {
151
  "id": 128016,
 
154
  "lstrip": false,
155
  "rstrip": false,
156
  "normalized": false,
157
+ "special": false
158
  },
159
  {
160
  "id": 128017,
tokenizer_config.json CHANGED
@@ -94,7 +94,7 @@
94
  "normalized": false,
95
  "rstrip": false,
96
  "single_word": false,
97
- "special": true
98
  },
99
  "128012": {
100
  "content": "</tools>",
@@ -102,7 +102,7 @@
102
  "normalized": false,
103
  "rstrip": false,
104
  "single_word": false,
105
- "special": true
106
  },
107
  "128013": {
108
  "content": "<tool_call>",
@@ -110,7 +110,7 @@
110
  "normalized": false,
111
  "rstrip": false,
112
  "single_word": false,
113
- "special": true
114
  },
115
  "128014": {
116
  "content": "</tool_call>",
@@ -118,7 +118,7 @@
118
  "normalized": false,
119
  "rstrip": false,
120
  "single_word": false,
121
- "special": true
122
  },
123
  "128015": {
124
  "content": "<tool_response>",
@@ -126,7 +126,7 @@
126
  "normalized": false,
127
  "rstrip": false,
128
  "single_word": false,
129
- "special": true
130
  },
131
  "128016": {
132
  "content": "</tool_response>",
@@ -134,7 +134,7 @@
134
  "normalized": false,
135
  "rstrip": false,
136
  "single_word": false,
137
- "special": true
138
  },
139
  "128017": {
140
  "content": "<|reserved_special_token_9|>",
 
94
  "normalized": false,
95
  "rstrip": false,
96
  "single_word": false,
97
+ "special": false
98
  },
99
  "128012": {
100
  "content": "</tools>",
 
102
  "normalized": false,
103
  "rstrip": false,
104
  "single_word": false,
105
+ "special": false
106
  },
107
  "128013": {
108
  "content": "<tool_call>",
 
110
  "normalized": false,
111
  "rstrip": false,
112
  "single_word": false,
113
+ "special": false
114
  },
115
  "128014": {
116
  "content": "</tool_call>",
 
118
  "normalized": false,
119
  "rstrip": false,
120
  "single_word": false,
121
+ "special": false
122
  },
123
  "128015": {
124
  "content": "<tool_response>",
 
126
  "normalized": false,
127
  "rstrip": false,
128
  "single_word": false,
129
+ "special": false
130
  },
131
  "128016": {
132
  "content": "</tool_response>",
 
134
  "normalized": false,
135
  "rstrip": false,
136
  "single_word": false,
137
+ "special": false
138
  },
139
  "128017": {
140
  "content": "<|reserved_special_token_9|>",