Hugging Face's logo

minpeter
/
Llama-3.2-1B-AlternateTokenizer-tool-chatml

Text Generation
Transformers
Safetensors
PyTorch
llama
facebook
meta
llama-3
text-generation-inference
Inference Endpoints
Model card Files Community
minpeter commited on
Commit
f2528b7
·
verified ·
1 Parent(s): cf35ef0

Upload folder using huggingface_hub

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +1 -1
  2. tokenizer.json +5 -5
  3. tokenizer_config.json +5 -5
special_tokens_map.json CHANGED
@@ -7,7 +7,7 @@
7
  "single_word": false
8
  },
9
  "eos_token": {
10
- "content": "<|end_of_text|>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
 
7
  "single_word": false
8
  },
9
  "eos_token": {
10
+ "content": "<|im_end|>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
tokenizer.json CHANGED
@@ -14,7 +14,7 @@
14
  },
15
  {
16
  "id": 128001,
17
- "content": "<|end_of_text|>",
18
  "single_word": false,
19
  "lstrip": false,
20
  "rstrip": false,
@@ -23,16 +23,16 @@
23
  },
24
  {
25
  "id": 128002,
26
- "content": "<|reserved_special_token_0|>",
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
30
  "normalized": false,
31
- "special": true
32
  },
33
  {
34
  "id": 128003,
35
- "content": "<|reserved_special_token_1|>",
36
  "single_word": false,
37
  "lstrip": false,
38
  "rstrip": false,
@@ -410560,4 +410560,4 @@
410560
  "éĶ ¦"
410561
  ]
410562
  }
410563
- }
 
14
  },
15
  {
16
  "id": 128001,
17
+ "content": "<|im_end|>",
18
  "single_word": false,
19
  "lstrip": false,
20
  "rstrip": false,
 
23
  },
24
  {
25
  "id": 128002,
26
+ "content": "<|im_start|>",
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
30
  "normalized": false,
31
+ "special": false
32
  },
33
  {
34
  "id": 128003,
35
+ "content": "<|end_of_text|>",
36
  "single_word": false,
37
  "lstrip": false,
38
  "rstrip": false,
 
410560
  "éĶ ¦"
410561
  ]
410562
  }
410563
+ }
tokenizer_config.json CHANGED
@@ -9,7 +9,7 @@
9
  "special": true
10
  },
11
  "128001": {
12
- "content": "<|end_of_text|>",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
@@ -17,15 +17,15 @@
17
  "special": true
18
  },
19
  "128002": {
20
- "content": "<|reserved_special_token_0|>",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
24
  "single_word": false,
25
- "special": true
26
  },
27
  "128003": {
28
- "content": "<|reserved_special_token_1|>",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
@@ -2051,7 +2051,7 @@
2051
  },
2052
  "bos_token": "<|begin_of_text|>",
2053
  "clean_up_tokenization_spaces": true,
2054
- "eos_token": "<|end_of_text|>",
2055
  "model_input_names": [
2056
  "input_ids",
2057
  "attention_mask"
 
9
  "special": true
10
  },
11
  "128001": {
12
+ "content": "<|im_end|>",
13
  "lstrip": false,
14
  "normalized": false,
15
  "rstrip": false,
 
17
  "special": true
18
  },
19
  "128002": {
20
+ "content": "<|im_start|>",
21
  "lstrip": false,
22
  "normalized": false,
23
  "rstrip": false,
24
  "single_word": false,
25
+ "special": false
26
  },
27
  "128003": {
28
+ "content": "<|end_of_text|>",
29
  "lstrip": false,
30
  "normalized": false,
31
  "rstrip": false,
 
2051
  },
2052
  "bos_token": "<|begin_of_text|>",
2053
  "clean_up_tokenization_spaces": true,
2054
+ "eos_token": "<|im_end|>",
2055
  "model_input_names": [
2056
  "input_ids",
2057
  "attention_mask"