minpeter commited on
Commit
5cd6b69
·
verified ·
1 Parent(s): 5f0aeb7

Upload folder using huggingface_hub

Browse files
Files changed (3) hide show
  1. config.json +11 -3
  2. special_tokens_map.json +9 -3
  3. tokenizer_config.json +3 -3
config.json CHANGED
@@ -4,9 +4,17 @@
4
  ],
5
  "attention_bias": false,
6
  "attention_dropout": 0.0,
7
- "bos_token_id": 100257,
 
 
 
8
  "end_token_id": 100257,
9
- "eos_token_id": 100257,
 
 
 
 
 
10
  "head_dim": 128,
11
  "hidden_act": "silu",
12
  "hidden_size": 3072,
@@ -30,4 +38,4 @@
30
  "transformers_version": "4.51.3",
31
  "use_cache": true,
32
  "vocab_size": 110592
33
- }
 
4
  ],
5
  "attention_bias": false,
6
  "attention_dropout": 0.0,
7
+ "bos_token_id": [
8
+ 100257,
9
+ 100272
10
+ ],
11
  "end_token_id": 100257,
12
+ "eos_token_id": [
13
+ 100257,
14
+ 100273,
15
+ 100274,
16
+ 100275
17
+ ],
18
  "head_dim": 128,
19
  "hidden_act": "silu",
20
  "hidden_size": 3072,
 
38
  "transformers_version": "4.51.3",
39
  "use_cache": true,
40
  "vocab_size": 110592
41
+ }
special_tokens_map.json CHANGED
@@ -56,13 +56,19 @@
56
  "<PASSWORD>"
57
  ],
58
  "bos_token": {
59
- "content": "<|endoftext|>",
 
 
 
 
 
 
 
60
  "lstrip": false,
61
  "normalized": false,
62
  "rstrip": false,
63
  "single_word": false
64
  },
65
- "eos_token": "<|endofturn|>",
66
  "pad_token": {
67
  "content": "<|endoftext|>",
68
  "lstrip": false,
@@ -77,4 +83,4 @@
77
  "rstrip": false,
78
  "single_word": false
79
  }
80
- }
 
56
  "<PASSWORD>"
57
  ],
58
  "bos_token": {
59
+ "content": "<|im_start|>",
60
+ "lstrip": false,
61
+ "normalized": false,
62
+ "rstrip": false,
63
+ "single_word": false
64
+ },
65
+ "eos_token": {
66
+ "content": "<|im_end|>",
67
  "lstrip": false,
68
  "normalized": false,
69
  "rstrip": false,
70
  "single_word": false
71
  },
 
72
  "pad_token": {
73
  "content": "<|endoftext|>",
74
  "lstrip": false,
 
83
  "rstrip": false,
84
  "single_word": false
85
  }
86
+ }
tokenizer_config.json CHANGED
@@ -490,13 +490,13 @@
490
  "<KEY>",
491
  "<PASSWORD>"
492
  ],
493
- "bos_token": "<|endoftext|>",
494
  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
495
  "clean_up_tokenization_spaces": true,
496
- "eos_token": "<|endofturn|>",
497
  "extra_special_tokens": {},
498
  "model_max_length": 1000000000000000019884624838656,
499
  "pad_token": "<|endoftext|>",
500
  "tokenizer_class": "GPT2Tokenizer",
501
  "unk_token": "<|endoftext|>"
502
- }
 
490
  "<KEY>",
491
  "<PASSWORD>"
492
  ],
493
+ "bos_token": "<|im_start|>",
494
  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
495
  "clean_up_tokenization_spaces": true,
496
+ "eos_token": "<|im_end|>",
497
  "extra_special_tokens": {},
498
  "model_max_length": 1000000000000000019884624838656,
499
  "pad_token": "<|endoftext|>",
500
  "tokenizer_class": "GPT2Tokenizer",
501
  "unk_token": "<|endoftext|>"
502
+ }