Hugging Face's logo

chujiezheng
/
tulu-2-dpo-13b

Text Generation
Transformers
Safetensors
English
llama
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
2

Update tokenizer_config.json

#2
by chujiezheng - opened
base: refs/heads/main
←
from: refs/pr/2
Discussion Files changed
+33
-39
Files changed (1) hide show
  1. tokenizer_config.json +33 -39
tokenizer_config.json CHANGED
@@ -1,42 +1,36 @@
1
  {
2
- "add_bos_token": true,
3
- "add_eos_token": false,
4
- "add_prefix_space": true,
5
- "added_tokens_decoder": {
6
- "0": {
7
- "content": "<unk>",
8
- "lstrip": false,
9
- "normalized": true,
10
- "rstrip": false,
11
- "single_word": false,
12
- "special": true
13
  },
14
- "1": {
15
- "content": "<s>",
16
- "lstrip": false,
17
- "normalized": true,
18
- "rstrip": false,
19
- "single_word": false,
20
- "special": true
 
 
21
  },
22
- "2": {
23
- "content": "</s>",
24
- "lstrip": false,
25
- "normalized": true,
26
- "rstrip": false,
27
- "single_word": false,
28
- "special": true
29
- }
30
- },
31
- "bos_token": "<s>",
32
- "clean_up_tokenization_spaces": false,
33
- "eos_token": "</s>",
34
- "legacy": true,
35
- "model_max_length": 2048,
36
- "pad_token": null,
37
- "sp_model_kwargs": {},
38
- "spaces_between_special_tokens": false,
39
- "tokenizer_class": "LlamaTokenizer",
40
- "unk_token": "<unk>",
41
- "use_default_system_prompt": false
42
- }
 
1
  {
2
+ "add_bos_token":true,
3
+ "add_eos_token":false,
4
+ "model_max_length":2048,
5
+ "pad_token":null,
6
+ "sp_model_kwargs":{
7
+
 
 
 
 
 
8
  },
9
+ "tokenizer_class":"LlamaTokenizer",
10
+ "clean_up_tokenization_spaces":false,
11
+ "bos_token":{
12
+ "__type":"AddedToken",
13
+ "content":"<s>",
14
+ "lstrip":false,
15
+ "normalized":true,
16
+ "rstrip":false,
17
+ "single_word":false
18
  },
19
+ "eos_token":{
20
+ "__type":"AddedToken",
21
+ "content":"</s>",
22
+ "lstrip":false,
23
+ "normalized":true,
24
+ "rstrip":false,
25
+ "single_word":false
26
+ },
27
+ "unk_token":{
28
+ "__type":"AddedToken",
29
+ "content":"<unk>",
30
+ "lstrip":false,
31
+ "normalized":true,
32
+ "rstrip":false,
33
+ "single_word":false
34
+ },
35
+ "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}"
36
+ }