vladislavbro
/

llama_backbone_0.5

Text Generation

Model card Files Files and versions

vladislavbro commited on 12 days ago

Commit

a6c48da

·

verified ·

1 Parent(s): bbf55f1

Update tokenizer_config.json

Files changed (1) hide show

tokenizer_config.json +4 -3

tokenizer_config.json CHANGED Viewed

@@ -2049,9 +2049,10 @@
       "special": true
     }
   },
-  "bos_token": "<|begin_of_text|>",
-  "clean_up_tokenization_spaces": true,
-  "eos_token": "<|end_of_text|>",
   "model_input_names": [
     "input_ids",
     "attention_mask"

       "special": true
     }
   },
+  "add_bos_token": true,
+  "add_eos_token": true,
+  "bos_token": "<s>",
+  "eos_token": "</s>",
   "model_input_names": [
     "input_ids",
     "attention_mask"