jonsaadfalcon commited on
Commit
8df67fb
·
verified ·
1 Parent(s): 95dfe8c

Upload tokenizer_config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +5 -4
tokenizer_config.json CHANGED
@@ -931,14 +931,15 @@
931
  },
932
  "clean_up_tokenization_spaces": true,
933
  "cls_token": "[CLS]",
 
934
  "mask_token": "[MASK]",
935
- "model_max_length": 8192,
936
- "pad_token": "[PAD]",
937
- "sep_token": "[SEP]",
938
- "tokenizer_class": "PreTrainedTokenizerFast",
939
  "model_input_names": [
940
  "input_ids",
941
  "attention_mask"
942
  ],
 
 
 
 
943
  "unk_token": "[UNK]"
944
  }
 
931
  },
932
  "clean_up_tokenization_spaces": true,
933
  "cls_token": "[CLS]",
934
+ "extra_special_tokens": {},
935
  "mask_token": "[MASK]",
 
 
 
 
936
  "model_input_names": [
937
  "input_ids",
938
  "attention_mask"
939
  ],
940
+ "model_max_length": 4096,
941
+ "pad_token": "[PAD]",
942
+ "sep_token": "[SEP]",
943
+ "tokenizer_class": "PreTrainedTokenizerFast",
944
  "unk_token": "[UNK]"
945
  }