Upload 4 files

tokenizer/special_tokens_map.json

@@ -9,15 +9,21 @@
   "eos_token": {
     "content": "<|endoftext|>",
     "lstrip": false,
-    "normalized": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<|endoftext|>",
   "unk_token": {
     "content": "<|endoftext|>",
     "lstrip": false,
-    "normalized": true,
+    "normalized": false,
     "rstrip": false,
     "single_word": false
   }

tokenizer/tokenizer_config.json

@@ -12,7 +12,7 @@
     "49407": {
       "content": "<|endoftext|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
@@ -23,7 +23,6 @@
   "do_lower_case": true,
   "eos_token": "<|endoftext|>",
   "errors": "replace",
-  "extra_special_tokens": {},
   "model_max_length": 77,
   "pad_token": "<|endoftext|>",
   "tokenizer_class": "CLIPTokenizer",