Update processor (#2)

- update (4c9178e7e178c01330fdde94209b405a3c9db114)


Co-authored-by: Raushan Turganbay <[email protected]>

added_tokens.json

@@ -10,6 +10,7 @@
   "<quad>": 151668,
   "<ref>": 151670,
   "<tool_call>": 151657,
+  "<video>": 151674,
   "<|box_end|>": 151649,
   "<|box_start|>": 151648,
   "<|endoftext|>": 151643,

chat_template.jinja

@@ -1,5 +1,5 @@
 {% for message in messages %}{{'<|im_start|>' + message['role'] + '
-'}}{% if message['content'] is string %}{{ message['content'] }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' %}{{ '<image>
+'}}{% if message['content'] is string %}{{ message['content'] }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' %}{{ '<IMG_CONTEXT>
 ' }}{% elif content['type'] == 'video' %}{{ '<video>
 ' }}{% elif content['type'] == 'text' %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}{{'<|im_end|>
 '}}{% endfor %}{% if add_generation_prompt %}{{'<|im_start|>assistant

processor_config.json

@@ -1,6 +1,4 @@
 {
-  "fake_image_token": "<image>",
-  "fake_video_token": "<video>",
   "image_seq_length": 256,
   "processor_class": "InternVLProcessor"
 }

special_tokens_map.json

@@ -13,69 +13,15 @@
     "<|vision_pad|>",
     "<|image_pad|>",
     "<|video_pad|>",
-    {
-      "content": "<img>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</img>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<IMG_CONTEXT>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<quad>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</quad>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<ref>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</ref>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "<box>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    },
-    {
-      "content": "</box>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false
-    }
+    "<img>",
+    "</img>",
+    "<IMG_CONTEXT>",
+    "<quad>",
+    "</quad>",
+    "<ref>",
+    "</ref>",
+    "<box>",
+    "</box>"
   ],
   "context_image_token": "<IMG_CONTEXT>",
   "end_image_token": "</img>",
@@ -93,5 +39,6 @@
     "rstrip": false,
     "single_word": false
   },
-  "start_image_token": "<img>"
+  "start_image_token": "<img>",
+  "video_token": "<video>"
 }

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6f9ba4b4a6625b5047a1356f6081b641c3e4e6a4a198facbd4bef217747d1685
-size 11423548
+oid sha256:7cc80b7e20adf8bf6f6ca442bf1abfac8056bb3b7d3e0b11c9d497d3e79398c9
+size 11423732

tokenizer_config.json

@@ -249,6 +249,14 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "151674": {
+      "content": "<video>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "additional_special_tokens": [
@@ -284,7 +292,8 @@
   "extra_special_tokens": {
     "context_image_token": "<IMG_CONTEXT>",
     "end_image_token": "</img>",
-    "start_image_token": "<img>"
+    "start_image_token": "<img>",
+    "video_token": "<video>"
   },
   "model_max_length": 8192,
   "pad_token": "<|endoftext|>",
@@ -292,5 +301,6 @@
   "split_special_tokens": false,
   "start_image_token": "<img>",
   "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
+  "unk_token": null,
+  "video_token": "<video>"
 }