Upload 4bit dynamic double quant model

README.md

@@ -19,6 +19,11 @@ BitsAndBytesConfig(
   bnb_4bit_use_double_quant=True,
   bnb_4bit_compute_dtype=torch.bfloat16,
   bnb_4bit_quant_storage=torch.bfloat16,
+  llm_int8_skip_modules=[
+    "multi_modal_projector",
+    "vision_tower",
+    "vision_model",
+  ],
 )
 ```
 
@@ -544,4 +549,4 @@ alternatives.
 [jax]: https://github.com/jax-ml/jax
 [ml-pathways]: https://blog.google/technology/ai/introducing-pathways-next-generation-ai-architecture/
 [sustainability]: https://sustainability.google/operating-sustainably/
-[gemini-2-paper]: https://arxiv.org/abs/2312.11805
+[gemini-2-paper]: https://arxiv.org/abs/2312.11805

config.json

@@ -21,7 +21,11 @@
     "bnb_4bit_use_double_quant": true,
     "llm_int8_enable_fp32_cpu_offload": false,
     "llm_int8_has_fp16_weight": false,
-    "llm_int8_skip_modules": null,
+    "llm_int8_skip_modules": [
+      "multi_modal_projector",
+      "vision_tower",
+      "vision_model"
+    ],
     "llm_int8_threshold": 6.0,
     "load_in_4bit": true,
     "load_in_8bit": false,

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dfe45368cb5ec8657ef24078447964d440389613e1aad51445558a0d4cf69c0e
-size 4992770880
+oid sha256:3c6ed0512682d19ffee93bae2d3e1d681fe1e0af1247aec039099c64145bc9e4
+size 5605791454