14b32c0b36f821cc883a829d4f54eb53e9147bf8d5e0da555ac776f1e81d82a3

README.md

@@ -1,5 +1,6 @@
 ---
 thumbnail: "https://assets-global.website-files.com/646b351987a8d8ce158d1940/64ec9e96b4334c0e1ac41504_Logo%20with%20white%20text.svg"
+base_model: ORIGINAL_REPO_NAME
 metrics:
 - memory_disk
 - memory_inference
@@ -30,7 +31,7 @@ tags:
 - Contact us and tell us which model to compress next [here](https://www.pruna.ai/contact).
 - Request access to easily compress your *own* AI models [here](https://z0halsaff74.typeform.com/pruna-access?typeform-source=www.pruna.ai).
 - Read the documentations to know more [here](https://pruna-ai-pruna.readthedocs-hosted.com/en/latest/)
-- Join Pruna AI community on Discord [here](https://discord.gg/rskEr4BZJx) to share feedback/suggestions or get help.
+- Join Pruna AI community on Discord [here](https://discord.gg/CP4VSgck) to share feedback/suggestions or get help.
 
 ## Results
 
@@ -39,7 +40,7 @@ tags:
 **Frequently Asked Questions**
 - ***How does the compression work?*** The model is compressed with llm-int8.
 - ***How does the model quality change?*** The quality of the model output might vary compared to the base model.
-- ***How is the model efficiency evaluated?*** These results were obtained on NVIDIA A100-PCIE-40GB with configuration described in `model/smash_config.json` and are obtained after a hardware warmup. The smashed model is directly compared to the original base model. Efficiency results may vary in other settings (e.g. other hardware, image size, batch size, ...). We recommend to directly run them in the use-case conditions to know if the smashed model can benefit you.
+- ***How is the model efficiency evaluated?*** These results were obtained with configuration described in `model/smash_config.json` and are obtained after a hardware warmup. The smashed model is directly compared to the original base model. Efficiency results may vary in other settings (e.g. other hardware, image size, batch size, ...). We recommend to directly run them in the use-case conditions to know if the smashed model can benefit you.
 - ***What is the model format?*** We use safetensors.
 - ***What calibration data has been used?*** If needed by the compression method, we used WikiText as the calibration data.
 - ***What is the naming convention for Pruna Huggingface models?*** We take the original model name and append "turbo", "tiny", or "green" if the smashed model has a measured inference speed, inference memory, or inference energy consumption which is less than 90% of the original base model.
@@ -51,7 +52,7 @@ tags:
 
 You can run the smashed model with these steps:
 
-0. Check requirements from the original repo tiiuae/falcon-7b-instruct installed. In particular, check python, cuda, and transformers versions.
+0. Check requirements from the original repo ORIGINAL_REPO_NAME installed. In particular, check python, cuda, and transformers versions.
 1. Make sure that you have installed quantization related packages.
     ```bash
     pip install transformers accelerate bitsandbytes>0.37.0
@@ -59,15 +60,15 @@ You can run the smashed model with these steps:
 2. Load & run the model.
     ```python 
    from transformers import AutoModelForCausalLM, AutoTokenizer
+   
 
-    model = AutoModelForCausalLM.from_pretrained("PrunaAI/tiiuae-falcon-7b-instruct-bnb-8bit-smashed",
-                                                 trust_remote_code=True, device_map='auto')
-    tokenizer = AutoTokenizer.from_pretrained("tiiuae/falcon-7b-instruct")
+   model = AutoModelForCausalLM.from_pretrained("PrunaAI/tiiuae-falcon-7b-instruct-bnb-8bit-smashed", trust_remote_code=True, device_map='auto')
+   tokenizer = AutoTokenizer.from_pretrained("ORIGINAL_REPO_NAME")
     
-    input_ids = tokenizer("What is the color of prunes?,", return_tensors='pt').to(model.device)["input_ids"]
+   input_ids = tokenizer("What is the color of prunes?,", return_tensors='pt').to(model.device)["input_ids"]
     
-    outputs = model.generate(input_ids, max_new_tokens=216)
-    tokenizer.decode(outputs[0])
+   outputs = model.generate(input_ids, max_new_tokens=216)
+   tokenizer.decode(outputs[0])
     ```
 
 ## Configurations
@@ -76,7 +77,7 @@ The configuration info are in `smash_config.json`.
 
 ## Credits & License
 
-The license of the smashed model follows the license of the original model. Please check the license of the original model tiiuae/falcon-7b-instruct before using this model which provided the base model. The license  of the `pruna-engine` is [here](https://pypi.org/project/pruna-engine/) on Pypi.
+The license of the smashed model follows the license of the original model. Please check the license of the original model ORIGINAL_REPO_NAME before using this model which provided the base model. The license  of the `pruna-engine` is [here](https://pypi.org/project/pruna-engine/) on Pypi.
 
 ## Want to compress other models?
 

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/tmp/tmp3kejc0fb",
+  "_name_or_path": "/tmp/models/tmpvkrarsxd110wktzv",
   "alibi": false,
   "apply_residual_connection_post_layernorm": false,
   "architectures": [
@@ -9,7 +9,7 @@
   "auto_map": {
     "AutoConfig": "configuration_falcon.FalconConfig",
     "AutoModel": "tiiuae/falcon-7b-instruct--modeling_falcon.FalconModel",
-    "AutoModelForCausalLM": "modeling_falcon.FalconForCausalLM",
+    "AutoModelForCausalLM": "tiiuae/falcon-7b-instruct--modeling_falcon.FalconForCausalLM",
     "AutoModelForQuestionAnswering": "tiiuae/falcon-7b-instruct--modeling_falcon.FalconForQuestionAnswering",
     "AutoModelForSequenceClassification": "tiiuae/falcon-7b-instruct--modeling_falcon.FalconForSequenceClassification",
     "AutoModelForTokenClassification": "tiiuae/falcon-7b-instruct--modeling_falcon.FalconForTokenClassification"
@@ -29,7 +29,10 @@
   "num_kv_heads": 71,
   "parallel_attn": true,
   "quantization_config": {
+    "_load_in_4bit": false,
+    "_load_in_8bit": true,
     "bnb_4bit_compute_dtype": "bfloat16",
+    "bnb_4bit_quant_storage": "uint8",
     "bnb_4bit_quant_type": "fp4",
     "bnb_4bit_use_double_quant": false,
     "llm_int8_enable_fp32_cpu_offload": false,
@@ -42,8 +45,8 @@
     "load_in_8bit": true,
     "quant_method": "bitsandbytes"
   },
-  "torch_dtype": "float16",
-  "transformers_version": "4.37.1",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.48.2",
   "use_cache": true,
   "vocab_size": 65024
 }

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 11,
   "eos_token_id": 11,
-  "transformers_version": "4.37.1"
+  "transformers_version": "4.48.2"
 }

model.safetensors.index.json

@@ -1,328 +1,456 @@
 {
   "metadata": {
-    "total_size": 7221577472
+    "total_size": 7221577600
   },
   "weight_map": {
     "transformer.h.0.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.0.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.0.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.0.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.0.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.0.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.0.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.0.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.0.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.0.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.1.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.1.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.1.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.1.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.1.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.1.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.1.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.1.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.1.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.1.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.10.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.10.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.10.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.10.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.10.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.10.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.10.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.10.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.10.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.10.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.11.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.11.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.11.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.11.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.11.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.11.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.11.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.11.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.11.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.11.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.12.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.12.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.12.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.12.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.12.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.12.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.12.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.12.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.12.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.12.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.13.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.13.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.13.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.13.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.13.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.13.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.13.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.13.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.13.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.13.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.14.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.14.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.14.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.14.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.14.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.14.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.14.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.14.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.14.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.14.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.15.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.15.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.15.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.15.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.15.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.15.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.15.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.15.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.15.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.15.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.16.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.16.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.16.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.16.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.16.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.16.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.16.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.16.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.16.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.16.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.17.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.17.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.17.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.17.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.17.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.17.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.17.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.17.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.17.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.17.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.18.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.18.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.18.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.18.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.18.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.18.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.18.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.18.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.18.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.18.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.19.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.19.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.19.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.19.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.19.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.19.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.19.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.19.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.19.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.19.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.2.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.2.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.2.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.2.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.2.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.2.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.2.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.2.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.2.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.2.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.20.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.20.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.20.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.20.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.20.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.20.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.20.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.20.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.20.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.20.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.21.input_layernorm.bias": "model-00002-of-00002.safetensors",
     "transformer.h.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "transformer.h.21.mlp.dense_4h_to_h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.21.mlp.dense_4h_to_h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.21.mlp.dense_4h_to_h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.21.mlp.dense_h_to_4h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.21.mlp.dense_h_to_4h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.21.mlp.dense_h_to_4h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.21.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.21.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.21.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.21.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.21.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.22.input_layernorm.bias": "model-00002-of-00002.safetensors",
     "transformer.h.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "transformer.h.22.mlp.dense_4h_to_h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.22.mlp.dense_4h_to_h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.22.mlp.dense_4h_to_h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.22.mlp.dense_h_to_4h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.22.mlp.dense_h_to_4h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.22.mlp.dense_h_to_4h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.22.self_attention.dense.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.22.self_attention.dense.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.22.self_attention.dense.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.22.self_attention.query_key_value.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.22.self_attention.query_key_value.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.22.self_attention.query_key_value.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.23.input_layernorm.bias": "model-00002-of-00002.safetensors",
     "transformer.h.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "transformer.h.23.mlp.dense_4h_to_h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.23.mlp.dense_4h_to_h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.23.mlp.dense_4h_to_h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.23.mlp.dense_h_to_4h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.23.mlp.dense_h_to_4h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.23.mlp.dense_h_to_4h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.23.self_attention.dense.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.23.self_attention.dense.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.23.self_attention.dense.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.23.self_attention.query_key_value.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.23.self_attention.query_key_value.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.23.self_attention.query_key_value.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.24.input_layernorm.bias": "model-00002-of-00002.safetensors",
     "transformer.h.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "transformer.h.24.mlp.dense_4h_to_h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.24.mlp.dense_4h_to_h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.24.mlp.dense_4h_to_h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.24.mlp.dense_h_to_4h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.24.mlp.dense_h_to_4h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.24.mlp.dense_h_to_4h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.24.self_attention.dense.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.24.self_attention.dense.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.24.self_attention.dense.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.24.self_attention.query_key_value.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.24.self_attention.query_key_value.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.24.self_attention.query_key_value.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.25.input_layernorm.bias": "model-00002-of-00002.safetensors",
     "transformer.h.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "transformer.h.25.mlp.dense_4h_to_h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.25.mlp.dense_4h_to_h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.25.mlp.dense_4h_to_h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.25.mlp.dense_h_to_4h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.25.mlp.dense_h_to_4h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.25.mlp.dense_h_to_4h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.25.self_attention.dense.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.25.self_attention.dense.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.25.self_attention.dense.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.25.self_attention.query_key_value.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.25.self_attention.query_key_value.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.25.self_attention.query_key_value.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.26.input_layernorm.bias": "model-00002-of-00002.safetensors",
     "transformer.h.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "transformer.h.26.mlp.dense_4h_to_h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.26.mlp.dense_4h_to_h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.26.mlp.dense_4h_to_h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.26.mlp.dense_h_to_4h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.26.mlp.dense_h_to_4h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.26.mlp.dense_h_to_4h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.26.self_attention.dense.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.26.self_attention.dense.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.26.self_attention.dense.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.26.self_attention.query_key_value.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.26.self_attention.query_key_value.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.26.self_attention.query_key_value.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.27.input_layernorm.bias": "model-00002-of-00002.safetensors",
     "transformer.h.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "transformer.h.27.mlp.dense_4h_to_h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.27.mlp.dense_4h_to_h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.27.mlp.dense_4h_to_h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.27.mlp.dense_h_to_4h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.27.mlp.dense_h_to_4h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.27.mlp.dense_h_to_4h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.27.self_attention.dense.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.27.self_attention.dense.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.27.self_attention.dense.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.27.self_attention.query_key_value.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.27.self_attention.query_key_value.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.27.self_attention.query_key_value.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.28.input_layernorm.bias": "model-00002-of-00002.safetensors",
     "transformer.h.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "transformer.h.28.mlp.dense_4h_to_h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.28.mlp.dense_4h_to_h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.28.mlp.dense_4h_to_h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.28.mlp.dense_h_to_4h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.28.mlp.dense_h_to_4h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.28.mlp.dense_h_to_4h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.28.self_attention.dense.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.28.self_attention.dense.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.28.self_attention.dense.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.28.self_attention.query_key_value.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.28.self_attention.query_key_value.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.28.self_attention.query_key_value.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.29.input_layernorm.bias": "model-00002-of-00002.safetensors",
     "transformer.h.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "transformer.h.29.mlp.dense_4h_to_h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.29.mlp.dense_4h_to_h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.29.mlp.dense_4h_to_h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.29.mlp.dense_h_to_4h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.29.mlp.dense_h_to_4h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.29.mlp.dense_h_to_4h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.29.self_attention.dense.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.29.self_attention.dense.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.29.self_attention.dense.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.29.self_attention.query_key_value.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.29.self_attention.query_key_value.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.29.self_attention.query_key_value.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.3.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.3.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.3.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.3.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.3.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.3.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.3.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.3.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.3.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.3.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.30.input_layernorm.bias": "model-00002-of-00002.safetensors",
     "transformer.h.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "transformer.h.30.mlp.dense_4h_to_h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.30.mlp.dense_4h_to_h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.30.mlp.dense_4h_to_h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.30.mlp.dense_h_to_4h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.30.mlp.dense_h_to_4h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.30.mlp.dense_h_to_4h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.30.self_attention.dense.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.30.self_attention.dense.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.30.self_attention.dense.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.30.self_attention.query_key_value.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.30.self_attention.query_key_value.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.30.self_attention.query_key_value.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.31.input_layernorm.bias": "model-00002-of-00002.safetensors",
     "transformer.h.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
     "transformer.h.31.mlp.dense_4h_to_h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.31.mlp.dense_4h_to_h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.31.mlp.dense_4h_to_h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.31.mlp.dense_h_to_4h.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.31.mlp.dense_h_to_4h.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.31.mlp.dense_h_to_4h.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.31.self_attention.dense.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.31.self_attention.dense.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.31.self_attention.dense.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.31.self_attention.query_key_value.SCB": "model-00002-of-00002.safetensors",
     "transformer.h.31.self_attention.query_key_value.weight": "model-00002-of-00002.safetensors",
+    "transformer.h.31.self_attention.query_key_value.weight_format": "model-00002-of-00002.safetensors",
     "transformer.h.4.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.4.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.4.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.4.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.4.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.4.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.4.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.4.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.4.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.4.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.5.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.5.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.5.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.5.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.5.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.5.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.5.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.5.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.5.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.5.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.6.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.6.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.6.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.6.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.6.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.6.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.6.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.6.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.6.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.6.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.7.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.7.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.7.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.7.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.7.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.7.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.7.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.7.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.7.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.7.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.8.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.8.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.8.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.8.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.8.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.8.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.8.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.8.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.8.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.8.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.9.input_layernorm.bias": "model-00001-of-00002.safetensors",
     "transformer.h.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
     "transformer.h.9.mlp.dense_4h_to_h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.9.mlp.dense_4h_to_h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.mlp.dense_4h_to_h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.9.mlp.dense_h_to_4h.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.9.mlp.dense_h_to_4h.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.mlp.dense_h_to_4h.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.9.self_attention.dense.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.9.self_attention.dense.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.self_attention.dense.weight_format": "model-00001-of-00002.safetensors",
     "transformer.h.9.self_attention.query_key_value.SCB": "model-00001-of-00002.safetensors",
     "transformer.h.9.self_attention.query_key_value.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.9.self_attention.query_key_value.weight_format": "model-00001-of-00002.safetensors",
     "transformer.ln_f.bias": "model-00002-of-00002.safetensors",
     "transformer.ln_f.weight": "model-00002-of-00002.safetensors",
     "transformer.word_embeddings.weight": "model-00001-of-00002.safetensors"

smash_config.json

@@ -1,27 +1,23 @@
 {
-    "api_key": null,
-    "verify_url": "http://johnrachwan.pythonanywhere.com",
-    "smash_config": {
-        "pruners": "None",
-        "factorizers": "None",
-        "quantizers": "['llm-int8']",
-        "compilers": "None",
-        "task": "text_text_generation",
-        "device": "cuda",
-        "cache_dir": "/ceph/hdd/staff/charpent/.cache/modelsqfyelaoa",
-        "batch_size": 1,
-        "model_name": "tiiuae/falcon-7b-instruct",
-        "pruning_ratio": 0.0,
-        "n_quantization_bits": 8,
-        "output_deviation": 0.005,
-        "max_batch_size": 1,
-        "qtype_weight": "torch.qint8",
-        "qtype_activation": "torch.quint8",
-        "qobserver": "<class 'torch.ao.quantization.observer.MinMaxObserver'>",
-        "qscheme": "torch.per_tensor_symmetric",
-        "qconfig": "x86",
-        "group_size": 128,
-        "damp_percent": 0.1,
-        "save_load_fn": "bitsandbytes"
-    }
+    "batchers": null,
+    "cachers": null,
+    "compilers": null,
+    "distillers": null,
+    "pruners": null,
+    "quantizers": "llm-int8",
+    "recoverers": null,
+    "quant_llm-int8_compute_dtype": "bfloat16",
+    "quant_llm-int8_double_quant": false,
+    "quant_llm-int8_enable_fp32_cpu_offload": false,
+    "quant_llm-int8_has_fp16_weight": false,
+    "quant_llm-int8_quant_type": "fp4",
+    "quant_llm-int8_threshold": 6.0,
+    "quant_llm-int8_weight_bits": 8,
+    "max_batch_size": 1,
+    "device": "cuda",
+    "cache_dir": "/tmp/models/tmpvkrarsxd",
+    "task": "",
+    "save_load_fn": "llm-int8",
+    "save_load_fn_args": {},
+    "api_key": null
 }