Regenerate the model to ensure successful execution on vLLM

config.json

@@ -25,13 +25,12 @@
   "num_key_value_heads": 4,
   "output_router_logits": false,
   "quantization_config": {
-    "autoround_version": "0.6.0",
+    "autoround_version": "0.6.1.dev",
     "bits": 4,
     "data_type": "int",
     "extra_config": {
       "model.layers.0.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.0.self_attn.k_proj": {
         "bits": 8,
@@ -50,8 +49,7 @@
         "group_size": 128
       },
       "model.layers.1.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.1.self_attn.k_proj": {
         "bits": 8,
@@ -70,8 +68,7 @@
         "group_size": 128
       },
       "model.layers.10.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.10.self_attn.k_proj": {
         "bits": 8,
@@ -90,8 +87,7 @@
         "group_size": 128
       },
       "model.layers.11.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.11.self_attn.k_proj": {
         "bits": 8,
@@ -110,8 +106,7 @@
         "group_size": 128
       },
       "model.layers.12.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.12.self_attn.k_proj": {
         "bits": 8,
@@ -130,8 +125,7 @@
         "group_size": 128
       },
       "model.layers.13.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.13.self_attn.k_proj": {
         "bits": 8,
@@ -150,8 +144,7 @@
         "group_size": 128
       },
       "model.layers.14.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.14.self_attn.k_proj": {
         "bits": 8,
@@ -170,8 +163,7 @@
         "group_size": 128
       },
       "model.layers.15.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.15.self_attn.k_proj": {
         "bits": 8,
@@ -190,8 +182,7 @@
         "group_size": 128
       },
       "model.layers.16.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.16.self_attn.k_proj": {
         "bits": 8,
@@ -210,8 +201,7 @@
         "group_size": 128
       },
       "model.layers.17.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.17.self_attn.k_proj": {
         "bits": 8,
@@ -230,8 +220,7 @@
         "group_size": 128
       },
       "model.layers.18.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.18.self_attn.k_proj": {
         "bits": 8,
@@ -250,8 +239,7 @@
         "group_size": 128
       },
       "model.layers.19.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.19.self_attn.k_proj": {
         "bits": 8,
@@ -270,8 +258,7 @@
         "group_size": 128
       },
       "model.layers.2.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.2.self_attn.k_proj": {
         "bits": 8,
@@ -290,8 +277,7 @@
         "group_size": 128
       },
       "model.layers.20.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.20.self_attn.k_proj": {
         "bits": 8,
@@ -310,8 +296,7 @@
         "group_size": 128
       },
       "model.layers.21.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.21.self_attn.k_proj": {
         "bits": 8,
@@ -330,8 +315,7 @@
         "group_size": 128
       },
       "model.layers.22.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.22.self_attn.k_proj": {
         "bits": 8,
@@ -350,8 +334,7 @@
         "group_size": 128
       },
       "model.layers.23.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.23.self_attn.k_proj": {
         "bits": 8,
@@ -370,8 +353,7 @@
         "group_size": 128
       },
       "model.layers.24.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.24.self_attn.k_proj": {
         "bits": 8,
@@ -390,8 +372,7 @@
         "group_size": 128
       },
       "model.layers.25.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.25.self_attn.k_proj": {
         "bits": 8,
@@ -410,8 +391,7 @@
         "group_size": 128
       },
       "model.layers.26.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.26.self_attn.k_proj": {
         "bits": 8,
@@ -430,8 +410,7 @@
         "group_size": 128
       },
       "model.layers.27.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.27.self_attn.k_proj": {
         "bits": 8,
@@ -450,8 +429,7 @@
         "group_size": 128
       },
       "model.layers.28.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.28.self_attn.k_proj": {
         "bits": 8,
@@ -470,8 +448,7 @@
         "group_size": 128
       },
       "model.layers.29.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.29.self_attn.k_proj": {
         "bits": 8,
@@ -490,8 +467,7 @@
         "group_size": 128
       },
       "model.layers.3.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.3.self_attn.k_proj": {
         "bits": 8,
@@ -510,8 +486,7 @@
         "group_size": 128
       },
       "model.layers.30.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.30.self_attn.k_proj": {
         "bits": 8,
@@ -530,8 +505,7 @@
         "group_size": 128
       },
       "model.layers.31.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.31.self_attn.k_proj": {
         "bits": 8,
@@ -550,8 +524,7 @@
         "group_size": 128
       },
       "model.layers.32.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.32.self_attn.k_proj": {
         "bits": 8,
@@ -570,8 +543,7 @@
         "group_size": 128
       },
       "model.layers.33.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.33.self_attn.k_proj": {
         "bits": 8,
@@ -590,8 +562,7 @@
         "group_size": 128
       },
       "model.layers.34.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.34.self_attn.k_proj": {
         "bits": 8,
@@ -610,8 +581,7 @@
         "group_size": 128
       },
       "model.layers.35.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.35.self_attn.k_proj": {
         "bits": 8,
@@ -630,8 +600,7 @@
         "group_size": 128
       },
       "model.layers.36.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.36.self_attn.k_proj": {
         "bits": 8,
@@ -650,8 +619,7 @@
         "group_size": 128
       },
       "model.layers.37.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.37.self_attn.k_proj": {
         "bits": 8,
@@ -670,8 +638,7 @@
         "group_size": 128
       },
       "model.layers.38.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.38.self_attn.k_proj": {
         "bits": 8,
@@ -690,8 +657,7 @@
         "group_size": 128
       },
       "model.layers.39.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.39.self_attn.k_proj": {
         "bits": 8,
@@ -710,8 +676,7 @@
         "group_size": 128
       },
       "model.layers.4.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.4.self_attn.k_proj": {
         "bits": 8,
@@ -730,8 +695,7 @@
         "group_size": 128
       },
       "model.layers.40.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.40.self_attn.k_proj": {
         "bits": 8,
@@ -750,8 +714,7 @@
         "group_size": 128
       },
       "model.layers.41.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.41.self_attn.k_proj": {
         "bits": 8,
@@ -770,8 +733,7 @@
         "group_size": 128
       },
       "model.layers.42.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.42.self_attn.k_proj": {
         "bits": 8,
@@ -790,8 +752,7 @@
         "group_size": 128
       },
       "model.layers.43.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.43.self_attn.k_proj": {
         "bits": 8,
@@ -810,8 +771,7 @@
         "group_size": 128
       },
       "model.layers.44.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.44.self_attn.k_proj": {
         "bits": 8,
@@ -830,8 +790,7 @@
         "group_size": 128
       },
       "model.layers.45.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.45.self_attn.k_proj": {
         "bits": 8,
@@ -850,8 +809,7 @@
         "group_size": 128
       },
       "model.layers.46.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.46.self_attn.k_proj": {
         "bits": 8,
@@ -870,8 +828,7 @@
         "group_size": 128
       },
       "model.layers.47.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.47.self_attn.k_proj": {
         "bits": 8,
@@ -890,8 +847,7 @@
         "group_size": 128
       },
       "model.layers.48.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.48.self_attn.k_proj": {
         "bits": 8,
@@ -910,8 +866,7 @@
         "group_size": 128
       },
       "model.layers.49.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.49.self_attn.k_proj": {
         "bits": 8,
@@ -930,8 +885,7 @@
         "group_size": 128
       },
       "model.layers.5.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.5.self_attn.k_proj": {
         "bits": 8,
@@ -950,8 +904,7 @@
         "group_size": 128
       },
       "model.layers.50.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.50.self_attn.k_proj": {
         "bits": 8,
@@ -970,8 +923,7 @@
         "group_size": 128
       },
       "model.layers.51.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.51.self_attn.k_proj": {
         "bits": 8,
@@ -990,8 +942,7 @@
         "group_size": 128
       },
       "model.layers.52.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.52.self_attn.k_proj": {
         "bits": 8,
@@ -1010,8 +961,7 @@
         "group_size": 128
       },
       "model.layers.53.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.53.self_attn.k_proj": {
         "bits": 8,
@@ -1030,8 +980,7 @@
         "group_size": 128
       },
       "model.layers.54.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.54.self_attn.k_proj": {
         "bits": 8,
@@ -1050,8 +999,7 @@
         "group_size": 128
       },
       "model.layers.55.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.55.self_attn.k_proj": {
         "bits": 8,
@@ -1070,8 +1018,7 @@
         "group_size": 128
       },
       "model.layers.56.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.56.self_attn.k_proj": {
         "bits": 8,
@@ -1090,8 +1037,7 @@
         "group_size": 128
       },
       "model.layers.57.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.57.self_attn.k_proj": {
         "bits": 8,
@@ -1110,8 +1056,7 @@
         "group_size": 128
       },
       "model.layers.58.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.58.self_attn.k_proj": {
         "bits": 8,
@@ -1130,8 +1075,7 @@
         "group_size": 128
       },
       "model.layers.59.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.59.self_attn.k_proj": {
         "bits": 8,
@@ -1150,8 +1094,7 @@
         "group_size": 128
       },
       "model.layers.6.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.6.self_attn.k_proj": {
         "bits": 8,
@@ -1170,8 +1113,7 @@
         "group_size": 128
       },
       "model.layers.60.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.60.self_attn.k_proj": {
         "bits": 8,
@@ -1190,8 +1132,7 @@
         "group_size": 128
       },
       "model.layers.61.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.61.self_attn.k_proj": {
         "bits": 8,
@@ -1210,8 +1151,7 @@
         "group_size": 128
       },
       "model.layers.62.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.62.self_attn.k_proj": {
         "bits": 8,
@@ -1230,8 +1170,7 @@
         "group_size": 128
       },
       "model.layers.63.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.63.self_attn.k_proj": {
         "bits": 8,
@@ -1250,8 +1189,7 @@
         "group_size": 128
       },
       "model.layers.64.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.64.self_attn.k_proj": {
         "bits": 8,
@@ -1270,8 +1208,7 @@
         "group_size": 128
       },
       "model.layers.65.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.65.self_attn.k_proj": {
         "bits": 8,
@@ -1290,8 +1227,7 @@
         "group_size": 128
       },
       "model.layers.66.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.66.self_attn.k_proj": {
         "bits": 8,
@@ -1310,8 +1246,7 @@
         "group_size": 128
       },
       "model.layers.67.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.67.self_attn.k_proj": {
         "bits": 8,
@@ -1330,8 +1265,7 @@
         "group_size": 128
       },
       "model.layers.68.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.68.self_attn.k_proj": {
         "bits": 8,
@@ -1350,8 +1284,7 @@
         "group_size": 128
       },
       "model.layers.69.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.69.self_attn.k_proj": {
         "bits": 8,
@@ -1370,8 +1303,7 @@
         "group_size": 128
       },
       "model.layers.7.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.7.self_attn.k_proj": {
         "bits": 8,
@@ -1390,8 +1322,7 @@
         "group_size": 128
       },
       "model.layers.70.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.70.self_attn.k_proj": {
         "bits": 8,
@@ -1410,8 +1341,7 @@
         "group_size": 128
       },
       "model.layers.71.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.71.self_attn.k_proj": {
         "bits": 8,
@@ -1430,8 +1360,7 @@
         "group_size": 128
       },
       "model.layers.72.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.72.self_attn.k_proj": {
         "bits": 8,
@@ -1450,8 +1379,7 @@
         "group_size": 128
       },
       "model.layers.73.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.73.self_attn.k_proj": {
         "bits": 8,
@@ -1470,8 +1398,7 @@
         "group_size": 128
       },
       "model.layers.74.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.74.self_attn.k_proj": {
         "bits": 8,
@@ -1490,8 +1417,7 @@
         "group_size": 128
       },
       "model.layers.75.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.75.self_attn.k_proj": {
         "bits": 8,
@@ -1510,8 +1436,7 @@
         "group_size": 128
       },
       "model.layers.76.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.76.self_attn.k_proj": {
         "bits": 8,
@@ -1530,8 +1455,7 @@
         "group_size": 128
       },
       "model.layers.77.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.77.self_attn.k_proj": {
         "bits": 8,
@@ -1550,8 +1474,7 @@
         "group_size": 128
       },
       "model.layers.78.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.78.self_attn.k_proj": {
         "bits": 8,
@@ -1570,8 +1493,7 @@
         "group_size": 128
       },
       "model.layers.79.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.79.self_attn.k_proj": {
         "bits": 8,
@@ -1590,8 +1512,7 @@
         "group_size": 128
       },
       "model.layers.8.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.8.self_attn.k_proj": {
         "bits": 8,
@@ -1610,8 +1531,7 @@
         "group_size": 128
       },
       "model.layers.80.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.80.self_attn.k_proj": {
         "bits": 8,
@@ -1630,8 +1550,7 @@
         "group_size": 128
       },
       "model.layers.81.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.81.self_attn.k_proj": {
         "bits": 8,
@@ -1650,8 +1569,7 @@
         "group_size": 128
       },
       "model.layers.82.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.82.self_attn.k_proj": {
         "bits": 8,
@@ -1670,8 +1588,7 @@
         "group_size": 128
       },
       "model.layers.83.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.83.self_attn.k_proj": {
         "bits": 8,
@@ -1690,8 +1607,7 @@
         "group_size": 128
       },
       "model.layers.84.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.84.self_attn.k_proj": {
         "bits": 8,
@@ -1710,8 +1626,7 @@
         "group_size": 128
       },
       "model.layers.85.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.85.self_attn.k_proj": {
         "bits": 8,
@@ -1730,8 +1645,7 @@
         "group_size": 128
       },
       "model.layers.86.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.86.self_attn.k_proj": {
         "bits": 8,
@@ -1750,8 +1664,7 @@
         "group_size": 128
       },
       "model.layers.87.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.87.self_attn.k_proj": {
         "bits": 8,
@@ -1770,8 +1683,7 @@
         "group_size": 128
       },
       "model.layers.88.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.88.self_attn.k_proj": {
         "bits": 8,
@@ -1790,8 +1702,7 @@
         "group_size": 128
       },
       "model.layers.89.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.89.self_attn.k_proj": {
         "bits": 8,
@@ -1810,8 +1721,7 @@
         "group_size": 128
       },
       "model.layers.9.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.9.self_attn.k_proj": {
         "bits": 8,
@@ -1830,8 +1740,7 @@
         "group_size": 128
       },
       "model.layers.90.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.90.self_attn.k_proj": {
         "bits": 8,
@@ -1850,8 +1759,7 @@
         "group_size": 128
       },
       "model.layers.91.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.91.self_attn.k_proj": {
         "bits": 8,
@@ -1870,8 +1778,7 @@
         "group_size": 128
       },
       "model.layers.92.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.92.self_attn.k_proj": {
         "bits": 8,
@@ -1890,8 +1797,7 @@
         "group_size": 128
       },
       "model.layers.93.mlp.gate": {
-        "bits": 8,
-        "group_size": 128
+        "bits": 16
       },
       "model.layers.93.self_attn.k_proj": {
         "bits": 8,
@@ -1908,6 +1814,382 @@
       "model.layers.93.self_attn.v_proj": {
         "bits": 8,
         "group_size": 128
+      },
+      "model.layers.0.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.1.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.2.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.3.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.4.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.5.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.6.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.7.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.8.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.9.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.10.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.11.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.12.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.13.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.14.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.15.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.16.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.17.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.18.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.19.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.20.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.21.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.22.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.23.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.24.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.25.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.26.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.27.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.28.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.29.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.30.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.31.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.32.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.33.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.34.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.35.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.36.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.37.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.38.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.39.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.40.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.41.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.42.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.43.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.44.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.45.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.46.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.47.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.48.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.49.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.50.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.51.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.52.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.53.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.54.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.55.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.56.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.57.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.58.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.59.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.60.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.61.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.62.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.63.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.64.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.65.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.66.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.67.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.68.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.69.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.70.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.71.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.72.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.73.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.74.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.75.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.76.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.77.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.78.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.79.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.80.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.81.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.82.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.83.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.84.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.85.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.86.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.87.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.88.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.89.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.90.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.91.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.92.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
+      },
+      "model.layers.93.self_attn.qkv_proj": {
+        "bits": 8,
+        "group_size": 128
       }
     },
     "group_size": 64,
@@ -1927,4 +2209,4 @@
   "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151936
-}
+}

model-00001-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2db4454021cbb415af060909c07d7e867237e798074255d5a00f28935e44f4a6
-size 4999542768
+oid sha256:709da264a348a57aed3d46e32b8b82f59936729748ea12d67f8640cd82bcc858
+size 4997686288

model-00002-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d9abe67c97534e243edf48f7afe97ae866fb2a7fdc43d47ea1abd3a642ef421
-size 4998619208
+oid sha256:2f6cc4f5228b2e72cd77956322d78f2e4682d52e999a380cf77c78674725f032
+size 5000469648

model-00003-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:de6f6dcb5b823e0d03a31132f7b7f6aa4f72d1435c6ecb59a39a35f532846875
-size 4999710112
+oid sha256:4b800b474eabd6517790d29c25a770e04ed0ef19951314cfee4012acb949e0e3
+size 4998050352

model-00004-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69db0731313b75802f8aa8ed4c3c3463ceb86fe321125db8083e748bd3af4894
-size 4998623192
+oid sha256:7625687947be1c92f45f5759aea6e6c8daa2f2c003b8b599c848a44bf6ae3ff0
+size 5000473624

model-00005-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3bda66646cd162bb0c1eaa20a15b97f0f7be3015089da3c21145f626bc179438
-size 4998623456
+oid sha256:f67b73e401c2306ae6c69fb3a3567e0f0d466d13233faf3cc5b02c28a711e76f
+size 4997475480

model-00006-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6606e710e95b0e2bbeeb4754b1b78df51331189c84483b0ac7a21889b9b8e207
-size 4999714272
+oid sha256:31b6e169a2927f79284433c0d09c3fae2dc70c3c82bc46f5dc85ea0bc85b3aba
+size 4997857776

model-00007-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d545a9d33959ac17bb553bb890df43985e41932206680ca1770a7b176bcc19c
-size 4998623328
+oid sha256:1854cf497f8a84e2ae95d1412a22f2004b351c71b753b2cac88dd2273b451530
+size 5000473760

model-00008-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:625f92aeb6b6126cfe7e6ef5b6219886cf25f79b1319d2e739f2ffaa792202e3
-size 4998623456
+oid sha256:670c80d0fb33f18555477b711f8d2ce5f33c77b74109636c7f3b1f5823516285
+size 4997475480

model-00009-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a72a2bf31b6c03c3e5e2a2328aae8590b83c2681512c0ffeae3ce798babe8b3
-size 4999714144
+oid sha256:33fe8f8ac4cfe9a986fd6952543c5c40429a7f626d88202515bf88359153349f
+size 4997857664

model-00010-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:64e95499bf82f134f87e2c05ae7380c18150c71f4697baf942dab8c1203d0184
-size 4998623456
+oid sha256:0676ba2db25493eb09b04cb26e29b7b64de60b5ea1ecfc2eac7758602ad28b09
+size 5000473880

model-00011-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:46bc13560510498aa0fc9bd83f9815256f4622370531cc76d9a87cf3d40327a0
-size 4998623528
+oid sha256:df7f3210476cf18ff837c6a0d488f134925f41714f79c36468efb175421734c8
+size 4979044216

model-00012-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f5ba0da091b5866788b8a350f0a90eb84c3e24ea24459412cef59ab4294392a
-size 4999714072
+oid sha256:e851692e845c7ba89b897c4f80f29ffe438600ce32e9363b2290aa0e17167ae9
+size 4999329640

model-00013-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ccf5369e611cc507eb538a183727a62643a3cbf829ce0548d5b818a7ce6b32f9
-size 4998623448
+oid sha256:964971f1c221f88a72a4486b959e460ccc56a90dcd98aafc6e6e98336aec703f
+size 5000473880

model-00014-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f7509d5f435abaa2d31c4a6e796135c4aa30bbdd430688cb1110f95e15fe51d
-size 4999714352
+oid sha256:5d0e15cfaf2acf5408c15228b28572bf7ef609f54d07578b2defc4dcd0fe0696
+size 4998054544

model-00015-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9efe25b316fb6b3d699245e3d5a5c5027c8c136a123c3138e4d499e91cd7b376
-size 4998623248
+oid sha256:f6b69923387f0fbf303b742683fc93f907c9c36467120c5dbac84308ffe2c54a
+size 5000473720

model-00016-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9345bded90fbe487712c7aa4500f7da635c850d392c02e0eb5d4146ba3c02964
-size 4998623456
+oid sha256:402be03a708628f4858dbcf72aca8129b758a257a066e3c23d12086dc633767a
+size 4997475480

model-00017-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:459b3ebefe1c381215f757889bdc1b23cb5fa59bd51c690188fe3df3243a437a
-size 4999714224
+oid sha256:a36e86af947290ff8731709b57b77cc7243d1de3950d0a6b57a06333567bbe35
+size 4997857696

model-00018-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84341a7c112b0016989ba68a21b993027f0f222ce06b3a5b59a33af2fd7bbb55
-size 4998623376
+oid sha256:ff72b29a06af55b80df3fec195527f5768be66062a5159a014e1d608ef05c566
+size 5000473840

model-00019-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6aa129e9a82157d02d9ef3a0884e17d81a3c9b120a477d545d112b0b64201dae
-size 4998623456
+oid sha256:b58b3cfdcd64748de5305eff0088a4361fc179c1bba9351d2e5a22417b4db060
+size 4997475536

model-00020-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:06f3481304c2b888f77354970248badb5af6998b21669801992117e26660de16
-size 4999714144
+oid sha256:cd1252267a4b0cb460e249218b7a7f2fb92f6dc23117cc0832e8d64074c53c00
+size 4997857608

model-00021-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cef8fe568b1a8dff29723b48649fa8d9f85e9bd459c34c011d47b7c3ba2481cf
-size 4998623456
+oid sha256:af9ca368691115d938cb090812bd1e4bf99b2be9288d9e56a7ee89b21e2d94de
+size 5000473880

model-00022-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5bacc4e28d075a2b26da4045afc3dfc3ce7c5bf4d43c5a8795147dc4d22d243f
-size 4969204312
+oid sha256:e6692a6a622d3ff5cdf2ab6306d6b902b05738693d36ecc6395c4eb00746fba7
+size 4998054608

model-00023-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:382ee5ce9ef03ece231055f63464a43c13aa8495d3901f52d7b75703af582d8c
-size 4998606584
+oid sha256:bdc587274e59ed106b411be6d8a32c2bda0ed0f688a9995b4e30592182e48175
+size 5000473656

model-00024-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac04d2d937796544cade0381c2ef8869ff6335e50bec0d12aaa18489c7072b44
-size 4998623448
+oid sha256:40204216497cfc2ad0b106281e942a5394cd8479a95b3d875469beaf4402b873
+size 4997475480

model-00025-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09f16a68a303a1aadc7efc9450c9f13e11bb5d2f792aae1cac73e68288f61099
-size 4999714280
+oid sha256:b1f38345ad63ad35ec89aaa0892ca70f419fb33691466929632c80fae45bc3a6
+size 4997857752

model-00026-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ff89e38ff71fd4db6c6e5b081ad1dd0eaa98b3f76913f4961879de43f68ddcc
-size 4998623320
+oid sha256:228366122aa9fa751c301f9d75af2e8efb0f6c59ae5a07baac7dc18ff47da9cc
+size 5000473784

model-00027-of-00027.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b6437f564ed0b86a445e61cc7f07fb3d6be42d3d155944e3abc4f2d87fad2f86
-size 1889136736
+oid sha256:722e40d3b7a0f7755937dd90e5325818c9c5e4019c6605f79435b10ab8bdbbc9
+size 1930035696

quantization_config.json

@@ -4,7 +4,7 @@
   "sym": true,
   "data_type": "int",
   "iters": 0,
-  "autoround_version": "0.6.0",
+  "autoround_version": "0.6.1.dev",
   "quant_method": "auto-round",
   "packing_format": "auto_round:auto_gptq",
   "extra_config": {
@@ -25,8 +25,7 @@
       "group_size": 128
     },
     "model.layers.0.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.1.self_attn.q_proj": {
       "bits": 8,
@@ -45,8 +44,7 @@
       "group_size": 128
     },
     "model.layers.1.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.2.self_attn.q_proj": {
       "bits": 8,
@@ -65,8 +63,7 @@
       "group_size": 128
     },
     "model.layers.2.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.3.self_attn.q_proj": {
       "bits": 8,
@@ -85,8 +82,7 @@
       "group_size": 128
     },
     "model.layers.3.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.4.self_attn.q_proj": {
       "bits": 8,
@@ -105,8 +101,7 @@
       "group_size": 128
     },
     "model.layers.4.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.5.self_attn.q_proj": {
       "bits": 8,
@@ -125,8 +120,7 @@
       "group_size": 128
     },
     "model.layers.5.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.6.self_attn.q_proj": {
       "bits": 8,
@@ -145,8 +139,7 @@
       "group_size": 128
     },
     "model.layers.6.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.7.self_attn.q_proj": {
       "bits": 8,
@@ -165,8 +158,7 @@
       "group_size": 128
     },
     "model.layers.7.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.8.self_attn.q_proj": {
       "bits": 8,
@@ -185,8 +177,7 @@
       "group_size": 128
     },
     "model.layers.8.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.9.self_attn.q_proj": {
       "bits": 8,
@@ -205,8 +196,7 @@
       "group_size": 128
     },
     "model.layers.9.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.10.self_attn.q_proj": {
       "bits": 8,
@@ -225,8 +215,7 @@
       "group_size": 128
     },
     "model.layers.10.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.11.self_attn.q_proj": {
       "bits": 8,
@@ -245,8 +234,7 @@
       "group_size": 128
     },
     "model.layers.11.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.12.self_attn.q_proj": {
       "bits": 8,
@@ -265,8 +253,7 @@
       "group_size": 128
     },
     "model.layers.12.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.13.self_attn.q_proj": {
       "bits": 8,
@@ -285,8 +272,7 @@
       "group_size": 128
     },
     "model.layers.13.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.14.self_attn.q_proj": {
       "bits": 8,
@@ -305,8 +291,7 @@
       "group_size": 128
     },
     "model.layers.14.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.15.self_attn.q_proj": {
       "bits": 8,
@@ -325,8 +310,7 @@
       "group_size": 128
     },
     "model.layers.15.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.16.self_attn.q_proj": {
       "bits": 8,
@@ -345,8 +329,7 @@
       "group_size": 128
     },
     "model.layers.16.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.17.self_attn.q_proj": {
       "bits": 8,
@@ -365,8 +348,7 @@
       "group_size": 128
     },
     "model.layers.17.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.18.self_attn.q_proj": {
       "bits": 8,
@@ -385,8 +367,7 @@
       "group_size": 128
     },
     "model.layers.18.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.19.self_attn.q_proj": {
       "bits": 8,
@@ -405,8 +386,7 @@
       "group_size": 128
     },
     "model.layers.19.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.20.self_attn.q_proj": {
       "bits": 8,
@@ -425,8 +405,7 @@
       "group_size": 128
     },
     "model.layers.20.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.21.self_attn.q_proj": {
       "bits": 8,
@@ -445,8 +424,7 @@
       "group_size": 128
     },
     "model.layers.21.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.22.self_attn.q_proj": {
       "bits": 8,
@@ -465,8 +443,7 @@
       "group_size": 128
     },
     "model.layers.22.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.23.self_attn.q_proj": {
       "bits": 8,
@@ -485,8 +462,7 @@
       "group_size": 128
     },
     "model.layers.23.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.24.self_attn.q_proj": {
       "bits": 8,
@@ -505,8 +481,7 @@
       "group_size": 128
     },
     "model.layers.24.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.25.self_attn.q_proj": {
       "bits": 8,
@@ -525,8 +500,7 @@
       "group_size": 128
     },
     "model.layers.25.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.26.self_attn.q_proj": {
       "bits": 8,
@@ -545,8 +519,7 @@
       "group_size": 128
     },
     "model.layers.26.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.27.self_attn.q_proj": {
       "bits": 8,
@@ -565,8 +538,7 @@
       "group_size": 128
     },
     "model.layers.27.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.28.self_attn.q_proj": {
       "bits": 8,
@@ -585,8 +557,7 @@
       "group_size": 128
     },
     "model.layers.28.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.29.self_attn.q_proj": {
       "bits": 8,
@@ -605,8 +576,7 @@
       "group_size": 128
     },
     "model.layers.29.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.30.self_attn.q_proj": {
       "bits": 8,
@@ -625,8 +595,7 @@
       "group_size": 128
     },
     "model.layers.30.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.31.self_attn.q_proj": {
       "bits": 8,
@@ -645,8 +614,7 @@
       "group_size": 128
     },
     "model.layers.31.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.32.self_attn.q_proj": {
       "bits": 8,
@@ -665,8 +633,7 @@
       "group_size": 128
     },
     "model.layers.32.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.33.self_attn.q_proj": {
       "bits": 8,
@@ -685,8 +652,7 @@
       "group_size": 128
     },
     "model.layers.33.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.34.self_attn.q_proj": {
       "bits": 8,
@@ -705,8 +671,7 @@
       "group_size": 128
     },
     "model.layers.34.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.35.self_attn.q_proj": {
       "bits": 8,
@@ -725,8 +690,7 @@
       "group_size": 128
     },
     "model.layers.35.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.36.self_attn.q_proj": {
       "bits": 8,
@@ -745,8 +709,7 @@
       "group_size": 128
     },
     "model.layers.36.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.37.self_attn.q_proj": {
       "bits": 8,
@@ -765,8 +728,7 @@
       "group_size": 128
     },
     "model.layers.37.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.38.self_attn.q_proj": {
       "bits": 8,
@@ -785,8 +747,7 @@
       "group_size": 128
     },
     "model.layers.38.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.39.self_attn.q_proj": {
       "bits": 8,
@@ -805,8 +766,7 @@
       "group_size": 128
     },
     "model.layers.39.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.40.self_attn.q_proj": {
       "bits": 8,
@@ -825,8 +785,7 @@
       "group_size": 128
     },
     "model.layers.40.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.41.self_attn.q_proj": {
       "bits": 8,
@@ -845,8 +804,7 @@
       "group_size": 128
     },
     "model.layers.41.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.42.self_attn.q_proj": {
       "bits": 8,
@@ -865,8 +823,7 @@
       "group_size": 128
     },
     "model.layers.42.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.43.self_attn.q_proj": {
       "bits": 8,
@@ -885,8 +842,7 @@
       "group_size": 128
     },
     "model.layers.43.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.44.self_attn.q_proj": {
       "bits": 8,
@@ -905,8 +861,7 @@
       "group_size": 128
     },
     "model.layers.44.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.45.self_attn.q_proj": {
       "bits": 8,
@@ -925,8 +880,7 @@
       "group_size": 128
     },
     "model.layers.45.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.46.self_attn.q_proj": {
       "bits": 8,
@@ -945,8 +899,7 @@
       "group_size": 128
     },
     "model.layers.46.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.47.self_attn.q_proj": {
       "bits": 8,
@@ -965,8 +918,7 @@
       "group_size": 128
     },
     "model.layers.47.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.48.self_attn.q_proj": {
       "bits": 8,
@@ -985,8 +937,7 @@
       "group_size": 128
     },
     "model.layers.48.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.49.self_attn.q_proj": {
       "bits": 8,
@@ -1005,8 +956,7 @@
       "group_size": 128
     },
     "model.layers.49.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.50.self_attn.q_proj": {
       "bits": 8,
@@ -1025,8 +975,7 @@
       "group_size": 128
     },
     "model.layers.50.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.51.self_attn.q_proj": {
       "bits": 8,
@@ -1045,8 +994,7 @@
       "group_size": 128
     },
     "model.layers.51.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.52.self_attn.q_proj": {
       "bits": 8,
@@ -1065,8 +1013,7 @@
       "group_size": 128
     },
     "model.layers.52.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.53.self_attn.q_proj": {
       "bits": 8,
@@ -1085,8 +1032,7 @@
       "group_size": 128
     },
     "model.layers.53.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.54.self_attn.q_proj": {
       "bits": 8,
@@ -1105,8 +1051,7 @@
       "group_size": 128
     },
     "model.layers.54.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.55.self_attn.q_proj": {
       "bits": 8,
@@ -1125,8 +1070,7 @@
       "group_size": 128
     },
     "model.layers.55.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.56.self_attn.q_proj": {
       "bits": 8,
@@ -1145,8 +1089,7 @@
       "group_size": 128
     },
     "model.layers.56.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.57.self_attn.q_proj": {
       "bits": 8,
@@ -1165,8 +1108,7 @@
       "group_size": 128
     },
     "model.layers.57.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.58.self_attn.q_proj": {
       "bits": 8,
@@ -1185,8 +1127,7 @@
       "group_size": 128
     },
     "model.layers.58.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.59.self_attn.q_proj": {
       "bits": 8,
@@ -1205,8 +1146,7 @@
       "group_size": 128
     },
     "model.layers.59.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.60.self_attn.q_proj": {
       "bits": 8,
@@ -1225,8 +1165,7 @@
       "group_size": 128
     },
     "model.layers.60.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.61.self_attn.q_proj": {
       "bits": 8,
@@ -1245,8 +1184,7 @@
       "group_size": 128
     },
     "model.layers.61.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.62.self_attn.q_proj": {
       "bits": 8,
@@ -1265,8 +1203,7 @@
       "group_size": 128
     },
     "model.layers.62.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.63.self_attn.q_proj": {
       "bits": 8,
@@ -1285,8 +1222,7 @@
       "group_size": 128
     },
     "model.layers.63.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.64.self_attn.q_proj": {
       "bits": 8,
@@ -1305,8 +1241,7 @@
       "group_size": 128
     },
     "model.layers.64.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.65.self_attn.q_proj": {
       "bits": 8,
@@ -1325,8 +1260,7 @@
       "group_size": 128
     },
     "model.layers.65.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.66.self_attn.q_proj": {
       "bits": 8,
@@ -1345,8 +1279,7 @@
       "group_size": 128
     },
     "model.layers.66.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.67.self_attn.q_proj": {
       "bits": 8,
@@ -1365,8 +1298,7 @@
       "group_size": 128
     },
     "model.layers.67.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.68.self_attn.q_proj": {
       "bits": 8,
@@ -1385,8 +1317,7 @@
       "group_size": 128
     },
     "model.layers.68.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.69.self_attn.q_proj": {
       "bits": 8,
@@ -1405,8 +1336,7 @@
       "group_size": 128
     },
     "model.layers.69.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.70.self_attn.q_proj": {
       "bits": 8,
@@ -1425,8 +1355,7 @@
       "group_size": 128
     },
     "model.layers.70.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.71.self_attn.q_proj": {
       "bits": 8,
@@ -1445,8 +1374,7 @@
       "group_size": 128
     },
     "model.layers.71.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.72.self_attn.q_proj": {
       "bits": 8,
@@ -1465,8 +1393,7 @@
       "group_size": 128
     },
     "model.layers.72.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.73.self_attn.q_proj": {
       "bits": 8,
@@ -1485,8 +1412,7 @@
       "group_size": 128
     },
     "model.layers.73.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.74.self_attn.q_proj": {
       "bits": 8,
@@ -1505,8 +1431,7 @@
       "group_size": 128
     },
     "model.layers.74.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.75.self_attn.q_proj": {
       "bits": 8,
@@ -1525,8 +1450,7 @@
       "group_size": 128
     },
     "model.layers.75.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.76.self_attn.q_proj": {
       "bits": 8,
@@ -1545,8 +1469,7 @@
       "group_size": 128
     },
     "model.layers.76.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.77.self_attn.q_proj": {
       "bits": 8,
@@ -1565,8 +1488,7 @@
       "group_size": 128
     },
     "model.layers.77.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.78.self_attn.q_proj": {
       "bits": 8,
@@ -1585,8 +1507,7 @@
       "group_size": 128
     },
     "model.layers.78.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.79.self_attn.q_proj": {
       "bits": 8,
@@ -1605,8 +1526,7 @@
       "group_size": 128
     },
     "model.layers.79.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.80.self_attn.q_proj": {
       "bits": 8,
@@ -1625,8 +1545,7 @@
       "group_size": 128
     },
     "model.layers.80.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.81.self_attn.q_proj": {
       "bits": 8,
@@ -1645,8 +1564,7 @@
       "group_size": 128
     },
     "model.layers.81.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.82.self_attn.q_proj": {
       "bits": 8,
@@ -1665,8 +1583,7 @@
       "group_size": 128
     },
     "model.layers.82.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.83.self_attn.q_proj": {
       "bits": 8,
@@ -1685,8 +1602,7 @@
       "group_size": 128
     },
     "model.layers.83.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.84.self_attn.q_proj": {
       "bits": 8,
@@ -1705,8 +1621,7 @@
       "group_size": 128
     },
     "model.layers.84.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.85.self_attn.q_proj": {
       "bits": 8,
@@ -1725,8 +1640,7 @@
       "group_size": 128
     },
     "model.layers.85.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.86.self_attn.q_proj": {
       "bits": 8,
@@ -1745,8 +1659,7 @@
       "group_size": 128
     },
     "model.layers.86.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.87.self_attn.q_proj": {
       "bits": 8,
@@ -1765,8 +1678,7 @@
       "group_size": 128
     },
     "model.layers.87.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.88.self_attn.q_proj": {
       "bits": 8,
@@ -1785,8 +1697,7 @@
       "group_size": 128
     },
     "model.layers.88.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.89.self_attn.q_proj": {
       "bits": 8,
@@ -1805,8 +1716,7 @@
       "group_size": 128
     },
     "model.layers.89.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.90.self_attn.q_proj": {
       "bits": 8,
@@ -1825,8 +1735,7 @@
       "group_size": 128
     },
     "model.layers.90.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.91.self_attn.q_proj": {
       "bits": 8,
@@ -1845,8 +1754,7 @@
       "group_size": 128
     },
     "model.layers.91.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.92.self_attn.q_proj": {
       "bits": 8,
@@ -1865,8 +1773,7 @@
       "group_size": 128
     },
     "model.layers.92.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     },
     "model.layers.93.self_attn.q_proj": {
       "bits": 8,
@@ -1885,8 +1792,7 @@
       "group_size": 128
     },
     "model.layers.93.mlp.gate": {
-      "bits": 8,
-      "group_size": 128
+      "bits": 16
     }
   }
 }