Upload folder using huggingface_hub

chat_template.jinja

@@ -1,9 +1,21 @@
-{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>
-' + message['content'] + '
-' }}{% elif message['role'] == 'user' %}{{ '<|user|>
-' + message['content'] + '
-' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>
-'  + message['content'] + eos_token + '
-' }}{% else %}{{ '<|assistant|>
-'  + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>
-' }}{% endif %}{% endfor %}
+
+    {{- bos_token }}
+    {%- for message in messages %}
+        {%- if message['role'] == 'assistant' %}
+            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
+' }}
+            {%- generation %}
+            {{- message['content'] + '<|eot_id|>' }}
+            {%- endgeneration %}
+            {{- '
+' }}
+        {%- else %}
+            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
+' + message['content'] + '<|eot_id|>' + '
+' }}
+        {%- endif %}
+    {%- endfor %}
+    {%- if add_generation_prompt %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>
+' }}
+    {%- endif %}

config.json

@@ -6,16 +6,16 @@
   "attention_dropout": 0.0,
   "bos_token_id": 128000,
   "eos_token_id": 128001,
-  "head_dim": 128,
+  "head_dim": 64,
   "hidden_act": "silu",
-  "hidden_size": 4096,
+  "hidden_size": 2048,
   "initializer_range": 0.02,
-  "intermediate_size": 14336,
+  "intermediate_size": 8192,
   "max_position_embeddings": 131072,
   "mlp_bias": false,
   "model_type": "llama",
   "num_attention_heads": 32,
-  "num_hidden_layers": 32,
+  "num_hidden_layers": 16,
   "num_key_value_heads": 8,
   "pretraining_tp": 1,
   "quantization_config": {
@@ -41,16 +41,16 @@
   },
   "rms_norm_eps": 1e-05,
   "rope_scaling": {
-    "factor": 8.0,
+    "factor": 32.0,
     "high_freq_factor": 4.0,
     "low_freq_factor": 1.0,
     "original_max_position_embeddings": 8192,
     "rope_type": "llama3"
   },
   "rope_theta": 500000.0,
-  "tie_word_embeddings": false,
+  "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.53.1",
-  "use_cache": true,
-  "vocab_size": 128264
+  "use_cache": false,
+  "vocab_size": 128256
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db35c00ca313f38797f303234a579e81dd4d7d4c17fa8321045e975c99ee13ff
-size 3974612224
+oid sha256:88f0bb257c743062340499ea96abff61e5b9ceb8b077aacaee691c1f6be38494
+size 787209792

quant_log.csv

@@ -1,225 +1,113 @@
 layer,module,loss,samples,damp,time
-0,self_attn.k_proj,11.50315285,0.01000,1.200
-0,self_attn.v_proj,0.08434783,0.01000,0.942
-0,self_attn.q_proj,18.04431343,0.01000,0.974
-0,self_attn.o_proj,0.00746359,0.01000,0.954
-0,mlp.up_proj,6.94379950,0.01000,0.970
-0,mlp.gate_proj,8.43739128,0.01000,0.971
-0,mlp.down_proj,0.03982410,0.01000,4.106
-1,self_attn.k_proj,9.79508400,0.01000,0.958
-1,self_attn.v_proj,0.30334821,0.01000,0.964
-1,self_attn.q_proj,17.00190735,0.01000,0.977
-1,self_attn.o_proj,0.02521426,0.01000,0.967
-1,mlp.up_proj,14.35952663,0.01000,0.979
-1,mlp.gate_proj,17.07050514,0.01000,0.983
-1,mlp.down_proj,9.23333359,0.01000,4.086
-2,self_attn.k_proj,30.43294144,0.01000,0.950
-2,self_attn.v_proj,1.15037918,0.01000,0.970
-2,self_attn.q_proj,45.58643723,0.01000,0.964
-2,self_attn.o_proj,0.03051591,0.01000,0.967
-2,mlp.up_proj,18.03928757,0.01000,0.982
-2,mlp.gate_proj,22.90150261,0.01000,0.980
-2,mlp.down_proj,0.10555533,0.01000,4.078
-3,self_attn.k_proj,27.10592270,0.01000,1.535
-3,self_attn.v_proj,2.30466604,0.01000,0.947
-3,self_attn.q_proj,47.55756378,0.01000,0.963
-3,self_attn.o_proj,0.05661678,0.01000,0.964
-3,mlp.up_proj,27.88656998,0.01000,0.980
-3,mlp.gate_proj,40.16275024,0.01000,0.975
-3,mlp.down_proj,0.23143937,0.01000,4.022
-4,self_attn.k_proj,27.37422943,0.01000,0.948
-4,self_attn.v_proj,2.49580097,0.01000,0.970
-4,self_attn.q_proj,47.32018280,0.01000,0.967
-4,self_attn.o_proj,0.11866941,0.01000,0.966
-4,mlp.up_proj,34.95981979,0.01000,0.982
-4,mlp.gate_proj,57.44453812,0.01000,0.976
-4,mlp.down_proj,0.39879456,0.01000,4.011
-5,self_attn.k_proj,44.15615082,0.01000,0.947
-5,self_attn.v_proj,2.45797062,0.01000,0.943
-5,self_attn.q_proj,69.22374725,0.01000,0.957
-5,self_attn.o_proj,0.13216227,0.01000,0.961
-5,mlp.up_proj,44.49318695,0.01000,0.983
-5,mlp.gate_proj,71.99307251,0.01000,0.977
-5,mlp.down_proj,0.57819533,0.01000,4.072
-6,self_attn.k_proj,39.04605484,0.01000,0.945
-6,self_attn.v_proj,3.02949667,0.01000,0.941
-6,self_attn.q_proj,68.57389832,0.01000,0.968
-6,self_attn.o_proj,0.20356959,0.01000,0.961
-6,mlp.up_proj,51.34634399,0.01000,0.985
-6,mlp.gate_proj,83.72372437,0.01000,0.980
-6,mlp.down_proj,0.74748993,0.01000,3.988
-7,self_attn.k_proj,41.18905640,0.01000,0.951
-7,self_attn.v_proj,3.36715841,0.01000,0.944
-7,self_attn.q_proj,67.75445557,0.01000,1.042
-7,self_attn.o_proj,0.25776845,0.01000,0.961
-7,mlp.up_proj,54.56769562,0.01000,0.981
-7,mlp.gate_proj,83.44071960,0.01000,0.979
-7,mlp.down_proj,0.84187621,0.01000,4.054
-8,self_attn.k_proj,51.15142441,0.01000,0.957
-8,self_attn.v_proj,4.34867573,0.01000,0.976
-8,self_attn.q_proj,82.86448669,0.01000,0.967
-8,self_attn.o_proj,0.30111048,0.01000,0.960
-8,mlp.up_proj,57.04904175,0.01000,0.985
-8,mlp.gate_proj,88.49418640,0.01000,0.980
-8,mlp.down_proj,0.88172758,0.01000,4.088
-9,self_attn.k_proj,50.66983795,0.01000,0.950
-9,self_attn.v_proj,6.26596260,0.01000,1.005
-9,self_attn.q_proj,84.79583740,0.01000,0.964
-9,self_attn.o_proj,0.39540437,0.01000,0.961
-9,mlp.up_proj,59.31522369,0.01000,1.601
-9,mlp.gate_proj,92.27600861,0.01000,0.976
-9,mlp.down_proj,0.90846139,0.01000,4.001
-10,self_attn.k_proj,60.17391205,0.01000,0.953
-10,self_attn.v_proj,4.89551830,0.01000,0.994
-10,self_attn.q_proj,98.11512756,0.01000,0.967
-10,self_attn.o_proj,0.34281918,0.01000,0.960
-10,mlp.up_proj,62.74011230,0.01000,0.982
-10,mlp.gate_proj,90.72794342,0.01000,0.980
-10,mlp.down_proj,0.96149510,0.01000,3.990
-11,self_attn.k_proj,55.94424438,0.01000,0.952
-11,self_attn.v_proj,5.04172277,0.01000,0.943
-11,self_attn.q_proj,86.99108887,0.01000,0.971
-11,self_attn.o_proj,0.33490360,0.01000,0.991
-11,mlp.up_proj,65.00627899,0.01000,0.994
-11,mlp.gate_proj,90.79618835,0.01000,0.993
-11,mlp.down_proj,1.04935539,0.01000,4.030
-12,self_attn.k_proj,42.39485168,0.01000,0.950
-12,self_attn.v_proj,5.80718994,0.01000,0.952
-12,self_attn.q_proj,74.00858307,0.01000,0.970
-12,self_attn.o_proj,0.50482625,0.01000,0.958
-12,mlp.up_proj,66.45251465,0.01000,0.981
-12,mlp.gate_proj,87.68470764,0.01000,0.979
-12,mlp.down_proj,1.16635084,0.01000,4.013
-13,self_attn.k_proj,66.97894287,0.01000,0.957
-13,self_attn.v_proj,6.70645237,0.01000,0.947
-13,self_attn.q_proj,102.37200928,0.01000,0.969
-13,self_attn.o_proj,0.47240034,0.01000,0.969
-13,mlp.up_proj,72.47052765,0.01000,0.991
-13,mlp.gate_proj,95.45094299,0.01000,0.989
-13,mlp.down_proj,1.29506612,0.01000,4.084
-14,self_attn.k_proj,65.34709930,0.01000,0.953
-14,self_attn.v_proj,6.54276228,0.01000,0.978
-14,self_attn.q_proj,95.75817871,0.01000,0.963
-14,self_attn.o_proj,0.56570506,0.01000,0.988
-14,mlp.up_proj,77.22135162,0.01000,0.983
-14,mlp.gate_proj,107.80537415,0.01000,0.976
-14,mlp.down_proj,1.62657499,0.01000,4.082
-15,self_attn.k_proj,60.99359131,0.01000,0.948
-15,self_attn.v_proj,7.83463526,0.01000,1.000
-15,self_attn.q_proj,114.51682281,0.01000,1.005
-15,self_attn.o_proj,0.98620689,0.01000,0.966
-15,mlp.up_proj,82.91996765,0.01000,0.985
-15,mlp.gate_proj,122.00700378,0.01000,0.991
-15,mlp.down_proj,1.85101867,0.01000,4.006
-16,self_attn.k_proj,61.05425644,0.01000,0.950
-16,self_attn.v_proj,6.67351246,0.01000,0.953
-16,self_attn.q_proj,100.83403015,0.01000,0.961
-16,self_attn.o_proj,0.69056815,0.01000,0.962
-16,mlp.up_proj,88.04904938,0.01000,0.977
-16,mlp.gate_proj,137.32760620,0.01000,0.976
-16,mlp.down_proj,2.13290906,0.01000,4.007
-17,self_attn.k_proj,66.14538574,0.01000,0.956
-17,self_attn.v_proj,7.93916798,0.01000,0.963
-17,self_attn.q_proj,109.25076294,0.01000,0.973
-17,self_attn.o_proj,0.67304528,0.01000,0.972
-17,mlp.up_proj,92.86845398,0.01000,0.984
-17,mlp.gate_proj,148.43232727,0.01000,0.978
-17,mlp.down_proj,2.42646503,0.01000,4.064
-18,self_attn.k_proj,70.76748657,0.01000,0.952
-18,self_attn.v_proj,7.24831295,0.01000,0.944
-18,self_attn.q_proj,106.46402740,0.01000,0.962
-18,self_attn.o_proj,0.36137423,0.01000,0.962
-18,mlp.up_proj,93.72953796,0.01000,0.983
-18,mlp.gate_proj,151.20054626,0.01000,0.977
-18,mlp.down_proj,2.41029239,0.01000,4.024
-19,self_attn.k_proj,62.90951157,0.01000,0.952
-19,self_attn.v_proj,8.11475182,0.01000,0.975
-19,self_attn.q_proj,106.72672272,0.01000,0.960
-19,self_attn.o_proj,0.45973969,0.01000,0.959
-19,mlp.up_proj,96.43962860,0.01000,0.983
-19,mlp.gate_proj,158.58799744,0.01000,0.977
-19,mlp.down_proj,2.60704017,0.01000,4.036
-20,self_attn.k_proj,66.16004944,0.01000,0.952
-20,self_attn.v_proj,8.60017872,0.01000,0.960
-20,self_attn.q_proj,106.42929077,0.01000,0.975
-20,self_attn.o_proj,0.42740995,0.01000,0.966
-20,mlp.up_proj,104.03691864,0.01000,0.986
-20,mlp.gate_proj,169.39122009,0.01000,2.004
-20,mlp.down_proj,2.89748049,0.01000,4.059
-21,self_attn.k_proj,64.93406677,0.01000,0.948
-21,self_attn.v_proj,9.05480289,0.01000,0.959
-21,self_attn.q_proj,102.93681335,0.01000,0.966
-21,self_attn.o_proj,0.57246387,0.01000,0.967
-21,mlp.up_proj,109.80873108,0.01000,0.985
-21,mlp.gate_proj,180.10443115,0.01000,0.980
-21,mlp.down_proj,3.18497372,0.01000,4.002
-22,self_attn.k_proj,68.19435120,0.01000,0.956
-22,self_attn.v_proj,10.42789459,0.01000,0.948
-22,self_attn.q_proj,102.62083435,0.01000,0.959
-22,self_attn.o_proj,0.82003343,0.01000,0.959
-22,mlp.up_proj,115.58087921,0.01000,0.992
-22,mlp.gate_proj,187.88652039,0.01000,0.987
-22,mlp.down_proj,3.17831731,0.01000,3.990
-23,self_attn.k_proj,65.93269348,0.01000,0.962
-23,self_attn.v_proj,11.55161858,0.01000,0.955
-23,self_attn.q_proj,106.37304688,0.01000,0.966
-23,self_attn.o_proj,0.79565775,0.01000,0.963
-23,mlp.up_proj,122.20275116,0.01000,0.984
-23,mlp.gate_proj,197.79428101,0.01000,0.987
-23,mlp.down_proj,3.40926170,0.01000,4.092
-24,self_attn.k_proj,65.81439209,0.01000,0.960
-24,self_attn.v_proj,14.61132812,0.01000,0.975
-24,self_attn.q_proj,109.37326050,0.01000,0.963
-24,self_attn.o_proj,0.96981573,0.01000,0.967
-24,mlp.up_proj,131.75790405,0.01000,0.978
-24,mlp.gate_proj,213.20829773,0.01000,0.975
-24,mlp.down_proj,3.76813459,0.01000,4.002
-25,self_attn.k_proj,62.32714081,0.01000,0.951
-25,self_attn.v_proj,15.46690750,0.01000,0.965
-25,self_attn.q_proj,109.55595398,0.01000,0.976
-25,self_attn.o_proj,1.38681269,0.01000,0.964
-25,mlp.up_proj,140.07965088,0.01000,1.005
-25,mlp.gate_proj,226.35734558,0.01000,0.983
-25,mlp.down_proj,4.22223282,0.01000,4.021
-26,self_attn.k_proj,69.03659821,0.01000,0.956
-26,self_attn.v_proj,14.82448673,0.01000,0.947
-26,self_attn.q_proj,109.22200775,0.01000,1.008
-26,self_attn.o_proj,1.21192455,0.01000,0.973
-26,mlp.up_proj,151.80865479,0.01000,0.987
-26,mlp.gate_proj,245.63934326,0.01000,0.980
-26,mlp.down_proj,4.96812248,0.01000,3.998
-27,self_attn.k_proj,73.81638336,0.01000,0.952
-27,self_attn.v_proj,21.00315094,0.01000,0.944
-27,self_attn.q_proj,113.85871887,0.01000,0.963
-27,self_attn.o_proj,1.86845124,0.01000,0.964
-27,mlp.up_proj,164.77142334,0.01000,0.989
-27,mlp.gate_proj,265.11877441,0.01000,0.984
-27,mlp.down_proj,5.92129993,0.01000,4.004
-28,self_attn.k_proj,61.08584976,0.01000,0.954
-28,self_attn.v_proj,19.68507385,0.01000,0.968
-28,self_attn.q_proj,108.52185822,0.01000,0.962
-28,self_attn.o_proj,2.66862583,0.01000,0.966
-28,mlp.up_proj,185.00190735,0.01000,0.984
-28,mlp.gate_proj,284.06256104,0.01000,0.983
-28,mlp.down_proj,7.90922022,0.01000,4.070
-29,self_attn.k_proj,65.70452881,0.01000,0.952
-29,self_attn.v_proj,23.99268150,0.01000,0.972
-29,self_attn.q_proj,115.29597473,0.01000,0.973
-29,self_attn.o_proj,2.71498299,0.01000,0.969
-29,mlp.up_proj,195.88598633,0.01000,0.988
-29,mlp.gate_proj,286.53732300,0.01000,0.981
-29,mlp.down_proj,10.72382736,0.01000,4.052
-30,self_attn.k_proj,65.22695923,0.01000,0.952
-30,self_attn.v_proj,32.39274597,0.01000,0.952
-30,self_attn.q_proj,109.58857727,0.01000,0.975
-30,self_attn.o_proj,6.09502459,0.01000,0.970
-30,mlp.up_proj,210.70892334,0.01000,0.988
-30,mlp.gate_proj,316.56787109,0.01000,0.985
-30,mlp.down_proj,18.89646912,0.01000,4.010
-31,self_attn.k_proj,47.18705368,0.01000,0.956
-31,self_attn.v_proj,20.02719879,0.01000,0.950
-31,self_attn.q_proj,90.60032654,0.01000,0.971
-31,self_attn.o_proj,7.11097431,0.01000,0.965
-31,mlp.up_proj,202.48239136,0.01000,0.984
-31,mlp.gate_proj,296.04345703,0.01000,0.980
-31,mlp.down_proj,60.59933090,0.01000,4.000
+0,self_attn.k_proj,8.79510689,0.01000,0.659
+0,self_attn.v_proj,0.21052508,0.01000,0.466
+0,self_attn.q_proj,18.25806046,0.01000,0.473
+0,self_attn.o_proj,0.02193195,0.01000,0.470
+0,mlp.up_proj,15.79567146,0.01000,0.476
+0,mlp.gate_proj,20.15350723,0.01000,0.474
+0,mlp.down_proj,0.15333621,0.01000,2.037
+1,self_attn.k_proj,18.12412453,0.01000,0.472
+1,self_attn.v_proj,0.98016936,0.01000,0.471
+1,self_attn.q_proj,33.08531570,0.01000,0.476
+1,self_attn.o_proj,0.11857565,0.01000,0.472
+1,mlp.up_proj,24.87150764,0.01000,0.482
+1,mlp.gate_proj,34.42536926,0.01000,0.479
+1,mlp.down_proj,20.08703995,0.01000,1.996
+2,self_attn.k_proj,28.00672913,0.01000,0.469
+2,self_attn.v_proj,1.93891215,0.01000,0.468
+2,self_attn.q_proj,55.34558868,0.01000,0.478
+2,self_attn.o_proj,0.09205560,0.01000,0.470
+2,mlp.up_proj,30.34683800,0.01000,0.479
+2,mlp.gate_proj,49.06833267,0.01000,0.501
+2,mlp.down_proj,0.42902470,0.01000,2.014
+3,self_attn.k_proj,21.30574417,0.01000,0.473
+3,self_attn.v_proj,2.84672379,0.01000,0.469
+3,self_attn.q_proj,46.99731827,0.01000,0.474
+3,self_attn.o_proj,0.16826847,0.01000,0.472
+3,mlp.up_proj,38.18422699,0.01000,0.479
+3,mlp.gate_proj,76.93240356,0.01000,0.477
+3,mlp.down_proj,0.58045858,0.01000,1.992
+4,self_attn.k_proj,24.07534409,0.01000,0.467
+4,self_attn.v_proj,2.88128042,0.01000,0.468
+4,self_attn.q_proj,50.19567108,0.01000,0.471
+4,self_attn.o_proj,0.22788243,0.01000,0.471
+4,mlp.up_proj,40.49008560,0.01000,0.480
+4,mlp.gate_proj,88.19657898,0.01000,0.475
+4,mlp.down_proj,0.64514506,0.01000,1.998
+5,self_attn.k_proj,35.62877274,0.01000,0.473
+5,self_attn.v_proj,2.56014800,0.01000,0.469
+5,self_attn.q_proj,64.17327118,0.01000,0.478
+5,self_attn.o_proj,0.23322049,0.01000,0.507
+5,mlp.up_proj,42.68757629,0.01000,0.481
+5,mlp.gate_proj,79.94951630,0.01000,0.480
+5,mlp.down_proj,0.71073961,0.01000,2.018
+6,self_attn.k_proj,32.50390625,0.01000,0.467
+6,self_attn.v_proj,3.48658395,0.01000,0.467
+6,self_attn.q_proj,50.43141556,0.01000,0.477
+6,self_attn.o_proj,0.32279876,0.01000,0.473
+6,mlp.up_proj,43.27687836,0.01000,0.481
+6,mlp.gate_proj,79.23511505,0.01000,0.478
+6,mlp.down_proj,0.72877872,0.01000,2.031
+7,self_attn.k_proj,33.91743851,0.01000,0.475
+7,self_attn.v_proj,4.19138622,0.01000,0.469
+7,self_attn.q_proj,63.09676743,0.01000,0.482
+7,self_attn.o_proj,0.31201273,0.01000,0.481
+7,mlp.up_proj,48.48965454,0.01000,0.484
+7,mlp.gate_proj,79.57895660,0.01000,0.493
+7,mlp.down_proj,0.84213424,0.01000,1.988
+8,self_attn.k_proj,42.15065765,0.01000,0.471
+8,self_attn.v_proj,4.18408966,0.01000,0.469
+8,self_attn.q_proj,69.34562683,0.01000,0.479
+8,self_attn.o_proj,0.36488774,0.01000,0.819
+8,mlp.up_proj,56.41041946,0.01000,0.480
+8,mlp.gate_proj,89.70135498,0.01000,0.482
+8,mlp.down_proj,1.05656636,0.01000,1.997
+9,self_attn.k_proj,35.16567993,0.01000,0.469
+9,self_attn.v_proj,5.07817173,0.01000,0.469
+9,self_attn.q_proj,86.08435059,0.01000,0.470
+9,self_attn.o_proj,0.62493443,0.01000,0.472
+9,mlp.up_proj,64.02466583,0.01000,0.482
+9,mlp.gate_proj,104.88742828,0.01000,0.485
+9,mlp.down_proj,1.33367336,0.01000,2.080
+10,self_attn.k_proj,43.94557190,0.01000,0.473
+10,self_attn.v_proj,5.46747303,0.01000,0.471
+10,self_attn.q_proj,92.83677673,0.01000,0.472
+10,self_attn.o_proj,0.59127998,0.01000,0.470
+10,mlp.up_proj,79.12550354,0.01000,0.480
+10,mlp.gate_proj,125.22569275,0.01000,0.480
+10,mlp.down_proj,1.87667823,0.01000,2.018
+11,self_attn.k_proj,51.54523468,0.01000,0.472
+11,self_attn.v_proj,5.58656693,0.01000,0.466
+11,self_attn.q_proj,90.90484619,0.01000,0.474
+11,self_attn.o_proj,0.48921397,0.01000,0.472
+11,mlp.up_proj,96.23802185,0.01000,0.478
+11,mlp.gate_proj,149.39210510,0.01000,0.478
+11,mlp.down_proj,2.70367146,0.01000,2.027
+12,self_attn.k_proj,49.63491821,0.01000,0.471
+12,self_attn.v_proj,5.90558290,0.01000,0.466
+12,self_attn.q_proj,93.67311096,0.01000,0.479
+12,self_attn.o_proj,0.51877552,0.01000,0.469
+12,mlp.up_proj,112.33209229,0.01000,0.478
+12,mlp.gate_proj,165.25785828,0.01000,0.480
+12,mlp.down_proj,4.04239559,0.01000,2.022
+13,self_attn.k_proj,52.68299866,0.01000,0.470
+13,self_attn.v_proj,10.03530312,0.01000,0.470
+13,self_attn.q_proj,107.74049377,0.01000,0.477
+13,self_attn.o_proj,0.80708027,0.01000,0.469
+13,mlp.up_proj,133.44732666,0.01000,0.480
+13,mlp.gate_proj,179.16419983,0.01000,0.479
+13,mlp.down_proj,5.91810417,0.01000,2.132
+14,self_attn.k_proj,49.46713257,0.01000,0.467
+14,self_attn.v_proj,18.70763016,0.01000,0.465
+14,self_attn.q_proj,103.55518341,0.01000,0.468
+14,self_attn.o_proj,1.71897697,0.01000,0.470
+14,mlp.up_proj,148.42242432,0.01000,0.479
+14,mlp.gate_proj,214.55715942,0.01000,0.480
+14,mlp.down_proj,7.16841698,0.01000,2.001
+15,self_attn.k_proj,46.09174347,0.01000,0.473
+15,self_attn.v_proj,20.09922600,0.01000,0.469
+15,self_attn.q_proj,97.27299500,0.01000,0.473
+15,self_attn.o_proj,4.08784580,0.01000,0.472
+15,mlp.up_proj,185.49623108,0.01000,0.479
+15,mlp.gate_proj,249.19700623,0.01000,0.481
+15,mlp.down_proj,15.86927128,0.01000,2.031

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9400df98529060210393c40f08cb127f7c0df584338b3fbfdba8cf82a33c1ade
-size 17210102
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

tokenizer_config.json

@@ -2047,14 +2047,6 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "128256": {
-      "content": "<pad>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
   "bos_token": "<|begin_of_text|>",