Upload folder using huggingface_hub

chat_template.jinja

@@ -1,21 +1,9 @@
-
-    {{- bos_token }}
-    {%- for message in messages %}
-        {%- if message['role'] == 'assistant' %}
-            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
-' }}
-            {%- generation %}
-            {{- message['content'] + '<|eot_id|>' }}
-            {%- endgeneration %}
-            {{- '
-' }}
-        {%- else %}
-            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
-' + message['content'] + '<|eot_id|>' + '
-' }}
-        {%- endif %}
-    {%- endfor %}
-    {%- if add_generation_prompt %}
-        {{- '<|start_header_id|>assistant<|end_header_id|>
-' }}
-    {%- endif %}
+{% for message in messages %}{% if message['role'] == 'system' %}{{ '<|system|>
+' + message['content'] + '
+' }}{% elif message['role'] == 'user' %}{{ '<|user|>
+' + message['content'] + '
+' }}{% elif message['role'] == 'assistant' %}{% if not loop.last %}{{ '<|assistant|>
+'  + message['content'] + eos_token + '
+' }}{% else %}{{ '<|assistant|>
+'  + message['content'] + eos_token }}{% endif %}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|assistant|>
+' }}{% endif %}{% endfor %}

config.json

@@ -6,16 +6,16 @@
   "attention_dropout": 0.0,
   "bos_token_id": 128000,
   "eos_token_id": 128001,
-  "head_dim": 64,
+  "head_dim": 128,
   "hidden_act": "silu",
-  "hidden_size": 2048,
+  "hidden_size": 4096,
   "initializer_range": 0.02,
-  "intermediate_size": 8192,
+  "intermediate_size": 14336,
   "max_position_embeddings": 131072,
   "mlp_bias": false,
   "model_type": "llama",
   "num_attention_heads": 32,
-  "num_hidden_layers": 16,
+  "num_hidden_layers": 32,
   "num_key_value_heads": 8,
   "pretraining_tp": 1,
   "quantization_config": {
@@ -41,16 +41,16 @@
   },
   "rms_norm_eps": 1e-05,
   "rope_scaling": {
-    "factor": 32.0,
+    "factor": 8.0,
     "high_freq_factor": 4.0,
     "low_freq_factor": 1.0,
     "original_max_position_embeddings": 8192,
     "rope_type": "llama3"
   },
   "rope_theta": 500000.0,
-  "tie_word_embeddings": true,
+  "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.53.1",
-  "use_cache": false,
-  "vocab_size": 128256
+  "use_cache": true,
+  "vocab_size": 128264
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:88f0bb257c743062340499ea96abff61e5b9ceb8b077aacaee691c1f6be38494
-size 787209792
+oid sha256:db35c00ca313f38797f303234a579e81dd4d7d4c17fa8321045e975c99ee13ff
+size 3974612224

quant_log.csv

@@ -1,113 +1,225 @@
 layer,module,loss,samples,damp,time
-0,self_attn.k_proj,8.79510689,0.01000,0.659
-0,self_attn.v_proj,0.21052508,0.01000,0.466
-0,self_attn.q_proj,18.25806046,0.01000,0.473
-0,self_attn.o_proj,0.02193195,0.01000,0.470
-0,mlp.up_proj,15.79567146,0.01000,0.476
-0,mlp.gate_proj,20.15350723,0.01000,0.474
-0,mlp.down_proj,0.15333621,0.01000,2.037
-1,self_attn.k_proj,18.12412453,0.01000,0.472
-1,self_attn.v_proj,0.98016936,0.01000,0.471
-1,self_attn.q_proj,33.08531570,0.01000,0.476
-1,self_attn.o_proj,0.11857565,0.01000,0.472
-1,mlp.up_proj,24.87150764,0.01000,0.482
-1,mlp.gate_proj,34.42536926,0.01000,0.479
-1,mlp.down_proj,20.08703995,0.01000,1.996
-2,self_attn.k_proj,28.00672913,0.01000,0.469
-2,self_attn.v_proj,1.93891215,0.01000,0.468
-2,self_attn.q_proj,55.34558868,0.01000,0.478
-2,self_attn.o_proj,0.09205560,0.01000,0.470
-2,mlp.up_proj,30.34683800,0.01000,0.479
-2,mlp.gate_proj,49.06833267,0.01000,0.501
-2,mlp.down_proj,0.42902470,0.01000,2.014
-3,self_attn.k_proj,21.30574417,0.01000,0.473
-3,self_attn.v_proj,2.84672379,0.01000,0.469
-3,self_attn.q_proj,46.99731827,0.01000,0.474
-3,self_attn.o_proj,0.16826847,0.01000,0.472
-3,mlp.up_proj,38.18422699,0.01000,0.479
-3,mlp.gate_proj,76.93240356,0.01000,0.477
-3,mlp.down_proj,0.58045858,0.01000,1.992
-4,self_attn.k_proj,24.07534409,0.01000,0.467
-4,self_attn.v_proj,2.88128042,0.01000,0.468
-4,self_attn.q_proj,50.19567108,0.01000,0.471
-4,self_attn.o_proj,0.22788243,0.01000,0.471
-4,mlp.up_proj,40.49008560,0.01000,0.480
-4,mlp.gate_proj,88.19657898,0.01000,0.475
-4,mlp.down_proj,0.64514506,0.01000,1.998
-5,self_attn.k_proj,35.62877274,0.01000,0.473
-5,self_attn.v_proj,2.56014800,0.01000,0.469
-5,self_attn.q_proj,64.17327118,0.01000,0.478
-5,self_attn.o_proj,0.23322049,0.01000,0.507
-5,mlp.up_proj,42.68757629,0.01000,0.481
-5,mlp.gate_proj,79.94951630,0.01000,0.480
-5,mlp.down_proj,0.71073961,0.01000,2.018
-6,self_attn.k_proj,32.50390625,0.01000,0.467
-6,self_attn.v_proj,3.48658395,0.01000,0.467
-6,self_attn.q_proj,50.43141556,0.01000,0.477
-6,self_attn.o_proj,0.32279876,0.01000,0.473
-6,mlp.up_proj,43.27687836,0.01000,0.481
-6,mlp.gate_proj,79.23511505,0.01000,0.478
-6,mlp.down_proj,0.72877872,0.01000,2.031
-7,self_attn.k_proj,33.91743851,0.01000,0.475
-7,self_attn.v_proj,4.19138622,0.01000,0.469
-7,self_attn.q_proj,63.09676743,0.01000,0.482
-7,self_attn.o_proj,0.31201273,0.01000,0.481
-7,mlp.up_proj,48.48965454,0.01000,0.484
-7,mlp.gate_proj,79.57895660,0.01000,0.493
-7,mlp.down_proj,0.84213424,0.01000,1.988
-8,self_attn.k_proj,42.15065765,0.01000,0.471
-8,self_attn.v_proj,4.18408966,0.01000,0.469
-8,self_attn.q_proj,69.34562683,0.01000,0.479
-8,self_attn.o_proj,0.36488774,0.01000,0.819
-8,mlp.up_proj,56.41041946,0.01000,0.480
-8,mlp.gate_proj,89.70135498,0.01000,0.482
-8,mlp.down_proj,1.05656636,0.01000,1.997
-9,self_attn.k_proj,35.16567993,0.01000,0.469
-9,self_attn.v_proj,5.07817173,0.01000,0.469
-9,self_attn.q_proj,86.08435059,0.01000,0.470
-9,self_attn.o_proj,0.62493443,0.01000,0.472
-9,mlp.up_proj,64.02466583,0.01000,0.482
-9,mlp.gate_proj,104.88742828,0.01000,0.485
-9,mlp.down_proj,1.33367336,0.01000,2.080
-10,self_attn.k_proj,43.94557190,0.01000,0.473
-10,self_attn.v_proj,5.46747303,0.01000,0.471
-10,self_attn.q_proj,92.83677673,0.01000,0.472
-10,self_attn.o_proj,0.59127998,0.01000,0.470
-10,mlp.up_proj,79.12550354,0.01000,0.480
-10,mlp.gate_proj,125.22569275,0.01000,0.480
-10,mlp.down_proj,1.87667823,0.01000,2.018
-11,self_attn.k_proj,51.54523468,0.01000,0.472
-11,self_attn.v_proj,5.58656693,0.01000,0.466
-11,self_attn.q_proj,90.90484619,0.01000,0.474
-11,self_attn.o_proj,0.48921397,0.01000,0.472
-11,mlp.up_proj,96.23802185,0.01000,0.478
-11,mlp.gate_proj,149.39210510,0.01000,0.478
-11,mlp.down_proj,2.70367146,0.01000,2.027
-12,self_attn.k_proj,49.63491821,0.01000,0.471
-12,self_attn.v_proj,5.90558290,0.01000,0.466
-12,self_attn.q_proj,93.67311096,0.01000,0.479
-12,self_attn.o_proj,0.51877552,0.01000,0.469
-12,mlp.up_proj,112.33209229,0.01000,0.478
-12,mlp.gate_proj,165.25785828,0.01000,0.480
-12,mlp.down_proj,4.04239559,0.01000,2.022
-13,self_attn.k_proj,52.68299866,0.01000,0.470
-13,self_attn.v_proj,10.03530312,0.01000,0.470
-13,self_attn.q_proj,107.74049377,0.01000,0.477
-13,self_attn.o_proj,0.80708027,0.01000,0.469
-13,mlp.up_proj,133.44732666,0.01000,0.480
-13,mlp.gate_proj,179.16419983,0.01000,0.479
-13,mlp.down_proj,5.91810417,0.01000,2.132
-14,self_attn.k_proj,49.46713257,0.01000,0.467
-14,self_attn.v_proj,18.70763016,0.01000,0.465
-14,self_attn.q_proj,103.55518341,0.01000,0.468
-14,self_attn.o_proj,1.71897697,0.01000,0.470
-14,mlp.up_proj,148.42242432,0.01000,0.479
-14,mlp.gate_proj,214.55715942,0.01000,0.480
-14,mlp.down_proj,7.16841698,0.01000,2.001
-15,self_attn.k_proj,46.09174347,0.01000,0.473
-15,self_attn.v_proj,20.09922600,0.01000,0.469
-15,self_attn.q_proj,97.27299500,0.01000,0.473
-15,self_attn.o_proj,4.08784580,0.01000,0.472
-15,mlp.up_proj,185.49623108,0.01000,0.479
-15,mlp.gate_proj,249.19700623,0.01000,0.481
-15,mlp.down_proj,15.86927128,0.01000,2.031
+0,self_attn.k_proj,11.50315285,0.01000,1.200
+0,self_attn.v_proj,0.08434783,0.01000,0.942
+0,self_attn.q_proj,18.04431343,0.01000,0.974
+0,self_attn.o_proj,0.00746359,0.01000,0.954
+0,mlp.up_proj,6.94379950,0.01000,0.970
+0,mlp.gate_proj,8.43739128,0.01000,0.971
+0,mlp.down_proj,0.03982410,0.01000,4.106
+1,self_attn.k_proj,9.79508400,0.01000,0.958
+1,self_attn.v_proj,0.30334821,0.01000,0.964
+1,self_attn.q_proj,17.00190735,0.01000,0.977
+1,self_attn.o_proj,0.02521426,0.01000,0.967
+1,mlp.up_proj,14.35952663,0.01000,0.979
+1,mlp.gate_proj,17.07050514,0.01000,0.983
+1,mlp.down_proj,9.23333359,0.01000,4.086
+2,self_attn.k_proj,30.43294144,0.01000,0.950
+2,self_attn.v_proj,1.15037918,0.01000,0.970
+2,self_attn.q_proj,45.58643723,0.01000,0.964
+2,self_attn.o_proj,0.03051591,0.01000,0.967
+2,mlp.up_proj,18.03928757,0.01000,0.982
+2,mlp.gate_proj,22.90150261,0.01000,0.980
+2,mlp.down_proj,0.10555533,0.01000,4.078
+3,self_attn.k_proj,27.10592270,0.01000,1.535
+3,self_attn.v_proj,2.30466604,0.01000,0.947
+3,self_attn.q_proj,47.55756378,0.01000,0.963
+3,self_attn.o_proj,0.05661678,0.01000,0.964
+3,mlp.up_proj,27.88656998,0.01000,0.980
+3,mlp.gate_proj,40.16275024,0.01000,0.975
+3,mlp.down_proj,0.23143937,0.01000,4.022
+4,self_attn.k_proj,27.37422943,0.01000,0.948
+4,self_attn.v_proj,2.49580097,0.01000,0.970
+4,self_attn.q_proj,47.32018280,0.01000,0.967
+4,self_attn.o_proj,0.11866941,0.01000,0.966
+4,mlp.up_proj,34.95981979,0.01000,0.982
+4,mlp.gate_proj,57.44453812,0.01000,0.976
+4,mlp.down_proj,0.39879456,0.01000,4.011
+5,self_attn.k_proj,44.15615082,0.01000,0.947
+5,self_attn.v_proj,2.45797062,0.01000,0.943
+5,self_attn.q_proj,69.22374725,0.01000,0.957
+5,self_attn.o_proj,0.13216227,0.01000,0.961
+5,mlp.up_proj,44.49318695,0.01000,0.983
+5,mlp.gate_proj,71.99307251,0.01000,0.977
+5,mlp.down_proj,0.57819533,0.01000,4.072
+6,self_attn.k_proj,39.04605484,0.01000,0.945
+6,self_attn.v_proj,3.02949667,0.01000,0.941
+6,self_attn.q_proj,68.57389832,0.01000,0.968
+6,self_attn.o_proj,0.20356959,0.01000,0.961
+6,mlp.up_proj,51.34634399,0.01000,0.985
+6,mlp.gate_proj,83.72372437,0.01000,0.980
+6,mlp.down_proj,0.74748993,0.01000,3.988
+7,self_attn.k_proj,41.18905640,0.01000,0.951
+7,self_attn.v_proj,3.36715841,0.01000,0.944
+7,self_attn.q_proj,67.75445557,0.01000,1.042
+7,self_attn.o_proj,0.25776845,0.01000,0.961
+7,mlp.up_proj,54.56769562,0.01000,0.981
+7,mlp.gate_proj,83.44071960,0.01000,0.979
+7,mlp.down_proj,0.84187621,0.01000,4.054
+8,self_attn.k_proj,51.15142441,0.01000,0.957
+8,self_attn.v_proj,4.34867573,0.01000,0.976
+8,self_attn.q_proj,82.86448669,0.01000,0.967
+8,self_attn.o_proj,0.30111048,0.01000,0.960
+8,mlp.up_proj,57.04904175,0.01000,0.985
+8,mlp.gate_proj,88.49418640,0.01000,0.980
+8,mlp.down_proj,0.88172758,0.01000,4.088
+9,self_attn.k_proj,50.66983795,0.01000,0.950
+9,self_attn.v_proj,6.26596260,0.01000,1.005
+9,self_attn.q_proj,84.79583740,0.01000,0.964
+9,self_attn.o_proj,0.39540437,0.01000,0.961
+9,mlp.up_proj,59.31522369,0.01000,1.601
+9,mlp.gate_proj,92.27600861,0.01000,0.976
+9,mlp.down_proj,0.90846139,0.01000,4.001
+10,self_attn.k_proj,60.17391205,0.01000,0.953
+10,self_attn.v_proj,4.89551830,0.01000,0.994
+10,self_attn.q_proj,98.11512756,0.01000,0.967
+10,self_attn.o_proj,0.34281918,0.01000,0.960
+10,mlp.up_proj,62.74011230,0.01000,0.982
+10,mlp.gate_proj,90.72794342,0.01000,0.980
+10,mlp.down_proj,0.96149510,0.01000,3.990
+11,self_attn.k_proj,55.94424438,0.01000,0.952
+11,self_attn.v_proj,5.04172277,0.01000,0.943
+11,self_attn.q_proj,86.99108887,0.01000,0.971
+11,self_attn.o_proj,0.33490360,0.01000,0.991
+11,mlp.up_proj,65.00627899,0.01000,0.994
+11,mlp.gate_proj,90.79618835,0.01000,0.993
+11,mlp.down_proj,1.04935539,0.01000,4.030
+12,self_attn.k_proj,42.39485168,0.01000,0.950
+12,self_attn.v_proj,5.80718994,0.01000,0.952
+12,self_attn.q_proj,74.00858307,0.01000,0.970
+12,self_attn.o_proj,0.50482625,0.01000,0.958
+12,mlp.up_proj,66.45251465,0.01000,0.981
+12,mlp.gate_proj,87.68470764,0.01000,0.979
+12,mlp.down_proj,1.16635084,0.01000,4.013
+13,self_attn.k_proj,66.97894287,0.01000,0.957
+13,self_attn.v_proj,6.70645237,0.01000,0.947
+13,self_attn.q_proj,102.37200928,0.01000,0.969
+13,self_attn.o_proj,0.47240034,0.01000,0.969
+13,mlp.up_proj,72.47052765,0.01000,0.991
+13,mlp.gate_proj,95.45094299,0.01000,0.989
+13,mlp.down_proj,1.29506612,0.01000,4.084
+14,self_attn.k_proj,65.34709930,0.01000,0.953
+14,self_attn.v_proj,6.54276228,0.01000,0.978
+14,self_attn.q_proj,95.75817871,0.01000,0.963
+14,self_attn.o_proj,0.56570506,0.01000,0.988
+14,mlp.up_proj,77.22135162,0.01000,0.983
+14,mlp.gate_proj,107.80537415,0.01000,0.976
+14,mlp.down_proj,1.62657499,0.01000,4.082
+15,self_attn.k_proj,60.99359131,0.01000,0.948
+15,self_attn.v_proj,7.83463526,0.01000,1.000
+15,self_attn.q_proj,114.51682281,0.01000,1.005
+15,self_attn.o_proj,0.98620689,0.01000,0.966
+15,mlp.up_proj,82.91996765,0.01000,0.985
+15,mlp.gate_proj,122.00700378,0.01000,0.991
+15,mlp.down_proj,1.85101867,0.01000,4.006
+16,self_attn.k_proj,61.05425644,0.01000,0.950
+16,self_attn.v_proj,6.67351246,0.01000,0.953
+16,self_attn.q_proj,100.83403015,0.01000,0.961
+16,self_attn.o_proj,0.69056815,0.01000,0.962
+16,mlp.up_proj,88.04904938,0.01000,0.977
+16,mlp.gate_proj,137.32760620,0.01000,0.976
+16,mlp.down_proj,2.13290906,0.01000,4.007
+17,self_attn.k_proj,66.14538574,0.01000,0.956
+17,self_attn.v_proj,7.93916798,0.01000,0.963
+17,self_attn.q_proj,109.25076294,0.01000,0.973
+17,self_attn.o_proj,0.67304528,0.01000,0.972
+17,mlp.up_proj,92.86845398,0.01000,0.984
+17,mlp.gate_proj,148.43232727,0.01000,0.978
+17,mlp.down_proj,2.42646503,0.01000,4.064
+18,self_attn.k_proj,70.76748657,0.01000,0.952
+18,self_attn.v_proj,7.24831295,0.01000,0.944
+18,self_attn.q_proj,106.46402740,0.01000,0.962
+18,self_attn.o_proj,0.36137423,0.01000,0.962
+18,mlp.up_proj,93.72953796,0.01000,0.983
+18,mlp.gate_proj,151.20054626,0.01000,0.977
+18,mlp.down_proj,2.41029239,0.01000,4.024
+19,self_attn.k_proj,62.90951157,0.01000,0.952
+19,self_attn.v_proj,8.11475182,0.01000,0.975
+19,self_attn.q_proj,106.72672272,0.01000,0.960
+19,self_attn.o_proj,0.45973969,0.01000,0.959
+19,mlp.up_proj,96.43962860,0.01000,0.983
+19,mlp.gate_proj,158.58799744,0.01000,0.977
+19,mlp.down_proj,2.60704017,0.01000,4.036
+20,self_attn.k_proj,66.16004944,0.01000,0.952
+20,self_attn.v_proj,8.60017872,0.01000,0.960
+20,self_attn.q_proj,106.42929077,0.01000,0.975
+20,self_attn.o_proj,0.42740995,0.01000,0.966
+20,mlp.up_proj,104.03691864,0.01000,0.986
+20,mlp.gate_proj,169.39122009,0.01000,2.004
+20,mlp.down_proj,2.89748049,0.01000,4.059
+21,self_attn.k_proj,64.93406677,0.01000,0.948
+21,self_attn.v_proj,9.05480289,0.01000,0.959
+21,self_attn.q_proj,102.93681335,0.01000,0.966
+21,self_attn.o_proj,0.57246387,0.01000,0.967
+21,mlp.up_proj,109.80873108,0.01000,0.985
+21,mlp.gate_proj,180.10443115,0.01000,0.980
+21,mlp.down_proj,3.18497372,0.01000,4.002
+22,self_attn.k_proj,68.19435120,0.01000,0.956
+22,self_attn.v_proj,10.42789459,0.01000,0.948
+22,self_attn.q_proj,102.62083435,0.01000,0.959
+22,self_attn.o_proj,0.82003343,0.01000,0.959
+22,mlp.up_proj,115.58087921,0.01000,0.992
+22,mlp.gate_proj,187.88652039,0.01000,0.987
+22,mlp.down_proj,3.17831731,0.01000,3.990
+23,self_attn.k_proj,65.93269348,0.01000,0.962
+23,self_attn.v_proj,11.55161858,0.01000,0.955
+23,self_attn.q_proj,106.37304688,0.01000,0.966
+23,self_attn.o_proj,0.79565775,0.01000,0.963
+23,mlp.up_proj,122.20275116,0.01000,0.984
+23,mlp.gate_proj,197.79428101,0.01000,0.987
+23,mlp.down_proj,3.40926170,0.01000,4.092
+24,self_attn.k_proj,65.81439209,0.01000,0.960
+24,self_attn.v_proj,14.61132812,0.01000,0.975
+24,self_attn.q_proj,109.37326050,0.01000,0.963
+24,self_attn.o_proj,0.96981573,0.01000,0.967
+24,mlp.up_proj,131.75790405,0.01000,0.978
+24,mlp.gate_proj,213.20829773,0.01000,0.975
+24,mlp.down_proj,3.76813459,0.01000,4.002
+25,self_attn.k_proj,62.32714081,0.01000,0.951
+25,self_attn.v_proj,15.46690750,0.01000,0.965
+25,self_attn.q_proj,109.55595398,0.01000,0.976
+25,self_attn.o_proj,1.38681269,0.01000,0.964
+25,mlp.up_proj,140.07965088,0.01000,1.005
+25,mlp.gate_proj,226.35734558,0.01000,0.983
+25,mlp.down_proj,4.22223282,0.01000,4.021
+26,self_attn.k_proj,69.03659821,0.01000,0.956
+26,self_attn.v_proj,14.82448673,0.01000,0.947
+26,self_attn.q_proj,109.22200775,0.01000,1.008
+26,self_attn.o_proj,1.21192455,0.01000,0.973
+26,mlp.up_proj,151.80865479,0.01000,0.987
+26,mlp.gate_proj,245.63934326,0.01000,0.980
+26,mlp.down_proj,4.96812248,0.01000,3.998
+27,self_attn.k_proj,73.81638336,0.01000,0.952
+27,self_attn.v_proj,21.00315094,0.01000,0.944
+27,self_attn.q_proj,113.85871887,0.01000,0.963
+27,self_attn.o_proj,1.86845124,0.01000,0.964
+27,mlp.up_proj,164.77142334,0.01000,0.989
+27,mlp.gate_proj,265.11877441,0.01000,0.984
+27,mlp.down_proj,5.92129993,0.01000,4.004
+28,self_attn.k_proj,61.08584976,0.01000,0.954
+28,self_attn.v_proj,19.68507385,0.01000,0.968
+28,self_attn.q_proj,108.52185822,0.01000,0.962
+28,self_attn.o_proj,2.66862583,0.01000,0.966
+28,mlp.up_proj,185.00190735,0.01000,0.984
+28,mlp.gate_proj,284.06256104,0.01000,0.983
+28,mlp.down_proj,7.90922022,0.01000,4.070
+29,self_attn.k_proj,65.70452881,0.01000,0.952
+29,self_attn.v_proj,23.99268150,0.01000,0.972
+29,self_attn.q_proj,115.29597473,0.01000,0.973
+29,self_attn.o_proj,2.71498299,0.01000,0.969
+29,mlp.up_proj,195.88598633,0.01000,0.988
+29,mlp.gate_proj,286.53732300,0.01000,0.981
+29,mlp.down_proj,10.72382736,0.01000,4.052
+30,self_attn.k_proj,65.22695923,0.01000,0.952
+30,self_attn.v_proj,32.39274597,0.01000,0.952
+30,self_attn.q_proj,109.58857727,0.01000,0.975
+30,self_attn.o_proj,6.09502459,0.01000,0.970
+30,mlp.up_proj,210.70892334,0.01000,0.988
+30,mlp.gate_proj,316.56787109,0.01000,0.985
+30,mlp.down_proj,18.89646912,0.01000,4.010
+31,self_attn.k_proj,47.18705368,0.01000,0.956
+31,self_attn.v_proj,20.02719879,0.01000,0.950
+31,self_attn.q_proj,90.60032654,0.01000,0.971
+31,self_attn.o_proj,7.11097431,0.01000,0.965
+31,mlp.up_proj,202.48239136,0.01000,0.984
+31,mlp.gate_proj,296.04345703,0.01000,0.980
+31,mlp.down_proj,60.59933090,0.01000,4.000

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
-size 17209920
+oid sha256:9400df98529060210393c40f08cb127f7c0df584338b3fbfdba8cf82a33c1ade
+size 17210102

tokenizer_config.json

@@ -2047,6 +2047,14 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "128256": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "bos_token": "<|begin_of_text|>",